context-doctor 0.31.0 → 0.32.0

package/dist/optimize.js

@@ -105,8 +105,13 @@ function stableCutoff(messageCount, keepRecent, step = TRIM_BOUNDARY_STEP) {
         return raw;
     return Math.floor(raw / step) * step;
 }
+/**
+ * Exact content, ends trimmed. Whitespace inside is meaningful: two versions of
+ * a Python file that differ only in indentation are different programs, and
+ * calling the later one "identical" would delete the fix.
+ */
 function hash(text) {
-    return createHash("sha1").update(text.replace(/\s+/g, " ").trim()).digest("hex");
+    return createHash("sha1").update(text.trim()).digest("hex");
 }
 /** Extract all text from a message content value (string or block array). */
 function textOf(content) {
@@ -129,36 +134,74 @@ function textOf(content) {
         .join("\n");
 }
 /**
- * Replace the text of a message content value, preserving block structure.
- * tool_result blocks must keep their type and tool_use_id — the Anthropic API
- * rejects conversations where a tool_use has no matching tool_result — so the
- * replacement text goes INSIDE the first tool_result/text block rather than
- * replacing the block itself. Later text blocks are dropped; other block types
- * (tool_use, image) pass through untouched.
+ * Rewrite each piece of text in a message where it sits: string content, each
+ * text block, and the text of each tool_result block. Block order, types and
+ * tool_use_ids never change, so each of several parallel results stays with
+ * its own call and a note the user typed beside them stays as it was. A
+ * result's text parts are one piece (a result is trimmed as a whole) and its
+ * images stay. tool_use inputs and images are not text here; trim-tool-calls
+ * and stripInputBlobs handle inputs. An OpenAI `role: "tool"` message is a
+ * tool result throughout.
  */
-function replaceText(content, newText) {
-    if (typeof content === "string" || !Array.isArray(content))
-        return newText;
-    let placed = false;
-    const out = content
-        .map((b) => {
-        if (b?.type === "tool_result") {
-            const replaced = { ...b, content: placed ? "[removed]" : newText };
-            placed = true;
-            return replaced;
-        }
-        if (b?.type === "text") {
+function mapPieces(m, fn) {
+    const content = m?.content;
+    const own = m?.role === "tool" ? "tool_result" : "text";
+    const textParts = (parts, piece) => {
+        const texts = parts.filter((p) => p?.type === "text" && typeof p.text === "string");
+        if (texts.length === 0)
+            return parts;
+        const joined = texts.map((p) => p.text).join("\n");
+        const next = fn(joined, piece);
+        if (next === joined)
+            return parts;
+        // The rewritten text takes the first text part's place; other text parts
+        // were folded into it, everything else (images) keeps its position.
+        let placed = false;
+        return parts
+            .map((p) => {
+            if (!(p?.type === "text" && typeof p.text === "string"))
+                return p;
             if (placed)
                 return null;
             placed = true;
-            return { ...b, text: newText };
+            return { ...p, text: next };
+        })
+            .filter((p) => p !== null);
+    };
+    if (typeof content === "string")
+        return fn(content, own);
+    if (!Array.isArray(content))
+        return content;
+    if (own === "tool_result")
+        return textParts(content, own);
+    return content.map((b) => {
+        if (b?.type === "text" && typeof b.text === "string") {
+            const t = fn(b.text, "text");
+            return t === b.text ? b : { ...b, text: t };
+        }
+        if (b?.type === "tool_result") {
+            if (typeof b.content === "string") {
+                const t = fn(b.content, "tool_result");
+                return t === b.content ? b : { ...b, content: t };
+            }
+            if (Array.isArray(b.content)) {
+                const parts = textParts(b.content, "tool_result");
+                return parts === b.content ? b : { ...b, content: parts };
+            }
         }
         return b;
-    })
-        .filter(Boolean);
-    if (!placed)
-        out.push({ type: "text", text: newText });
-    return out;
+    });
+}
+/** Strip base64 blobs from the string values of a tool call's input, at any depth. */
+function stripInputBlobs(value) {
+    if (typeof value === "string")
+        return hasBase64Blob(value) ? stripBase64Blobs(value) : value;
+    if (Array.isArray(value))
+        return value.map(stripInputBlobs);
+    if (value && typeof value === "object") {
+        return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, stripInputBlobs(v)]));
+    }
+    return value;
 }
 function truncateToTokens(text, maxTokens, model) {
     const approxChars = Math.round(maxTokens * charsPerTokenFor(text, model));
@@ -219,7 +262,7 @@ export function optimizeConversation(input, options = {}) {
     catch {
         throw new Error("optimize requires a JSON conversation (message array, or object with a `messages` field)");
     }
-    const messages = Array.isArray(data) ? data : data.messages;
+    const messages = Array.isArray(data) ? data : data?.messages;
     if (!Array.isArray(messages)) {
         throw new Error("No `messages` array found in input");
     }
@@ -248,10 +291,13 @@ export function optimizeConversation(input, options = {}) {
             if (!hasBase64Blob(text))
                 return;
             const before = tok(text);
-            const cleaned = stripBase64Blobs(text);
-            const saved = before - tok(cleaned);
+            let content = mapPieces(m, (t) => (hasBase64Blob(t) ? stripBase64Blobs(t) : t));
+            if (Array.isArray(content)) {
+                content = content.map((b) => (b?.type === "tool_use" && b.input != null ? { ...b, input: stripInputBlobs(b.input) } : b));
+            }
+            const saved = before - tok(textOf(content));
             if (saved > 50) {
-                m.content = replaceText(m.content, cleaned);
+                m.content = content;
                 applied.push({ strategy: "strip-base64", messageIndex: i, tokensSaved: saved, note: "Removed inline base64 data" });
             }
         });
@@ -265,26 +311,34 @@ export function optimizeConversation(input, options = {}) {
         // and just sees a worse answer with no explanation. Older copies are fair
         // game; the live turn is not.
         const cutoff = stableCutoff(messages.length, opts.keepRecent, opts.trimBoundaryStep);
+        // Per piece (a text block or one tool's result), not per message: a whole
+        // message rarely repeats, and a message-level replacement had to flatten
+        // its blocks, which put one tool's output under another's id.
         const seen = new Map();
         messages.forEach((m, i) => {
-            const text = textOf(m.content);
-            if (text.length < 300)
-                return;
-            if (i >= cutoff) {
-                // Still record it, so a later duplicate can point back here.
-                if (!seen.has(hash(text)))
-                    seen.set(hash(text), i);
-                return;
-            }
-            const h = hash(text);
-            const first = seen.get(h);
-            if (first === undefined) {
-                seen.set(h, i);
+            let saved = 0;
+            let firstOf;
+            const content = mapPieces(m, (t) => {
+                if (t.length < 300)
+                    return t;
+                const h = hash(t);
+                const first = seen.get(h);
+                if (first === undefined) {
+                    seen.set(h, i);
+                    return t;
+                }
+                // Still recorded above, so a later duplicate can point back here.
+                if (i >= cutoff)
+                    return t;
+                const pointer = `[context-doctor: identical to message #${first} — content removed]`;
+                saved += tok(t) - tok(pointer);
+                firstOf ??= first;
+                return pointer;
+            });
+            if (saved <= 0)
                 return;
-            }
-            const saved = tok(text);
-            m.content = replaceText(m.content, `[context-doctor: identical to message #${first} — content removed]`);
-            applied.push({ strategy: "dedupe", messageIndex: i, tokensSaved: saved, note: `Duplicate of message #${first}` });
+            m.content = content;
+            applied.push({ strategy: "dedupe", messageIndex: i, tokensSaved: saved, note: `Duplicate of message #${firstOf}` });
         });
     }
     // -- trim-tool-results: shrink stale tool output ------------------------------
@@ -293,21 +347,32 @@ export function optimizeConversation(input, options = {}) {
         messages.forEach((m, i) => {
             if (i >= cutoff || !isToolResultMessage(m))
                 return;
-            const text = textOf(m.content);
-            const before = tok(text);
-            if (before <= opts.maxToolResultTokens)
-                return;
-            const trimmed = truncateToTokens(text, opts.maxToolResultTokens, model);
-            // The truncation notice has a length of its own, so a result only just
-            // over the budget can come back LARGER than it went in. Measured on a
-            // real session: 2,941 tokens "optimized" to 2,947.
-            if (tok(trimmed) >= before)
+            // Each result on its own budget; text the user typed beside them is not
+            // tool output and stays whole.
+            let saved = 0;
+            const content = mapPieces(m, (t, piece) => {
+                if (piece !== "tool_result")
+                    return t;
+                const before = tok(t);
+                if (before <= opts.maxToolResultTokens)
+                    return t;
+                const trimmed = truncateToTokens(t, opts.maxToolResultTokens, model);
+                // The truncation notice has a length of its own, so a result only just
+                // over the budget can come back LARGER than it went in. Measured on a
+                // real session: 2,941 tokens "optimized" to 2,947.
+                const after = tok(trimmed);
+                if (after >= before)
+                    return t;
+                saved += before - after;
+                return trimmed;
+            });
+            if (saved <= 0)
                 return;
-            m.content = replaceText(m.content, trimmed);
+            m.content = content;
             applied.push({
                 strategy: "trim-tool-results",
                 messageIndex: i,
-                tokensSaved: before - tok(trimmed),
+                tokensSaved: saved,
                 note: "Stale tool result truncated",
             });
         });
@@ -345,6 +410,8 @@ export function optimizeConversation(input, options = {}) {
                 }
             }
             // OpenAI shape: tool_calls[].function.arguments is a JSON string.
+            // Cut inside the parsed arguments, as above: a JSON string cut in the
+            // middle is no longer JSON, and the next request would carry it as such.
             for (const tc of m.tool_calls ?? []) {
                 const args = tc?.function?.arguments;
                 if (typeof args !== "string")
@@ -352,8 +419,21 @@ export function optimizeConversation(input, options = {}) {
                 const before = tok(args);
                 if (before <= opts.maxToolResultTokens)
                     continue;
-                tc.function.arguments = truncateToTokens(args, opts.maxToolResultTokens, model);
-                saved += before - tok(tc.function.arguments);
+                let parsed;
+                try {
+                    parsed = JSON.parse(args);
+                }
+                catch {
+                    continue;
+                }
+                if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
+                    continue;
+                const trimmedArgs = JSON.stringify(trimCallArguments(parsed, opts.maxToolResultTokens, model));
+                const after = tok(trimmedArgs);
+                if (after >= before)
+                    continue;
+                tc.function.arguments = trimmedArgs;
+                saved += before - after;
             }
             if (saved > 0) {
                 applied.push({
@@ -368,7 +448,12 @@ export function optimizeConversation(input, options = {}) {
     // -- prune-history: replace the older half with a stub ------------------------
     // Opt-in only: it is lossy, so it is not in the default strategy set.
     let prunedDigest;
-    if (opts.strategies.includes("prune-history") && messages.length > opts.keepRecent * 2) {
+    // OpenAI-shaped requests carry the system prompt as leading system/developer
+    // messages; those are instructions, not history, and are never pruned.
+    let lead = 0;
+    while (lead < messages.length && (messages[lead]?.role === "system" || messages[lead]?.role === "developer"))
+        lead++;
+    if (opts.strategies.includes("prune-history") && messages.length - lead > opts.keepRecent * 2) {
         let keepFrom = messages.length - opts.keepRecent;
         // Never let the kept tail START with a tool result: its matching tool_use
         // would be pruned away and both the Anthropic and OpenAI APIs reject
@@ -384,22 +469,22 @@ export function optimizeConversation(input, options = {}) {
             // APIs reject a conversation containing an orphan. Measured on a real
             // 1,011-message session, which pruned to 7 messages with one orphan.
             dropOrphanedToolResults(messages.slice(keepFrom));
-            const pruned = messages.slice(0, keepFrom);
+            const pruned = messages.slice(lead, keepFrom);
             const prunedTokens = pruned.reduce((s, m) => s + tok(textOf(m.content)), 0);
             // Digest: first ~200 chars of each pruned turn — enough for a host LLM to
             // write a faithful summary, small enough not to defeat the pruning.
             prunedDigest = pruned
-                .map((m, i) => `[${i}:${m.role}] ${textOf(m.content).replace(/\s+/g, " ").slice(0, 200)}`)
+                .map((m, i) => `[${lead + i}:${m.role}] ${textOf(m.content).replace(/\s+/g, " ").slice(0, 200)}`)
                 .join("\n");
             const stub = {
                 role: "user",
                 content: `[context-doctor: ${pruned.length} earlier messages (~${prunedTokens} tokens) pruned. ` +
                     `Replace this stub with an LLM-written summary of those turns for best results.]`,
             };
-            messages.splice(0, keepFrom, stub);
+            messages.splice(lead, keepFrom - lead, stub);
             applied.push({
                 strategy: "prune-history",
-                messageIndex: 0,
+                messageIndex: lead,
                 tokensSaved: prunedTokens - tok(stub.content),
                 note: `Pruned ${pruned.length} old messages`,
             });

package/dist/overhead.js

@@ -17,9 +17,9 @@
 import { readdirSync, readFileSync, statSync } from "node:fs";
 import { homedir } from "node:os";
 import { dirname, isAbsolute, join, relative, resolve } from "node:path";
-import { forEachLine, listSessions } from "./session.js";
-import { estimateTokens, formatTokens, providerFor } from "./tokens.js";
-import { formatUsd, pricingFor } from "./pricing.js";
+import { forEachLine, listSessions, replyCounter } from "./session.js";
+import { estimateTokens, formatTokens, providerFor, realModel } from "./tokens.js";
+import { cacheWriteSplit, formatUsd, pricingFor, writeMultiplier as ttlWriteMultiplier } from "./pricing.js";
 import { isGeminiChat, readGeminiMessages } from "./gemini.js";
 import { claudeMcpConfigs, mcpUsage, measureServer, toolPrefixName } from "./mcpschema.js";
 const TTL_MS = 3_600_000;
@@ -147,11 +147,17 @@ function textOf(content) {
         return "";
     return content.map((b) => (typeof b === "string" ? b : typeof b?.text === "string" ? b.text : "")).filter(Boolean).join("\n");
 }
-/** Claude Code: usage rows on assistant messages (one per API reply id). */
-function claudeUsage(path) {
-    const out = { agent: "Claude Code", reqs: [] };
+/**
+ * Claude Code: usage rows on assistant messages (one per API reply id).
+ * `runRows` is shared by every transcript of one report: a forked session
+ * begins with a verbatim copy of its parent's history, whose requests (and
+ * first request) the parent already counted.
+ */
+function claudeUsage(path, runRows) {
+    const out = { agent: "Claude Code", reqs: [], writes5m: 0, writes1h: 0 };
     let firstUser;
-    let lastId;
+    let started = false;
+    const sighting = replyCounter(runRows);
     forEachLine(path, (line) => {
         let e;
         try {
@@ -162,22 +168,32 @@ function claudeUsage(path) {
         }
         if (!e || e.isSidechain)
             return;
-        if (e.type === "user" && out.first === undefined && firstUser === undefined && !e.isMeta) {
+        if (e.type === "user" && !started && firstUser === undefined && !e.isMeta) {
             const text = textOf(e.message?.content);
             if (text)
                 firstUser = text;
             return;
         }
-        if (e.type !== "assistant" || !e.message?.usage || e.message.id === lastId)
+        if (e.type !== "assistant" || !e.message?.usage)
+            return;
+        const seen = sighting(e);
+        if (seen === "repeat")
             return;
-        lastId = e.message.id;
         const u = e.message.usage;
         const prompt = (u.input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0);
         const at = Date.parse(e.timestamp);
         if (!(prompt > 0) || !Number.isFinite(at))
             return;
-        out.first ??= Math.max(0, prompt - estimateTokens(firstUser ?? "", CLAUDE));
-        out.reqs.push({ at, model: e.message.model });
+        // A fork's first request is its parent's: not a second session start.
+        if (!started && seen === "new")
+            out.first = Math.max(0, prompt - estimateTokens(firstUser ?? "", CLAUDE));
+        started = true;
+        if (seen === "copy")
+            return;
+        const w = cacheWriteSplit(u);
+        out.writes5m += w.fiveMinute;
+        out.writes1h += w.oneHour;
+        out.reqs.push({ at, model: realModel(e.message.model) });
     });
     return out;
 }
@@ -237,32 +253,39 @@ function geminiUsage(path) {
     }
     return out;
 }
-function usageOf(path) {
+function usageOf(path, runRows) {
     if (isGeminiChat(path))
         return geminiUsage(path);
     if (/[\\/]\.codex[\\/]/.test(path))
         return codexUsage(path);
-    return claudeUsage(path);
+    return claudeUsage(path, runRows);
 }
-/** Cache writes cost 1.25x input on Anthropic; OpenAI and Google charge no write premium. */
-function writeMultiplier(model) {
-    return providerFor(model) === "anthropic" ? 1.25 : 1;
+/**
+ * Cache writes cost 1.25x input on Anthropic at the 5-minute TTL and 2x at 1
+ * hour, which Claude Code's main thread mostly uses: the agent's own
+ * recorded mix decides. OpenAI and Google charge no write premium.
+ */
+function writeMultiplier(model, writes5m, writes1h) {
+    return providerFor(model) === "anthropic" ? ttlWriteMultiplier(writes5m, writes1h) : 1;
 }
 function aggregate(agent, sessions, since, days) {
     const firsts = [];
-    let requests = 0, coldStarts = 0, warm = 0;
+    let requests = 0, coldStarts = 0, writes5m = 0, writes1h = 0;
     const models = new Map();
+    const starts = [];
     for (const s of sessions) {
         if (s.first !== undefined && s.first > 1000)
             firsts.push(s.first);
+        writes5m += s.writes5m ?? 0;
+        writes1h += s.writes1h ?? 0;
         let prevAt;
         for (const r of s.reqs) {
             if (r.at >= since) {
                 requests++;
-                if (prevAt === undefined || r.at - prevAt > TTL_MS)
+                const cold = prevAt === undefined || r.at - prevAt > TTL_MS;
+                if (cold)
                     coldStarts++;
-                else
-                    warm++;
+                starts.push({ cold, model: r.model });
                 if (r.model)
                     models.set(r.model, (models.get(r.model) ?? 0) + 1);
             }
@@ -273,12 +296,18 @@ function aggregate(agent, sessions, since, days) {
         return undefined;
     firsts.sort((a, b) => a - b);
     const model = [...models.entries()].sort((a, b) => b[1] - a[1])[0]?.[0];
-    const price = pricingFor(model);
     // Each overhead token is a cache read on warm requests and a full-price
-    // write (with the provider's premium) after every cold start.
-    const usdPerKPerMonth = price
-        ? ((1000 * (warm * price.cacheReadPerM + coldStarts * writeMultiplier(model) * price.inputPerM)) / 1e6) * (30 / days)
-        : undefined;
+    // write (with the provider's premium) after every cold start, each request
+    // at its own model's price: one month can mix a $4 and a $10 model.
+    let usdPerK = 0, priced = 0;
+    for (const r of starts) {
+        const price = pricingFor(r.model ?? model);
+        if (!price)
+            continue;
+        priced++;
+        usdPerK += (1000 * (r.cold ? writeMultiplier(r.model ?? model, writes5m, writes1h) * price.inputPerM : price.cacheReadPerM)) / 1e6;
+    }
+    const usdPerKPerMonth = priced > 0 ? usdPerK * (30 / days) : undefined;
     return {
         agent,
         sessions: firsts.length,
@@ -301,10 +330,11 @@ export function measureBaselines(days = 30, paths) {
     const since = Date.now() - days * 86_400_000;
     const files = paths ?? listSessions(10_000).filter((s) => s.modifiedAt.getTime() >= since).map((s) => s.path);
     const byAgent = new Map();
+    const runRows = new Set();
     for (const path of files) {
         let u;
         try {
-            u = usageOf(path);
+            u = usageOf(path, runRows);
         }
         catch {
             continue;

package/dist/pack.d.ts

@@ -94,6 +94,17 @@ export declare const PACK_LIMITS: {
     fileBytes: number;
     totalBytes: number;
 };
+/**
+ * Read files and directories (recursively, skipping build and dependency
+ * folders, binaries, and anything past the size limits) into sources.
+ * `skipped` says what was left out and why, so nothing vanishes silently.
+ *
+ * A path named here is read even through a symlink; a symlink met inside a
+ * folder is not followed, as git does not follow one either. A link out of
+ * the folder would pack files nobody named (a tracked link to /etc/hosts, or
+ * to a secret beside the repo), a link to a parent loops, and what a link
+ * inside points at is read under its own name (or left out by .gitignore).
+ */
 export declare function readSources(paths: string[], cwd?: string): {
     sources: PackSource[];
     skipped: string[];

package/dist/pack.js

@@ -14,8 +14,8 @@
  * comes with every result lets the model pick sections by heading when it does.
  */
 import { spawnSync } from "node:child_process";
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { basename, extname, join, relative } from "node:path";
+import { existsSync, lstatSync, readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
+import { basename, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { estimateTokens, formatTokens } from "./tokens.js";
 import { EXTRACTABLE, extractText } from "./extract.js";
 const DEFAULT_BUDGET = 4000;
@@ -92,6 +92,8 @@ function units(lines, kind) {
     }
     return out;
 }
+const isHigh = (c) => c >= 0xd800 && c <= 0xdbff;
+const isLow = (c) => c >= 0xdc00 && c <= 0xdfff;
 /** Split one source into chunks of at most `maxTokens`, along its own structure. */
 export function chunkSource(source, opts = {}) {
     const maxTokens = Math.max(50, opts.maxChunkTokens ?? DEFAULT_CHUNK_TOKENS);
@@ -105,7 +107,11 @@ export function chunkSource(source, opts = {}) {
     source.text.replace(/\r\n?/g, "\n").split("\n").forEach((line, n) => {
         while (line.length > maxChars) {
             const cut = line.lastIndexOf(" ", maxChars);
-            const at = cut > maxChars / 2 ? cut + 1 : maxChars;
+            let at = cut > maxChars / 2 ? cut + 1 : maxChars;
+            // Indexes are UTF-16 units: a cut between the two halves of an emoji or
+            // other astral character would leave a lone surrogate on each side.
+            if (isHigh(line.charCodeAt(at - 1)) && isLow(line.charCodeAt(at)))
+                at--;
             lines.push(line.slice(0, at));
             lineNo.push(n + 1);
             line = line.slice(at);
@@ -329,11 +335,6 @@ export function renderPack(r, { outlineLimit = 60 } = {}) {
 }
 const SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", ".next", "target", "__pycache__", ".venv", "venv", "vendor"]);
 export const PACK_LIMITS = { files: 500, fileBytes: 5 * 1024 * 1024, totalBytes: 50 * 1024 * 1024 };
-/**
- * Read files and directories (recursively, skipping build and dependency
- * folders, binaries, and anything past the size limits) into sources.
- * `skipped` says what was left out and why, so nothing vanishes silently.
- */
 /** Files git would consider part of the repository under `dir` (relative to it), or undefined outside git. */
 function gitFiles(dir) {
     try {
@@ -347,10 +348,37 @@ function gitFiles(dir) {
         return undefined;
     }
 }
+/** Git ignores something under `dir`: why a folder that has files can list none. */
+function gitIgnoresSome(dir) {
+    // --directory names an ignored folder once instead of every file in it.
+    const r = spawnSync("git", ["-C", dir, "ls-files", "-z", "--others", "--ignored", "--exclude-standard", "--directory"], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 30_000 });
+    return r.status === 0 && typeof r.stdout === "string" && r.stdout.length > 0;
+}
+/** `inner` is `root` itself or somewhere under it. */
+function within(root, inner) {
+    const rel = relative(root, inner);
+    return rel === "" || (rel !== ".." && !rel.startsWith(".." + sep) && !isAbsolute(rel));
+}
+/**
+ * Read files and directories (recursively, skipping build and dependency
+ * folders, binaries, and anything past the size limits) into sources.
+ * `skipped` says what was left out and why, so nothing vanishes silently.
+ *
+ * A path named here is read even through a symlink; a symlink met inside a
+ * folder is not followed, as git does not follow one either. A link out of
+ * the folder would pack files nobody named (a tracked link to /etc/hosts, or
+ * to a secret beside the repo), a link to a parent loops, and what a link
+ * inside points at is read under its own name (or left out by .gitignore).
+ */
 export function readSources(paths, cwd = process.cwd()) {
     const sources = [];
     const skipped = [];
     let bytes = 0;
+    // Real paths already packed: a file reached twice ("." and "guide.md") is packed once, not twice under one id.
+    // Only packed ones: a binary passed over inside a folder and then named outright still gets its message.
+    const seen = new Set();
+    // Real location of the path being read: a link out of it is reported, not just dropped.
+    let root = "";
     const visit = (p, top) => {
         if (sources.length >= PACK_LIMITS.files) {
             if (top || skipped.length < 20)
@@ -359,12 +387,24 @@ export function readSources(paths, cwd = process.cwd()) {
         }
         let st;
         try {
-            st = statSync(p);
+            st = top ? statSync(p) : lstatSync(p);
         }
         catch {
             skipped.push(`${p}: not found`);
             return;
         }
+        if (st.isSymbolicLink()) {
+            let target;
+            try {
+                target = realpathSync(p);
+            }
+            catch {
+                return; // dangling: nothing to read
+            }
+            if (!within(root, target))
+                skipped.push(`${p}: symlink pointing outside the folder (not followed)`);
+            return;
+        }
         if (st.isDirectory()) {
             if (!top && (SKIP_DIRS.has(basename(p)) || basename(p).startsWith(".")))
                 return;
@@ -373,6 +413,11 @@ export function readSources(paths, cwd = process.cwd()) {
             // data dumps and secrets stay out. Elsewhere, walk with SKIP_DIRS.
             const listed = top ? gitFiles(p) : undefined;
             if (listed) {
+                // A folder named outright that git lists nothing in is still not read
+                // (that is the .gitignore guarantee), but the user hears why.
+                if (listed.length === 0 && gitIgnoresSome(p)) {
+                    skipped.push(`${p}: ignored by git (.gitignore); pack reads folders as git does, so ignored files stay out. Name a file to pack it anyway`);
+                }
                 for (const f of listed)
                     visit(join(p, f), false);
                 return;
@@ -383,6 +428,15 @@ export function readSources(paths, cwd = process.cwd()) {
         }
         if (!st.isFile())
             return;
+        let real;
+        try {
+            real = realpathSync(p);
+        }
+        catch {
+            real = resolve(p);
+        }
+        if (seen.has(real))
+            return;
         if (st.size > PACK_LIMITS.fileBytes || bytes + st.size > PACK_LIMITS.totalBytes) {
             skipped.push(`${p}: too large (${Math.round(st.size / 1024)} KB)`);
             return;
@@ -395,6 +449,7 @@ export function readSources(paths, cwd = process.cwd()) {
                 skipped.push(`${p}: ${got.error}`);
             else {
                 bytes += st.size;
+                seen.add(real);
                 sources.push({ name, text: got.text });
             }
             return;
@@ -406,13 +461,21 @@ export function readSources(paths, cwd = process.cwd()) {
             return;
         }
         bytes += st.size;
+        seen.add(real);
         sources.push({ name, text: buf.toString("utf8") });
     };
     for (const p of paths) {
-        if (!existsSync(p))
+        if (!existsSync(p)) {
             skipped.push(`${p}: not found`);
-        else
-            visit(p, true);
+            continue;
+        }
+        try {
+            root = realpathSync(p);
+        }
+        catch {
+            root = resolve(p);
+        }
+        visit(p, true);
     }
     return { sources, skipped };
 }