@mcarvin/smart-diff 2.0.0 → 2.1.0

package/README.md

@@ -122,8 +122,30 @@ const markdown = await summarizeGitDiff({
 | `provider` | `LlmProviderId` — wins over `LLM_PROVIDER` env and auto-detection. |
 | `model` | Chat model id; overrides `LLM_MODEL` and the provider default. |
 | `maxDiffChars` | Caps unified diff size for the request. |
+| `contextLines` | Number of context lines around each change (`git diff -U<n>`). Lower values (1 or 0) are the single biggest token saver on modification-heavy diffs. |
+| `ignoreWhitespace` | Passes `-w` / `--ignore-all-space` to `git diff` so pure-whitespace hunks don't consume tokens. Also applies to `--numstat` / `--name-status` so counts stay consistent. |
+| `stripDiffPreamble` | Removes low-value lines from the unified diff (`diff --git`, `index`, mode changes, `similarity/rename/copy` metadata). `--- a/…`, `+++ b/…`, and `@@` hunk headers are kept. |
+| `maxHunkLines` | Caps the body of each hunk; anything past the limit is replaced with a single elision marker. The `@@` header and `DiffSummary` totals are preserved. |
+| `excludeDefaultNoise` | Merges the built-in `DEFAULT_NOISE_EXCLUDES` list (lockfiles, `dist`, `build`, `out`, `coverage`, `node_modules`, `__snapshots__`) into `excludeFolders`. |
 | `llmModelProvider` | `() => Promise<LanguageModel>` — bypass env-based resolution entirely; hand-wire a Vercel AI SDK `LanguageModel` (required in tests or custom setups). |
 
+#### Reducing tokens
+
+For most repos, the cheapest wins are:
+
+```ts
+await summarizeGitDiff({
+  from: 'origin/main',
+  contextLines: 1,          // -U1 cuts 30-60% of tokens on typical diffs
+  ignoreWhitespace: true,   // drop pure-whitespace hunks entirely
+  stripDiffPreamble: true,  // kill `index`/`mode`/`similarity` lines
+  maxHunkLines: 400,        // truncate monster hunks but keep the @@ header
+  excludeDefaultNoise: true // skip lockfiles, dist/, coverage/, node_modules/
+});
+```
+
+These options only reshape the *unified diff text* — the structured `DiffSummary` still reports true file counts and line totals, so the model always sees the full change inventory.
+
 ### Injecting your own `LanguageModel`
 
 If you want full control — for example, to configure retries, middlewares, or hit an in-process mock — pass `llmModelProvider`:
@@ -150,7 +172,7 @@ const md = await summarizeGitDiff({
 
 The package also exports helpers for building a custom pipeline on top of the same git and LLM behavior:
 
-- **Git**: `createGitClient`, `getRepoRoot`, `getCommits`, `getDiff`, `getDiffSummary`, `getChangedFiles`, `filterCommitsByMessageRegexes`, `buildDiffPathspecs`
+- **Git**: `createGitClient`, `getRepoRoot`, `getCommits`, `getDiff`, `getDiffSummary`, `getChangedFiles`, `filterCommitsByMessageRegexes`, `buildDiffPathspecs`, `buildDiffShapingGitArgs`, `shapeUnifiedDiff`, `DEFAULT_NOISE_EXCLUDES`
 - **AI**: `generateSummary`, `resolveLlmMaxDiffChars`, `truncateUnifiedDiffForLlm`
 - **Provider resolution**: `resolveLanguageModel`, `detectLlmProvider`, `isLlmProviderConfigured`, `defaultModelForProvider`, `resolveLlmBaseUrl`, `parseLlmDefaultHeadersFromEnv`
 - **Constants / types**: `DEFAULT_GIT_DIFF_SYSTEM_PROMPT`, `LLM_GATEWAY_REQUIRED_MESSAGE`, `LlmProviderId`, `LlmModelProvider`, `ResolveLanguageModelOptions`, `GenerateSummaryInput`, `SummarizeFlags`

package/dist/index.cjs

@@ -401,7 +401,6 @@ function buildUserContent(flags, commits, fileNames, diffText, diffSummary) {
 }
 function callLlm(userContent, systemPrompt, maxOutputTokens, llmModelProvider, flags) {
     return __awaiter(this, void 0, void 0, function* () {
-        var _a, _b;
         const model = llmModelProvider
             ? yield llmModelProvider()
             : yield resolveLanguageModel(Object.assign(Object.assign({}, (flags.provider ? { provider: flags.provider } : {})), (flags.model ? { model: flags.model } : {})));
@@ -412,7 +411,7 @@ function callLlm(userContent, systemPrompt, maxOutputTokens, llmModelProvider, f
             temperature: 0.2,
             maxOutputTokens,
         });
-        const text = (_b = (_a = result.text) === null || _a === void 0 ? void 0 : _a.trim()) !== null && _b !== void 0 ? _b : "";
+        const text = result.text.trim();
         return text.length > 0 ? text : "No summary generated by the model.";
     });
 }
@@ -495,6 +494,130 @@ function filterCommitsByMessageRegexes(commits, includePatterns, excludePatterns
     return commits.filter((c) => commitMessagePassesFilters(c.message, includeRes, excludeRes));
 }
 
+const DEFAULT_NOISE_EXCLUDES = [
+    "package-lock.json",
+    "yarn.lock",
+    "pnpm-lock.yaml",
+    "npm-shrinkwrap.json",
+    "bun.lockb",
+    "go.sum",
+    "Cargo.lock",
+    "Gemfile.lock",
+    "composer.lock",
+    "Pipfile.lock",
+    "poetry.lock",
+    "uv.lock",
+    "Podfile.lock",
+    "node_modules",
+    "dist",
+    "build",
+    "out",
+    "coverage",
+    "__snapshots__",
+];
+function normalizeContextLines(raw) {
+    if (!Number.isFinite(raw) || raw < 0)
+        return 0;
+    return Math.trunc(raw);
+}
+function buildDiffShapingGitArgs(shaping) {
+    const args = [];
+    if ((shaping === null || shaping === void 0 ? void 0 : shaping.contextLines) !== undefined) {
+        args.push(`-U${normalizeContextLines(shaping.contextLines)}`);
+    }
+    if (shaping === null || shaping === void 0 ? void 0 : shaping.ignoreWhitespace) {
+        args.push("-w");
+    }
+    return args;
+}
+const PREAMBLE_NOISE_PREFIXES = [
+    "diff --git ",
+    "index ",
+    "new file mode ",
+    "deleted file mode ",
+    "old mode ",
+    "new mode ",
+    "similarity index ",
+    "dissimilarity index ",
+    "rename from ",
+    "rename to ",
+    "copy from ",
+    "copy to ",
+];
+function isPreambleNoiseLine(line) {
+    for (const prefix of PREAMBLE_NOISE_PREFIXES) {
+        if (line.startsWith(prefix))
+            return true;
+    }
+    return false;
+}
+function stripPreambleLines(text) {
+    return text
+        .split(/\r?\n/)
+        .filter((line) => !isPreambleNoiseLine(line))
+        .join("\n");
+}
+function isFileHeaderLine(line) {
+    return (/^--- (a\/|b\/|"a\/|"b\/|\/dev\/null)/.test(line) ||
+        /^\+\+\+ (a\/|b\/|"a\/|"b\/|\/dev\/null)/.test(line));
+}
+function elideLargeHunks(text, maxHunkLines) {
+    const limit = normalizeContextLines(maxHunkLines);
+    const lines = text.split(/\r?\n/);
+    const out = [];
+    let inHunk = false;
+    let hunkBuf = [];
+    const flushHunk = () => {
+        if (hunkBuf.length > limit) {
+            const elided = hunkBuf.length - limit;
+            out.push(...hunkBuf.slice(0, limit));
+            out.push(`[... ${elided} diff line${elided === 1 ? "" : "s"} elided ...]`);
+        }
+        else {
+            out.push(...hunkBuf);
+        }
+        hunkBuf = [];
+        inHunk = false;
+    };
+    for (const line of lines) {
+        if (line.startsWith("@@")) {
+            if (inHunk)
+                flushHunk();
+            out.push(line);
+            inHunk = true;
+            continue;
+        }
+        if (line.startsWith("diff --git ") || isFileHeaderLine(line)) {
+            if (inHunk)
+                flushHunk();
+            out.push(line);
+            continue;
+        }
+        if (inHunk) {
+            hunkBuf.push(line);
+        }
+        else {
+            out.push(line);
+        }
+    }
+    if (inHunk)
+        flushHunk();
+    return out.join("\n");
+}
+function shapeUnifiedDiff(text, shaping) {
+    if (!(shaping === null || shaping === void 0 ? void 0 : shaping.stripDiffPreamble) && (shaping === null || shaping === void 0 ? void 0 : shaping.maxHunkLines) === undefined) {
+        return text;
+    }
+    let out = text;
+    if (shaping.stripDiffPreamble) {
+        out = stripPreambleLines(out);
+    }
+    if (shaping.maxHunkLines !== undefined) {
+        out = elideLargeHunks(out, shaping.maxHunkLines);
+    }
+    return out;
+}
+
 const GIT_STATUS_BY_FIRST_CHAR = {
     A: "added",
     D: "deleted",
@@ -525,25 +648,17 @@ function mergeStatus(existing, next) {
 }
 
 function parseNameStatusLine(line) {
-    var _a;
     const parts = line.split("\t");
     let entry = null;
     if (parts.length >= 2) {
-        const statusToken = (_a = parts[0]) !== null && _a !== void 0 ? _a : "";
+        const statusToken = parts[0];
         const status = mapGitStatus(statusToken);
         const isRenameOrCopy = statusToken.startsWith("R") || statusToken.startsWith("C");
         if (isRenameOrCopy && parts.length >= 3) {
-            const oldPath = parts[1];
-            const newPath = parts[2];
-            if (oldPath !== undefined && newPath !== undefined) {
-                entry = { path: newPath, status, oldPath };
-            }
+            entry = { path: parts[2], status, oldPath: parts[1] };
         }
         else if (!isRenameOrCopy) {
-            const pathOnly = parts[1];
-            if (pathOnly !== undefined) {
-                entry = { path: pathOnly, status };
-            }
+            entry = { path: parts[1], status };
         }
     }
     return entry;
@@ -588,12 +703,11 @@ function numStatPathToLookupKey(pathField) {
     return `${dirRaw}${toSeg}`;
 }
 function parseNumStatLine(line) {
-    var _a, _b;
     const parts = line.split("\t");
     if (parts.length < 3)
         return null;
-    const addStr = (_a = parts[0]) !== null && _a !== void 0 ? _a : "";
-    const delStr = (_b = parts[1]) !== null && _b !== void 0 ? _b : "";
+    const addStr = parts[0];
+    const delStr = parts[1];
     const pathField = parts.slice(2).join("\t");
     const additions = addStr !== "-" ? Number.parseInt(addStr, 10) || 0 : 0;
     const deletions = delStr !== "-" ? Number.parseInt(delStr, 10) || 0 : 0;
@@ -618,11 +732,10 @@ function accumulateNumStat(numStatOutput, into) {
 }
 
 function parseTabDiffSummaryLine(line) {
-    var _a;
     const parts = line.split("\t");
     if (parts.length < 3)
         return null;
-    const statusToken = (_a = parts.shift()) !== null && _a !== void 0 ? _a : "";
+    const statusToken = parts.shift();
     const status = mapGitStatus(statusToken);
     const add0 = parts[0];
     const del0 = parts[1];
@@ -740,31 +853,48 @@ function getDiffPathContext(git, pathFilter, repoRootOverride) {
 }
 function getDiff(git, query) {
     return __awaiter(this, void 0, void 0, function* () {
-        const { from, to, commits, filterByCommits, pathFilter, repoRootOverride } = query;
+        const { from, to, commits, filterByCommits, pathFilter, repoRootOverride, shaping, } = query;
         const { specs } = yield getDiffPathContext(git, pathFilter, repoRootOverride);
+        const shapingArgs = buildDiffShapingGitArgs(shaping);
         if (!filterByCommits) {
-            return git.diff([`${from}..${to}`, "--", ...specs]);
+            const raw = yield git.diff([
+                ...shapingArgs,
+                `${from}..${to}`,
+                "--",
+                ...specs,
+            ]);
+            return shapeUnifiedDiff(raw, shaping);
         }
-        const patches = yield Promise.all(commits.map((c) => git.diff([`${c.hash}^!`, "--", ...specs])));
-        return patches.filter(Boolean).join("\n");
+        const patches = yield Promise.all(commits.map((c) => git.diff([...shapingArgs, `${c.hash}^!`, "--", ...specs])));
+        return patches
+            .map((p) => shapeUnifiedDiff(p, shaping))
+            .filter(Boolean)
+            .join("\n");
     });
 }
 function getDiffSummary(git, query) {
     return __awaiter(this, void 0, void 0, function* () {
-        const { from, to, commits, filterByCommits, pathFilter, repoRootOverride } = query;
+        const { from, to, commits, filterByCommits, pathFilter, repoRootOverride, shaping, } = query;
         const { specs } = yield getDiffPathContext(git, pathFilter, repoRootOverride);
+        const whitespaceArgs = (shaping === null || shaping === void 0 ? void 0 : shaping.ignoreWhitespace) ? ["-w"] : [];
         if (!filterByCommits) {
             const [numOutput, nameOutput] = yield Promise.all([
-                git.diff(["--numstat", `${from}..${to}`, "--", ...specs]),
-                git.diff(["--name-status", `${from}..${to}`, "--", ...specs]),
+                git.diff([...whitespaceArgs, "--numstat", `${from}..${to}`, "--", ...specs]),
+                git.diff([
+                    ...whitespaceArgs,
+                    "--name-status",
+                    `${from}..${to}`,
+                    "--",
+                    ...specs,
+                ]),
             ]);
             return buildDiffSummaryFromGitOutputs(nameOutput, numOutput);
         }
         const pairs = yield Promise.all(commits.map((c) => __awaiter(this, void 0, void 0, function* () {
             const range = `${c.hash}^!`;
             const [numOutput, nameOutput] = yield Promise.all([
-                git.diff(["--numstat", range, "--", ...specs]),
-                git.diff(["--name-status", range, "--", ...specs]),
+                git.diff([...whitespaceArgs, "--numstat", range, "--", ...specs]),
+                git.diff([...whitespaceArgs, "--name-status", range, "--", ...specs]),
             ]);
             return { numOutput, nameOutput };
         })));
@@ -814,6 +944,37 @@ function getChangedFiles(git, query) {
     });
 }
 
+function buildShapingFromOptions(options) {
+    const shaping = {};
+    if (options.contextLines !== undefined) {
+        shaping.contextLines = options.contextLines;
+    }
+    if (options.ignoreWhitespace)
+        shaping.ignoreWhitespace = true;
+    if (options.stripDiffPreamble)
+        shaping.stripDiffPreamble = true;
+    if (options.maxHunkLines !== undefined) {
+        shaping.maxHunkLines = options.maxHunkLines;
+    }
+    return Object.keys(shaping).length > 0 ? shaping : undefined;
+}
+function buildEffectiveExcludeFolders(options) {
+    var _a;
+    const userExcludes = (_a = options.excludeFolders) !== null && _a !== void 0 ? _a : [];
+    if (!options.excludeDefaultNoise) {
+        return userExcludes.length > 0 ? userExcludes : undefined;
+    }
+    const seen = new Set();
+    const merged = [];
+    for (const p of [...DEFAULT_NOISE_EXCLUDES, ...userExcludes]) {
+        const key = p.trim();
+        if (!key || seen.has(key))
+            continue;
+        seen.add(key);
+        merged.push(p);
+    }
+    return merged;
+}
 function hasNonEmptyTrimmed(arr) {
     return (arr !== null && arr !== void 0 ? arr : []).some((s) => s.trim().length > 0);
 }
@@ -830,22 +991,25 @@ function summarizeGitDiff(options) {
         const git = (_a = options.git) !== null && _a !== void 0 ? _a : createGitClient(options.cwd);
         const from = options.from;
         const to = (_b = options.to) !== null && _b !== void 0 ? _b : "HEAD";
+        const effectiveExcludeFolders = buildEffectiveExcludeFolders(options);
         const pathFilter = hasNonEmptyTrimmed(options.includeFolders) ||
-            hasNonEmptyTrimmed(options.excludeFolders)
+            hasNonEmptyTrimmed(effectiveExcludeFolders)
             ? {
                 includeFolders: options.includeFolders,
-                excludeFolders: options.excludeFolders,
+                excludeFolders: effectiveExcludeFolders,
             }
             : undefined;
         const allCommits = yield getCommits(git, from, to);
         const filteredCommits = filterCommitsByMessageRegexes(allCommits, options.commitMessageIncludeRegexes, options.commitMessageExcludeRegexes);
         const filterByCommits = shouldFilterByCommits(allCommits, filteredCommits, options);
+        const shaping = buildShapingFromOptions(options);
         const rangeQuery = {
             from,
             to,
             commits: filteredCommits,
             filterByCommits,
             pathFilter,
+            shaping,
         };
         const [diffText, fileNames, diffSummary] = yield Promise.all([
             getDiff(git, rangeQuery),
@@ -875,8 +1039,10 @@ function summarizeGitDiff(options) {
 }
 
 exports.DEFAULT_GIT_DIFF_SYSTEM_PROMPT = DEFAULT_GIT_DIFF_SYSTEM_PROMPT;
+exports.DEFAULT_NOISE_EXCLUDES = DEFAULT_NOISE_EXCLUDES;
 exports.LLM_GATEWAY_REQUIRED_MESSAGE = LLM_GATEWAY_REQUIRED_MESSAGE;
 exports.buildDiffPathspecs = buildDiffPathspecs;
+exports.buildDiffShapingGitArgs = buildDiffShapingGitArgs;
 exports.createGitClient = createGitClient;
 exports.defaultModelForProvider = defaultModelForProvider;
 exports.detectLlmProvider = detectLlmProvider;
@@ -892,6 +1058,7 @@ exports.parseLlmDefaultHeadersFromEnv = parseLlmDefaultHeadersFromEnv;
 exports.resolveLanguageModel = resolveLanguageModel;
 exports.resolveLlmBaseUrl = resolveLlmBaseUrl;
 exports.resolveLlmMaxDiffChars = resolveLlmMaxDiffChars;
+exports.shapeUnifiedDiff = shapeUnifiedDiff;
 exports.summarizeGitDiff = summarizeGitDiff;
 exports.truncateUnifiedDiffForLlm = truncateUnifiedDiffForLlm;
 //# sourceMappingURL=index.cjs.map