diffdoc 0.4.3 → 0.5.0

package/.diffdocrc.example

@@ -9,6 +9,7 @@
   "cloudChatModel": "gpt-4o-mini",
   "cloudEmbedModel": "text-embedding-3-small",
   "embedBatchSize": 25,
+  "summarizeConcurrency": 2,
   "openaiApiKey": "",
   "includeGlobs": [],
   "excludeGlobs": [],

package/README.md

@@ -125,6 +125,7 @@ Example `.diffdocrc` for local models:
   "localChatModel": "qwen2.5-coder:7b",
   "localEmbedModel": "nomic-embed-code",
   "embedBatchSize": 25,
+  "summarizeConcurrency": 2,
   "includeGlobs": [],
   "excludeGlobs": [],
   "ignoreFile": ".diffdocignore"
@@ -141,6 +142,7 @@ Example `.diffdocrc` for a cloud OpenAI-compatible endpoint:
   "cloudChatModel": "gpt-4o-mini",
   "cloudEmbedModel": "text-embedding-3-small",
   "embedBatchSize": 25,
+  "summarizeConcurrency": 2,
   "includeGlobs": [],
   "excludeGlobs": [],
   "ignoreFile": ".diffdocignore"
@@ -159,6 +161,7 @@ Supported environment variables:
 AI_PROVIDER
 DIFFDOC_BASE_DIR
 DIFFDOC_EMBED_BATCH_SIZE
+DIFFDOC_SUMMARIZE_CONCURRENCY
 DIFFDOC_INCLUDE_GLOBS
 DIFFDOC_EXCLUDE_GLOBS
 DIFFDOC_IGNORE_FILE
@@ -227,8 +230,11 @@ Summarize files into `.diffdoc/manifest.json` and `.diffdoc/summaries/*.json`:
 npx diffdoc summarize --path . --mode all
 npx diffdoc summarize --path . --mode delta
 npx diffdoc summarize --path . --mode delta --json
+npx diffdoc summarize --path . --mode all --summarize-concurrency 4
 ```
 
+Summarization runs with bounded concurrency. The default is `2`; use `1` for strict rate limits, `2-4` for most providers, and higher values only when your local model server or API quota can handle the request volume.
+
 Store raw code snapshots in summary assets when you want retrieved results to include source text:
 
 ```bash

package/dist/commands/init.js

@@ -19,6 +19,7 @@ const DEFAULT_CONFIG = {
     cloudChatModel: "gpt-4o-mini",
     cloudEmbedModel: "text-embedding-3-small",
     openaiApiKey: "",
+    summarizeConcurrency: 2,
     includeGlobs: [],
     excludeGlobs: [],
     ignoreFile: ".diffdocignore"

package/dist/commands/summarize.js

@@ -223,6 +223,25 @@ async function ensureSummaryAsset(summaryDir, hash, summaryText, rawCodeSnapshot
     };
     await writeSummaryAsset(summaryPath, summary);
 }
+async function runWithConcurrency(items, concurrency, worker) {
+    let nextIndex = 0;
+    const workerCount = Math.min(concurrency, items.length);
+    await Promise.all(Array.from({ length: workerCount }, async () => {
+        while (nextIndex < items.length) {
+            const item = items[nextIndex];
+            nextIndex += 1;
+            await worker(item);
+        }
+    }));
+}
+function createManifestLock() {
+    let queue = Promise.resolve();
+    return async function withManifestLock(task) {
+        const run = queue.then(task, task);
+        queue = run.then(() => undefined, () => undefined);
+        return run;
+    };
+}
 async function pruneOrphanedSummaries(summaryDir, manifest) {
     const activeHashes = new Set(Object.values(manifest.files));
     let entries = [];
@@ -269,6 +288,26 @@ async function runSummarize(options, config) {
     const totals = { scanned: 0, skipped: 0, updated: 0, failed: 0, pruned: 0 };
     const failures = [];
     const isJson = options.json;
+    const concurrency = config.summarize.concurrency;
+    const withManifestLock = createManifestLock();
+    const summaryAssetTasks = new Map();
+    async function ensureSummaryAssetForFile(filePath, hash, rawCodeSnapshot) {
+        const summaryPath = getSummaryPath(summaryDir, hash);
+        if (await fileExists(summaryPath)) {
+            return;
+        }
+        let task = summaryAssetTasks.get(hash);
+        if (!task) {
+            task = (async () => {
+                const summaryText = await (0, llm_1.generateFunctionalSummary)(filePath, rawCodeSnapshot, config.chat);
+                await ensureSummaryAsset(summaryDir, hash, summaryText, rawCodeSnapshot, options.includeCodeSnapshot);
+            })().finally(() => {
+                summaryAssetTasks.delete(hash);
+            });
+            summaryAssetTasks.set(hash, task);
+        }
+        await task;
+    }
     if (!isJson) {
         console.log(`Starting summarize run`);
         console.log(`Mode: ${options.mode}`);
@@ -283,44 +322,51 @@ async function runSummarize(options, config) {
         await writeManifest(manifestPath, manifest);
         const files = await walkCodeFiles(repoPath, includePatterns, excludePatterns, ignoreMatcher);
         const totalFiles = files.length;
+        let completedFiles = 0;
         if (!isJson) {
             console.log(`Candidates: ${totalFiles}`);
+            console.log(`Concurrency: ${concurrency}`);
         }
-        for (let i = 0; i < files.length; i += 1) {
-            const filePath = files[i];
-            totals.scanned += 1;
+        await runWithConcurrency(files, concurrency, async (filePath) => {
+            await withManifestLock(async () => {
+                totals.scanned += 1;
+            });
             try {
                 const absolutePath = node_path_1.default.join(repoPath, filePath);
                 const rawCodeSnapshot = await promises_1.default.readFile(absolutePath, "utf8");
                 const hash = (0, hashing_1.hashFileContent)(rawCodeSnapshot);
-                const summaryPath = getSummaryPath(summaryDir, hash);
-                if (!await fileExists(summaryPath)) {
-                    const summaryText = await (0, llm_1.generateFunctionalSummary)(filePath, rawCodeSnapshot, config.chat);
-                    await ensureSummaryAsset(summaryDir, hash, summaryText, rawCodeSnapshot, options.includeCodeSnapshot);
-                }
-                manifest.files[filePath] = hash;
-                refs.set(hash, (refs.get(hash) || 0) + 1);
-                await writeManifest(manifestPath, manifest);
-                totals.updated += 1;
-                if (!isJson) {
-                    console.log(`[${i + 1}/${totalFiles}] summarized ${filePath}`);
-                }
+                await ensureSummaryAssetForFile(filePath, hash, rawCodeSnapshot);
+                await withManifestLock(async () => {
+                    manifest.files[filePath] = hash;
+                    refs.set(hash, (refs.get(hash) || 0) + 1);
+                    await writeManifest(manifestPath, manifest);
+                    totals.updated += 1;
+                    completedFiles += 1;
+                    if (!isJson) {
+                        console.log(`[${completedFiles}/${totalFiles}] summarized ${filePath}`);
+                    }
+                });
             }
             catch (error) {
                 const message = error instanceof Error ? error.message : String(error);
-                failures.push({ filePath, message });
-                totals.failed += 1;
-                if (!isJson) {
-                    console.error(`[${i + 1}/${totalFiles}] failed ${filePath}: ${message}`);
-                }
+                await withManifestLock(async () => {
+                    failures.push({ filePath, message });
+                    totals.failed += 1;
+                    completedFiles += 1;
+                    if (!isJson) {
+                        console.error(`[${completedFiles}/${totalFiles}] failed ${filePath}: ${message}`);
+                    }
+                });
             }
-        }
+        });
     }
     else {
         const deltas = await (0, git_1.getGitDeltas)(repoPath, manifest.lastSyncedCommit);
         const totalCandidates = deltas.modifiedOrAdded.length + deltas.deleted.length;
+        let completedModified = 0;
         if (!isJson) {
             console.log(`Candidates: ${totalCandidates} (${deltas.modifiedOrAdded.length} modified/added, ${deltas.deleted.length} deleted)`);
+            console.log(`Concurrency: ${concurrency}`);
         }
         for (const deletedPath of deltas.deleted) {
             const removed = await removeManifestPath(deletedPath, manifest, manifestPath, summaryDir, refs);
@@ -331,73 +377,85 @@ async function runSummarize(options, config) {
                 console.log(`pruned ${deletedPath}`);
             }
         }
-        for (let i = 0; i < deltas.modifiedOrAdded.length; i += 1) {
-            const filePath = deltas.modifiedOrAdded[i];
-            totals.scanned += 1;
+        await runWithConcurrency(deltas.modifiedOrAdded, concurrency, async (filePath) => {
+            await withManifestLock(async () => {
+                totals.scanned += 1;
+            });
             try {
                 if (!shouldIncludeFile(filePath, includePatterns, excludePatterns, ignoreMatcher)) {
-                    const removed = await removeManifestPath(filePath, manifest, manifestPath, summaryDir, refs);
-                    if (removed) {
-                        totals.pruned += 1;
-                    }
-                    else {
-                        totals.skipped += 1;
-                    }
-                    if (!isJson) {
-                        console.log(`[${i + 1}/${deltas.modifiedOrAdded.length}] excluded ${filePath}`);
-                    }
-                    continue;
+                    await withManifestLock(async () => {
+                        const removed = await removeManifestPath(filePath, manifest, manifestPath, summaryDir, refs);
+                        if (removed) {
+                            totals.pruned += 1;
+                        }
+                        else {
+                            totals.skipped += 1;
+                        }
+                        completedModified += 1;
+                        if (!isJson) {
+                            console.log(`[${completedModified}/${deltas.modifiedOrAdded.length}] excluded ${filePath}`);
+                        }
+                    });
+                    return;
                 }
                 const previousHash = manifest.files[filePath];
                 const absolutePath = node_path_1.default.join(repoPath, filePath);
                 const rawCodeSnapshot = await promises_1.default.readFile(absolutePath, "utf8");
                 const hash = (0, hashing_1.hashFileContent)(rawCodeSnapshot);
                 if (previousHash === hash) {
-                    totals.skipped += 1;
-                    if (!isJson) {
-                        console.log(`[${i + 1}/${deltas.modifiedOrAdded.length}] unchanged ${filePath}`);
-                    }
-                    continue;
-                }
-                const summaryPath = getSummaryPath(summaryDir, hash);
-                if (!await fileExists(summaryPath)) {
-                    const summaryText = await (0, llm_1.generateFunctionalSummary)(filePath, rawCodeSnapshot, config.chat);
-                    await ensureSummaryAsset(summaryDir, hash, summaryText, rawCodeSnapshot, options.includeCodeSnapshot);
-                }
-                const changed = await setManifestPathHash(filePath, hash, manifest, manifestPath, summaryDir, refs);
-                if (changed) {
-                    totals.updated += 1;
-                }
-                else {
-                    totals.skipped += 1;
-                }
-                if (!isJson) {
-                    console.log(`[${i + 1}/${deltas.modifiedOrAdded.length}] updated ${filePath}`);
+                    await withManifestLock(async () => {
+                        totals.skipped += 1;
+                        completedModified += 1;
+                        if (!isJson) {
+                            console.log(`[${completedModified}/${deltas.modifiedOrAdded.length}] unchanged ${filePath}`);
+                        }
+                    });
+                    return;
                 }
-            }
-            catch (error) {
-                const nodeError = error;
-                if (nodeError.code === "ENOENT") {
-                    const removed = await removeManifestPath(filePath, manifest, manifestPath, summaryDir, refs);
-                    if (removed) {
-                        totals.pruned += 1;
+                await ensureSummaryAssetForFile(filePath, hash, rawCodeSnapshot);
+                await withManifestLock(async () => {
+                    const changed = await setManifestPathHash(filePath, hash, manifest, manifestPath, summaryDir, refs);
+                    if (changed) {
+                        totals.updated += 1;
                     }
                     else {
                         totals.skipped += 1;
                     }
+                    completedModified += 1;
                     if (!isJson) {
-                        console.log(`[${i + 1}/${deltas.modifiedOrAdded.length}] missing ${filePath}`);
+                        console.log(`[${completedModified}/${deltas.modifiedOrAdded.length}] updated ${filePath}`);
                     }
-                    continue;
+                });
+            }
+            catch (error) {
+                const nodeError = error;
+                if (nodeError.code === "ENOENT") {
+                    await withManifestLock(async () => {
+                        const removed = await removeManifestPath(filePath, manifest, manifestPath, summaryDir, refs);
+                        if (removed) {
+                            totals.pruned += 1;
+                        }
+                        else {
+                            totals.skipped += 1;
+                        }
+                        completedModified += 1;
+                        if (!isJson) {
+                            console.log(`[${completedModified}/${deltas.modifiedOrAdded.length}] missing ${filePath}`);
+                        }
+                    });
+                    return;
                 }
                 const message = error instanceof Error ? error.message : String(error);
-                failures.push({ filePath, message });
-                totals.failed += 1;
-                if (!isJson) {
-                    console.error(`[${i + 1}/${deltas.modifiedOrAdded.length}] failed ${filePath}: ${message}`);
-                }
+                await withManifestLock(async () => {
+                    failures.push({ filePath, message });
+                    totals.failed += 1;
+                    completedModified += 1;
+                    if (!isJson) {
+                        console.error(`[${completedModified}/${deltas.modifiedOrAdded.length}] failed ${filePath}: ${message}`);
+                    }
+                });
             }
-        }
+        });
     }
     manifest.lastSyncedCommit = await (0, git_1.getCurrentCommit)(repoPath);
     await writeManifest(manifestPath, manifest);
@@ -413,7 +471,7 @@ async function runSummarize(options, config) {
         finishedAt: finishedAt.toISOString(),
         durationMs,
         totals,
-        failures
+        failures: failures.sort((a, b) => a.filePath.localeCompare(b.filePath))
     };
     if (isJson) {
         console.log(JSON.stringify(report, null, 2));

package/dist/config.js

@@ -72,6 +72,7 @@ function buildRuntimeConfig(options, needs = { chat: true, embeddings: true }) {
     const includeGlobs = readListOption(mergedOptions.includeGlobs, "DIFFDOC_INCLUDE_GLOBS");
     const excludeGlobs = readListOption(mergedOptions.excludeGlobs, "DIFFDOC_EXCLUDE_GLOBS");
     const ignoreFile = readOption(mergedOptions.ignoreFile, "DIFFDOC_IGNORE_FILE", ".diffdocignore");
+    const summarizeConcurrency = readPositiveIntegerOption(mergedOptions.summarizeConcurrency, "DIFFDOC_SUMMARIZE_CONCURRENCY", 2);
     const chatBaseURL = provider === "cloud"
         ? readOption(mergedOptions.cloudLlmEndpoint, "CLOUD_LLM_ENDPOINT", "https://api.openai.com/v1")
         : readOption(mergedOptions.localLlmEndpoint, "LOCAL_LLM_ENDPOINT");
@@ -116,7 +117,8 @@ function buildRuntimeConfig(options, needs = { chat: true, embeddings: true }) {
         summarize: {
             includeGlobs,
             excludeGlobs,
-            ignoreFile
+            ignoreFile,
+            concurrency: summarizeConcurrency
         }
     };
 }

package/dist/index.js

@@ -42,7 +42,7 @@ function addCloudEndpointAndKeyOptions(command) {
 program
     .name("diffdoc")
     .description("Translate repository code shifts into plain-English business context")
-    .version("0.4.3");
+    .version("0.5.0");
 program
     .command("init")
     .description("Initialize DiffDoc configuration for this repository")
@@ -71,6 +71,7 @@ addChatOptions(addBaseOptions(program
     .option("--include-glob <pattern>", "include glob pattern (repeatable)", collectOption, [])
     .option("--exclude-glob <pattern>", "exclude glob pattern (repeatable)", collectOption, [])
     .option("--ignore-file <path>", "path to ignore pattern file relative to --path")
+    .option("--summarize-concurrency <count>", "number of files to summarize concurrently")
     .action(async (options) => {
     try {
         const config = (0, config_1.buildRuntimeConfig)(options, { chat: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "diffdoc",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "description": "Translate repository code shifts into plain-English business context",
   "license": "MIT",
   "author": "Christopher Sullivan",