@imenam/mcp-documentor 1.0.78 → 1.0.79

package/dist/src/core/concurrency.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Maps `items` with at most `limit` calls in flight, preserving input order in the result.
+ *
+ * Freshness work is almost entirely waiting on the network: one round trip per file that has to
+ * be read. Running those sequentially makes a whole-project scan take latency × file count, which
+ * is what pushes a large project past the MCP client's timeout. Order is preserved because the
+ * reports list files in the order the doc declares them.
+ *
+ * A rejection propagates once every worker has settled; callers that need per-item failures
+ * (freshness does) catch inside `fn` and return a value instead.
+ */
+export declare function mapWithConcurrency<T, R>(items: readonly T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;

package/dist/src/core/concurrency.js

@@ -0,0 +1,27 @@
+/**
+ * Maps `items` with at most `limit` calls in flight, preserving input order in the result.
+ *
+ * Freshness work is almost entirely waiting on the network: one round trip per file that has to
+ * be read. Running those sequentially makes a whole-project scan take latency × file count, which
+ * is what pushes a large project past the MCP client's timeout. Order is preserved because the
+ * reports list files in the order the doc declares them.
+ *
+ * A rejection propagates once every worker has settled; callers that need per-item failures
+ * (freshness does) catch inside `fn` and return a value instead.
+ */
+export async function mapWithConcurrency(items, limit, fn) {
+    if (items.length === 0)
+        return [];
+    const results = new Array(items.length);
+    let next = 0;
+    const worker = async () => {
+        while (true) {
+            const index = next++;
+            if (index >= items.length)
+                return;
+            results[index] = await fn(items[index], index);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
+    return results;
+}

package/dist/src/core/github-code-api.d.ts

@@ -52,8 +52,13 @@ export declare class GitHubCodeApi {
      * once within it, however many files that operation ends up reading.
      */
     resolveRef(): Promise<ResolvedRef>;
-    /** path → blob sha for every file in the commit's tree. */
+    /** path → blob sha for every file in the commit's tree. A commit's tree never changes. */
     getTree(commitSha: string): Promise<Map<string, string>>;
+    /**
+     * Memoises `compute` under `key`, sharing the in-flight promise with concurrent callers.
+     * A failure is evicted, so a transient network error does not poison the cache for good.
+     */
+    private cached;
     /** Whether the tree we hold for this commit is known to be incomplete. */
     isTreeTruncated(commitSha: string): boolean;
     /** Blob sha for one path at one commit, or undefined if the file does not exist there. */

package/dist/src/core/github-code-api.js

@@ -54,6 +54,9 @@ export class GitHubApiError extends Error {
 export class GitHubCodeApi {
     spec;
     token;
+    // In-flight requests are cached, not just their results: a whole-project scan checks many docs
+    // at once, and they all need the same tree — and often the same blobs. Caching the promise makes
+    // concurrent callers await one request instead of each firing its own.
     treeByCommit = new Map();
     truncatedCommits = new Set();
     blobCache = new Map();
@@ -113,25 +116,38 @@ export class GitHubCodeApi {
             };
         });
     }
-    /** path → blob sha for every file in the commit's tree. */
-    async getTree(commitSha) {
-        const cached = this.treeByCommit.get(commitSha);
-        if (cached)
-            return cached;
-        const tree = await this.request(`/repos/${this.spec.owner}/${this.spec.repo}/git/trees/${commitSha}?recursive=1`);
-        const map = new Map();
-        for (const entry of tree.tree ?? []) {
-            if (entry.type === "blob")
-                map.set(entry.path, entry.sha);
-        }
-        if (tree.truncated) {
-            // Past ~100k entries GitHub stops listing. Existence checks would then be wrong, so we
-            // remember it and fall back to a per-path lookup for anything absent from the partial tree.
-            this.truncatedCommits.add(commitSha);
-            logger.warn(`${this.label}: tree listing truncated at ${map.size} entries; falling back to per-path lookups`);
-        }
-        this.treeByCommit.set(commitSha, map);
-        return map;
+    /** path → blob sha for every file in the commit's tree. A commit's tree never changes. */
+    getTree(commitSha) {
+        return this.cached(this.treeByCommit, commitSha, async () => {
+            const tree = await this.request(`/repos/${this.spec.owner}/${this.spec.repo}/git/trees/${commitSha}?recursive=1`);
+            const map = new Map();
+            for (const entry of tree.tree ?? []) {
+                if (entry.type === "blob")
+                    map.set(entry.path, entry.sha);
+            }
+            if (tree.truncated) {
+                // Past ~100k entries GitHub stops listing. Existence checks would then be wrong, so we
+                // remember it and fall back to a per-path lookup for anything absent from the partial tree.
+                this.truncatedCommits.add(commitSha);
+                logger.warn(`${this.label}: tree listing truncated at ${map.size} entries; falling back to per-path lookups`);
+            }
+            return map;
+        });
+    }
+    /**
+     * Memoises `compute` under `key`, sharing the in-flight promise with concurrent callers.
+     * A failure is evicted, so a transient network error does not poison the cache for good.
+     */
+    cached(store, key, compute) {
+        const pending = store.get(key);
+        if (pending)
+            return pending;
+        const promise = compute().catch((err) => {
+            store.delete(key);
+            throw err;
+        });
+        store.set(key, promise);
+        return promise;
     }
     /** Whether the tree we hold for this commit is known to be incomplete. */
     isTreeTruncated(commitSha) {
@@ -154,14 +170,11 @@ export class GitHubCodeApi {
         }
     }
     /** A blob's bytes. Blobs are immutable, so this is cached for the life of the process. */
-    async getBlob(blobSha) {
-        const cached = this.blobCache.get(blobSha);
-        if (cached)
-            return cached;
-        const blob = await this.request(`/repos/${this.spec.owner}/${this.spec.repo}/git/blobs/${blobSha}`);
-        const bytes = Buffer.from(blob.content ?? "", blob.encoding === "base64" ? "base64" : "utf-8");
-        this.blobCache.set(blobSha, bytes);
-        return bytes;
+    getBlob(blobSha) {
+        return this.cached(this.blobCache, blobSha, async () => {
+            const blob = await this.request(`/repos/${this.spec.owner}/${this.spec.repo}/git/blobs/${blobSha}`);
+            return Buffer.from(blob.content ?? "", blob.encoding === "base64" ? "base64" : "utf-8");
+        });
     }
     /**
      * Unified diff of every file that changed between two commits, keyed by path.

package/dist/src/index.js

@@ -17,7 +17,8 @@ import { getDoc } from "./tools/get-doc.js";
 import { listDocs } from "./tools/list-docs.js";
 import { updateDoc } from "./tools/update-doc.js";
 import { searchDocs } from "./tools/search-docs.js";
-import { checkFreshness, computeFreshness } from "./tools/check-freshness.js";
+import { checkFreshness, computeFreshness, DOC_CONCURRENCY } from "./tools/check-freshness.js";
+import { mapWithConcurrency } from "./core/concurrency.js";
 import { findAndHashSection } from "./core/section-tracker.js";
 import { getCodeSource } from "./core/code-source.js";
 import { runInCodeOperation } from "./core/code-operation.js";
@@ -204,14 +205,14 @@ function requireProjectFromIPC(msg) {
 /** Loads one project's summaries and flags each entry as stale/broken. */
 async function listDocsWithFreshness(project) {
     const docs = await storageEngine.listDocs(project);
-    for (const doc of docs) {
+    await mapWithConcurrency(docs, DOC_CONCURRENCY, async (doc) => {
         const full = await storageEngine.getDoc(doc.id, project);
-        if (full && full.tracked_files.length > 0) {
-            const report = await computeFreshness(full.tracked_files, project);
-            doc.isStale = report.status === "stale" || report.status === "broken";
-            doc.isBroken = report.status === "broken";
-        }
-    }
+        if (!full || full.tracked_files.length === 0)
+            return;
+        const report = await computeFreshness(full.tracked_files, project);
+        doc.isStale = report.status === "stale" || report.status === "broken";
+        doc.isBroken = report.status === "broken";
+    });
     return docs;
 }
 /**

package/dist/src/tools/check-freshness.d.ts

@@ -1,5 +1,11 @@
 import type { TrackedFile, FreshnessReport } from "../../shared/schemas/doc.schema.js";
 import type { StorageEngine } from "../core/storage-engine.js";
+/**
+ * Docs checked at once during a whole-project scan. With FILE_CONCURRENCY this caps in-flight
+ * source reads at 32 — enough to hide network latency on a 160-doc project, far below anything
+ * GitHub objects to.
+ */
+export declare const DOC_CONCURRENCY = 8;
 export declare function computeFreshness(trackedFiles: TrackedFile[], project: string): Promise<FreshnessReport>;
 export declare function formatFreshnessReport(title: string, report: FreshnessReport): string;
 export declare function checkFreshness(storageEngine: StorageEngine, args: unknown): Promise<{

package/dist/src/tools/check-freshness.js

@@ -1,94 +1,74 @@
 import { CheckFreshnessArgsSchema } from "../../shared/schemas/doc.schema.js";
 import { findAndHashSection, fileContentMatches } from "../core/section-tracker.js";
 import { getCodeSource } from "../core/code-source.js";
+import { mapWithConcurrency } from "../core/concurrency.js";
 import { resolveDocInProject, isScopeError, scopeErrorResult } from "../core/project-scope.js";
+/** Files of one doc checked at once. Kept modest: docs are also scanned several at a time. */
+const FILE_CONCURRENCY = 4;
+/**
+ * Docs checked at once during a whole-project scan. With FILE_CONCURRENCY this caps in-flight
+ * source reads at 32 — enough to hide network latency on a 160-doc project, far below anything
+ * GitHub objects to.
+ */
+export const DOC_CONCURRENCY = 8;
+/** Compares one tracked file with the source, never throwing — the failure is part of the verdict. */
+async function checkTrackedFile(tracked, source) {
+    try {
+        // When the source can identify the content without transferring it and that identity is
+        // unchanged, the file is provably untouched — sections included — so nothing is read.
+        if (tracked.sourceBlobId && (await source.contentId(tracked.path)) === tracked.sourceBlobId) {
+            return { kind: "unchanged" };
+        }
+        const file = await source.read(tracked.path);
+        // A path that does not resolve at the source counts as absent, so a doc whose files are all
+        // gone ends up "unverifiable" rather than crying wolf with "stale".
+        if (!file)
+            return { kind: "missing" };
+        if (tracked.sections && tracked.sections.length > 0) {
+            for (const section of tracked.sections) {
+                const result = findAndHashSection(file.content, section);
+                if (!result.found)
+                    return { kind: "anchors_lost" };
+                if (!result.matchesStored)
+                    return { kind: "modified" };
+            }
+            return { kind: "unchanged" };
+        }
+        if (tracked.contentHash) {
+            return fileContentMatches(file.content, tracked.contentHash) ? { kind: "unchanged" } : { kind: "modified" };
+        }
+        // Hash comparison is the only freshness system. An entry with neither sections nor
+        // contentHash predates it and cannot be checked — it is reported as broken (never checked by
+        // any other means) so readers can still expose the doc and its tracked list for re-tracking.
+        return { kind: "broken" };
+    }
+    catch (err) {
+        // The source itself is unreachable (network, auth, rate limit). Nothing can be compared, and
+        // reporting what we did not manage to read as "modified" would be crying wolf.
+        return { kind: "source_error", message: err instanceof Error ? err.message : String(err) };
+    }
+}
 export async function computeFreshness(trackedFiles, project) {
     const source = getCodeSource(project);
     const empty = { modified: [], missing: [], unchanged: [], anchors_lost: [], broken: [], source: source.label };
     if (!trackedFiles || trackedFiles.length === 0) {
         return { status: "no_tracking", ...empty, total: 0 };
     }
+    const verdicts = await mapWithConcurrency(trackedFiles, FILE_CONCURRENCY, (tracked) => checkTrackedFile(tracked, source));
+    const failed = verdicts.find((verdict) => verdict.kind === "source_error");
+    if (failed) {
+        return { status: "unverifiable", ...empty, total: trackedFiles.length, sourceError: failed.message };
+    }
     const modified = [];
     const missing = [];
     const unchanged = [];
     const anchors_lost = [];
     const broken = [];
-    for (const tracked of trackedFiles) {
-        let file;
-        try {
-            // When the source can identify the content without transferring it and that identity is
-            // unchanged, the file is provably untouched — sections included — so nothing is read.
-            if (tracked.sourceBlobId && (await source.contentId(tracked.path)) === tracked.sourceBlobId) {
-                unchanged.push(tracked.path);
-                continue;
-            }
-            file = await source.read(tracked.path);
-        }
-        catch (err) {
-            // The source itself is unreachable (network, auth, rate limit). Nothing can be compared, and
-            // reporting what we did not manage to read as "modified" would be crying wolf.
-            const message = err instanceof Error ? err.message : String(err);
-            return {
-                status: "unverifiable",
-                modified: [],
-                missing: [],
-                unchanged: [],
-                anchors_lost: [],
-                broken: [],
-                total: trackedFiles.length,
-                source: source.label,
-                sourceError: message,
-            };
-        }
-        // A path that does not resolve at the source counts as absent, so we end up "unverifiable"
-        // rather than crying wolf with "stale".
-        if (!file) {
-            missing.push(tracked.path);
-            continue;
-        }
-        if (tracked.sections && tracked.sections.length > 0) {
-            let sectionStale = false;
-            let anchorLost = false;
-            for (const section of tracked.sections) {
-                const result = findAndHashSection(file.content, section);
-                if (!result.found) {
-                    sectionStale = true;
-                    anchorLost = true;
-                    break;
-                }
-                else if (!result.matchesStored) {
-                    sectionStale = true;
-                    break;
-                }
-            }
-            if (sectionStale) {
-                if (anchorLost) {
-                    anchors_lost.push(tracked.path);
-                }
-                else {
-                    modified.push(tracked.path);
-                }
-            }
-            else {
-                unchanged.push(tracked.path);
-            }
-        }
-        else if (tracked.contentHash) {
-            if (fileContentMatches(file.content, tracked.contentHash)) {
-                unchanged.push(tracked.path);
-            }
-            else {
-                modified.push(tracked.path);
-            }
-        }
-        else {
-            // Hash comparison is the only freshness system. An entry with neither
-            // sections nor contentHash predates it and cannot be checked — it is
-            // reported as broken (never checked by any other means) so readers can
-            // still expose the doc and its tracked list to the agent for re-tracking.
-            broken.push(tracked.path);
-        }
-    }
+    const buckets = { modified, missing, unchanged, anchors_lost, broken };
+    verdicts.forEach((verdict, i) => {
+        if (verdict.kind !== "source_error")
+            buckets[verdict.kind].push(trackedFiles[i].path);
+    });
     // When every tracked file is absent, the source code simply isn't present at the source
     // (e.g. docs cloned without the codebase, or a branch where those files don't exist) — we
     // can't verify freshness, so report "unverifiable" rather than crying wolf with "stale".
@@ -183,11 +163,14 @@ async function scanProjectFreshness(storageEngine, project) {
         unverifiable: 0,
         source: getCodeSource(project).label,
     };
-    for (const summary of summaries) {
+    const checked = await mapWithConcurrency(summaries, DOC_CONCURRENCY, async (summary) => {
         const doc = await storageEngine.getDoc(summary.id, project);
-        if (!doc)
+        return doc ? { doc, report: await computeFreshness(doc.tracked_files, project) } : undefined;
+    });
+    for (const entry of checked) {
+        if (!entry)
             continue;
-        const report = await computeFreshness(doc.tracked_files, project);
+        const { doc, report } = entry;
         if (report.sourceError)
             scan.sourceError = report.sourceError;
         if (report.status === "broken") {

package/dist/src/tools/list-docs.js

@@ -1,4 +1,19 @@
-import { computeFreshness } from "./check-freshness.js";
+import { computeFreshness, DOC_CONCURRENCY } from "./check-freshness.js";
+import { mapWithConcurrency } from "../core/concurrency.js";
+/**
+ * Fills in each summary's staleness flags, checking several docs at once — a listing is a
+ * whole-project scan, and doing it one doc at a time is what made large projects time out.
+ */
+async function flagStaleness(storageEngine, summaries, project) {
+    await mapWithConcurrency(summaries, DOC_CONCURRENCY, async (summary) => {
+        const doc = await storageEngine.getDoc(summary.id, project);
+        if (!doc || doc.tracked_files.length === 0)
+            return;
+        const report = await computeFreshness(doc.tracked_files, project);
+        summary.isStale = report.status === "stale";
+        summary.isBroken = report.status === "broken";
+    });
+}
 function renderProjectDocs(summaries, totalLabel) {
     if (summaries.length === 0)
         return "  (no entries)";
@@ -38,14 +53,7 @@ export async function listDocs(storageEngine, args) {
         if (!isMultiProject) {
             const proj = projectsToList[0];
             const summaries = await storageEngine.listDocs(proj);
-            for (const summary of summaries) {
-                const doc = await storageEngine.getDoc(summary.id, proj);
-                if (doc && doc.tracked_files.length > 0) {
-                    const report = await computeFreshness(doc.tracked_files, proj);
-                    summary.isStale = report.status === "stale";
-                    summary.isBroken = report.status === "broken";
-                }
-            }
+            await flagStaleness(storageEngine, summaries, proj);
             if (summaries.length === 0) {
                 return { content: [{ type: "text", text: `No documentation entries found in project "${proj}".` }] };
             }
@@ -57,14 +65,7 @@ export async function listDocs(storageEngine, args) {
         let grandTotal = 0;
         for (const proj of projectsToList) {
             const summaries = await storageEngine.listDocs(proj);
-            for (const summary of summaries) {
-                const doc = await storageEngine.getDoc(summary.id, proj);
-                if (doc && doc.tracked_files.length > 0) {
-                    const report = await computeFreshness(doc.tracked_files, proj);
-                    summary.isStale = report.status === "stale";
-                    summary.isBroken = report.status === "broken";
-                }
-            }
+            await flagStaleness(storageEngine, summaries, proj);
             grandTotal += summaries.length;
             lines.push(`\n🗂 ${proj} (${summaries.length})`);
             lines.push(renderProjectDocs(summaries, ""));

package/dist/src/version.d.ts

@@ -1 +1 @@
-export declare const APP_VERSION = "1.0.78";
+export declare const APP_VERSION = "1.0.79";

package/dist/src/version.js

@@ -1,2 +1,2 @@
 // Auto-generated version file - do not edit manually
-export const APP_VERSION = '1.0.78';
+export const APP_VERSION = '1.0.79';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@imenam/mcp-documentor",
-  "version": "1.0.78",
+  "version": "1.0.79",
   "main": "dist/src/index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",