@mcoda/mswarm 0.1.110 → 0.1.112

package/dist/vendor/codali/tools/docdex/DocdexTools.js

@@ -67,13 +67,146 @@ const hasWebFindings = (payload) => {
 const hasWebContent = (payload) => {
     if (!isRecord(payload))
         return false;
-    for (const key of ["web_context", "webContext", "hits", "results"]) {
+    // Fetched pages first. `hits` is not evidence of a successful fetch: with
+    // forceWeb it still carries local index matches, so judging by it alone
+    // reports success for a call that downloaded nothing.
+    const discovery = isRecord(payload.webDiscovery) ? payload.webDiscovery : undefined;
+    const fetches = Array.isArray(discovery?.fetches) ? discovery.fetches : [];
+    if (fetches.some((page) => isRecord(page) && (page.ai_digested_content || page.content))) {
+        return true;
+    }
+    for (const key of ["web_context", "webContext"]) {
         const value = payload[key];
         if (Array.isArray(value) && value.length > 0)
             return true;
     }
     return false;
 };
+/**
+ * Rewrites a path into the repo-relative form docdex accepts.
+ *
+ * Docdex answers `invalid path` for a leading slash, a `./` prefix, or an
+ * absolute path, and a model asked to list a repository reaches for exactly
+ * those. That took out every "which files are in X" question: the tool was
+ * chosen correctly and then rejected on a formatting detail. Normalising here
+ * is the adapter's job — the caller meant the same directory either way.
+ *
+ * Returns undefined for "the whole repository", which is what docdex wants when
+ * no path is given.
+ */
+export const toRepoRelativePath = (value, workspaceRoot) => {
+    if (typeof value !== "string")
+        return undefined;
+    let candidate = value.trim();
+    if (!candidate || candidate === "." || candidate === "./" || candidate === "/") {
+        return undefined;
+    }
+    if (workspaceRoot && candidate.startsWith(workspaceRoot)) {
+        candidate = candidate.slice(workspaceRoot.length);
+    }
+    candidate = candidate.replace(/^\.\//, "").replace(/^\/+/, "").replace(/\/{2,}/g, "/");
+    return candidate || undefined;
+};
+/** Pages to keep from a web research call, and how much of each. */
+const MAX_WEB_PAGES = 6;
+const MAX_WEB_PAGE_CHARS = 6000;
+/**
+ * Reduces a web-research response to the pages it actually fetched.
+ *
+ * The raw response is the wrong shape twice over. The fetched pages — the whole
+ * point of the call — are buried at `webDiscovery.fetches`, where the evidence
+ * normalizer does not look, while `hits` holds whatever the *local* index
+ * matched. For an external question those hits are repo source files, so a
+ * question about a central bank's interest rate came back citing eight files
+ * from this repository and none of the eight pages just downloaded.
+ *
+ * Suppressing local hits by asking docdex to skip its local search does not
+ * work: docdex indexes the pages it fetches and serves them back through that
+ * same path, so skipping it returns nothing at all. The response has to be
+ * reshaped here instead.
+ */
+const toWebResearchOutput = (payload) => {
+    if (!isRecord(payload))
+        return payload;
+    const discovery = isRecord(payload.webDiscovery) ? payload.webDiscovery : undefined;
+    const fetches = Array.isArray(discovery?.fetches) ? discovery.fetches : [];
+    if (fetches.length === 0)
+        return payload;
+    const pages = fetches
+        .filter(isRecord)
+        .filter((page) => typeof page.url === "string")
+        .sort((a, b) => Number(b.relevance_score ?? 0) - Number(a.relevance_score ?? 0))
+        .slice(0, MAX_WEB_PAGES)
+        .map((page) => {
+        // The digest is the model-written summary docdex already produced; it
+        // says more per character than the raw page and costs the run less.
+        const body = (typeof page.ai_digested_content === "string" && page.ai_digested_content) ||
+            (typeof page.content === "string" && page.content) ||
+            "";
+        return {
+            url: page.url,
+            content: body.slice(0, MAX_WEB_PAGE_CHARS),
+            ...(page.relevance_score !== undefined ? { relevanceScore: page.relevance_score } : {}),
+        };
+    })
+        .filter((page) => page.content.trim().length > 0);
+    if (pages.length === 0)
+        return payload;
+    return {
+        query: isRecord(discovery?.discovery) ? discovery.discovery.query : undefined,
+        pages,
+    };
+};
+/** Hits to re-snippet. Enough to cover the answer, few enough to stay quick. */
+const QUERY_SNIPPET_HITS = 3;
+const QUERY_SNIPPET_WINDOW = 8;
+/**
+ * Replaces each hit's snippet with the part of the file that matches the query.
+ *
+ * Search returns a window starting at line one, which for a code file is the
+ * import block. Asked for the value of a constant, a worker therefore received
+ * three files' worth of imports and reported that it could not find the value —
+ * correct about what it was given, useless as an answer. The follow-up tools
+ * exist (`docdex_open`, `docdex_symbols`) and the tool description recommends
+ * them, but a small model reliably searches once and stops, so the first result
+ * has to carry the content.
+ *
+ * Best-effort throughout: a failed re-snippet leaves the original hit alone,
+ * because a worse snippet is much better than a failed search.
+ */
+const withQueryCentredSnippets = async (client, query, payload) => {
+    if (!isRecord(payload))
+        return payload;
+    const key = ["hits", "results", "matches"].find((candidate) => Array.isArray(payload[candidate]));
+    if (!key)
+        return payload;
+    const hits = payload[key];
+    if (hits.length === 0)
+        return payload;
+    const enriched = await Promise.all(hits.map(async (hit, index) => {
+        if (index >= QUERY_SNIPPET_HITS || !isRecord(hit))
+            return hit;
+        const docId = hit.doc_id ?? hit.docId ?? hit.rel_path ?? hit.relPath;
+        if (typeof docId !== "string" || !docId)
+            return hit;
+        try {
+            const snippet = await client.openSnippet(docId, {
+                query,
+                window: QUERY_SNIPPET_WINDOW,
+            });
+            const text = isRecord(snippet) && isRecord(snippet.snippet)
+                ? snippet.snippet.text
+                : undefined;
+            if (typeof text !== "string" || text.trim().length === 0)
+                return hit;
+            return { ...hit, snippet: text, snippet_origin: "query_centred" };
+        }
+        catch {
+            return hit;
+        }
+    }));
+    return { ...payload, [key]: enriched };
+};
 const withDocdexMetadata = (tools) => tools.map((tool) => ({
     ...tool,
     readOnly: tool.readOnly ?? !DOCDEX_WRITE_TOOLS.has(tool.name),
@@ -111,9 +244,9 @@ const docdexToolDefinitions = (client, options) => [
         description: "Search the indexed repository and organizational memory for code, docs, and prior decisions. " +
             "Searches locally first and falls back to the web when the local index has nothing relevant, " +
             "so this is the default tool for both repo questions and general external questions. " +
-            "Returns file paths, doc ids, and a snippet from the START of each file — usually imports, " +
-            "not the part that answers the question. To explain what something does, follow up with " +
-            "docdex_symbols for its signature or docdex_open for the relevant lines.",
+            "Returns file paths, doc ids, and for the top results a snippet taken from the part of the " +
+            "file that matches the query. For more of a file, follow up with docdex_symbols for its " +
+            "signatures or docdex_open for a specific range of lines.",
         inputSchema: {
             type: "object",
             required: ["query"],
@@ -154,7 +287,7 @@ const docdexToolDefinitions = (client, options) => [
                     // simply found nothing. Return the local (empty) result instead.
                 }
             }
-            return toOutput(local);
+            return toOutput(await withQueryCentredSnippets(client, query, local));
         },
     },
     {
@@ -277,7 +410,7 @@ const docdexToolDefinitions = (client, options) => [
     },
     {
         name: "docdex_tree",
-        description: "Render the repository folder tree with build output and vendor directories excluded. Use to orient in an unfamiliar repo instead of listing directories manually.",
+        description: "List the files and folders in the repository, or under one directory, with build output and vendor directories excluded. This is the tool for \"which files are in X\" and \"what packages exist\" — a text search ranks files by content and will not enumerate them.",
         inputSchema: {
             type: "object",
             properties: {
@@ -288,10 +421,22 @@ const docdexToolDefinitions = (client, options) => [
                 extraExcludes: { type: "array", items: { type: "string" } },
             },
         },
-        handler: async (args) => {
+        handler: async (args, context) => {
             const { path, maxDepth, dirsOnly, includeHidden, extraExcludes } = args;
-            const result = await client.tree({ path, maxDepth, dirsOnly, includeHidden, extraExcludes });
-            return toOutput(result);
+            const scoped = toRepoRelativePath(path, context?.workspaceRoot);
+            const options = { maxDepth, dirsOnly, includeHidden, extraExcludes };
+            try {
+                return toOutput(await client.tree({ path: scoped, ...options }));
+            }
+            catch (error) {
+                // Docdex fails a directory that does not exist, and a model asked to
+                // list a repository guesses names — "mcoda", "monorepo", "root". The
+                // request was still "show me what is here", so answer that from the
+                // top rather than returning nothing.
+                if (!scoped)
+                    throw error;
+                return toOutput(await client.tree(options));
+            }
         },
     },
     {
@@ -396,7 +541,17 @@ const docdexToolDefinitions = (client, options) => [
         },
         handler: async (args) => {
             const { query, forceWeb, skipLocalSearch, webLimit, noCache } = args;
-            let result = await client.webResearch(query, { forceWeb, skipLocalSearch, webLimit, noCache });
+            // This tool means "go and look outside". Left to default, docdex searches
+            // the local index first and returns whatever it holds, so a question about
+            // a central bank's interest rate came back sourced to this repository's
+            // own documentation. Anything genuinely local is what `docdex_search` is
+            // for; a caller can still ask for the mixed behaviour explicitly.
+            let result = await client.webResearch(query, {
+                forceWeb: forceWeb ?? true,
+                skipLocalSearch,
+                webLimit,
+                noCache,
+            });
             // A cache hit returns discovery metadata with no page content at all —
             // no web_context, no hits — so the run sees that a search happened and
             // nothing it can quote. Retrying uncached fetches the pages the cache
@@ -404,7 +559,7 @@ const docdexToolDefinitions = (client, options) => [
             if (!noCache && !hasWebContent(result)) {
                 try {
                     const fresh = await client.webResearch(query, {
-                        forceWeb,
+                        forceWeb: forceWeb ?? true,
                         skipLocalSearch,
                         webLimit,
                         noCache: true,
@@ -417,7 +572,7 @@ const docdexToolDefinitions = (client, options) => [
                     // Keep the cached response rather than failing a search that worked.
                 }
             }
-            return toOutput(result);
+            return toOutput(toWebResearchOutput(result));
         },
     },
     {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mcoda/mswarm",
-  "version": "0.1.110",
+  "version": "0.1.112",
   "description": "Owner-run mswarm node for exposing local mcoda agents through the mswarm self-hosted gateway.",
   "type": "module",
   "main": "dist/server.js",
@@ -48,8 +48,8 @@
   "dependencies": {
     "fastify": "^5.7.4",
     "undici": "^6.28.0",
-    "@mcoda/db": "0.1.110",
-    "@mcoda/shared": "0.1.110"
+    "@mcoda/shared": "0.1.112",
+    "@mcoda/db": "0.1.112"
   },
   "scripts": {
     "build": "pnpm --filter @mcoda/shared run build && pnpm --filter @mcoda/codali run build && tsc -p tsconfig.json && node ../../scripts/copy-mswarm-codali-vendor.js",