@cohortapp/agent-sdk 2.16.0 → 2.18.4

package/lib/engine/tools/toolsearch.mjs

@@ -0,0 +1,217 @@
+/**
+ * lib/engine/tools/toolsearch.mjs — deferred tool loading and the ToolSearch tool.
+ *
+ * A run with many MCP tools pays for every definition on every turn. When the
+ * tool list is large — more than `DEFER_TOOL_COUNT` tools, or definitions worth
+ * more than `DEFER_WINDOW_SHARE` of the context window — MCP tools are
+ * DEFERRED: they stay callable, but their definitions are not sent. The model
+ * sees how many are held back (a line in the system prompt) and finds them with
+ * ToolSearch, which marks the matches loaded; their full definitions are sent
+ * from the next request on.
+ *
+ * Keeping the cached prefix stable: the sent list is the always-loaded tools
+ * in their fixed order, followed by loaded deferred tools IN THE ORDER THEY
+ * WERE LOADED. Nothing is ever unloaded or reordered (compaction included), so
+ * a load changes the request once — the cache is rewritten from the tool list
+ * on, one time — and every later request repeats the same bytes. ToolSearch
+ * results can name several tools at once, which keeps loads few.
+ *
+ * A deferred tool the model calls by name without searching runs (the guard
+ * still applies) and is loaded from then on.
+ *
+ * Authorship: the ToolSearch description was written for Cohort Engine by the
+ * Cohort team, 2026-09-14 (original text).
+ *
+ * @module lib/engine/tools/toolsearch
+ */
+
+import { canonicalJson } from "../context/cache.mjs";
+
+export const DEFER_TOOL_COUNT = 40;
+export const DEFER_WINDOW_SHARE = 0.12;
+export const DEFAULT_SEARCH_RESULTS = 5;
+export const MAX_SEARCH_RESULTS = 25;
+const DESCRIPTION_CHARS = 400;
+
+/**
+ * Should MCP tools be deferred? (pure)
+ * @param {{tools:Array<{name:string, description:string, input_schema:object, mcp?:unknown}>, contextWindow:number, charsPerToken?:number, maxTools?:number, maxShare?:number}} p
+ */
+export function shouldDeferTools({ tools, contextWindow, charsPerToken = 4, maxTools = DEFER_TOOL_COUNT, maxShare = DEFER_WINDOW_SHARE }) {
+  if (!tools.some((t) => t.mcp)) return false;
+  if (tools.length > maxTools) return true;
+  const chars = JSON.stringify(tools.map((t) => ({ name: t.name, description: t.description, input_schema: t.input_schema }))).length;
+  return chars / charsPerToken > contextWindow * maxShare;
+}
+
+/**
+ * @template {{name:string, description:string, input_schema:object, mcp?:any}} T
+ * @param {{tools:T[], isDeferrable?:(t:T)=>boolean}} p
+ */
+export function createDeferredTools({ tools, isDeferrable = (t) => Boolean(t.mcp) }) {
+  let all = [...tools];
+  /** @type {string[]} load order */
+  const loaded = [];
+  const deferrable = (/** @type T */ t) => isDeferrable(t);
+
+  const byName = () => new Map(all.map((t) => [t.name, t]));
+  return {
+    /** Every tool, callable. */
+    get all() {
+      return all;
+    },
+    isDeferred: (/** @type string */ name) => {
+      const t = byName().get(name);
+      return Boolean(t && deferrable(t));
+    },
+    isLoaded: (/** @type string */ name) => loaded.includes(name),
+    /** @returns {T[]} deferred tools not yet loaded */
+    pending: () => all.filter((t) => deferrable(t) && !loaded.includes(t.name)),
+    /** @returns {T[]} deferred tools, loaded or not */
+    deferred: () => all.filter((t) => deferrable(t)),
+    /**
+     * Mark tools loaded; returns the names that were newly loaded.
+     * @param {string[]} names
+     */
+    load(names) {
+      const map = byName();
+      const added = [];
+      for (const n of names) {
+        const t = map.get(n);
+        if (t && deferrable(t) && !loaded.includes(n)) {
+          loaded.push(n);
+          added.push(n);
+        }
+      }
+      return added;
+    },
+    /** The tools whose definitions are sent: fixed tools in order, then loaded ones in load order. @returns {T[]} */
+    active() {
+      const map = byName();
+      return [...all.filter((t) => !deferrable(t)), ...loaded.map((n) => map.get(n)).filter((t) => t !== undefined)];
+    },
+    loadedNames: () => [...loaded],
+    /** Replace the tool set (an MCP server's list changed). Loaded names that disappeared are dropped. @param {T[]} next */
+    replace(next) {
+      all = [...next];
+      const names = new Set(all.map((t) => t.name));
+      for (let i = loaded.length - 1; i >= 0; i--) if (!names.has(loaded[i])) loaded.splice(i, 1);
+    },
+    /** Server names that have deferred tools, sorted. */
+    servers() {
+      return [...new Set(all.filter((t) => deferrable(t)).map((t) => String(t.mcp?.server ?? "?")))].sort();
+    },
+  };
+}
+
+/**
+ * Parse a query: `select:a,b` (exact names), or words where `+word` must appear
+ * in the tool name.
+ * @param {string} query
+ * @returns {{kind:'select', names:string[]}|{kind:'search', required:string[], terms:string[]}}
+ */
+export function parseToolQuery(query) {
+  const q = String(query ?? "").trim();
+  if (/^select:/i.test(q)) return { kind: "select", names: q.slice(7).split(",").map((s) => s.trim()).filter(Boolean) };
+  const words = q.toLowerCase().split(/[\s,]+/).filter(Boolean);
+  return { kind: "search", required: words.filter((w) => w.startsWith("+") && w.length > 1).map((w) => w.slice(1)), terms: words.map((w) => w.replace(/^\+/, "")).filter(Boolean) };
+}
+
+/**
+ * Rank tools against a search query (pure).
+ * @template {{name:string, description:string}} T
+ * @param {T[]} candidates @param {string} query @param {number} max
+ * @returns {T[]}
+ */
+export function searchTools(candidates, query, max) {
+  const parsed = parseToolQuery(query);
+  if (parsed.kind === "select") {
+    const out = [];
+    for (const n of parsed.names) {
+      const t = candidates.find((c) => c.name === n) ?? candidates.find((c) => c.name.toLowerCase() === n.toLowerCase());
+      if (t && !out.includes(t)) out.push(t);
+    }
+    return out;
+  }
+  if (parsed.terms.length === 0) return [];
+  const scored = [];
+  for (const t of candidates) {
+    const name = t.name.toLowerCase();
+    if (!parsed.required.every((r) => name.includes(r))) continue;
+    const parts = name.split(/__|_|-|\./).filter(Boolean);
+    const desc = String(t.description ?? "").toLowerCase();
+    let score = 0;
+    for (const term of parsed.terms) {
+      if (name === term) score += 100;
+      if (parts.includes(term)) score += 10;
+      else if (name.includes(term)) score += 5;
+      if (desc.includes(term)) score += 2;
+    }
+    if (score > 0) scored.push({ t, score });
+  }
+  scored.sort((a, b) => b.score - a.score || (a.t.name < b.t.name ? -1 : a.t.name > b.t.name ? 1 : 0));
+  return scored.slice(0, max).map((s) => s.t);
+}
+
+/** One line per parameter. @param {any} schema */
+function describeParameters(schema) {
+  const props = schema && typeof schema.properties === "object" ? schema.properties : {};
+  const required = new Set(Array.isArray(schema?.required) ? schema.required : []);
+  const names = Object.keys(props);
+  if (names.length === 0) return "no parameters";
+  return names.map((n) => `${n} (${props[n]?.type ?? "any"}${required.has(n) ? ", required" : ""})`).join(", ");
+}
+
+/**
+ * The system-prompt note saying tools are held back (fixed for the run: it
+ * names the servers and the count at start, not what is loaded later).
+ * @param {ReturnType<typeof createDeferredTools>} state
+ */
+export function formatDeferredNote(state) {
+  const n = state.deferred().length;
+  if (n === 0) return null;
+  return `${n} more tools, from the MCP servers ${state.servers().join(", ")}, are available but their definitions are not loaded. Use ToolSearch to find and load the ones you need before calling them.`;
+}
+
+/**
+ * @param {ReturnType<typeof createDeferredTools>} state
+ * @returns {import('./index.mjs').Tool}
+ */
+export function createToolSearchTool(state) {
+  return {
+    name: "ToolSearch",
+    readOnly: true,
+    description:
+      "Find and load tools that are available in this run but whose definitions have not been sent to you yet (tools from MCP servers with many tools are held back to save context). " +
+      "Search with keywords that describe what you need, prefix a word with + to require it in the tool name, or pass `select:name1,name2` to load exact tool names. " +
+      "Matching tools are loaded, and you can call them from your next reply.",
+    input_schema: {
+      type: "object",
+      properties: {
+        query: { type: "string", description: 'Keywords, or "select:<tool name>[,<tool name>…]".' },
+        max_results: { type: "integer", minimum: 1, description: `How many matches to return (default ${DEFAULT_SEARCH_RESULTS}).` },
+      },
+      required: ["query"],
+    },
+    async run(input) {
+      const max = Math.min(MAX_SEARCH_RESULTS, input.max_results ?? DEFAULT_SEARCH_RESULTS);
+      const matches = searchTools(state.deferred(), input.query, max);
+      if (matches.length === 0) {
+        const pending = state.pending().length;
+        return {
+          ok: true,
+          content: `No held-back tool matches "${input.query}". ${pending} tool${pending === 1 ? " is" : "s are"} still unloaded, from: ${state.servers().join(", ") || "no servers"}. Try other words, or select:<exact tool name>.`,
+        };
+      }
+      const already = new Set(matches.filter((t) => state.isLoaded(t.name)).map((t) => t.name));
+      const added = state.load(matches.map((t) => t.name));
+      const lines = matches.map((t) => {
+        const d = String(t.description ?? "").replace(/\s+/g, " ").trim();
+        const desc = d.length > DESCRIPTION_CHARS ? `${d.slice(0, DESCRIPTION_CHARS - 1)}…` : d;
+        return `${t.name}${already.has(t.name) ? " (already loaded)" : ""}\n  ${desc || "(no description)"}\n  parameters: ${describeParameters(canonicalJson(t.input_schema))}`;
+      });
+      const head = added.length > 0 ? `Loaded ${added.length} tool${added.length === 1 ? "" : "s"}; ${added.length === 1 ? "it" : "they"} can be called from your next reply.` : "These tools are already loaded.";
+      return { ok: true, content: `${head}\n\n${lines.join("\n\n")}` };
+    },
+  };
+}

package/lib/engine/tools/walk.mjs

@@ -0,0 +1,193 @@
+/**
+ * lib/engine/tools/walk.mjs — glob matching, file walking, and the ripgrep seam.
+ *
+ * Glob and Grep prefer ripgrep when it is installed (fast, honours
+ * .gitignore) and fall back to a pure-JS walk when it is not. The two paths
+ * must agree on WHICH files match, so pattern matching is always ours:
+ * ripgrep only enumerates candidate files, and `globToRegExp` decides.
+ *
+ * Both paths skip dot-entries and `node_modules` (ripgrep skips hidden files by
+ * default and gets `node_modules` from RG_EXCLUDES, because a tree with no
+ * .gitignore would otherwise list it). Only ripgrep reads .gitignore. That
+ * difference is documented rather than hidden.
+ *
+ * @module lib/engine/tools/walk
+ */
+
+/** Exclusions ripgrep needs to agree with the fallback walk. */
+export const RG_EXCLUDES = Object.freeze(["--glob", "!**/node_modules/**"]);
+
+import { readdirSync, statSync } from "node:fs";
+import { spawn, spawnSync } from "node:child_process";
+import path from "node:path";
+
+/**
+ * Translate a glob to an anchored RegExp over POSIX-style relative paths.
+ * Supports `**`, `*`, `?`, `{a,b}` and `[...]` classes.
+ * @param {string} glob
+ * @returns {RegExp}
+ */
+export function globToRegExp(glob) {
+  let re = "";
+  let braceDepth = 0;
+  for (let i = 0; i < glob.length; i++) {
+    const c = glob[i];
+    if (c === "*") {
+      if (glob[i + 1] === "*") {
+        // `**/` matches zero or more directories; a bare `**` matches anything.
+        if (glob[i + 2] === "/") {
+          re += "(?:.*/)?";
+          i += 2;
+        } else {
+          re += ".*";
+          i += 1;
+        }
+      } else {
+        re += "[^/]*";
+      }
+    } else if (c === "?") re += "[^/]";
+    else if (c === "{") {
+      braceDepth++;
+      re += "(?:";
+    } else if (c === "}" && braceDepth > 0) {
+      braceDepth--;
+      re += ")";
+    } else if (c === "," && braceDepth > 0) re += "|";
+    else if (c === "[") {
+      const end = glob.indexOf("]", i + 1);
+      if (end === -1) re += "\\[";
+      else {
+        let cls = glob.slice(i + 1, end).replace(/\\/g, "\\\\");
+        if (cls.startsWith("!")) cls = "^" + cls.slice(1);
+        re += `[${cls}]`;
+        i = end;
+      }
+    } else if ("\\^$+.()|".includes(c)) re += "\\" + c;
+    else re += c;
+  }
+  return new RegExp(`^${re}$`);
+}
+
+/**
+ * Does a relative path match a glob? A glob with no `/` matches the basename
+ * at any depth (ripgrep `--glob` semantics); one with a `/` matches the whole
+ * relative path.
+ * @param {string} relPath POSIX separators
+ * @param {string} glob
+ */
+export function matchGlob(relPath, glob) {
+  const re = globToRegExp(glob);
+  if (!glob.includes("/")) return re.test(relPath.slice(relPath.lastIndexOf("/") + 1));
+  return re.test(relPath);
+}
+
+/**
+ * Pure-JS file enumeration under `root` (absolute paths). Bounded so a walk of
+ * `/` cannot hang the loop.
+ * @param {string} root
+ * @param {{ maxFiles?: number, signal?: AbortSignal }} [opts]
+ * @returns {string[]}
+ */
+export function walkFiles(root, opts = {}) {
+  const maxFiles = opts.maxFiles ?? 200_000;
+  const out = [];
+  const stack = [root];
+  while (stack.length > 0 && out.length < maxFiles) {
+    if (opts.signal?.aborted) break;
+    const dir = /** @type string */ (stack.pop());
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue; // unreadable directory (permissions, raced delete): skip it, keep walking
+    }
+    for (const e of entries) {
+      if (e.name.startsWith(".") || e.name === "node_modules") continue;
+      const p = path.join(dir, e.name);
+      if (e.isDirectory()) stack.push(p);
+      else if (e.isFile()) out.push(p);
+    }
+  }
+  return out;
+}
+
+let rgCache;
+/**
+ * Is ripgrep on PATH? Probed once per process. `COHORT_ENGINE_NO_RG=1` forces
+ * the fallback (tests, and machines with a broken rg).
+ * @param {NodeJS.ProcessEnv} [env]
+ */
+export function hasRipgrep(env = process.env) {
+  if (env.COHORT_ENGINE_NO_RG === "1") return false;
+  if (rgCache === undefined) {
+    try {
+      rgCache = spawnSync("rg", ["--version"], { stdio: "ignore" }).status === 0;
+    } catch {
+      rgCache = false; // spawn itself failed: treat as not installed
+    }
+  }
+  return rgCache;
+}
+
+/**
+ * Run ripgrep and collect stdout. Exit 1 means "no matches" and is success.
+ * @param {string[]} args
+ * @param {{ cwd: string, signal?: AbortSignal, maxBytes?: number }} opts
+ * @returns {Promise<{ ok:true, stdout:string, truncated:boolean } | { ok:false, error:string }>}
+ */
+export function runRipgrep(args, opts) {
+  const maxBytes = opts.maxBytes ?? 20_000_000;
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn("rg", args, { cwd: opts.cwd, stdio: ["ignore", "pipe", "pipe"], signal: opts.signal });
+    } catch (e) {
+      return resolve({ ok: false, error: `ripgrep failed to start: ${e instanceof Error ? e.message : e}` });
+    }
+    const chunks = [];
+    let size = 0;
+    let truncated = false;
+    let stderr = "";
+    child.stdout.on("data", (d) => {
+      if (size >= maxBytes) {
+        truncated = true;
+        child.kill();
+        return;
+      }
+      chunks.push(d);
+      size += d.length;
+    });
+    child.stderr.on("data", (d) => {
+      stderr += d.toString();
+    });
+    child.on("error", (e) => resolve({ ok: false, error: `ripgrep error: ${e.message}` }));
+    child.on("close", (code) => {
+      const stdout = Buffer.concat(chunks).toString("utf8");
+      if (code === 0 || code === 1 || truncated) resolve({ ok: true, stdout, truncated });
+      else resolve({ ok: false, error: stderr.trim() || `ripgrep exited ${code}` });
+    });
+  });
+}
+
+/**
+ * Sort absolute paths newest-first by mtime (ties by name, for determinism).
+ * @param {string[]} files
+ */
+export function sortByMtime(files) {
+  const withTime = files.map((f) => {
+    let t = 0;
+    try {
+      t = statSync(f).mtimeMs;
+    } catch {
+      /* vanished between listing and stat: sorts last */
+    }
+    return { f, t };
+  });
+  withTime.sort((a, b) => b.t - a.t || a.f.localeCompare(b.f));
+  return withTime.map((x) => x.f);
+}
+
+/** @param {string} p */
+export function toPosix(p) {
+  return p.split(path.sep).join("/");
+}

package/lib/engine/tools/web-switch.mjs

@@ -0,0 +1,31 @@
+/**
+ * lib/engine/tools/web-switch.mjs — whether a run offers the web tools.
+ *
+ * WebFetch and WebSearch are built-ins, on by default. They are also an
+ * outbound channel the outbound send gate does not screen: whatever a model
+ * puts in a URL or a search query leaves the machine (the SSRF guard protects
+ * internal hosts, not the data). A seat whose sends are gated can switch them
+ * off from the default tool set:
+ *
+ *   env       COHORT_ENGINE_WEB_TOOLS=0 (or false/off/no); 1/true/on/yes forces on
+ *   settings  cohort.webTools: false
+ *
+ * Environment wins over settings. An explicit `--tools` list that names a web
+ * tool still gets it: the switch shapes the default set, the invoker's list
+ * is deliberate. Deny rules (`WebFetch`, `WebSearch`) remove them regardless.
+ *
+ * @module lib/engine/tools/web-switch
+ */
+
+export const WEB_TOOL_NAMES = Object.freeze(["WebFetch", "WebSearch"]);
+
+/**
+ * @param {{env?:string, setting?:boolean}} p  COHORT_ENGINE_WEB_TOOLS and settings `cohort.webTools`
+ * @returns {boolean}
+ */
+export function webToolsEnabled({ env, setting }) {
+  const v = String(env ?? "").trim().toLowerCase();
+  if (["0", "false", "off", "no"].includes(v)) return false;
+  if (["1", "true", "on", "yes"].includes(v)) return true;
+  return setting !== false;
+}