loopctl-mcp-server 2.39.0 → 2.40.0

package/README.md

@@ -123,6 +123,10 @@ autonomous agent) can self-remediate without a human. Full agent-tenant lifecycl
 
 ## Tools (84)
 
+> Plus **per-tenant generated Context Retriever tools** (`cr_*`) appended
+> dynamically at runtime — see [Dynamic per-tenant Context Retriever
+> tools](#dynamic-per-tenant-context-retriever-tools-epic-30) below.
+
 ### Project Tools
 
 | Tool | Description |
@@ -280,6 +284,26 @@ it is enforced server-side and a no-op for a non-superadmin key — see below.)
 | `list_routes` | List all available API routes on the loopctl server. |
 | `get_system_articles` | List or fetch system-scoped (global, cross-tenant) wiki articles. Public — no auth required. Optional: `slug` (fetch one), `category`. |
 
+### Dynamic per-tenant Context Retriever tools (Epic 30)
+
+Beyond the static tools above, the server appends **per-tenant generated tools** to
+`ListTools` at runtime. When your tenant declares an **entity** (`POST
+/api/v1/entities`), loopctl auto-generates governed query tools over that entity's
+allowlisted columns, and this MCP server fetches them from `GET
+/api/v1/retrieve/tools` and lists them alongside the static tools:
+
+| Generated tool | What it does |
+|---|---|
+| `cr_filter_<entity>_by_<field>` | Filter that entity's records where `<field>` equals a value. Params: the field value + `limit`/`offset`. |
+| `cr_search_<entity>` | Full-text search across that entity's searchable text fields. Params: `query` + `limit`/`offset`. |
+
+These are **tenant-scoped**: the listing reflects only the tenant of the process
+key (`LOOPCTL_AGENT_KEY`), resolved server-side — you never pass a tenant. A
+`cr_`-prefixed call is dispatched generically to `POST /api/v1/retrieve/:entity`
+through the same authenticated + witness/STH path as every static read tool. If the
+`/retrieve/tools` fetch fails, listing degrades to the static tools (never errors).
+The generated-tool count per tenant is bounded by the per-tenant entity cap.
+
 ### Dispatch & Chain of Custody (v2) Tools
 
 Key distribution for the dispatch pattern (Epic 26): per-dispatch ephemeral keys and capability-token recovery. See `docs/chain-of-custody-v2.md`.

package/index.js

@@ -25,6 +25,10 @@ import {
   createWitnessClient,
   resolveSthStatePath,
 } from "./lib/witness-sth.js";
+import {
+  createGeneratedToolsRuntime,
+  GENERATED_TOOL_PREFIX,
+} from "./lib/generated-tools.js";
 
 // Single source of truth for the server version: the package.json this file
 // ships with (npm always includes package.json in the published tarball).
@@ -61,6 +65,11 @@ const SERVER_VERSION = JSON.parse(
 // transparent retry above). The write is atomic + symlink-safe (temp + rename).
 const WITNESS_FS = { readFileSync, writeFileSync, renameSync, lstatSync, unlinkSync };
 
+// Default per-request HTTP timeout. Individual calls may pass a SHORTER budget via
+// `apiCall(..., { timeoutMs })` — e.g. the init-time generated-tools listing fetch,
+// which must degrade to static tools quickly rather than block connection setup.
+const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+
 // One witness client PER API KEY (#298 review HIGH-2): the STH is per-tenant and
 // each key resolves to a tenant server-side, so distinct keys must NOT share an
 // in-memory cache or a state file (a collision causes spurious 409s + false
@@ -77,7 +86,7 @@ function witnessClientFor(apiKey) {
       fs: WITNESS_FS,
       getuid: typeof process.getuid === "function" ? () => process.getuid() : undefined,
       pid: process.pid,
-      timeoutMs: 30_000,
+      timeoutMs: DEFAULT_REQUEST_TIMEOUT_MS,
     });
     witnessClients.set(apiKey, client);
   }
@@ -104,7 +113,7 @@ function resolveKey(keyOverride) {
   );
 }
 
-async function apiCall(method, path, body, keyOverride, { exactKey = false } = {}) {
+async function apiCall(method, path, body, keyOverride, { exactKey = false, timeoutMs } = {}) {
   const url = `${getBaseUrl()}${path}`;
   // Secret-managing tools pass exactKey:true so the request uses the EXACT
   // role-pinned key (LOOPCTL_USER_KEY) and does NOT fall back to the global
@@ -137,10 +146,11 @@ async function apiCall(method, path, body, keyOverride, { exactKey = false } = {
   // attempt's Response; we keep body parsing / error shaping here.
   let response;
   try {
-    response = await witnessClientFor(key).send({ url, method, headers, serializedBody });
+    response = await witnessClientFor(key).send({ url, method, headers, serializedBody, timeoutMs });
   } catch (err) {
     if (err.name === "TimeoutError") {
-      return { error: true, status: 0, body: "Request timed out after 30s" };
+      const secs = Math.max(1, Math.round((timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS) / 1000));
+      return { error: true, status: 0, body: `Request timed out after ${secs}s` };
     }
     const cause = err.cause?.message ? ` (${err.cause.message})` : "";
     return { error: true, status: 0, body: `Network error: ${err.message}${cause}` };
@@ -4467,6 +4477,41 @@ const TOOLS = [
   },
 ];
 
+// ---------------------------------------------------------------------------
+// Context Retriever — dynamic per-tenant generated tools (US-30.5)
+// ---------------------------------------------------------------------------
+//
+// Epic 30 lets a tenant declare ENTITIES whose schema auto-generates agent tools
+// (`cr_filter_<entity>_by_<field>`, `cr_search_<entity>`) over governed loopctl
+// data. Those specs are generated SERVER-SIDE per tenant, so the MCP server can't
+// hard-code them in the static TOOLS array — it must fetch them at ListTools time
+// and append them, then dispatch any unknown `cr_`-prefixed CallTool to one
+// generic executor endpoint.
+//
+// Trust model: the stdio MCP process is ONE-TENANT-PER-PROCESS — the process key
+// (LOOPCTL_AGENT_KEY) resolves to a tenant server-side, and GET /retrieve/tools
+// returns ONLY that tenant's generated specs. The client never picks a tenant, so
+// cross-tenant listing/calling is impossible by construction (AC-30.5.3). Both the
+// listing fetch and the generic call ride the SAME `apiCall` path as static read
+// tools, so they carry identical auth + witness/STH headers (AC-30.5.4).
+
+// The generated-tools runtime (fetch + short-timeout/negative-TTL cache + generic
+// dispatch) lives in lib/generated-tools.js so its caching/TTL/negative-cache and
+// error-classification edge cases are exercised DIRECTLY by the test suite rather
+// than a hand-copied mirror. We wire the SHIPPED `apiCall` + static tool names +
+// `toContent` into it here. The process key (LOOPCTL_AGENT_KEY) is read per-call via
+// a getter so it rides the SAME auth + witness/STH path as every static read tool
+// (AC-30.5.4), and the static-name set powers the defense-in-depth drop of any spec
+// that isn't `cr_`-prefixed or that collides with a built-in tool name.
+const generatedToolsRuntime = createGeneratedToolsRuntime({
+  apiCall,
+  agentKey: () => process.env.LOOPCTL_AGENT_KEY,
+  staticToolNames: new Set(TOOLS.map((t) => t.name)),
+  toContent,
+});
+
+const { fetchGeneratedTools, callGeneratedTool } = generatedToolsRuntime;
+
 // ---------------------------------------------------------------------------
 // MCP Server
 // ---------------------------------------------------------------------------
@@ -4481,9 +4526,13 @@ const server = new Server(
   }
 );
 
-server.setRequestHandler(ListToolsRequestSchema, async () => ({
-  tools: TOOLS,
-}));
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+  // Static hand-maintained tools PLUS the calling tenant's per-tenant generated
+  // Context Retriever tools (US-30.5). fetchGeneratedTools degrades to the static
+  // tools (returns []/cache) on any fetch failure, so listing never errors.
+  const generated = await fetchGeneratedTools();
+  return { tools: [...TOOLS, ...generated] };
+});
 
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
   const { name, arguments: args } = request.params;
@@ -4753,6 +4802,13 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
       return await getAcceptanceCriteria(args);
 
     default:
+      // Per-tenant generated Context Retriever tools (US-30.5) are not in the
+      // static switch — dispatch any unknown `cr_`-prefixed name generically to
+      // the /retrieve/:entity executor. Static tools are handled above, so this
+      // never regresses them (AC-30.5.2).
+      if (typeof name === "string" && name.startsWith(GENERATED_TOOL_PREFIX)) {
+        return await callGeneratedTool(name, args);
+      }
       throw new Error(`Unknown tool: ${name}`);
   }
 });

package/lib/generated-tools.js

@@ -0,0 +1,209 @@
+// Per-tenant generated Context Retriever tool runtime (US-30.5).
+//
+// index.js is a stdio entry point with top-level await (importing it boots the MCP
+// server), so its handlers can't be imported by tests. This module extracts the
+// generated-tools fetch / cache / dispatch runtime — the logic with real caching,
+// TTL, negative-cache and error-classification edge cases — into a side-effect-free
+// factory (like lib/http-helpers.js). index.js wires the SHIPPED `apiCall`, static
+// tool names, and `toContent` into `createGeneratedToolsRuntime`, and the test suite
+// exercises the SAME code, so a regression fails CI instead of silently passing
+// against a hand-copied mirror.
+
+import {
+  retrieveToolsPath,
+  retrieveEntityPath,
+  specToMcpTool,
+  buildRetrieveBody,
+} from "./http-helpers.js";
+
+export const GENERATED_TOOL_PREFIX = "cr_";
+
+// ListTools is an init-time, agent-blocking call (Claude Code blocks on it during
+// connection setup). So the generated-tools fetch uses BOTH a SHORT timeout and a
+// NEGATIVE TTL: during any Context-Retriever backend slowness/outage the listing
+// degrades to the static tool surface quickly, and a failed fetch is cached for a
+// short window so subsequent ListTools calls do NOT each re-issue a blocking fetch
+// before degrading (us_30.5.json technical_notes: "keep it cached/short-timeout and
+// degrade to static on failure").
+export const GENERATED_TOOLS_CACHE_TTL_MS = 30_000; // positive: last fetch succeeded
+export const GENERATED_TOOLS_NEGATIVE_TTL_MS = 5_000; // negative: last fetch failed
+export const GENERATED_TOOLS_FETCH_TIMEOUT_MS = 5_000; // short per-fetch AbortSignal budget
+
+/**
+ * Build a generated-tools runtime bound to injected dependencies. State (cache +
+ * name->metadata map) is private to the returned closure.
+ *
+ * @param {{
+ *   apiCall: (method: string, path: string, body: unknown, key: (string|undefined), opts?: object) => Promise<any>,
+ *   agentKey: () => (string|undefined),
+ *   staticToolNames?: Set<string>,
+ *   toContent: (result: any) => object,
+ *   now?: () => number,
+ *   log?: (msg: string) => void,
+ *   cacheTtlMs?: number,
+ *   negativeTtlMs?: number,
+ *   fetchTimeoutMs?: number,
+ *   prefix?: string,
+ * }} deps
+ */
+export function createGeneratedToolsRuntime({
+  apiCall,
+  agentKey,
+  staticToolNames = new Set(),
+  toContent,
+  now = () => Date.now(),
+  log = (msg) => console.error(msg),
+  cacheTtlMs = GENERATED_TOOLS_CACHE_TTL_MS,
+  negativeTtlMs = GENERATED_TOOLS_NEGATIVE_TTL_MS,
+  fetchTimeoutMs = GENERATED_TOOLS_FETCH_TIMEOUT_MS,
+  prefix = GENERATED_TOOL_PREFIX,
+}) {
+  // `ok` records the LAST fetch outcome so (a) the TTL guard applies the negative
+  // TTL after a failure and (b) an unresolved CallTool can tell "temporarily
+  // unavailable" from "genuinely unknown to this tenant". `tools`/`metadata` retain
+  // the last KNOWN-GOOD listing so a failure degrades to it, never poisons it.
+  let cache = { tools: [], fetchedAt: 0, ok: true };
+  let metadataByName = new Map();
+
+  function cacheFailure(at, message) {
+    log(message);
+    // Negative caching: stamp fetchedAt on failure (ok:false) WITHOUT touching the
+    // retained tools/metadata, so subsequent calls within the negative-TTL window
+    // return the degraded static surface instead of re-issuing a blocking fetch.
+    cache = { tools: cache.tools, fetchedAt: at, ok: false };
+    return cache.tools;
+  }
+
+  /**
+   * Fetch the calling tenant's generated tool specs from GET /retrieve/tools and
+   * map them to the MCP `{ name, description, inputSchema }` shape, (re)populating
+   * the name->metadata map. On ANY failure this logs, negative-caches, and returns
+   * the last-known (possibly empty) tools so ListTools degrades to static rather
+   * than erroring (AC-30.5.1). Honors the positive/negative TTL unless `force`.
+   *
+   * @param {{ force?: boolean }} [opts]
+   * @returns {Promise<Array<{ name: string, description: string, inputSchema: object }>>}
+   */
+  async function fetchGeneratedTools({ force = false } = {}) {
+    const at = now();
+    if (!force && cache.fetchedAt !== 0) {
+      const ttl = cache.ok ? cacheTtlMs : negativeTtlMs;
+      if (at - cache.fetchedAt < ttl) return cache.tools;
+    }
+
+    let result;
+    try {
+      result = await apiCall("GET", retrieveToolsPath(), null, agentKey(), {
+        timeoutMs: fetchTimeoutMs,
+      });
+    } catch (err) {
+      return cacheFailure(
+        at,
+        `loopctl: failed to fetch generated tools, degrading to static tools: ${err.message}`,
+      );
+    }
+
+    if (result && result.error === true) {
+      const detail = typeof result.body === "string" ? result.body : JSON.stringify(result.body);
+      return cacheFailure(
+        at,
+        `loopctl: failed to fetch generated tools (HTTP ${result.status}), degrading to static tools: ${detail}`,
+      );
+    }
+
+    const specs = result && Array.isArray(result.data) ? result.data : [];
+    const tools = [];
+    const metadata = new Map();
+    for (const spec of specs) {
+      const tool = specToMcpTool(spec);
+      if (!tool) continue;
+      // Defense-in-depth (US-30.5 review): the `cr_` prefix is a security-relevant
+      // property the CallTool dispatcher relies on and the server is trusted to
+      // enforce. Re-check it HERE so a non-cr_ spec (which would be listed but
+      // UNCALLABLE) or one colliding with a STATIC tool name (which would shadow a
+      // trusted built-in's description/inputSchema under tenant-controlled text —
+      // a tool-confusion/description-spoofing vector) can never be listed or
+      // registered under a trusted identity.
+      if (!tool.name.startsWith(prefix) || staticToolNames.has(tool.name)) {
+        log(
+          `loopctl: dropping generated tool spec with invalid name "${tool.name}" ` +
+            `(must be ${prefix}-prefixed and not collide with a static tool)`,
+        );
+        continue;
+      }
+      tools.push(tool);
+      if (spec.metadata && typeof spec.metadata === "object") {
+        metadata.set(tool.name, spec.metadata);
+      }
+    }
+
+    metadataByName = metadata;
+    cache = { tools, fetchedAt: at, ok: true };
+    return tools;
+  }
+
+  /**
+   * Resolve a generated tool name to its STRUCTURED dispatch metadata. On a miss
+   * (name not in the current map) force ONE fresh fetch — bypassing the warm-TTL
+   * short-circuit — so a tool created server-side since the last listing resolves
+   * instead of being reported as authoritatively unknown.
+   *
+   * @param {string} name
+   * @returns {Promise<object|null>}
+   */
+  async function resolveGeneratedToolMetadata(name) {
+    if (metadataByName.has(name)) return metadataByName.get(name);
+    await fetchGeneratedTools({ force: true });
+    return metadataByName.get(name) || null;
+  }
+
+  /**
+   * Generic dispatcher for a per-tenant generated tool. Resolves (entity, field,
+   * operation) from STRUCTURED metadata (never by name-splitting — AC-30.5.2),
+   * builds the US-30.4 body, and POSTs to /retrieve/:entity via the SAME `apiCall`
+   * path as static reads (same auth + witness/STH — AC-30.5.4). An unresolvable
+   * name after a forced refetch distinguishes a TRANSIENT backend failure (503,
+   * retryable) from a name genuinely UNKNOWN to this tenant (404).
+   *
+   * @param {string} name
+   * @param {object} [args]
+   */
+  async function callGeneratedTool(name, args = {}) {
+    const metadata = await resolveGeneratedToolMetadata(name);
+    if (metadata && typeof metadata.entity === "string" && typeof metadata.operation === "string") {
+      const result = await apiCall(
+        "POST",
+        retrieveEntityPath(metadata.entity),
+        buildRetrieveBody(metadata, args),
+        agentKey(),
+      );
+      return toContent(result);
+    }
+
+    // Unresolved. If the forced refetch above FAILED (cache.ok === false), this is a
+    // transient backend blip, not a definitive "unknown to this tenant" — say so, so
+    // an agent retries instead of abandoning a valid tool.
+    if (!metadata && !cache.ok) {
+      return toContent({
+        error: true,
+        status: 503,
+        body:
+          `Generated tool "${name}" could not be resolved: the Context Retriever ` +
+          `tool listing is temporarily unavailable (the tools fetch failed). Retry shortly.`,
+      });
+    }
+    return toContent({
+      error: true,
+      status: 404,
+      body: `Unknown tool: ${name}. No generated tool by that name is available to this tenant.`,
+    });
+  }
+
+  return {
+    fetchGeneratedTools,
+    resolveGeneratedToolMetadata,
+    callGeneratedTool,
+    // Introspection seam for tests (not used by the server).
+    cacheState: () => ({ ...cache }),
+  };
+}

package/lib/http-helpers.js

@@ -86,6 +86,98 @@ export function memoryPath({ limit, offset, include_superseded, all_subjects } =
   ])}`;
 }
 
+/**
+ * Path for the Context Retriever generated-tool specs (US-30.5, US-30.4).
+ * The literal `/retrieve/tools` route is fixed and carries no query params — the
+ * calling tenant is resolved SERVER-SIDE from the API key, never from the client,
+ * so there is nothing for the client to parameterize here (AC-30.5.3/AC-30.5.4).
+ *
+ * @returns {string}
+ */
+export function retrieveToolsPath() {
+  return "/api/v1/retrieve/tools";
+}
+
+/**
+ * Path for executing a generated tool against a single entity (US-30.5, US-30.4).
+ * `entity` comes from the STRUCTURED metadata on a generated tool spec (never by
+ * splitting the tool name — entity/field names contain underscores), so it is
+ * encoded defensively.
+ *
+ * @param {string} entity
+ * @returns {string}
+ */
+export function retrieveEntityPath(entity) {
+  return `/api/v1/retrieve/${encodeURIComponent(entity)}`;
+}
+
+/**
+ * Map a raw generated-tool spec (as returned by GET /api/v1/retrieve/tools —
+ * `{ name, description, input_schema, metadata }`) into the MCP tool shape the
+ * ListTools handler must return (`{ name, description, inputSchema }`). The server
+ * emits snake_case `input_schema`; MCP expects camelCase `inputSchema`.
+ *
+ * Returns null for a malformed spec (no string `name`) so a single bad entry is
+ * skipped rather than corrupting the whole listing.
+ *
+ * @param {{ name?: unknown, description?: unknown, input_schema?: unknown }} spec
+ * @returns {{ name: string, description: string, inputSchema: object } | null}
+ */
+export function specToMcpTool(spec) {
+  if (!spec || typeof spec.name !== "string") return null;
+  return {
+    name: spec.name,
+    description: typeof spec.description === "string" ? spec.description : "",
+    inputSchema:
+      spec.input_schema && typeof spec.input_schema === "object"
+        ? spec.input_schema
+        : { type: "object", properties: {} },
+  };
+}
+
+/**
+ * Build the POST /api/v1/retrieve/:entity request body for a generated-tool call
+ * from the tool's STRUCTURED dispatch metadata (US-30.2 — `{ entity, field,
+ * operation }`) plus the caller's runtime `args`. The (entity, field, operation)
+ * come from metadata, NOT from splitting the tool name (AC-30.5.2), so entity and
+ * field names containing underscores dispatch unambiguously.
+ *
+ * The body shape matches the US-30.4 controller (`RetrieveRequest`): `op` selects
+ * the operation; `filter` carries `field` + `value`, `search` carries `query`;
+ * `limit`/`offset` pagination pass through when present. Nullish args are omitted
+ * so unset params never override server defaults.
+ *
+ * Filter-value sourcing (US-30.5 fix): the US-30.2 ToolGenerator emits the filter
+ * tool's input_schema with the value argument under the FIELD-NAME key (e.g.
+ * `{status}`, `required: ["status"]`) — there is NO `value` property. A
+ * schema-compliant agent therefore calls `cr_filter_project_by_status({status:
+ * "active"})`. So read the value from the field-named arg FIRST, falling back to a
+ * literal `value` arg (the shape the controller also accepts) for tolerance.
+ * Reading only `args.value` would drop every real agent-supplied filter value and
+ * dispatch an empty filter that silently returns zero rows.
+ *
+ * @param {{ entity: string, field?: string|null, operation: string }} metadata
+ * @param {Record<string, unknown>} [args]
+ * @returns {object}
+ */
+export function buildRetrieveBody(metadata, args = {}) {
+  const body = { op: metadata.operation };
+
+  if (metadata.operation === "filter") {
+    body.field = metadata.field;
+    const fieldArg = metadata.field != null ? args[metadata.field] : undefined;
+    const filterValue = fieldArg !== undefined ? fieldArg : args.value;
+    if (filterValue !== undefined) body.value = filterValue;
+  } else if (metadata.operation === "search") {
+    if (args.query !== undefined) body.query = args.query;
+  }
+
+  if (args.limit != null) body.limit = args.limit;
+  if (args.offset != null) body.offset = args.offset;
+
+  return body;
+}
+
 /**
  * Defensively parse the raw text body of a JSON-content-type HTTP response
  * (#249, mcp-03).

package/lib/witness-sth.js

@@ -186,6 +186,7 @@ export function bootstrapRetrySth(status, currentSthHeader) {
  *   getuid?: () => number,
  *   pid?: number,
  *   timeoutMs?: number,
+ *   makeTimeoutSignal?: (ms: number) => (AbortSignal|undefined),
  * }} [deps]
  */
 export function createWitnessClient({
@@ -195,6 +196,10 @@ export function createWitnessClient({
   getuid,
   pid,
   timeoutMs = 30_000,
+  // Injectable so tests can observe the EFFECTIVE per-attempt timeout (an
+  // AbortSignal from AbortSignal.timeout does not expose its ms). Production uses
+  // the real timer.
+  makeTimeoutSignal = (ms) => AbortSignal.timeout(ms),
 } = {}) {
   // Load any persisted STH so a FRESH process sends a real header on request #1
   // and skips the bootstrap-grace 412 entirely.
@@ -213,10 +218,13 @@ export function createWitnessClient({
     if (statePath && fs) persistSth(statePath, sth, { fs, pid });
   }
 
-  async function attempt({ url, method, headers, serializedBody }) {
+  async function attempt({ url, method, headers, serializedBody, timeoutMs: reqTimeoutMs }) {
     // Witness protocol: echo the cached STH, or (never seen one) request a
     // one-time bootstrap. A fresh AbortSignal per attempt so the retry gets its
-    // own timeout budget.
+    // own timeout budget. A per-request `timeoutMs` (e.g. the SHORT budget the
+    // init-time ListTools generated-tools fetch uses so a backend blip degrades to
+    // static quickly instead of blocking connection setup for the full 30s)
+    // overrides the client default.
     const withWitness = { ...headers };
     if (lastKnownSTH) withWitness["X-Loopctl-Last-Known-STH"] = lastKnownSTH;
     else withWitness["X-Loopctl-STH-Bootstrap"] = "true";
@@ -224,7 +232,7 @@ export function createWitnessClient({
     const options = {
       method,
       headers: withWitness,
-      signal: AbortSignal.timeout(timeoutMs),
+      signal: makeTimeoutSignal(reqTimeoutMs ?? timeoutMs),
     };
     if (serializedBody !== undefined) options.body = serializedBody;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "loopctl-mcp-server",
-  "version": "2.39.0",
+  "version": "2.40.0",
   "description": "MCP server for loopctl — structural trust for AI development loops",
   "type": "module",
   "main": "index.js",