@stigmer/runner 3.10.0 → 3.11.1-dev.20260812192248

package/src/shared/datastore-attachment.ts

@@ -22,8 +22,12 @@
  *     OSS callers resolve as the local principal (T05 R2).
  *
  * Approval-free by construction (DD-001 SD-3): empty toolApprovals +
- * pinnedToolApprovals mean mergeApprovalPolicies emits no entries for
- * this server — zero classifier involvement. `discoveredCapabilitiesEmpty`
+ * pinnedToolApprovals + toolApprovalOverrides mean mergeApprovalPolicies
+ * emits no entries for this server — zero classifier involvement, and no
+ * proto surface can target it (the attachment has no McpServerUsage, and
+ * since issue #349 an agent's overrides are scoped to their own usage's
+ * server, so a same-named override elsewhere cannot reach this one).
+ * `discoveredCapabilitiesEmpty`
  * is false and the attachment has no McpServerUsage, so the connect
  * backfill (whose destructiveHint tightener would force-gate
  * delete_record — silently skipped on channels under UNATTENDED mode)
@@ -63,6 +67,7 @@ export function synthesizeDatastoreAttachment(
     slug: DATASTORE_ATTACHMENT_SLUG,
     toolApprovals: [],
     pinnedToolApprovals: [],
+    toolApprovalOverrides: [],
     discoveredCapabilitiesEmpty: false,
   };
 
@@ -90,25 +95,115 @@ export function synthesizeDatastoreAttachment(
 }
 
 /**
- * The `<available_datastores>` prompt section (DD-005 SD-5, the
- * skills-section precedent): names the attached datastores and points
- * the model at describe_datastore first. Shared by both harnesses so
- * the section text cannot drift (the sender-identity precedent).
+ * The five record tools the records roster always serves (DD-005; the
+ * mcp-server's records roster registers all five unconditionally for the
+ * agent audience — only the `org` argument shape varies by audience, see
+ * mcp-server/src/domains/records/tools.ts). Because the roster never
+ * legitimately narrows, any of these missing from the connected toolset
+ * means the store is degraded (issue #325) — the prompt section and the
+ * reconciliation both derive from this one list so they cannot drift.
  */
-export function formatDatastoresSection(datastoreUsages: DatastoreUsage[]): string {
-  const slugs = datastoreUsages
+export const EXPECTED_RECORD_TOOLS = [
+  "describe_datastore",
+  "find_records",
+  "insert_record",
+  "update_record",
+  "delete_record",
+] as const;
+
+/**
+ * Reconcile the record tools actually connected against the roster
+ * contract (issue #325). Returns the expected tools absent from
+ * `actualToolNames` — empty means healthy. Names are bare tool names
+ * exactly as reported by tools/list (the mcp-enabled-tools identity
+ * space); extraneous names are ignored.
+ */
+export function missingRecordTools(actualToolNames: Iterable<string>): string[] {
+  const present = new Set(actualToolNames);
+  return EXPECTED_RECORD_TOOLS.filter((name) => !present.has(name));
+}
+
+/**
+ * The operator-facing degradation notice (issue #325): pushed as a
+ * MESSAGE_SYSTEM row on the execution status so "declared N datastores,
+ * X/5 record tools connected" is visible without reading a transcript.
+ * Lives HERE with the rest of the datastore-attachment wording — the
+ * harness only threads it.
+ */
+export function formatDatastoreDegradationNotice(
+  declaredCount: number,
+  missing: readonly string[],
+): string {
+  const connected = EXPECTED_RECORD_TOOLS.length - missing.length;
+  return (
+    `Datastore record tools unavailable this turn: declared ` +
+    `${declaredCount} datastore(s), ${connected}/${EXPECTED_RECORD_TOOLS.length} ` +
+    `record tools connected (missing: ${missing.join(", ")}). The agent has ` +
+    `been instructed to disclose this instead of answering from memory.`
+  );
+}
+
+/**
+ * The datastores prompt section (DD-005 SD-5, the skills-section
+ * precedent). Shared by both harnesses so the section text cannot drift
+ * (the sender-identity precedent). Two honest renderings (issue #325):
+ *
+ * - Healthy (`missingToolNames` empty or omitted — the Cursor harness
+ *   always calls it this way: the Cursor SDK connects MCP itself, so
+ *   that harness can never observe the live roster):
+ *   `<available_datastores>` names the attached datastores, points the
+ *   model at describe_datastore first, and carries a standing
+ *   failure-disclosure instruction — the only mechanism that covers
+ *   tools which connected but fail at call time (the WhatsApp-pilot
+ *   outage shape).
+ * - Degraded (deep-agent only, from missingRecordTools against the
+ *   connected roster): `<unavailable_datastores>` instead — the section
+ *   must not promise tools the agent does not have. Names the declared
+ *   datastores, states which record tools are missing, and instructs
+ *   plain disclosure over improvisation.
+ */
+export function formatDatastoresSection(
+  datastoreUsages: DatastoreUsage[],
+  missingToolNames: readonly string[] = [],
+): string {
+  const entries = datastoreUsages
     .map((u) => u.datastoreRef?.slug)
-    .filter((s): s is string => !!s);
-  const entries = slugs.map((slug) => `- ${slug}`);
+    .filter((s): s is string => !!s)
+    .map((slug) => `- ${slug}`);
+
+  if (missingToolNames.length > 0) {
+    return [
+      "<unavailable_datastores>",
+      "This agent is configured to use the datastores listed below, but the",
+      "following record tools failed to connect this turn:",
+      missingToolNames.join(", ") + ".",
+      "Treat these datastores as unreachable for any operation that needs a",
+      "missing tool.",
+      "",
+      ...entries,
+      "",
+      "When the user's request depends on one of these datastores, tell them",
+      "plainly that the datastore cannot be reached right now. Never answer",
+      "from memory, guess, or improvise a substitute for what the datastore",
+      "would have returned.",
+      "</unavailable_datastores>",
+    ].join("\n");
+  }
+
   return [
     "<available_datastores>",
     "You have access to the following datastores through the record tools",
-    "(describe_datastore, find_records, insert_record, update_record, delete_record).",
+    `(${EXPECTED_RECORD_TOOLS.join(", ")}).`,
     "Before your first operation against a datastore, call describe_datastore to",
     "learn its collections, field encodings, and which operations you are allowed",
     "to perform.",
     "",
     ...entries,
+    "",
+    "If any record tool is missing from your toolset or a record tool call",
+    "fails, the datastore is unreachable: tell the user plainly that you cannot",
+    "reach it right now, and do not answer from memory, guess, or improvise a",
+    "substitute for what it would have returned.",
     "</available_datastores>",
   ].join("\n");
 }

package/src/shared/grpc-retry.ts

@@ -6,7 +6,11 @@
  * `updateStatus` should back off and retry (transient transport errors) or
  * fail fast (deterministic errors). This module is intentionally just the
  * classification policy + its options type, kept small and separately tested;
- * the persist loop that consumes it lives with the rest of the persist logic.
+ * the persist loop that consumes it lives with the rest of the persist logic
+ * in status.ts — a single consumer whose loop is interwoven with persist
+ * semantics. (The HTTP twin, http-retry.ts, diverged there deliberately: with
+ * three fetch clients needing a byte-identical loop it hosts the shared
+ * fetchWithRetry too — see its module doc.)
  */
 
 import { ConnectError, Code } from "@connectrpc/connect";

package/src/shared/http-retry.ts

@@ -0,0 +1,139 @@
+/**
+ * HTTP error classification AND the shared bounded-backoff loop for
+ * side-channel proxy retries.
+ *
+ * The runner reaches stigmer-service's Side-Channel Proxy (`/v1/proxy/...`)
+ * over plain `fetch` from several clients: the checkpoint saver, artifact
+ * storage, and the model-registry fetches (stigmer/stigmer#468). This module
+ * is the one place that answers "is this HTTP failure worth retrying?" — the
+ * `fetch` twin of grpc-retry.ts — and, since #468, also owns the
+ * {@link fetchWithRetry} loop those clients share. That is a deliberate
+ * divergence from grpc-retry.ts's policy-only split: the gRPC side has a
+ * single consumer whose loop is interwoven with persist-specific logic
+ * (status.ts), while the HTTP side grew three consumers needing a
+ * byte-identical loop — per-client copies were the drift risk, not the seam.
+ *
+ * The policy is grounded in what the proxy actually emits, not HTTP folklore.
+ * CheckpointerProxyController returns exactly four deterministic errors —
+ * 400 (malformed document), 403 (FGA deny), 404 (no such checkpoint), and
+ * 413 (the 4 MB cap) — and never a transient status. Transient failures
+ * reach the runner only as 5xx (Spring's error handler, ingress 502-504),
+ * 408/429 (infrastructure between runner and service), or as network-level
+ * fetch rejections. 401/403 stay terminal deliberately: retrying cannot fix
+ * an expired or rejected credential. The same classification holds for the
+ * presigned R2 URLs artifact storage talks to: their deterministic failures
+ * (403 expired signature, 404 missing object) are 4xx, their transient ones
+ * are 5xx/network.
+ */
+
+/**
+ * True for HTTP statuses that signal a transient condition worth retrying:
+ * 408 (request timeout), 429 (throttled), and all 5xx. Every other status is
+ * deterministic — the proxy's 4xx errors mean the same request will fail the
+ * same way forever.
+ */
+export function isRetryableHttpStatus(status: number): boolean {
+  return status === 408 || status === 429 || status >= 500;
+}
+
+/**
+ * True for `fetch` rejections that signal a transient transport condition:
+ *
+ * - `TypeError` — undici's network-failure shape (`fetch failed`, with the
+ *   ECONNREFUSED/ECONNRESET/DNS cause attached). A deterministic TypeError
+ *   (e.g. a malformed URL) also matches, which is accepted: such a bug fails
+ *   every attempt including the first test run, and the bounded budget caps
+ *   the wasted retries at seconds.
+ * - `TimeoutError` — the rejection shape of `AbortSignal.timeout()`, i.e. a
+ *   hung connection whose attempt we bounded ourselves.
+ *
+ * A manual abort (`AbortError`) is deliberately NOT retryable: the caller
+ * cancelled on purpose.
+ */
+export function isRetryableFetchError(err: unknown): boolean {
+  if (err instanceof TypeError) return true;
+  return err instanceof Error && err.name === "TimeoutError";
+}
+
+/**
+ * A fully-resolved retry policy for {@link fetchWithRetry}. Every field is
+ * required on purpose: defaults differ per client (the checkpointer and the
+ * presign endpoints run 30 s requests, artifact transfers 120 s, registry
+ * fetches 10 s), so each client resolves its own defaults once and this
+ * module never guesses.
+ */
+export interface FetchRetryPolicy {
+  /** Client name for retry logs, e.g. "HttpCheckpointSaver". */
+  readonly label: string;
+  /** Delay before the first retry (ms); doubles per attempt via the factor. */
+  readonly baseDelayMs: number;
+  readonly backoffFactor: number;
+  /** Retry attempts after the initial one. */
+  readonly maxRetries: number;
+  /**
+   * Milliseconds before an in-flight request is aborted and the attempt is
+   * classified retryable. Without it a hung connection (the realistic
+   * unplanned-pod-kill symptom) stalls forever and the retry never engages.
+   */
+  readonly requestTimeoutMs: number;
+  /** Injectable delay for tests; defaults to a real setTimeout sleep. */
+  readonly delayFn?: (ms: number) => Promise<void>;
+}
+
+function defaultDelay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * `fetch` with bounded exponential backoff on transient failures and a
+ * per-request abort timeout.
+ *
+ * Composition contract: on a non-retryable status — or when the budget is
+ * exhausted on a retryable one — the last `Response` is RETURNED, so every
+ * call site keeps its own `resp.ok` / 404 handling unchanged; this wrapper
+ * changes when a response arrives, never what call sites do with it. It
+ * throws only what `fetch` itself throws: a network-level rejection that is
+ * non-retryable or has exhausted the budget.
+ */
+export async function fetchWithRetry(
+  url: string,
+  init: RequestInit | undefined,
+  policy: FetchRetryPolicy,
+): Promise<Response> {
+  const delay = policy.delayFn ?? defaultDelay;
+  for (let attempt = 0; ; attempt++) {
+    const canRetry = attempt < policy.maxRetries;
+    let resp: Response;
+    try {
+      resp = await fetch(url, {
+        ...init,
+        signal: AbortSignal.timeout(policy.requestTimeoutMs),
+      });
+    } catch (err) {
+      if (canRetry && isRetryableFetchError(err)) {
+        await backOff(policy, delay, attempt, String(err));
+        continue;
+      }
+      throw err;
+    }
+    if (canRetry && isRetryableHttpStatus(resp.status)) {
+      await backOff(policy, delay, attempt, `HTTP ${resp.status} ${resp.statusText}`);
+      continue;
+    }
+    return resp;
+  }
+}
+
+async function backOff(
+  policy: FetchRetryPolicy,
+  delay: (ms: number) => Promise<void>,
+  attempt: number,
+  reason: string,
+): Promise<void> {
+  const delayMs = policy.baseDelayMs * Math.pow(policy.backoffFactor, attempt);
+  console.warn(
+    `[${policy.label}] retryable failure ` +
+    `(attempt ${attempt + 1}/${policy.maxRetries + 1}, retry in ${delayMs}ms): ${reason}`,
+  );
+  await delay(delayMs);
+}