@stigmer/runner 3.9.0 → 3.11.0

package/src/shared/checkpointer/__tests__/http-saver.test.ts

@@ -29,9 +29,17 @@ async function serializeForProxy(obj: unknown): Promise<{ type: string; binary:
 describe("HttpCheckpointSaver", () => {
   let saver: HttpCheckpointSaver;
   let fetchSpy: ReturnType<typeof vi.fn>;
+  let recordedDelays: number[];
 
   beforeEach(() => {
-    saver = new HttpCheckpointSaver(PROXY, TOKEN);
+    // delayFn injected so retry-path tests record the backoff schedule
+    // instead of sleeping through it (~7.75 s of real delays otherwise).
+    recordedDelays = [];
+    saver = new HttpCheckpointSaver(PROXY, TOKEN, {
+      delayFn: async (ms) => {
+        recordedDelays.push(ms);
+      },
+    });
     fetchSpy = vi.fn();
     vi.stubGlobal("fetch", fetchSpy);
   });
@@ -163,6 +171,193 @@ describe("HttpCheckpointSaver", () => {
     });
   });
 
+  // The bounded-backoff loop (cloud#188). Classification policy itself is
+  // covered by shared/__tests__/http-retry.test.ts; these cases pin the
+  // loop's behavior at the saver's call sites: transient failures recover,
+  // deterministic failures never retry, budget exhaustion still fails
+  // loudly, and 404 semantics survive unchanged.
+  describe("retry behavior", () => {
+    const ok404 = { status: 404, ok: false, statusText: "Not Found" };
+    const err503 = { status: 503, ok: false, statusText: "Service Unavailable" };
+    const err400 = { status: 400, ok: false, statusText: "Bad Request" };
+    const err401 = { status: 401, ok: false, statusText: "Unauthorized" };
+
+    function makeCheckpoint(): Checkpoint {
+      return {
+        v: 4,
+        id: "cp-retry",
+        ts: new Date().toISOString(),
+        channel_values: {},
+        channel_versions: {},
+        versions_seen: {},
+      };
+    }
+    const md: CheckpointMetadata = { source: "loop", step: 0, parents: {} };
+
+    it("recovers a put from a transient 503", async () => {
+      fetchSpy
+        .mockResolvedValueOnce(err503)
+        .mockResolvedValueOnce({ ok: true, json: () => Promise.resolve({}) });
+
+      const result = await saver.put(makeConfig(), makeCheckpoint(), md, {});
+      expect(result.configurable?.checkpoint_id).toBe("cp-retry");
+      expect(fetchSpy).toHaveBeenCalledTimes(2);
+      expect(recordedDelays).toEqual([250]);
+    });
+
+    it("recovers a put from a network-level rejection (undici TypeError)", async () => {
+      fetchSpy
+        .mockRejectedValueOnce(new TypeError("fetch failed"))
+        .mockResolvedValueOnce({ ok: true, json: () => Promise.resolve({}) });
+
+      await expect(saver.put(makeConfig(), makeCheckpoint(), md, {})).resolves.toBeDefined();
+      expect(fetchSpy).toHaveBeenCalledTimes(2);
+    });
+
+    it("backs off exponentially across consecutive transient failures", async () => {
+      fetchSpy
+        .mockResolvedValueOnce(err503)
+        .mockResolvedValueOnce(err503)
+        .mockResolvedValueOnce(err503)
+        .mockResolvedValueOnce({ ok: true, json: () => Promise.resolve({}) });
+
+      await saver.put(makeConfig(), makeCheckpoint(), md, {});
+      expect(recordedDelays).toEqual([250, 500, 1000]);
+    });
+
+    it("fails loudly when the budget is exhausted on a retryable status", async () => {
+      fetchSpy.mockResolvedValue(err503);
+
+      await expect(saver.put(makeConfig(), makeCheckpoint(), md, {}))
+        .rejects.toThrow("Checkpoint PUT failed: 503");
+      // 1 initial attempt + 5 retries (the default budget).
+      expect(fetchSpy).toHaveBeenCalledTimes(6);
+      expect(recordedDelays).toEqual([250, 500, 1000, 2000, 4000]);
+    });
+
+    it("rethrows the original error when the budget is exhausted on network rejections", async () => {
+      const netErr = new TypeError("fetch failed");
+      fetchSpy.mockRejectedValue(netErr);
+
+      await expect(saver.putWrites(makeConfig(), [["messages", { a: 1 }]], "t1"))
+        .rejects.toBe(netErr);
+      expect(fetchSpy).toHaveBeenCalledTimes(6);
+    });
+
+    it("never retries a deterministic 400", async () => {
+      fetchSpy.mockResolvedValue(err400);
+
+      await expect(saver.put(makeConfig(), makeCheckpoint(), md, {}))
+        .rejects.toThrow("Checkpoint PUT failed: 400");
+      expect(fetchSpy).toHaveBeenCalledTimes(1);
+      expect(recordedDelays).toEqual([]);
+    });
+
+    it("never retries a rejected credential (401)", async () => {
+      fetchSpy.mockResolvedValue(err401);
+
+      await expect(saver.getTuple(makeConfig())).rejects.toThrow("Checkpoint GET failed: 401");
+      expect(fetchSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it("never retries a manual abort (AbortError)", async () => {
+      const abortErr = new DOMException("This operation was aborted", "AbortError");
+      fetchSpy.mockRejectedValue(abortErr);
+
+      await expect(saver.getTuple(makeConfig())).rejects.toBe(abortErr);
+      expect(fetchSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it("retries a hung request aborted by the per-request timeout (TimeoutError)", async () => {
+      fetchSpy
+        .mockRejectedValueOnce(
+          new DOMException("The operation was aborted due to timeout", "TimeoutError"),
+        )
+        .mockResolvedValueOnce(ok404);
+
+      await expect(saver.getTuple(makeConfig())).resolves.toBeUndefined();
+      expect(fetchSpy).toHaveBeenCalledTimes(2);
+    });
+
+    it("preserves getTuple's 404 semantics without retrying", async () => {
+      fetchSpy.mockResolvedValue(ok404);
+
+      await expect(saver.getTuple(makeConfig())).resolves.toBeUndefined();
+      expect(fetchSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it("preserves deleteThread's 404 tolerance without retrying", async () => {
+      fetchSpy.mockResolvedValue(ok404);
+
+      await expect(saver.deleteThread("thread-1")).resolves.toBeUndefined();
+      expect(fetchSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it("bounds every request with an abort signal", async () => {
+      fetchSpy.mockResolvedValue(ok404);
+
+      await saver.getTuple(makeConfig());
+      const [, opts] = fetchSpy.mock.calls[0];
+      expect(opts.signal).toBeInstanceOf(AbortSignal);
+    });
+
+    it("surfaces a pending-writes read failure instead of silently degrading", async () => {
+      // The checkpoint GET succeeds; the follow-up writes GET fails on every
+      // attempt. Pre-#188 code mapped this to "no pending writes" and
+      // returned a tuple stripped of resume state.
+      const cp = makeCheckpoint();
+      const cpSer = await serializeForProxy(cp);
+      const mdSer = await serializeForProxy(md);
+      const checkpointDoc = {
+        thread_id: "thread-1",
+        checkpoint_ns: "",
+        checkpoint_id: cp.id,
+        type: cpSer.type,
+        checkpoint: cpSer.binary,
+        metadata_type: mdSer.type,
+        metadata: mdSer.binary,
+      };
+      fetchSpy.mockImplementation((url: string) => {
+        if (url.includes("/writes")) return Promise.resolve(err503);
+        return Promise.resolve({ ok: true, json: () => Promise.resolve(checkpointDoc) });
+      });
+
+      await expect(saver.getTuple(makeConfig())).rejects.toThrow(
+        "Checkpoint writes GET failed: 503",
+      );
+    });
+
+    it("recovers the pending-writes read from a transient failure", async () => {
+      const cp = makeCheckpoint();
+      const cpSer = await serializeForProxy(cp);
+      const mdSer = await serializeForProxy(md);
+      const checkpointDoc = {
+        thread_id: "thread-1",
+        checkpoint_ns: "",
+        checkpoint_id: cp.id,
+        type: cpSer.type,
+        checkpoint: cpSer.binary,
+        metadata_type: mdSer.type,
+        metadata: mdSer.binary,
+      };
+      let writesCalls = 0;
+      fetchSpy.mockImplementation((url: string) => {
+        if (url.includes("/writes")) {
+          writesCalls++;
+          return writesCalls === 1
+            ? Promise.resolve(err503)
+            : Promise.resolve({ ok: true, json: () => Promise.resolve({ writes: [] }) });
+        }
+        return Promise.resolve({ ok: true, json: () => Promise.resolve(checkpointDoc) });
+      });
+
+      const tuple = await saver.getTuple(makeConfig());
+      expect(tuple?.checkpoint.id).toBe(cp.id);
+      expect(tuple?.pendingWrites).toEqual([]);
+      expect(writesCalls).toBe(2);
+    });
+  });
+
   describe("URL construction", () => {
     it("strips trailing slashes from proxy endpoint", () => {
       const s = new HttpCheckpointSaver("https://proxy.test///", "tok");

package/src/shared/checkpointer/http-saver.ts

@@ -13,6 +13,19 @@
  *
  * The Java proxy calls Document.parse(json) which handles $binary
  * natively, and doc.toJson() emits the same format on reads.
+ *
+ * Every request runs through {@link HttpCheckpointSaver.fetchWithRetry}:
+ * bounded exponential backoff on transient failures (classified by
+ * http-retry.ts) with a per-request abort timeout. This loop is the ONLY
+ * retry layer between an agent execution and its checkpoints — the deep
+ * agent activity is deliberately non-retryable at the Temporal level
+ * (maximumAttempts: 1; replaying a whole agent run is not safe), so before
+ * this existed a single dropped request killed the execution
+ * (stigmer/stigmer-cloud#188). Retrying puts is safe by server contract:
+ * CheckpointStore documents putCheckpoint/putWrite as keyed save-or-replace
+ * upserts. Budget exhaustion still fails loudly — checkpoints are not
+ * lossy-tolerable (unlike status updates, see status.ts), and silently
+ * dropping one would corrupt resume state.
  */
 
 import type { RunnableConfig } from "@langchain/core/runnables";
@@ -25,6 +38,8 @@ import {
   type ChannelVersions,
 } from "@langchain/langgraph-checkpoint";
 import type { CheckpointMetadata, PendingWrite } from "@langchain/langgraph-checkpoint";
+import type { RetryOptions } from "../grpc-retry.js";
+import { isRetryableFetchError, isRetryableHttpStatus } from "../http-retry.js";
 
 function encodeB64(data: Uint8Array): string {
   if (typeof Buffer !== "undefined") {
@@ -77,17 +92,104 @@ function configOrg(config: RunnableConfig): string | undefined {
   return config.configurable?.org as string | undefined;
 }
 
+/** Retry policy plus the per-request bound; every field is defaulted. */
+export interface HttpCheckpointSaverOptions extends RetryOptions {
+  /**
+   * Milliseconds before an in-flight request is aborted and the attempt is
+   * classified retryable. Without it a hung connection (the realistic
+   * unplanned-pod-kill symptom) stalls forever and the retry never engages.
+   */
+  readonly requestTimeoutMs?: number;
+}
+
+/**
+ * Default transient-retry policy. Wider than status.ts's DEFAULT_PERSIST_RETRY
+ * (3 retries, ~700 ms) on purpose: a dropped status update is lossy-tolerable,
+ * a dropped checkpoint kills the execution, so the budget here is sized to
+ * span a pod-restart reroute (~7.75 s of delays across 5 retries). Worst case
+ * with every attempt hanging — 6 x 30 s + delays, ~3.7 min — stays well inside
+ * the deep-agent activity's 1 h startToCloseTimeout (no heartbeatTimeout).
+ */
+const DEFAULT_RETRY = {
+  baseDelayMs: 250,
+  backoffFactor: 2,
+  maxRetries: 5,
+  requestTimeoutMs: 30_000,
+} as const;
+
+function defaultDelay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 export class HttpCheckpointSaver extends BaseCheckpointSaver {
   private readonly baseUrl: string;
   private readonly headers: Record<string, string>;
-
-  constructor(proxyEndpoint: string, authToken: string) {
+  private readonly baseDelayMs: number;
+  private readonly backoffFactor: number;
+  private readonly maxRetries: number;
+  private readonly requestTimeoutMs: number;
+  private readonly delay: (ms: number) => Promise<void>;
+
+  constructor(
+    proxyEndpoint: string,
+    authToken: string,
+    options: HttpCheckpointSaverOptions = {},
+  ) {
     super();
     this.baseUrl = `${proxyEndpoint.replace(/\/+$/, "")}/v1/proxy/checkpoints`;
     this.headers = {
       Authorization: `Bearer ${authToken}`,
       "Content-Type": "application/json",
     };
+    this.baseDelayMs = options.baseDelayMs ?? DEFAULT_RETRY.baseDelayMs;
+    this.backoffFactor = options.backoffFactor ?? DEFAULT_RETRY.backoffFactor;
+    this.maxRetries = options.maxRetries ?? DEFAULT_RETRY.maxRetries;
+    this.requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_RETRY.requestTimeoutMs;
+    this.delay = options.delayFn ?? defaultDelay;
+  }
+
+  /**
+   * `fetch` with bounded exponential backoff on transient failures and a
+   * per-request abort timeout.
+   *
+   * Composition contract: on a non-retryable status — or when the budget is
+   * exhausted on a retryable one — the last `Response` is RETURNED, so every
+   * call site keeps its own `resp.ok` / 404 handling unchanged; this wrapper
+   * changes when a response arrives, never what call sites do with it. It
+   * throws only what `fetch` itself throws: a network-level rejection that is
+   * non-retryable or has exhausted the budget.
+   */
+  private async fetchWithRetry(url: string, init?: RequestInit): Promise<Response> {
+    for (let attempt = 0; ; attempt++) {
+      const canRetry = attempt < this.maxRetries;
+      let resp: Response;
+      try {
+        resp = await fetch(url, {
+          ...init,
+          signal: AbortSignal.timeout(this.requestTimeoutMs),
+        });
+      } catch (err) {
+        if (canRetry && isRetryableFetchError(err)) {
+          await this.backOff(attempt, String(err));
+          continue;
+        }
+        throw err;
+      }
+      if (canRetry && isRetryableHttpStatus(resp.status)) {
+        await this.backOff(attempt, `HTTP ${resp.status} ${resp.statusText}`);
+        continue;
+      }
+      return resp;
+    }
+  }
+
+  private async backOff(attempt: number, reason: string): Promise<void> {
+    const delayMs = this.baseDelayMs * Math.pow(this.backoffFactor, attempt);
+    console.warn(
+      `[HttpCheckpointSaver] retryable failure ` +
+      `(attempt ${attempt + 1}/${this.maxRetries + 1}, retry in ${delayMs}ms): ${reason}`,
+    );
+    await this.delay(delayMs);
   }
 
   private async serializeTyped(obj: unknown): Promise<[string, BinaryObj]> {
@@ -113,7 +215,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
       params.set("checkpoint_id", checkpointId);
     }
 
-    const resp = await fetch(`${this.baseUrl}/checkpoint?${params}`, {
+    const resp = await this.fetchWithRetry(`${this.baseUrl}/checkpoint?${params}`, {
       headers: this.headers,
     });
     if (resp.status === 404) return undefined;
@@ -143,7 +245,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
       params.set("before", beforeId);
     }
 
-    const resp = await fetch(`${this.baseUrl}/checkpoints?${params}`, {
+    const resp = await this.fetchWithRetry(`${this.baseUrl}/checkpoints?${params}`, {
       headers: this.headers,
     });
     if (!resp.ok) {
@@ -183,7 +285,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
     const orgId = configOrg(config);
     if (orgId) doc.org_id = orgId;
 
-    const resp = await fetch(`${this.baseUrl}/checkpoint`, {
+    const resp = await this.fetchWithRetry(`${this.baseUrl}/checkpoint`, {
       method: "PUT",
       headers: this.headers,
       body: JSON.stringify(doc),
@@ -203,7 +305,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
 
   async deleteThread(threadId: string): Promise<void> {
     const params = new URLSearchParams({ thread_id: threadId });
-    const resp = await fetch(`${this.baseUrl}/thread?${params}`, {
+    const resp = await this.fetchWithRetry(`${this.baseUrl}/thread?${params}`, {
       method: "DELETE",
       headers: this.headers,
     });
@@ -238,7 +340,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
       return doc;
     }));
 
-    const resp = await fetch(`${this.baseUrl}/writes`, {
+    const resp = await this.fetchWithRetry(`${this.baseUrl}/writes`, {
       method: "PUT",
       headers: this.headers,
       body: JSON.stringify({ writes: docs }),
@@ -272,7 +374,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
       };
     }
 
-    const writesResp = await fetch(
+    const writesResp = await this.fetchWithRetry(
       `${this.baseUrl}/writes?${new URLSearchParams({
         thread_id: threadId,
         checkpoint_ns: checkpointNs,
@@ -280,8 +382,14 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
       })}`,
       { headers: this.headers },
     );
+    // Fail loudly, never degrade: this used to map ANY non-ok response to "no
+    // pending writes", which silently stripped resume state on a transient
+    // 500 — a checkpoint would load without the writes recorded against it.
+    if (!writesResp.ok) {
+      throw new Error(`Checkpoint writes GET failed: ${writesResp.status} ${writesResp.statusText}`);
+    }
     const pendingWrites = await this.parseWrites(
-      writesResp.ok ? (await writesResp.json()) as Record<string, any> : {},
+      (await writesResp.json()) as Record<string, any>,
     );
 
     return {

package/src/shared/conversation-attachment.ts

@@ -105,6 +105,7 @@ export function synthesizeConversationAttachment(
     slug: CONVERSATION_ATTACHMENT_SLUG,
     toolApprovals: [],
     pinnedToolApprovals: [],
+    toolApprovalOverrides: [],
     discoveredCapabilitiesEmpty: false,
     connectionType: "http",
     url: options.bridgeEndpoint.replace(/\/+$/, "") + CONVERSATION_ROUTE,

package/src/shared/datastore-attachment.ts

@@ -22,8 +22,12 @@
  *     OSS callers resolve as the local principal (T05 R2).
  *
  * Approval-free by construction (DD-001 SD-3): empty toolApprovals +
- * pinnedToolApprovals mean mergeApprovalPolicies emits no entries for
- * this server — zero classifier involvement. `discoveredCapabilitiesEmpty`
+ * pinnedToolApprovals + toolApprovalOverrides mean mergeApprovalPolicies
+ * emits no entries for this server — zero classifier involvement, and no
+ * proto surface can target it (the attachment has no McpServerUsage, and
+ * since issue #349 an agent's overrides are scoped to their own usage's
+ * server, so a same-named override elsewhere cannot reach this one).
+ * `discoveredCapabilitiesEmpty`
  * is false and the attachment has no McpServerUsage, so the connect
  * backfill (whose destructiveHint tightener would force-gate
  * delete_record — silently skipped on channels under UNATTENDED mode)
@@ -63,6 +67,7 @@ export function synthesizeDatastoreAttachment(
     slug: DATASTORE_ATTACHMENT_SLUG,
     toolApprovals: [],
     pinnedToolApprovals: [],
+    toolApprovalOverrides: [],
     discoveredCapabilitiesEmpty: false,
   };
 
@@ -90,25 +95,115 @@ export function synthesizeDatastoreAttachment(
 }
 
 /**
- * The `<available_datastores>` prompt section (DD-005 SD-5, the
- * skills-section precedent): names the attached datastores and points
- * the model at describe_datastore first. Shared by both harnesses so
- * the section text cannot drift (the sender-identity precedent).
+ * The five record tools the records roster always serves (DD-005; the
+ * mcp-server's records roster registers all five unconditionally for the
+ * agent audience — only the `org` argument shape varies by audience, see
+ * mcp-server/src/domains/records/tools.ts). Because the roster never
+ * legitimately narrows, any of these missing from the connected toolset
+ * means the store is degraded (issue #325) — the prompt section and the
+ * reconciliation both derive from this one list so they cannot drift.
  */
-export function formatDatastoresSection(datastoreUsages: DatastoreUsage[]): string {
-  const slugs = datastoreUsages
+export const EXPECTED_RECORD_TOOLS = [
+  "describe_datastore",
+  "find_records",
+  "insert_record",
+  "update_record",
+  "delete_record",
+] as const;
+
+/**
+ * Reconcile the record tools actually connected against the roster
+ * contract (issue #325). Returns the expected tools absent from
+ * `actualToolNames` — empty means healthy. Names are bare tool names
+ * exactly as reported by tools/list (the mcp-enabled-tools identity
+ * space); extraneous names are ignored.
+ */
+export function missingRecordTools(actualToolNames: Iterable<string>): string[] {
+  const present = new Set(actualToolNames);
+  return EXPECTED_RECORD_TOOLS.filter((name) => !present.has(name));
+}
+
+/**
+ * The operator-facing degradation notice (issue #325): pushed as a
+ * MESSAGE_SYSTEM row on the execution status so "declared N datastores,
+ * X/5 record tools connected" is visible without reading a transcript.
+ * Lives HERE with the rest of the datastore-attachment wording — the
+ * harness only threads it.
+ */
+export function formatDatastoreDegradationNotice(
+  declaredCount: number,
+  missing: readonly string[],
+): string {
+  const connected = EXPECTED_RECORD_TOOLS.length - missing.length;
+  return (
+    `Datastore record tools unavailable this turn: declared ` +
+    `${declaredCount} datastore(s), ${connected}/${EXPECTED_RECORD_TOOLS.length} ` +
+    `record tools connected (missing: ${missing.join(", ")}). The agent has ` +
+    `been instructed to disclose this instead of answering from memory.`
+  );
+}
+
+/**
+ * The datastores prompt section (DD-005 SD-5, the skills-section
+ * precedent). Shared by both harnesses so the section text cannot drift
+ * (the sender-identity precedent). Two honest renderings (issue #325):
+ *
+ * - Healthy (`missingToolNames` empty or omitted — the Cursor harness
+ *   always calls it this way: the Cursor SDK connects MCP itself, so
+ *   that harness can never observe the live roster):
+ *   `<available_datastores>` names the attached datastores, points the
+ *   model at describe_datastore first, and carries a standing
+ *   failure-disclosure instruction — the only mechanism that covers
+ *   tools which connected but fail at call time (the WhatsApp-pilot
+ *   outage shape).
+ * - Degraded (deep-agent only, from missingRecordTools against the
+ *   connected roster): `<unavailable_datastores>` instead — the section
+ *   must not promise tools the agent does not have. Names the declared
+ *   datastores, states which record tools are missing, and instructs
+ *   plain disclosure over improvisation.
+ */
+export function formatDatastoresSection(
+  datastoreUsages: DatastoreUsage[],
+  missingToolNames: readonly string[] = [],
+): string {
+  const entries = datastoreUsages
     .map((u) => u.datastoreRef?.slug)
-    .filter((s): s is string => !!s);
-  const entries = slugs.map((slug) => `- ${slug}`);
+    .filter((s): s is string => !!s)
+    .map((slug) => `- ${slug}`);
+
+  if (missingToolNames.length > 0) {
+    return [
+      "<unavailable_datastores>",
+      "This agent is configured to use the datastores listed below, but the",
+      "following record tools failed to connect this turn:",
+      missingToolNames.join(", ") + ".",
+      "Treat these datastores as unreachable for any operation that needs a",
+      "missing tool.",
+      "",
+      ...entries,
+      "",
+      "When the user's request depends on one of these datastores, tell them",
+      "plainly that the datastore cannot be reached right now. Never answer",
+      "from memory, guess, or improvise a substitute for what the datastore",
+      "would have returned.",
+      "</unavailable_datastores>",
+    ].join("\n");
+  }
+
   return [
     "<available_datastores>",
     "You have access to the following datastores through the record tools",
-    "(describe_datastore, find_records, insert_record, update_record, delete_record).",
+    `(${EXPECTED_RECORD_TOOLS.join(", ")}).`,
     "Before your first operation against a datastore, call describe_datastore to",
     "learn its collections, field encodings, and which operations you are allowed",
     "to perform.",
     "",
     ...entries,
+    "",
+    "If any record tool is missing from your toolset or a record tool call",
+    "fails, the datastore is unreachable: tell the user plainly that you cannot",
+    "reach it right now, and do not answer from memory, guess, or improvise a",
+    "substitute for what it would have returned.",
     "</available_datastores>",
   ].join("\n");
 }

package/src/shared/http-retry.ts

@@ -0,0 +1,50 @@
+/**
+ * HTTP error classification for side-channel proxy retries.
+ *
+ * The runner reaches stigmer-service's Side-Channel Proxy (`/v1/proxy/...`)
+ * over plain `fetch` from several clients — the checkpoint saver today;
+ * artifact storage, the LLM proxy, and the registry endpoint are candidates
+ * (stigmer/stigmer#468). This module is the one place that answers "is this
+ * HTTP failure worth retrying?", the `fetch` twin of grpc-retry.ts: it holds
+ * the classification policy only, kept small and separately tested. The
+ * bounded-backoff loop that consumes it lives with each client (the
+ * grpc-retry/status.ts split).
+ *
+ * The policy is grounded in what the proxy actually emits, not HTTP folklore.
+ * CheckpointerProxyController returns exactly four deterministic errors —
+ * 400 (malformed document), 403 (FGA deny), 404 (no such checkpoint), and
+ * 413 (the 4 MB cap) — and never a transient status. Transient failures
+ * reach the runner only as 5xx (Spring's error handler, ingress 502-504),
+ * 408/429 (infrastructure between runner and service), or as network-level
+ * fetch rejections. 401/403 stay terminal deliberately: retrying cannot fix
+ * an expired or rejected credential.
+ */
+
+/**
+ * True for HTTP statuses that signal a transient condition worth retrying:
+ * 408 (request timeout), 429 (throttled), and all 5xx. Every other status is
+ * deterministic — the proxy's 4xx errors mean the same request will fail the
+ * same way forever.
+ */
+export function isRetryableHttpStatus(status: number): boolean {
+  return status === 408 || status === 429 || status >= 500;
+}
+
+/**
+ * True for `fetch` rejections that signal a transient transport condition:
+ *
+ * - `TypeError` — undici's network-failure shape (`fetch failed`, with the
+ *   ECONNREFUSED/ECONNRESET/DNS cause attached). A deterministic TypeError
+ *   (e.g. a malformed URL) also matches, which is accepted: such a bug fails
+ *   every attempt including the first test run, and the bounded budget caps
+ *   the wasted retries at seconds.
+ * - `TimeoutError` — the rejection shape of `AbortSignal.timeout()`, i.e. a
+ *   hung connection whose attempt we bounded ourselves.
+ *
+ * A manual abort (`AbortError`) is deliberately NOT retryable: the caller
+ * cancelled on purpose.
+ */
+export function isRetryableFetchError(err: unknown): boolean {
+  if (err instanceof TypeError) return true;
+  return err instanceof Error && err.name === "TimeoutError";
+}