@stigmer/runner 3.9.0 → 3.11.0

package/dist/shared/checkpointer/http-saver.d.ts

@@ -13,14 +13,55 @@
  *
  * The Java proxy calls Document.parse(json) which handles $binary
  * natively, and doc.toJson() emits the same format on reads.
+ *
+ * Every request runs through {@link HttpCheckpointSaver.fetchWithRetry}:
+ * bounded exponential backoff on transient failures (classified by
+ * http-retry.ts) with a per-request abort timeout. This loop is the ONLY
+ * retry layer between an agent execution and its checkpoints — the deep
+ * agent activity is deliberately non-retryable at the Temporal level
+ * (maximumAttempts: 1; replaying a whole agent run is not safe), so before
+ * this existed a single dropped request killed the execution
+ * (stigmer/stigmer-cloud#188). Retrying puts is safe by server contract:
+ * CheckpointStore documents putCheckpoint/putWrite as keyed save-or-replace
+ * upserts. Budget exhaustion still fails loudly — checkpoints are not
+ * lossy-tolerable (unlike status updates, see status.ts), and silently
+ * dropping one would corrupt resume state.
  */
 import type { RunnableConfig } from "@langchain/core/runnables";
 import { BaseCheckpointSaver, type Checkpoint, type CheckpointListOptions, type CheckpointTuple, type ChannelVersions } from "@langchain/langgraph-checkpoint";
 import type { CheckpointMetadata, PendingWrite } from "@langchain/langgraph-checkpoint";
+import type { RetryOptions } from "../grpc-retry.js";
+/** Retry policy plus the per-request bound; every field is defaulted. */
+export interface HttpCheckpointSaverOptions extends RetryOptions {
+    /**
+     * Milliseconds before an in-flight request is aborted and the attempt is
+     * classified retryable. Without it a hung connection (the realistic
+     * unplanned-pod-kill symptom) stalls forever and the retry never engages.
+     */
+    readonly requestTimeoutMs?: number;
+}
 export declare class HttpCheckpointSaver extends BaseCheckpointSaver {
     private readonly baseUrl;
     private readonly headers;
-    constructor(proxyEndpoint: string, authToken: string);
+    private readonly baseDelayMs;
+    private readonly backoffFactor;
+    private readonly maxRetries;
+    private readonly requestTimeoutMs;
+    private readonly delay;
+    constructor(proxyEndpoint: string, authToken: string, options?: HttpCheckpointSaverOptions);
+    /**
+     * `fetch` with bounded exponential backoff on transient failures and a
+     * per-request abort timeout.
+     *
+     * Composition contract: on a non-retryable status — or when the budget is
+     * exhausted on a retryable one — the last `Response` is RETURNED, so every
+     * call site keeps its own `resp.ok` / 404 handling unchanged; this wrapper
+     * changes when a response arrives, never what call sites do with it. It
+     * throws only what `fetch` itself throws: a network-level rejection that is
+     * non-retryable or has exhausted the budget.
+     */
+    private fetchWithRetry;
+    private backOff;
     private serializeTyped;
     private deserializeTyped;
     getTuple(config: RunnableConfig): Promise<CheckpointTuple | undefined>;

package/dist/shared/checkpointer/http-saver.js

@@ -13,8 +13,22 @@
  *
  * The Java proxy calls Document.parse(json) which handles $binary
  * natively, and doc.toJson() emits the same format on reads.
+ *
+ * Every request runs through {@link HttpCheckpointSaver.fetchWithRetry}:
+ * bounded exponential backoff on transient failures (classified by
+ * http-retry.ts) with a per-request abort timeout. This loop is the ONLY
+ * retry layer between an agent execution and its checkpoints — the deep
+ * agent activity is deliberately non-retryable at the Temporal level
+ * (maximumAttempts: 1; replaying a whole agent run is not safe), so before
+ * this existed a single dropped request killed the execution
+ * (stigmer/stigmer-cloud#188). Retrying puts is safe by server contract:
+ * CheckpointStore documents putCheckpoint/putWrite as keyed save-or-replace
+ * upserts. Budget exhaustion still fails loudly — checkpoints are not
+ * lossy-tolerable (unlike status updates, see status.ts), and silently
+ * dropping one would corrupt resume state.
  */
 import { BaseCheckpointSaver, } from "@langchain/langgraph-checkpoint";
+import { isRetryableFetchError, isRetryableHttpStatus } from "../http-retry.js";
 function encodeB64(data) {
     if (typeof Buffer !== "undefined") {
         return Buffer.from(data).toString("base64");
@@ -54,16 +68,84 @@ function configCheckpointId(config) {
 function configOrg(config) {
     return config.configurable?.org;
 }
+/**
+ * Default transient-retry policy. Wider than status.ts's DEFAULT_PERSIST_RETRY
+ * (3 retries, ~700 ms) on purpose: a dropped status update is lossy-tolerable,
+ * a dropped checkpoint kills the execution, so the budget here is sized to
+ * span a pod-restart reroute (~7.75 s of delays across 5 retries). Worst case
+ * with every attempt hanging — 6 x 30 s + delays, ~3.7 min — stays well inside
+ * the deep-agent activity's 1 h startToCloseTimeout (no heartbeatTimeout).
+ */
+const DEFAULT_RETRY = {
+    baseDelayMs: 250,
+    backoffFactor: 2,
+    maxRetries: 5,
+    requestTimeoutMs: 30_000,
+};
+function defaultDelay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
 export class HttpCheckpointSaver extends BaseCheckpointSaver {
     baseUrl;
     headers;
-    constructor(proxyEndpoint, authToken) {
+    baseDelayMs;
+    backoffFactor;
+    maxRetries;
+    requestTimeoutMs;
+    delay;
+    constructor(proxyEndpoint, authToken, options = {}) {
         super();
         this.baseUrl = `${proxyEndpoint.replace(/\/+$/, "")}/v1/proxy/checkpoints`;
         this.headers = {
             Authorization: `Bearer ${authToken}`,
             "Content-Type": "application/json",
         };
+        this.baseDelayMs = options.baseDelayMs ?? DEFAULT_RETRY.baseDelayMs;
+        this.backoffFactor = options.backoffFactor ?? DEFAULT_RETRY.backoffFactor;
+        this.maxRetries = options.maxRetries ?? DEFAULT_RETRY.maxRetries;
+        this.requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_RETRY.requestTimeoutMs;
+        this.delay = options.delayFn ?? defaultDelay;
+    }
+    /**
+     * `fetch` with bounded exponential backoff on transient failures and a
+     * per-request abort timeout.
+     *
+     * Composition contract: on a non-retryable status — or when the budget is
+     * exhausted on a retryable one — the last `Response` is RETURNED, so every
+     * call site keeps its own `resp.ok` / 404 handling unchanged; this wrapper
+     * changes when a response arrives, never what call sites do with it. It
+     * throws only what `fetch` itself throws: a network-level rejection that is
+     * non-retryable or has exhausted the budget.
+     */
+    async fetchWithRetry(url, init) {
+        for (let attempt = 0;; attempt++) {
+            const canRetry = attempt < this.maxRetries;
+            let resp;
+            try {
+                resp = await fetch(url, {
+                    ...init,
+                    signal: AbortSignal.timeout(this.requestTimeoutMs),
+                });
+            }
+            catch (err) {
+                if (canRetry && isRetryableFetchError(err)) {
+                    await this.backOff(attempt, String(err));
+                    continue;
+                }
+                throw err;
+            }
+            if (canRetry && isRetryableHttpStatus(resp.status)) {
+                await this.backOff(attempt, `HTTP ${resp.status} ${resp.statusText}`);
+                continue;
+            }
+            return resp;
+        }
+    }
+    async backOff(attempt, reason) {
+        const delayMs = this.baseDelayMs * Math.pow(this.backoffFactor, attempt);
+        console.warn(`[HttpCheckpointSaver] retryable failure ` +
+            `(attempt ${attempt + 1}/${this.maxRetries + 1}, retry in ${delayMs}ms): ${reason}`);
+        await this.delay(delayMs);
     }
     async serializeTyped(obj) {
         const [typeTag, payload] = await this.serde.dumpsTyped(obj);
@@ -84,7 +166,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
         if (checkpointId) {
             params.set("checkpoint_id", checkpointId);
         }
-        const resp = await fetch(`${this.baseUrl}/checkpoint?${params}`, {
+        const resp = await this.fetchWithRetry(`${this.baseUrl}/checkpoint?${params}`, {
             headers: this.headers,
         });
         if (resp.status === 404)
@@ -108,7 +190,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
         if (beforeId) {
             params.set("before", beforeId);
         }
-        const resp = await fetch(`${this.baseUrl}/checkpoints?${params}`, {
+        const resp = await this.fetchWithRetry(`${this.baseUrl}/checkpoints?${params}`, {
             headers: this.headers,
         });
         if (!resp.ok) {
@@ -138,7 +220,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
         const orgId = configOrg(config);
         if (orgId)
             doc.org_id = orgId;
-        const resp = await fetch(`${this.baseUrl}/checkpoint`, {
+        const resp = await this.fetchWithRetry(`${this.baseUrl}/checkpoint`, {
             method: "PUT",
             headers: this.headers,
             body: JSON.stringify(doc),
@@ -156,7 +238,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
     }
     async deleteThread(threadId) {
         const params = new URLSearchParams({ thread_id: threadId });
-        const resp = await fetch(`${this.baseUrl}/thread?${params}`, {
+        const resp = await this.fetchWithRetry(`${this.baseUrl}/thread?${params}`, {
             method: "DELETE",
             headers: this.headers,
         });
@@ -185,7 +267,7 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
                 doc.org_id = orgId;
             return doc;
         }));
-        const resp = await fetch(`${this.baseUrl}/writes`, {
+        const resp = await this.fetchWithRetry(`${this.baseUrl}/writes`, {
             method: "PUT",
             headers: this.headers,
             body: JSON.stringify({ writes: docs }),
@@ -211,12 +293,18 @@ export class HttpCheckpointSaver extends BaseCheckpointSaver {
                 },
             };
         }
-        const writesResp = await fetch(`${this.baseUrl}/writes?${new URLSearchParams({
+        const writesResp = await this.fetchWithRetry(`${this.baseUrl}/writes?${new URLSearchParams({
             thread_id: threadId,
             checkpoint_ns: checkpointNs,
             checkpoint_id: doc.checkpoint_id,
         })}`, { headers: this.headers });
-        const pendingWrites = await this.parseWrites(writesResp.ok ? (await writesResp.json()) : {});
+        // Fail loudly, never degrade: this used to map ANY non-ok response to "no
+        // pending writes", which silently stripped resume state on a transient
+        // 500 — a checkpoint would load without the writes recorded against it.
+        if (!writesResp.ok) {
+            throw new Error(`Checkpoint writes GET failed: ${writesResp.status} ${writesResp.statusText}`);
+        }
+        const pendingWrites = await this.parseWrites((await writesResp.json()));
         return {
             config: {
                 configurable: {

package/dist/shared/checkpointer/http-saver.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-saver.js","sourceRoot":"","sources":["../../../src/shared/checkpointer/http-saver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,EACL,mBAAmB,GAMpB,MAAM,iCAAiC,CAAC;AAGzC,SAAS,SAAS,CAAC,IAAgB;IACjC,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAMD,SAAS,YAAY,CAAC,OAAmB;IACvC,OAAO,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;AACpE,CAAC;AAED,SAAS,YAAY,CAAC,GAAc;IAClC,OAAO,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,YAAY,CAAC,MAAsB;IAC1C,OAAQ,MAAM,CAAC,YAAY,EAAE,SAAoB,IAAI,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,QAAQ,CAAC,MAAsB;IACtC,OAAQ,MAAM,CAAC,YAAY,EAAE,aAAwB,IAAI,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAsB;IAChD,OAAO,MAAM,CAAC,YAAY,EAAE,aAAmC,CAAC;AAClE,CAAC;AAED,SAAS,SAAS,CAAC,MAAsB;IACvC,OAAO,MAAM,CAAC,YAAY,EAAE,GAAyB,CAAC;AACxD,CAAC;AAED,MAAM,OAAO,mBAAoB,SAAQ,mBAAmB;IACzC,OAAO,CAAS;IAChB,OAAO,CAAyB;IAEjD,YAAY,aAAqB,EAAE,SAAiB;QAClD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,uBAAuB,CAAC;QAC3E,IAAI,CAAC,OAAO,GAAG;YACb,aAAa,EAAE,UAAU,SAAS,EAAE;YACpC,cAAc,EAAE,kBAAkB;SACnC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,GAAY;QACvC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5D,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAe,EAAE,SAAoB;QAClE,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAsB;QACnC,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;SAC5B,CAAC,CAAC;QACH,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,eAAe,MAAM,EAAE,EAAE;YAC/D,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,SAAS,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAwB,CAAC;QACvD,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,CAAC,IAAI,CACT,MAAsB,EACtB,OAA+B;QAE/B,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QAEnC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;YAC3B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;SACrB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,aAAmC,CAAC;QACpF,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,gBAAgB,MAAM,EAAE,EAAE;YAChE,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAA4C,CAAC;QAC5E,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;YACzC,MAAM,MAAM,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CACP,MAAsB,EACtB,UAAsB,EACtB,QAA4B,EAC5B,YAA6B;QAE7B,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,UAAU,CAAC,EAAE,CAAC;QAEnC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAE/D,MAAM,GAAG,GAA4B;YACnC,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;YAC3B,aAAa,EAAE,YAAY;YAC3B,oBAAoB,EAAE,kBAAkB,CAAC,MAAM,CAAC;YAChD,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,QAAQ;YACpB,aAAa,EAAE,MAAM;YACrB,QAAQ,EAAE,QAAQ;SACnB,CAAC;QAEF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,KAAK;YAAE,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC;QAE9B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,aAAa,EAAE;YACrD,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO;YACL,YAAY,EAAE;gBACZ,SAAS,EAAE,QAAQ;gBACnB,aAAa,EAAE,YAAY;gBAC3B,aAAa,EAAE,YAAY;aAC5B;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,QAAgB;QACjC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5D,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,WAAW,MAAM,EAAE,EAAE;YAC3D,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CACb,MAAsB,EACtB,MAAsB,EACtB,MAAc;QAEd,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAEhC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE;YACxE,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YAC9D,MAAM,GAAG,GAA4B;gBACnC,SAAS,EAAE,QAAQ;gBACnB,aAAa,EAAE,YAAY;gBAC3B,aAAa,EAAE,YAAY;gBAC3B,OAAO,EAAE,MAAM;gBACf,GAAG;gBACH,OAAO;gBACP,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,SAAS;aACjB,CAAC;YACF,IAAI,KAAK;gBAAE,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC;YAC9B,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,SAAS,EAAE;YACjD,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SACvC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,GAAwB,EACxB,QAAgB,EAChB,YAAoB;QAEpB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC;QAClC,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,CAAC,CAAe,CAAC;QAEvF,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,IAAI,MAAM,CAAC;QAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ;YAC3B,CAAC,CAAE,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAwB;YAC7E,CAAC,CAAC,SAAS,CAAC;QAEd,IAAI,YAAwC,CAAC;QAC7C,IAAI,GAAG,CAAC,oBAAoB,EAAE,CAAC;YAC7B,YAAY,GAAG;gBACb,YAAY,EAAE;oBACZ,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE;oBACtC,aAAa,EAAE,GAAG,CAAC,oBAAoB;iBACxC;aACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,KAAK,CAC5B,GAAG,IAAI,CAAC,OAAO,WAAW,IAAI,eAAe,CAAC;YAC5C,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;YAC3B,aAAa,EAAE,GAAG,CAAC,aAAa;SACjC,CAAC,EAAE,EACJ,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;QACF,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAC1C,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,EAAE,CAAwB,CAAC,CAAC,CAAC,EAAE,CACtE,CAAC;QAEF,OAAO;YACL,MAAM,EAAE;gBACN,YAAY,EAAE;oBACZ,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE;oBACtC,aAAa,EAAE,GAAG,CAAC,aAAa;iBACjC;aACF;YACD,UAAU;YACV,QAAQ;YACR,YAAY;YACZ,aAAa;SACd,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,+BAA+B,CAC3C,GAAwB;QAExB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC;QAClC,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAe,CAAC;QAErG,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,IAAI,MAAM,CAAC;QAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ;YAC3B,CAAC,CAAE,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAwB;YAC3F,CAAC,CAAC,SAAS,CAAC;QAEd,IAAI,YAAwC,CAAC;QAC7C,IAAI,GAAG,CAAC,oBAAoB,EAAE,CAAC;YAC7B,YAAY,GAAG;gBACb,YAAY,EAAE;oBACZ,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE;oBACtC,aAAa,EAAE,GAAG,CAAC,oBAAoB;iBACxC;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,MAAM,EAAE;gBACN,YAAY,EAAE;oBACZ,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE;oBACtC,aAAa,EAAE,GAAG,CAAC,aAAa;iBACjC;aACF;YACD,UAAU;YACV,QAAQ;YACR,YAAY;SACb,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,IAAyB;QAEzB,MAAM,MAAM,GAA6B,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC;YAC/B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"http-saver.js","sourceRoot":"","sources":["../../../src/shared/checkpointer/http-saver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAGH,OAAO,EACL,mBAAmB,GAMpB,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEhF,SAAS,SAAS,CAAC,IAAgB;IACjC,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAMD,SAAS,YAAY,CAAC,OAAmB;IACvC,OAAO,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;AACpE,CAAC;AAED,SAAS,YAAY,CAAC,GAAc;IAClC,OAAO,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,YAAY,CAAC,MAAsB;IAC1C,OAAQ,MAAM,CAAC,YAAY,EAAE,SAAoB,IAAI,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,QAAQ,CAAC,MAAsB;IACtC,OAAQ,MAAM,CAAC,YAAY,EAAE,aAAwB,IAAI,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAsB;IAChD,OAAO,MAAM,CAAC,YAAY,EAAE,aAAmC,CAAC;AAClE,CAAC;AAED,SAAS,SAAS,CAAC,MAAsB;IACvC,OAAO,MAAM,CAAC,YAAY,EAAE,GAAyB,CAAC;AACxD,CAAC;AAYD;;;;;;;GAOG;AACH,MAAM,aAAa,GAAG;IACpB,WAAW,EAAE,GAAG;IAChB,aAAa,EAAE,CAAC;IAChB,UAAU,EAAE,CAAC;IACb,gBAAgB,EAAE,MAAM;CAChB,CAAC;AAEX,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,OAAO,mBAAoB,SAAQ,mBAAmB;IACzC,OAAO,CAAS;IAChB,OAAO,CAAyB;IAChC,WAAW,CAAS;IACpB,aAAa,CAAS;IACtB,UAAU,CAAS;IACnB,gBAAgB,CAAS;IACzB,KAAK,CAAgC;IAEtD,YACE,aAAqB,EACrB,SAAiB,EACjB,UAAsC,EAAE;QAExC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,uBAAuB,CAAC;QAC3E,IAAI,CAAC,OAAO,GAAG;YACb,aAAa,EAAE,UAAU,SAAS,EAAE;YACpC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,aAAa,CAAC,WAAW,CAAC;QACpE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,aAAa,CAAC,aAAa,CAAC;QAC1E,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,aAAa,CAAC,UAAU,CAAC;QACjE,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,CAAC;QACnF,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,OAAO,IAAI,YAAY,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;OAUG;IACK,KAAK,CAAC,cAAc,CAAC,GAAW,EAAE,IAAkB;QAC1D,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;YAC3C,IAAI,IAAc,CAAC;YACnB,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBACtB,GAAG,IAAI;oBACP,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC;iBACnD,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;oBACzC,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,IAAI,QAAQ,IAAI,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnD,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;gBACtE,SAAS;YACX,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAe,EAAE,MAAc;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CACV,0CAA0C;YAC1C,YAAY,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,cAAc,OAAO,QAAQ,MAAM,EAAE,CACpF,CAAC;QACF,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,GAAY;QACvC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5D,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAe,EAAE,SAAoB;QAClE,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAsB;QACnC,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;SAC5B,CAAC,CAAC;QACH,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,OAAO,eAAe,MAAM,EAAE,EAAE;YAC7E,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,SAAS,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAwB,CAAC;QACvD,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,CAAC,IAAI,CACT,MAAsB,EACtB,OAA+B;QAE/B,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QAEnC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;YAC3B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;SACrB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,aAAmC,CAAC;QACpF,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,OAAO,gBAAgB,MAAM,EAAE,EAAE;YAC9E,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAA4C,CAAC;QAC5E,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;YACzC,MAAM,MAAM,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CACP,MAAsB,EACtB,UAAsB,EACtB,QAA4B,EAC5B,YAA6B;QAE7B,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,UAAU,CAAC,EAAE,CAAC;QAEnC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAE/D,MAAM,GAAG,GAA4B;YACnC,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;YAC3B,aAAa,EAAE,YAAY;YAC3B,oBAAoB,EAAE,kBAAkB,CAAC,MAAM,CAAC;YAChD,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,QAAQ;YACpB,aAAa,EAAE,MAAM;YACrB,QAAQ,EAAE,QAAQ;SACnB,CAAC;QAEF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,KAAK;YAAE,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC;QAE9B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,OAAO,aAAa,EAAE;YACnE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO;YACL,YAAY,EAAE;gBACZ,SAAS,EAAE,QAAQ;gBACnB,aAAa,EAAE,YAAY;gBAC3B,aAAa,EAAE,YAAY;aAC5B;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,QAAgB;QACjC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,OAAO,WAAW,MAAM,EAAE,EAAE;YACzE,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CACb,MAAsB,EACtB,MAAsB,EACtB,MAAc;QAEd,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAEhC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE;YACxE,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YAC9D,MAAM,GAAG,GAA4B;gBACnC,SAAS,EAAE,QAAQ;gBACnB,aAAa,EAAE,YAAY;gBAC3B,aAAa,EAAE,YAAY;gBAC3B,OAAO,EAAE,MAAM;gBACf,GAAG;gBACH,OAAO;gBACP,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,SAAS;aACjB,CAAC;YACF,IAAI,KAAK;gBAAE,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC;YAC9B,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,OAAO,SAAS,EAAE;YAC/D,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SACvC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,GAAwB,EACxB,QAAgB,EAChB,YAAoB;QAEpB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC;QAClC,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,CAAC,CAAe,CAAC;QAEvF,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,IAAI,MAAM,CAAC;QAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ;YAC3B,CAAC,CAAE,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAwB;YAC7E,CAAC,CAAC,SAAS,CAAC;QAEd,IAAI,YAAwC,CAAC;QAC7C,IAAI,GAAG,CAAC,oBAAoB,EAAE,CAAC;YAC7B,YAAY,GAAG;gBACb,YAAY,EAAE;oBACZ,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE;oBACtC,aAAa,EAAE,GAAG,CAAC,oBAAoB;iBACxC;aACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,cAAc,CAC1C,GAAG,IAAI,CAAC,OAAO,WAAW,IAAI,eAAe,CAAC;YAC5C,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;YAC3B,aAAa,EAAE,GAAG,CAAC,aAAa;SACjC,CAAC,EAAE,EACJ,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;QACF,0EAA0E;QAC1E,uEAAuE;QACvE,wEAAwE;QACxE,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,iCAAiC,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC,CAAC;QACjG,CAAC;QACD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAC1C,CAAC,MAAM,UAAU,CAAC,IAAI,EAAE,CAAwB,CACjD,CAAC;QAEF,OAAO;YACL,MAAM,EAAE;gBACN,YAAY,EAAE;oBACZ,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE;oBACtC,aAAa,EAAE,GAAG,CAAC,aAAa;iBACjC;aACF;YACD,UAAU;YACV,QAAQ;YACR,YAAY;YACZ,aAAa;SACd,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,+BAA+B,CAC3C,GAAwB;QAExB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC;QAClC,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAe,CAAC;QAErG,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,IAAI,MAAM,CAAC;QAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ;YAC3B,CAAC,CAAE,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAwB;YAC3F,CAAC,CAAC,SAAS,CAAC;QAEd,IAAI,YAAwC,CAAC;QAC7C,IAAI,GAAG,CAAC,oBAAoB,EAAE,CAAC;YAC7B,YAAY,GAAG;gBACb,YAAY,EAAE;oBACZ,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE;oBACtC,aAAa,EAAE,GAAG,CAAC,oBAAoB;iBACxC;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,MAAM,EAAE;gBACN,YAAY,EAAE;oBACZ,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE;oBACtC,aAAa,EAAE,GAAG,CAAC,aAAa;iBACjC;aACF;YACD,UAAU;YACV,QAAQ;YACR,YAAY;SACb,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,IAAyB;QAEzB,MAAM,MAAM,GAA6B,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC;YAC/B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/shared/conversation-attachment.js

@@ -91,6 +91,7 @@ export function synthesizeConversationAttachment(channelId, options) {
         slug: CONVERSATION_ATTACHMENT_SLUG,
         toolApprovals: [],
         pinnedToolApprovals: [],
+        toolApprovalOverrides: [],
         discoveredCapabilitiesEmpty: false,
         connectionType: "http",
         url: options.bridgeEndpoint.replace(/\/+$/, "") + CONVERSATION_ROUTE,

package/dist/shared/conversation-attachment.js.map

@@ -1 +1 @@
-{"version":3,"file":"conversation-attachment.js","sourceRoot":"","sources":["../../src/shared/conversation-attachment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAKH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,sBAAsB,CAAC;AAEnE;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC;AAElD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;AAExD;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CACvC,MAA0C;IAE1C,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,CAAC;IACrD,OAAO,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAC9C,SAA6B,EAC7B,OAAqC;IAErC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,OAAO,CAAC,cAAc,KAAK,IAAI,IAAI,OAAO,CAAC,cAAc,KAAK,EAAE,EAAE,CAAC;QACrE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,mEAAmE;IACnE,OAAO;QACL,IAAI,EAAE,4BAA4B;QAClC,aAAa,EAAE,EAAE;QACjB,mBAAmB,EAAE,EAAE;QACvB,2BAA2B,EAAE,KAAK;QAClC,cAAc,EAAE,MAAM;QACtB,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,kBAAkB;QACpE,OAAO,EAAE,OAAO,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,CAAC,UAAU,KAAK,EAAE;YAC/D,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,OAAO,CAAC,UAAU,EAAE,EAAE;YACnD,CAAC,CAAC,SAAS;KACd,CAAC;AACJ,CAAC"}
+{"version":3,"file":"conversation-attachment.js","sourceRoot":"","sources":["../../src/shared/conversation-attachment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAKH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,sBAAsB,CAAC;AAEnE;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC;AAElD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;AAExD;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CACvC,MAA0C;IAE1C,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,CAAC;IACrD,OAAO,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAC9C,SAA6B,EAC7B,OAAqC;IAErC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,OAAO,CAAC,cAAc,KAAK,IAAI,IAAI,OAAO,CAAC,cAAc,KAAK,EAAE,EAAE,CAAC;QACrE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,mEAAmE;IACnE,OAAO;QACL,IAAI,EAAE,4BAA4B;QAClC,aAAa,EAAE,EAAE;QACjB,mBAAmB,EAAE,EAAE;QACvB,qBAAqB,EAAE,EAAE;QACzB,2BAA2B,EAAE,KAAK;QAClC,cAAc,EAAE,MAAM;QACtB,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,kBAAkB;QACpE,OAAO,EAAE,OAAO,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,CAAC,UAAU,KAAK,EAAE;YAC/D,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,OAAO,CAAC,UAAU,EAAE,EAAE;YACnD,CAAC,CAAC,SAAS;KACd,CAAC;AACJ,CAAC"}

package/dist/shared/datastore-attachment.d.ts

@@ -22,8 +22,12 @@
  *     OSS callers resolve as the local principal (T05 R2).
  *
  * Approval-free by construction (DD-001 SD-3): empty toolApprovals +
- * pinnedToolApprovals mean mergeApprovalPolicies emits no entries for
- * this server — zero classifier involvement. `discoveredCapabilitiesEmpty`
+ * pinnedToolApprovals + toolApprovalOverrides mean mergeApprovalPolicies
+ * emits no entries for this server — zero classifier involvement, and no
+ * proto surface can target it (the attachment has no McpServerUsage, and
+ * since issue #349 an agent's overrides are scoped to their own usage's
+ * server, so a same-named override elsewhere cannot reach this one).
+ * `discoveredCapabilitiesEmpty`
  * is false and the attachment has no McpServerUsage, so the connect
  * backfill (whose destructiveHint tightener would force-gate
  * delete_record — silently skipped on channels under UNATTENDED mode)
@@ -48,9 +52,48 @@ export declare const RECORDS_ROUTE = "/records";
  */
 export declare function synthesizeDatastoreAttachment(datastoreUsages: DatastoreUsage[], options: SynthesizedAttachmentOptions): ResolvedMcpServer | undefined;
 /**
- * The `<available_datastores>` prompt section (DD-005 SD-5, the
- * skills-section precedent): names the attached datastores and points
- * the model at describe_datastore first. Shared by both harnesses so
- * the section text cannot drift (the sender-identity precedent).
+ * The five record tools the records roster always serves (DD-005; the
+ * mcp-server's records roster registers all five unconditionally for the
+ * agent audience — only the `org` argument shape varies by audience, see
+ * mcp-server/src/domains/records/tools.ts). Because the roster never
+ * legitimately narrows, any of these missing from the connected toolset
+ * means the store is degraded (issue #325) — the prompt section and the
+ * reconciliation both derive from this one list so they cannot drift.
  */
-export declare function formatDatastoresSection(datastoreUsages: DatastoreUsage[]): string;
+export declare const EXPECTED_RECORD_TOOLS: readonly ["describe_datastore", "find_records", "insert_record", "update_record", "delete_record"];
+/**
+ * Reconcile the record tools actually connected against the roster
+ * contract (issue #325). Returns the expected tools absent from
+ * `actualToolNames` — empty means healthy. Names are bare tool names
+ * exactly as reported by tools/list (the mcp-enabled-tools identity
+ * space); extraneous names are ignored.
+ */
+export declare function missingRecordTools(actualToolNames: Iterable<string>): string[];
+/**
+ * The operator-facing degradation notice (issue #325): pushed as a
+ * MESSAGE_SYSTEM row on the execution status so "declared N datastores,
+ * X/5 record tools connected" is visible without reading a transcript.
+ * Lives HERE with the rest of the datastore-attachment wording — the
+ * harness only threads it.
+ */
+export declare function formatDatastoreDegradationNotice(declaredCount: number, missing: readonly string[]): string;
+/**
+ * The datastores prompt section (DD-005 SD-5, the skills-section
+ * precedent). Shared by both harnesses so the section text cannot drift
+ * (the sender-identity precedent). Two honest renderings (issue #325):
+ *
+ * - Healthy (`missingToolNames` empty or omitted — the Cursor harness
+ *   always calls it this way: the Cursor SDK connects MCP itself, so
+ *   that harness can never observe the live roster):
+ *   `<available_datastores>` names the attached datastores, points the
+ *   model at describe_datastore first, and carries a standing
+ *   failure-disclosure instruction — the only mechanism that covers
+ *   tools which connected but fail at call time (the WhatsApp-pilot
+ *   outage shape).
+ * - Degraded (deep-agent only, from missingRecordTools against the
+ *   connected roster): `<unavailable_datastores>` instead — the section
+ *   must not promise tools the agent does not have. Names the declared
+ *   datastores, states which record tools are missing, and instructs
+ *   plain disclosure over improvisation.
+ */
+export declare function formatDatastoresSection(datastoreUsages: DatastoreUsage[], missingToolNames?: readonly string[]): string;

package/dist/shared/datastore-attachment.js

@@ -22,8 +22,12 @@
  *     OSS callers resolve as the local principal (T05 R2).
  *
  * Approval-free by construction (DD-001 SD-3): empty toolApprovals +
- * pinnedToolApprovals mean mergeApprovalPolicies emits no entries for
- * this server — zero classifier involvement. `discoveredCapabilitiesEmpty`
+ * pinnedToolApprovals + toolApprovalOverrides mean mergeApprovalPolicies
+ * emits no entries for this server — zero classifier involvement, and no
+ * proto surface can target it (the attachment has no McpServerUsage, and
+ * since issue #349 an agent's overrides are scoped to their own usage's
+ * server, so a same-named override elsewhere cannot reach this one).
+ * `discoveredCapabilitiesEmpty`
  * is false and the attachment has no McpServerUsage, so the connect
  * backfill (whose destructiveHint tightener would force-gate
  * delete_record — silently skipped on channels under UNATTENDED mode)
@@ -53,6 +57,7 @@ export function synthesizeDatastoreAttachment(datastoreUsages, options) {
         slug: DATASTORE_ATTACHMENT_SLUG,
         toolApprovals: [],
         pinnedToolApprovals: [],
+        toolApprovalOverrides: [],
         discoveredCapabilitiesEmpty: false,
     };
     if (options.bridgeEndpoint !== null && options.bridgeEndpoint !== "") {
@@ -77,25 +82,102 @@ export function synthesizeDatastoreAttachment(datastoreUsages, options) {
     };
 }
 /**
- * The `<available_datastores>` prompt section (DD-005 SD-5, the
- * skills-section precedent): names the attached datastores and points
- * the model at describe_datastore first. Shared by both harnesses so
- * the section text cannot drift (the sender-identity precedent).
+ * The five record tools the records roster always serves (DD-005; the
+ * mcp-server's records roster registers all five unconditionally for the
+ * agent audience — only the `org` argument shape varies by audience, see
+ * mcp-server/src/domains/records/tools.ts). Because the roster never
+ * legitimately narrows, any of these missing from the connected toolset
+ * means the store is degraded (issue #325) — the prompt section and the
+ * reconciliation both derive from this one list so they cannot drift.
  */
-export function formatDatastoresSection(datastoreUsages) {
-    const slugs = datastoreUsages
+export const EXPECTED_RECORD_TOOLS = [
+    "describe_datastore",
+    "find_records",
+    "insert_record",
+    "update_record",
+    "delete_record",
+];
+/**
+ * Reconcile the record tools actually connected against the roster
+ * contract (issue #325). Returns the expected tools absent from
+ * `actualToolNames` — empty means healthy. Names are bare tool names
+ * exactly as reported by tools/list (the mcp-enabled-tools identity
+ * space); extraneous names are ignored.
+ */
+export function missingRecordTools(actualToolNames) {
+    const present = new Set(actualToolNames);
+    return EXPECTED_RECORD_TOOLS.filter((name) => !present.has(name));
+}
+/**
+ * The operator-facing degradation notice (issue #325): pushed as a
+ * MESSAGE_SYSTEM row on the execution status so "declared N datastores,
+ * X/5 record tools connected" is visible without reading a transcript.
+ * Lives HERE with the rest of the datastore-attachment wording — the
+ * harness only threads it.
+ */
+export function formatDatastoreDegradationNotice(declaredCount, missing) {
+    const connected = EXPECTED_RECORD_TOOLS.length - missing.length;
+    return (`Datastore record tools unavailable this turn: declared ` +
+        `${declaredCount} datastore(s), ${connected}/${EXPECTED_RECORD_TOOLS.length} ` +
+        `record tools connected (missing: ${missing.join(", ")}). The agent has ` +
+        `been instructed to disclose this instead of answering from memory.`);
+}
+/**
+ * The datastores prompt section (DD-005 SD-5, the skills-section
+ * precedent). Shared by both harnesses so the section text cannot drift
+ * (the sender-identity precedent). Two honest renderings (issue #325):
+ *
+ * - Healthy (`missingToolNames` empty or omitted — the Cursor harness
+ *   always calls it this way: the Cursor SDK connects MCP itself, so
+ *   that harness can never observe the live roster):
+ *   `<available_datastores>` names the attached datastores, points the
+ *   model at describe_datastore first, and carries a standing
+ *   failure-disclosure instruction — the only mechanism that covers
+ *   tools which connected but fail at call time (the WhatsApp-pilot
+ *   outage shape).
+ * - Degraded (deep-agent only, from missingRecordTools against the
+ *   connected roster): `<unavailable_datastores>` instead — the section
+ *   must not promise tools the agent does not have. Names the declared
+ *   datastores, states which record tools are missing, and instructs
+ *   plain disclosure over improvisation.
+ */
+export function formatDatastoresSection(datastoreUsages, missingToolNames = []) {
+    const entries = datastoreUsages
         .map((u) => u.datastoreRef?.slug)
-        .filter((s) => !!s);
-    const entries = slugs.map((slug) => `- ${slug}`);
+        .filter((s) => !!s)
+        .map((slug) => `- ${slug}`);
+    if (missingToolNames.length > 0) {
+        return [
+            "<unavailable_datastores>",
+            "This agent is configured to use the datastores listed below, but the",
+            "following record tools failed to connect this turn:",
+            missingToolNames.join(", ") + ".",
+            "Treat these datastores as unreachable for any operation that needs a",
+            "missing tool.",
+            "",
+            ...entries,
+            "",
+            "When the user's request depends on one of these datastores, tell them",
+            "plainly that the datastore cannot be reached right now. Never answer",
+            "from memory, guess, or improvise a substitute for what the datastore",
+            "would have returned.",
+            "</unavailable_datastores>",
+        ].join("\n");
+    }
     return [
         "<available_datastores>",
         "You have access to the following datastores through the record tools",
-        "(describe_datastore, find_records, insert_record, update_record, delete_record).",
+        `(${EXPECTED_RECORD_TOOLS.join(", ")}).`,
         "Before your first operation against a datastore, call describe_datastore to",
         "learn its collections, field encodings, and which operations you are allowed",
         "to perform.",
         "",
         ...entries,
+        "",
+        "If any record tool is missing from your toolset or a record tool call",
+        "fails, the datastore is unreachable: tell the user plainly that you cannot",
+        "reach it right now, and do not answer from memory, guess, or improvise a",
+        "substitute for what it would have returned.",
         "</available_datastores>",
     ].join("\n");
 }

package/dist/shared/datastore-attachment.js.map

@@ -1 +1 @@
-{"version":3,"file":"datastore-attachment.js","sourceRoot":"","sources":["../../src/shared/datastore-attachment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAIH,OAAO,EAAE,UAAU,EAAqC,MAAM,6BAA6B,CAAC;AAE5F;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,iBAAiB,CAAC;AAE3D,4EAA4E;AAC5E,MAAM,CAAC,MAAM,aAAa,GAAG,UAAU,CAAC;AAExC;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAC3C,eAAiC,EACjC,OAAqC;IAErC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,mEAAmE;IACnE,MAAM,IAAI,GAAG;QACX,IAAI,EAAE,yBAAyB;QAC/B,aAAa,EAAE,EAAE;QACjB,mBAAmB,EAAE,EAAE;QACvB,2BAA2B,EAAE,KAAK;KACnC,CAAC;IAEF,IAAI,OAAO,CAAC,cAAc,KAAK,IAAI,IAAI,OAAO,CAAC,cAAc,KAAK,EAAE,EAAE,CAAC;QACrE,OAAO;YACL,GAAG,IAAI;YACP,cAAc,EAAE,MAAM;YACtB,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,aAAa;YAC/D,OAAO,EAAE,OAAO,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,CAAC,UAAU,KAAK,EAAE;gBAC/D,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,OAAO,CAAC,UAAU,EAAE,EAAE;gBACnD,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;IAED,OAAO;QACL,GAAG,IAAI;QACP,cAAc,EAAE,OAAO;QACvB,OAAO,EAAE,SAAS;QAClB,IAAI,EAAE,CAAC,YAAY,CAAC;QACpB,GAAG,EAAE;YACH,kBAAkB,EAAE,SAAS;YAC7B,sBAAsB,EAAE,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC;SAC5D;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CAAC,eAAiC;IACvE,MAAM,KAAK,GAAG,eAAe;SAC1B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,EAAE,IAAI,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACjD,OAAO;QACL,wBAAwB;QACxB,sEAAsE;QACtE,kFAAkF;QAClF,6EAA6E;QAC7E,8EAA8E;QAC9E,aAAa;QACb,EAAE;QACF,GAAG,OAAO;QACV,yBAAyB;KAC1B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC"}
+{"version":3,"file":"datastore-attachment.js","sourceRoot":"","sources":["../../src/shared/datastore-attachment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAIH,OAAO,EAAE,UAAU,EAAqC,MAAM,6BAA6B,CAAC;AAE5F;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,iBAAiB,CAAC;AAE3D,4EAA4E;AAC5E,MAAM,CAAC,MAAM,aAAa,GAAG,UAAU,CAAC;AAExC;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAC3C,eAAiC,EACjC,OAAqC;IAErC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,mEAAmE;IACnE,MAAM,IAAI,GAAG;QACX,IAAI,EAAE,yBAAyB;QAC/B,aAAa,EAAE,EAAE;QACjB,mBAAmB,EAAE,EAAE;QACvB,qBAAqB,EAAE,EAAE;QACzB,2BAA2B,EAAE,KAAK;KACnC,CAAC;IAEF,IAAI,OAAO,CAAC,cAAc,KAAK,IAAI,IAAI,OAAO,CAAC,cAAc,KAAK,EAAE,EAAE,CAAC;QACrE,OAAO;YACL,GAAG,IAAI;YACP,cAAc,EAAE,MAAM;YACtB,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,aAAa;YAC/D,OAAO,EAAE,OAAO,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,CAAC,UAAU,KAAK,EAAE;gBAC/D,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,OAAO,CAAC,UAAU,EAAE,EAAE;gBACnD,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;IAED,OAAO;QACL,GAAG,IAAI;QACP,cAAc,EAAE,OAAO;QACvB,OAAO,EAAE,SAAS;QAClB,IAAI,EAAE,CAAC,YAAY,CAAC;QACpB,GAAG,EAAE;YACH,kBAAkB,EAAE,SAAS;YAC7B,sBAAsB,EAAE,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC;SAC5D;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,oBAAoB;IACpB,cAAc;IACd,eAAe;IACf,eAAe;IACf,eAAe;CACP,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,eAAiC;IAClE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,CAAC;IACzC,OAAO,qBAAqB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AACpE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gCAAgC,CAC9C,aAAqB,EACrB,OAA0B;IAE1B,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAChE,OAAO,CACL,yDAAyD;QACzD,GAAG,aAAa,kBAAkB,SAAS,IAAI,qBAAqB,CAAC,MAAM,GAAG;QAC9E,oCAAoC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB;QACzE,oEAAoE,CACrE,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,uBAAuB,CACrC,eAAiC,EACjC,mBAAsC,EAAE;IAExC,MAAM,OAAO,GAAG,eAAe;SAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,EAAE,IAAI,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;SAC/B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAE9B,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,OAAO;YACL,0BAA0B;YAC1B,sEAAsE;YACtE,qDAAqD;YACrD,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG;YACjC,sEAAsE;YACtE,eAAe;YACf,EAAE;YACF,GAAG,OAAO;YACV,EAAE;YACF,uEAAuE;YACvE,sEAAsE;YACtE,sEAAsE;YACtE,sBAAsB;YACtB,2BAA2B;SAC5B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAED,OAAO;QACL,wBAAwB;QACxB,sEAAsE;QACtE,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;QACxC,6EAA6E;QAC7E,8EAA8E;QAC9E,aAAa;QACb,EAAE;QACF,GAAG,OAAO;QACV,EAAE;QACF,uEAAuE;QACvE,4EAA4E;QAC5E,0EAA0E;QAC1E,6CAA6C;QAC7C,yBAAyB;KAC1B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC"}

package/dist/shared/http-retry.d.ts

@@ -0,0 +1,43 @@
+/**
+ * HTTP error classification for side-channel proxy retries.
+ *
+ * The runner reaches stigmer-service's Side-Channel Proxy (`/v1/proxy/...`)
+ * over plain `fetch` from several clients — the checkpoint saver today;
+ * artifact storage, the LLM proxy, and the registry endpoint are candidates
+ * (stigmer/stigmer#468). This module is the one place that answers "is this
+ * HTTP failure worth retrying?", the `fetch` twin of grpc-retry.ts: it holds
+ * the classification policy only, kept small and separately tested. The
+ * bounded-backoff loop that consumes it lives with each client (the
+ * grpc-retry/status.ts split).
+ *
+ * The policy is grounded in what the proxy actually emits, not HTTP folklore.
+ * CheckpointerProxyController returns exactly four deterministic errors —
+ * 400 (malformed document), 403 (FGA deny), 404 (no such checkpoint), and
+ * 413 (the 4 MB cap) — and never a transient status. Transient failures
+ * reach the runner only as 5xx (Spring's error handler, ingress 502-504),
+ * 408/429 (infrastructure between runner and service), or as network-level
+ * fetch rejections. 401/403 stay terminal deliberately: retrying cannot fix
+ * an expired or rejected credential.
+ */
+/**
+ * True for HTTP statuses that signal a transient condition worth retrying:
+ * 408 (request timeout), 429 (throttled), and all 5xx. Every other status is
+ * deterministic — the proxy's 4xx errors mean the same request will fail the
+ * same way forever.
+ */
+export declare function isRetryableHttpStatus(status: number): boolean;
+/**
+ * True for `fetch` rejections that signal a transient transport condition:
+ *
+ * - `TypeError` — undici's network-failure shape (`fetch failed`, with the
+ *   ECONNREFUSED/ECONNRESET/DNS cause attached). A deterministic TypeError
+ *   (e.g. a malformed URL) also matches, which is accepted: such a bug fails
+ *   every attempt including the first test run, and the bounded budget caps
+ *   the wasted retries at seconds.
+ * - `TimeoutError` — the rejection shape of `AbortSignal.timeout()`, i.e. a
+ *   hung connection whose attempt we bounded ourselves.
+ *
+ * A manual abort (`AbortError`) is deliberately NOT retryable: the caller
+ * cancelled on purpose.
+ */
+export declare function isRetryableFetchError(err: unknown): boolean;

package/dist/shared/http-retry.js

@@ -0,0 +1,50 @@
+/**
+ * HTTP error classification for side-channel proxy retries.
+ *
+ * The runner reaches stigmer-service's Side-Channel Proxy (`/v1/proxy/...`)
+ * over plain `fetch` from several clients — the checkpoint saver today;
+ * artifact storage, the LLM proxy, and the registry endpoint are candidates
+ * (stigmer/stigmer#468). This module is the one place that answers "is this
+ * HTTP failure worth retrying?", the `fetch` twin of grpc-retry.ts: it holds
+ * the classification policy only, kept small and separately tested. The
+ * bounded-backoff loop that consumes it lives with each client (the
+ * grpc-retry/status.ts split).
+ *
+ * The policy is grounded in what the proxy actually emits, not HTTP folklore.
+ * CheckpointerProxyController returns exactly four deterministic errors —
+ * 400 (malformed document), 403 (FGA deny), 404 (no such checkpoint), and
+ * 413 (the 4 MB cap) — and never a transient status. Transient failures
+ * reach the runner only as 5xx (Spring's error handler, ingress 502-504),
+ * 408/429 (infrastructure between runner and service), or as network-level
+ * fetch rejections. 401/403 stay terminal deliberately: retrying cannot fix
+ * an expired or rejected credential.
+ */
+/**
+ * True for HTTP statuses that signal a transient condition worth retrying:
+ * 408 (request timeout), 429 (throttled), and all 5xx. Every other status is
+ * deterministic — the proxy's 4xx errors mean the same request will fail the
+ * same way forever.
+ */
+export function isRetryableHttpStatus(status) {
+    return status === 408 || status === 429 || status >= 500;
+}
+/**
+ * True for `fetch` rejections that signal a transient transport condition:
+ *
+ * - `TypeError` — undici's network-failure shape (`fetch failed`, with the
+ *   ECONNREFUSED/ECONNRESET/DNS cause attached). A deterministic TypeError
+ *   (e.g. a malformed URL) also matches, which is accepted: such a bug fails
+ *   every attempt including the first test run, and the bounded budget caps
+ *   the wasted retries at seconds.
+ * - `TimeoutError` — the rejection shape of `AbortSignal.timeout()`, i.e. a
+ *   hung connection whose attempt we bounded ourselves.
+ *
+ * A manual abort (`AbortError`) is deliberately NOT retryable: the caller
+ * cancelled on purpose.
+ */
+export function isRetryableFetchError(err) {
+    if (err instanceof TypeError)
+        return true;
+    return err instanceof Error && err.name === "TimeoutError";
+}
+//# sourceMappingURL=http-retry.js.map

package/dist/shared/http-retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http-retry.js","sourceRoot":"","sources":["../../src/shared/http-retry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAc;IAClD,OAAO,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,IAAI,GAAG,CAAC;AAC3D,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY;IAChD,IAAI,GAAG,YAAY,SAAS;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC;AAC7D,CAAC"}