@stigmer/runner 3.10.0 → 3.11.1-dev.20260812192248

package/src/shared/__tests__/vertex-adapter.test.ts

@@ -0,0 +1,169 @@
+/**
+ * Adapter test for the vertex backend branch of buildChatModel — the
+ * complement of vertex-seam.test.ts.
+ *
+ * The seam test pins the REAL AnthropicVertex client's wire behavior under
+ * a hand-built ChatAnthropic. This file pins OUR production wiring: a real
+ * `buildChatModel` and a real `ChatAnthropic` drive a mocked
+ * `@anthropic-ai/vertex-sdk`, so LangChain's lazy `createClient` invocation
+ * path (client constructed on first request, once per cached client) is
+ * exercised exactly as production does it. Together the two files cover the
+ * whole chain without any test-only injection seam in production code.
+ *
+ * Pinned here:
+ * - the factory forwards LangChain's `maxRetries: 0` to the client (a
+ *   factory that drops it nests SDK retries inside LangChain's own loop),
+ * - the factory forwards the request timeout (clientOptions.timeout ->
+ *   factory options -> SDK constructor; how STIGMER_LLM_REQUEST_TIMEOUT_MS
+ *   bounds this backend),
+ * - construction and invocation succeed with NO ANTHROPIC_API_KEY (Vertex
+ *   auth is Google's, and ChatAnthropic waives the key when createClient
+ *   is provided),
+ * - the translated `@date` id is what crosses the wire while the returned
+ *   apiModelId stays canonical (the canonical-id invariant),
+ * - usage_metadata survives the adapter (what billing reads).
+ */
+
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { HumanMessage, AIMessage } from "@langchain/core/messages";
+
+const { vertexCtorArgs, createRequests } = vi.hoisted(() => ({
+  vertexCtorArgs: [] as Array<{ maxRetries?: number; timeout?: number }>,
+  createRequests: [] as Array<Record<string, unknown>>,
+}));
+
+vi.mock("@anthropic-ai/vertex-sdk", () => ({
+  AnthropicVertex: class {
+    readonly maxRetries: number | undefined;
+    readonly messages = {
+      create: async (request: Record<string, unknown>) => {
+        createRequests.push(request);
+        return {
+          id: "msg_vertex_adapter_01",
+          type: "message",
+          role: "assistant",
+          model: request.model,
+          content: [{ type: "text", text: "Namaste from Vertex." }],
+          stop_reason: "end_turn",
+          stop_sequence: null,
+          usage: { input_tokens: 7, output_tokens: 5 },
+        };
+      },
+    };
+
+    constructor(opts: { maxRetries?: number; timeout?: number }) {
+      vertexCtorArgs.push(opts);
+      this.maxRetries = opts.maxRetries;
+    }
+  },
+}));
+
+import { ChatAnthropic } from "@langchain/anthropic";
+
+import { buildChatModel } from "../model-client.js";
+import { _resetRegistryCache } from "../model-registry.js";
+import { toVertexModelId } from "../llm-backend.js";
+
+describe("buildChatModel vertex adapter", () => {
+  const savedEnv = { ...process.env };
+
+  beforeEach(() => {
+    _resetRegistryCache();
+    vertexCtorArgs.length = 0;
+    createRequests.length = 0;
+    delete process.env.ANTHROPIC_API_KEY;
+    process.env.STIGMER_ANTHROPIC_BACKEND = "vertex";
+    process.env.CLOUD_ML_REGION = "asia-south1";
+    // Registry offline → resolveToApiModelId passes the id through, keeping
+    // this test free of registry fixtures (that path has its own tests).
+    vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("registry offline"));
+  });
+
+  afterEach(() => {
+    _resetRegistryCache();
+    vi.restoreAllMocks();
+    process.env = { ...savedEnv };
+  });
+
+  it("drives the mocked client through the real createClient path with the invariants intact", async () => {
+    const { model, apiModelId } = await buildChatModel({
+      modelName: "claude-sonnet-4-5-20250929",
+      maxTokens: 1024,
+    });
+
+    // Canonical id returned; the client is not constructed yet (lazy factory).
+    expect(apiModelId).toBe("claude-sonnet-4-5-20250929");
+    expect(vertexCtorArgs).toHaveLength(0);
+
+    const result = await model.invoke([new HumanMessage("Weather in Chennai?")]);
+
+    // First request constructed exactly one (batch) client, with LangChain's
+    // maxRetries: 0 honored — LangChain owns retrying.
+    expect(vertexCtorArgs).toHaveLength(1);
+    expect(vertexCtorArgs[0].maxRetries).toBe(0);
+
+    // The wire carries the Vertex-translated id; billing already got the
+    // canonical one above.
+    expect(createRequests).toHaveLength(1);
+    expect(createRequests[0].model).toBe("claude-sonnet-4-5@20250929");
+    expect(createRequests[0].max_tokens).toBe(1024);
+
+    // Response and usage flow back through LangChain untouched.
+    expect(result).toBeInstanceOf(AIMessage);
+    expect(result.text).toBe("Namaste from Vertex.");
+    expect((result as AIMessage).usage_metadata).toMatchObject({
+      input_tokens: 7,
+      output_tokens: 5,
+      total_tokens: 12,
+    });
+  });
+
+  it("translated ids inherit the same default maxTokens as canonical ids", () => {
+    // setup.ts deliberately omits maxTokens, so the LangChain per-model
+    // default applies. That default prefix-matches the model string — and the
+    // vertex branch hands ChatAnthropic the TRANSLATED string. This pins the
+    // equality the zero-behavior-change criterion rests on (verified 16384 ==
+    // 16384 for 4.5-generation ids at @langchain/anthropic 1.4.0); a future
+    // bump that diverges the two forms fails here instead of silently capping
+    // vertex deployments differently from public ones.
+    for (const canonical of [
+      "claude-sonnet-4-5-20250929",
+      "claude-haiku-4-5-20251001",
+      "claude-sonnet-4-6",
+    ]) {
+      const publicModel = new ChatAnthropic({ model: canonical, apiKey: "probe" });
+      const vertexModel = new ChatAnthropic({ model: toVertexModelId(canonical), apiKey: "probe" });
+      expect(vertexModel.maxTokens, canonical).toBe(publicModel.maxTokens);
+    }
+  });
+
+  it("forwards the request timeout to the SDK client (STIGMER_LLM_REQUEST_TIMEOUT_MS path)", async () => {
+    // The timeout rides clientOptions -> factory options -> SDK constructor.
+    // A factory that drops it silently unbounds the operator's timeout on
+    // this backend (the regression T06 repaired for the public path).
+    const { model } = await buildChatModel({
+      modelName: "claude-sonnet-4-6",
+      maxTokens: 256,
+      timeoutMs: 5000,
+    });
+    await model.invoke([new HumanMessage("hi")]);
+
+    expect(vertexCtorArgs).toHaveLength(1);
+    expect(vertexCtorArgs[0].timeout).toBe(5000);
+    expect(vertexCtorArgs[0].maxRetries).toBe(0);
+  });
+
+  it("constructs and invokes with no ANTHROPIC_API_KEY anywhere in the environment", async () => {
+    expect(process.env.ANTHROPIC_API_KEY).toBeUndefined();
+
+    const { model } = await buildChatModel({
+      modelName: "claude-sonnet-4-6",
+      maxTokens: 256,
+    });
+    const result = await model.invoke([new HumanMessage("hi")]);
+
+    expect(result).toBeInstanceOf(AIMessage);
+    // Dateless 4.6-generation id crossed the wire untranslated.
+    expect(createRequests.at(-1)?.model).toBe("claude-sonnet-4-6");
+  });
+});

package/src/shared/__tests__/vertex-seam.test.ts

@@ -0,0 +1,295 @@
+/**
+ * Characterization test for the ChatAnthropic `createClient` -> AnthropicVertex
+ * seam — the integration T02's vertex backend adapter will be built on.
+ *
+ * This is NOT a unit test of our code (there is no production vertex code
+ * yet). It pins the exact cross-package behavior production will rely on:
+ * the REAL `ChatAnthropic` (@langchain/anthropic, bundling @anthropic-ai/sdk
+ * 0.95.x) driving the REAL `AnthropicVertex` client (@anthropic-ai/vertex-sdk,
+ * bundling its own nested @anthropic-ai/sdk >=0.115). If a future bump of
+ * either side changes request shaping, streaming event handling, tool-call
+ * assembly, or usage accounting across this seam, this suite fails in CI
+ * instead of production. See scripts/check-langchain-deps.sh for why two
+ * @anthropic-ai/sdk copies coexist and when they collapse to one.
+ *
+ * Determinism: zero credentials, zero network. Google auth is bypassed by
+ * injecting a fake `authClient` (the SDK's supported constructor option —
+ * `accessToken` alone does NOT skip the auth client, adaptRequest always
+ * awaits it), and transport is a recording `fetch` injected through the
+ * SDK's own `fetch` option. No global patching, no SDK mocks.
+ */
+
+import { describe, it, expect } from "vitest";
+import { ChatAnthropic } from "@langchain/anthropic";
+import { AnthropicVertex } from "@anthropic-ai/vertex-sdk";
+import type { AuthClient } from "google-auth-library";
+import { HumanMessage, AIMessage, AIMessageChunk } from "@langchain/core/messages";
+
+// ─── Fixtures ────────────────────────────────────────────────────────────────
+
+/** Dated pre-4.6 id in Vertex's `@date` form (see llm-backend.ts translation). */
+const VERTEX_MODEL_ID = "claude-sonnet-4-5@20250929";
+const PROJECT_ID = "test-project";
+const REGION = "asia-south1";
+
+const WEATHER_TOOL = {
+  name: "get_weather",
+  description: "Get the current weather for a city.",
+  input_schema: {
+    type: "object" as const,
+    properties: { city: { type: "string" } },
+    required: ["city"],
+  },
+};
+
+/** Non-streaming (`:rawPredict`) response: text + tool_use + usage. */
+const MESSAGE_RESPONSE = {
+  id: "msg_vertex_test_01",
+  type: "message",
+  role: "assistant",
+  model: VERTEX_MODEL_ID,
+  content: [
+    { type: "text", text: "I'll check the weather." },
+    { type: "tool_use", id: "toolu_test_01", name: "get_weather", input: { city: "Chennai" } },
+  ],
+  stop_reason: "tool_use",
+  stop_sequence: null,
+  usage: { input_tokens: 25, output_tokens: 17 },
+};
+
+/**
+ * Streaming (`:streamRawPredict`) response as Anthropic SSE: a text block,
+ * a tool_use block assembled from input_json_delta, then cumulative usage in
+ * message_delta — the exact event grammar the nested SDK parses.
+ */
+const SSE_EVENTS: ReadonlyArray<[string, object]> = [
+  ["message_start", {
+    type: "message_start",
+    message: {
+      id: "msg_vertex_test_02", type: "message", role: "assistant",
+      model: VERTEX_MODEL_ID, content: [], stop_reason: null, stop_sequence: null,
+      usage: { input_tokens: 25, output_tokens: 1 },
+    },
+  }],
+  ["content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }],
+  ["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "I'll check the weather." } }],
+  ["content_block_stop", { type: "content_block_stop", index: 0 }],
+  ["content_block_start", { type: "content_block_start", index: 1, content_block: { type: "tool_use", id: "toolu_test_02", name: "get_weather", input: {} } }],
+  ["content_block_delta", { type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: '{"city":"Chennai"}' } }],
+  ["content_block_stop", { type: "content_block_stop", index: 1 }],
+  ["message_delta", { type: "message_delta", delta: { stop_reason: "tool_use", stop_sequence: null }, usage: { output_tokens: 17 } }],
+  ["message_stop", { type: "message_stop" }],
+];
+
+function sseBody(): string {
+  return SSE_EVENTS
+    .map(([event, data]) => `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`)
+    .join("");
+}
+
+// ─── Harness ─────────────────────────────────────────────────────────────────
+
+interface RecordedRequest {
+  url: string;
+  method: string;
+  headers: Record<string, string>;
+  body: Record<string, unknown>;
+}
+
+interface SeamHarness {
+  model: ChatAnthropic;
+  requests: RecordedRequest[];
+  /** Options ChatAnthropic passed to the createClient factory. */
+  factoryOptions: Array<{ maxRetries?: number }>;
+  /** The AnthropicVertex instances the factory constructed. */
+  clients: AnthropicVertex[];
+}
+
+/**
+ * Build a real ChatAnthropic wired to a real AnthropicVertex through the
+ * `createClient` seam, with transport replaced by a recording fetch.
+ *
+ * The factory honors `maxRetries` from the incoming options: LangChain owns
+ * retrying (its AsyncCaller wraps every request) and passes `maxRetries: 0`
+ * so the underlying SDK must not retry underneath it. A factory that ignored
+ * this would nest the Vertex SDK's default 2 retries inside LangChain's loop,
+ * multiplying every transient failure — the T02 adapter must preserve this.
+ */
+function buildSeamHarness(): SeamHarness {
+  const requests: RecordedRequest[] = [];
+  const factoryOptions: SeamHarness["factoryOptions"] = [];
+  const clients: AnthropicVertex[] = [];
+
+  const recordingFetch: typeof fetch = async (input, init) => {
+    const headers: Record<string, string> = {};
+    new Headers(init?.headers).forEach((value, key) => {
+      headers[key] = value;
+    });
+    requests.push({
+      url: String(input),
+      method: init?.method ?? "GET",
+      headers,
+      body: JSON.parse(String(init?.body)) as Record<string, unknown>,
+    });
+    const streaming = String(input).includes(":streamRawPredict");
+    return streaming
+      ? new Response(sseBody(), {
+          status: 200,
+          headers: { "Content-Type": "text/event-stream" },
+        })
+      : new Response(JSON.stringify(MESSAGE_RESPONSE), {
+          status: 200,
+          headers: { "Content-Type": "application/json" },
+        });
+  };
+
+  // Minimal structural fake of google-auth-library's AuthClient — the Vertex
+  // SDK's adaptRequest calls exactly `getRequestHeaders()` (merged into the
+  // outbound request) and reads `projectId`. The cast is test-only: building
+  // a real AuthClient would require credentials, defeating determinism.
+  const fakeAuthClient = {
+    projectId: PROJECT_ID,
+    getRequestHeaders: async () => new Headers({ authorization: "Bearer test-token" }),
+  } as unknown as AuthClient;
+
+  const model = new ChatAnthropic({
+    model: VERTEX_MODEL_ID,
+    temperature: 0,
+    maxTokens: 1024,
+    createClient: (options) => {
+      factoryOptions.push({ maxRetries: options.maxRetries });
+      const client = new AnthropicVertex({
+        projectId: PROJECT_ID,
+        region: REGION,
+        authClient: fakeAuthClient,
+        fetch: recordingFetch,
+        maxRetries: options.maxRetries,
+      });
+      clients.push(client);
+      return client;
+    },
+  });
+
+  return { model, requests, factoryOptions, clients };
+}
+
+const expectedUrl = (specifier: "rawPredict" | "streamRawPredict") =>
+  `https://${REGION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}` +
+  `/locations/${REGION}/publishers/anthropic/models/${VERTEX_MODEL_ID}:${specifier}`;
+
+// ─── Tests ───────────────────────────────────────────────────────────────────
+
+describe("ChatAnthropic createClient -> AnthropicVertex seam", () => {
+  it("constructs without an Anthropic API key when createClient is provided", () => {
+    // chat_models.js waives the "Anthropic API key not found" check for
+    // factory-constructed clients — the waiver the vertex backend depends on.
+    expect(() => buildSeamHarness()).not.toThrow();
+  });
+
+  it("shapes a non-streaming request into Vertex wire form (:rawPredict)", async () => {
+    const h = buildSeamHarness();
+
+    const result = await h.model.invoke([new HumanMessage("Weather in Chennai?")]);
+
+    expect(h.requests).toHaveLength(1);
+    const req = h.requests[0];
+
+    // The model id rides in the URL path — with `@` intact — not in the body.
+    expect(req.url).toBe(expectedUrl("rawPredict"));
+    expect(req.method).toBe("POST");
+    expect(req.body).not.toHaveProperty("model");
+    expect(req.body.anthropic_version).toBe("vertex-2023-10-16");
+    expect(req.body.max_tokens).toBe(1024);
+
+    // The fake auth client's OAuth header reached the wire.
+    expect(req.headers.authorization).toBe("Bearer test-token");
+
+    expect(result).toBeInstanceOf(AIMessage);
+  });
+
+  it("round-trips tool definitions and tool_use blocks into tool_calls", async () => {
+    const h = buildSeamHarness();
+    const withTools = h.model.bindTools([WEATHER_TOOL]);
+
+    const result = (await withTools.invoke([
+      new HumanMessage("Weather in Chennai?"),
+    ])) as AIMessage;
+
+    // Tool definition survived request shaping through the Vertex adapter.
+    const tools = h.requests[0].body.tools as Array<{ name: string }>;
+    expect(tools).toHaveLength(1);
+    expect(tools[0].name).toBe("get_weather");
+
+    // The tool_use response block became a LangChain tool_call with its id —
+    // the identity the HITL approval flow keys on.
+    expect(result.tool_calls).toHaveLength(1);
+    expect(result.tool_calls?.[0]).toMatchObject({
+      id: "toolu_test_01",
+      name: "get_weather",
+      args: { city: "Chennai" },
+    });
+  });
+
+  it("reports usage_metadata on non-streaming responses (what billing reads)", async () => {
+    const h = buildSeamHarness();
+
+    const result = (await h.model.invoke([
+      new HumanMessage("Weather in Chennai?"),
+    ])) as AIMessage;
+
+    expect(result.usage_metadata).toMatchObject({
+      input_tokens: 25,
+      output_tokens: 17,
+      total_tokens: 42,
+    });
+  });
+
+  it("streams via :streamRawPredict, assembling tool_calls and usage from SSE", async () => {
+    const h = buildSeamHarness();
+
+    let final: AIMessageChunk | undefined;
+    for await (const chunk of await h.model.stream([
+      new HumanMessage("Weather in Chennai?"),
+    ])) {
+      final = final === undefined ? chunk : final.concat(chunk);
+    }
+
+    expect(h.requests).toHaveLength(1);
+    expect(h.requests[0].url).toBe(expectedUrl("streamRawPredict"));
+    expect(h.requests[0].body.stream).toBe(true);
+    expect(h.requests[0].body).not.toHaveProperty("model");
+
+    expect(final).toBeDefined();
+    expect(final?.text).toBe("I'll check the weather.");
+    expect(final?.tool_calls).toHaveLength(1);
+    expect(final?.tool_calls?.[0]).toMatchObject({
+      id: "toolu_test_02",
+      name: "get_weather",
+      args: { city: "Chennai" },
+    });
+
+    // input from message_start; output accumulated across message events.
+    // Pinned to the observed accumulation so a LangChain bump that changes
+    // usage math (billing input) fails here first.
+    expect(final?.usage_metadata).toMatchObject({
+      input_tokens: 25,
+      output_tokens: 18,
+    });
+  });
+
+  it("passes maxRetries: 0 to the factory and the honored value reaches the client", async () => {
+    const h = buildSeamHarness();
+
+    await h.model.invoke([new HumanMessage("hi")]);
+    for await (const chunk of await h.model.stream([new HumanMessage("hi")])) {
+      void chunk;
+    }
+
+    // LangChain owns retrying: it must hand the factory maxRetries: 0, once
+    // per cached client (batch + streaming are constructed independently).
+    expect(h.factoryOptions).toHaveLength(2);
+    expect(h.factoryOptions.every((o) => o.maxRetries === 0)).toBe(true);
+    expect(h.clients).toHaveLength(2);
+    expect(h.clients.every((c) => c.maxRetries === 0)).toBe(true);
+  });
+});

package/src/shared/approval-policy.ts

@@ -16,7 +16,6 @@
  */
 
 import type { ToolApprovalPolicy } from "@stigmer/protos/ai/stigmer/agentic/mcpserver/v1/spec_pb";
-import type { ToolApprovalOverride } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/spec_pb";
 import type { AgentExecution } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
 import { ApprovalAction, ApprovalMode, ApprovalPolicySource } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
 import { toolApprovalCategory, type ToolApprovalCategory } from "./tool-kind.js";
@@ -321,9 +320,16 @@ export interface MergedToolPolicy {
  * Policy chain (each level overrides the previous):
  * 1. status.toolApprovals — system-generated defaults; presence = requires approval
  * 2. spec.pinnedToolApprovals — manual overrides; presence = requires approval
- * 3. agent tool_approval_overrides — explicit boolean per tool (enable OR disable)
+ * 3. usage tool_approval_overrides — explicit boolean per tool (enable OR disable)
  * 4. active leases — runtime bypass (highest priority), now scoped
  *
+ * Layer 3 is read from {@link ResolvedMcpServer.toolApprovalOverrides} — the
+ * overrides ride each server from the usage that resolved it (issue #349), so
+ * an override is STRUCTURALLY scoped to its own server. There is deliberately
+ * no cross-server override input: a flat list applied inside this per-server
+ * loop is how an override once leaked onto — or silently un-gated — a
+ * same-named tool on another server.
+ *
  * The map carries ONLY the tools that require approval — a tool's absence means
  * "auto-approved". Leases shape that absence:
  * - On a global pre-arm ({@link ActiveLeases.global}) the map is empty.
@@ -341,7 +347,6 @@ export interface MergedToolPolicy {
  */
 export function mergeApprovalPolicies(
   resolvedServers: ResolvedMcpServer[],
-  agentOverrides: ToolApprovalOverride[],
   leases: ActiveLeases,
 ): Map<string, MergedToolPolicy> {
   const merged = new Map<string, MergedToolPolicy>();
@@ -382,10 +387,12 @@ export function mergeApprovalPolicies(
       });
     }
 
-    // Layer 3: per-agent overrides (explicit boolean, can enable or disable).
-    // Touching a tool here makes the per-agent layer the responsible source,
-    // whether it enables, disables, or re-messages the gate.
-    for (const override of agentOverrides) {
+    // Layer 3: this usage's overrides (explicit boolean, can enable or
+    // disable) — scoped to THIS server because they arrived on it (see the
+    // function doc). Touching a tool here makes the per-agent layer the
+    // responsible source, whether it enables, disables, or re-messages the
+    // gate.
+    for (const override of server.toolApprovalOverrides) {
       if (!override.toolName) continue;
       const existing = serverPolicies.get(override.toolName);
       if (existing) {

package/src/shared/artifact-storage.ts

@@ -13,6 +13,11 @@
  * The runner never holds R2/S3 credentials — in cloud mode it calls the proxy
  * to obtain a presigned upload URL, then PUTs content over plain HTTPS.
  *
+ * Every proxy-backend request rides the shared bounded-backoff loop
+ * (http-retry.ts) with a per-request abort timeout, so a momentary
+ * stigmer-service interruption is retried and a hung connection cannot
+ * stall an agent turn unboundedly (stigmer/stigmer#468).
+ *
  * DD-6: No direct R2 backend. Local + Proxy only.
  */
 
@@ -20,6 +25,8 @@ import { mkdir, writeFile, readFile, access, rm } from "node:fs/promises";
 import { dirname, join, resolve, sep } from "node:path";
 import { homedir } from "node:os";
 import type { Config } from "../config.js";
+import type { RetryOptions } from "./grpc-retry.js";
+import { fetchWithRetry, type FetchRetryPolicy } from "./http-retry.js";
 
 // ── Interface ────────────────────────────────────────────────────────
 
@@ -117,13 +124,64 @@ export class LocalArtifactStorage implements ArtifactStorage {
 
 // ── Proxy Backend ────────────────────────────────────────────────────
 
+/** Retry policy plus the two per-request bounds; every field is defaulted. */
+export interface ProxyArtifactStorageOptions extends RetryOptions {
+  /** Bound for proxy presign calls and the 1-byte exists probe. Default 30 s. */
+  readonly requestTimeoutMs?: number;
+  /**
+   * Bound for the presigned R2 PUT/GET, which move whole artifact payloads
+   * (claim-check offloads, tool outputs, file-review captures — MB scale).
+   * Deliberately wider than requestTimeoutMs so a legitimate slow transfer
+   * is never aborted by a bound sized for JSON round-trips. Default 120 s.
+   */
+  readonly transferTimeoutMs?: number;
+}
+
+/**
+ * Default transient-retry policy (stigmer/stigmer#468): the checkpointer's
+ * budget — sized to span a pod-restart reroute, ~7.75 s of delays across
+ * 5 retries. Retrying is safe at every site: the presign POSTs are
+ * idempotent mints, the PUT re-sends identical bytes to the same key (a
+ * keyed overwrite), and the GETs are reads.
+ */
+const DEFAULT_RETRY = {
+  baseDelayMs: 250,
+  backoffFactor: 2,
+  maxRetries: 5,
+  requestTimeoutMs: 30_000,
+  transferTimeoutMs: 120_000,
+} as const;
+
 export class ProxyArtifactStorage implements ArtifactStorage {
   private readonly baseUrl: string;
   private readonly authTokenSource: ProxyAuthTokenSource;
-
-  constructor(proxyEndpoint: string, authToken: ProxyAuthTokenSource) {
+  /** Governs the proxy presign calls and the exists probe. */
+  private readonly requestPolicy: FetchRetryPolicy;
+  /** Governs the presigned R2 transfers; same budget, wider timeout. */
+  private readonly transferPolicy: FetchRetryPolicy;
+
+  constructor(
+    proxyEndpoint: string,
+    authToken: ProxyAuthTokenSource,
+    options: ProxyArtifactStorageOptions = {},
+  ) {
     this.baseUrl = `${proxyEndpoint.replace(/\/+$/, "")}/v1/proxy/artifacts`;
     this.authTokenSource = authToken;
+    const shared = {
+      label: "ProxyArtifactStorage",
+      baseDelayMs: options.baseDelayMs ?? DEFAULT_RETRY.baseDelayMs,
+      backoffFactor: options.backoffFactor ?? DEFAULT_RETRY.backoffFactor,
+      maxRetries: options.maxRetries ?? DEFAULT_RETRY.maxRetries,
+      delayFn: options.delayFn,
+    };
+    this.requestPolicy = {
+      ...shared,
+      requestTimeoutMs: options.requestTimeoutMs ?? DEFAULT_RETRY.requestTimeoutMs,
+    };
+    this.transferPolicy = {
+      ...shared,
+      requestTimeoutMs: options.transferTimeoutMs ?? DEFAULT_RETRY.transferTimeoutMs,
+    };
   }
 
   /**
@@ -141,14 +199,14 @@ export class ProxyArtifactStorage implements ArtifactStorage {
   async upload(key: string, content: Buffer, contentType?: string): Promise<string> {
     const ct = contentType ?? "application/octet-stream";
 
-    const presignResp = await fetch(`${this.baseUrl}/presigned-upload-url`, {
+    const presignResp = await fetchWithRetry(`${this.baseUrl}/presigned-upload-url`, {
       method: "POST",
       headers: {
         "Authorization": `Bearer ${this.authToken}`,
         "Content-Type": "application/json",
       },
       body: JSON.stringify({ key, content_type: ct }),
-    });
+    }, this.requestPolicy);
     if (!presignResp.ok) {
       throw new Error(
         `Failed to get presigned upload URL (HTTP ${presignResp.status}): ` +
@@ -180,11 +238,14 @@ export class ProxyArtifactStorage implements ArtifactStorage {
       uploadHeaders["Content-Type"] = ct;
     }
 
-    const putResp = await fetch(data.url, {
+    // Retrying the PUT alone (not the presign+PUT pair) is deliberate: the
+    // retry budget's few seconds of delays sit far inside presign validity,
+    // and re-sending the same Buffer to the same key is a keyed overwrite.
+    const putResp = await fetchWithRetry(data.url, {
       method: "PUT",
       headers: uploadHeaders,
       body: content,
-    });
+    }, this.transferPolicy);
     if (!putResp.ok) {
       throw new Error(
         `Presigned upload failed (HTTP ${putResp.status}): ` +
@@ -196,14 +257,14 @@ export class ProxyArtifactStorage implements ArtifactStorage {
   }
 
   async getDownloadUrl(key: string): Promise<string> {
-    const resp = await fetch(`${this.baseUrl}/presigned-download-url`, {
+    const resp = await fetchWithRetry(`${this.baseUrl}/presigned-download-url`, {
       method: "POST",
       headers: {
         "Authorization": `Bearer ${this.authToken}`,
         "Content-Type": "application/json",
       },
       body: JSON.stringify({ key }),
-    });
+    }, this.requestPolicy);
     if (!resp.ok) {
       throw new Error(
         `Failed to get presigned download URL (HTTP ${resp.status}): ` +
@@ -216,7 +277,7 @@ export class ProxyArtifactStorage implements ArtifactStorage {
 
   async download(key: string): Promise<Buffer> {
     const url = await this.getDownloadUrl(key);
-    const resp = await fetch(url);
+    const resp = await fetchWithRetry(url, undefined, this.transferPolicy);
     if (!resp.ok) {
       throw new Error(
         `Artifact download failed (HTTP ${resp.status}) for key '${key}': ` +
@@ -242,7 +303,11 @@ export class ProxyArtifactStorage implements ArtifactStorage {
       // addressed CAS blob dedup) then re-uploads, which is idempotent.
       return false;
     }
-    const resp = await fetch(url, { headers: { Range: "bytes=0-0" } });
+    const resp = await fetchWithRetry(
+      url,
+      { headers: { Range: "bytes=0-0" } },
+      this.requestPolicy,
+    );
     await resp.arrayBuffer().catch(() => undefined); // drain the <=1-byte body
     if (resp.status === 404) return false;
     if (resp.status === 200 || resp.status === 206 || resp.status === 416) return true;