@indigoai-us/hq-cloud 5.46.0 → 5.47.0

package/src/object-io.ts

@@ -0,0 +1,782 @@
+/**
+ * ObjectIO — transport seam for vault object byte/metadata movement.
+ *
+ * s3.ts holds the *semantics* of sync (symlink-record encoding, mode/mtime
+ * stamping, created-at preservation, directory-marker filtering). Those never
+ * change. What CAN change is the *wire transport* underneath them:
+ *
+ *   - `S3SdkObjectIO` — the historical path. STS-vended credentials + the AWS
+ *     S3 SDK talking directly to the per-company bucket. No policy-size
+ *     ceiling concern for the BYTES, but the STS session policy that grants
+ *     access has the 2048-char IAM limit that motivated the presigned model.
+ *
+ *   - `PresignObjectIO` — the presigned-URL path. The vault-service decides
+ *     access as a runtime DDB check (no IAM policy ceiling) and hands back
+ *     short-lived presigned GET/PUT/DELETE URLs + (for PUT) the exact headers
+ *     to replay. The client never holds AWS credentials — it just `fetch`es
+ *     the signed URLs.
+ *
+ * The seam is a per-EntityContext factory resolved INSIDE s3.ts, so every
+ * existing call site (`uploadFile(ctx, …)`, `downloadFile(ctx, …)`, …) keeps
+ * its signature. `runRunner` selects the transport once per session via
+ * {@link setObjectIOFactory}; absent any selection the default is the S3 SDK,
+ * preserving today's behavior for every non-gated caller.
+ */
+
+import {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand,
+  ListObjectsV2Command,
+  DeleteObjectCommand,
+  HeadObjectCommand,
+} from "@aws-sdk/client-s3";
+import type { EntityContext } from "./types.js";
+import type {
+  PresignOp,
+  PresignKeyInput,
+  PresignResultRow,
+  VaultListedObject,
+} from "./vault-client.js";
+import { VaultClientError } from "./vault-client.js";
+
+/**
+ * The slice of {@link VaultClient} the presigned transport needs. Narrowed to
+ * just `presign` + `listFiles` so the factory accepts any caller that exposes
+ * those two (the real VaultClient, or a stub in tests) without depending on
+ * the full 20-method surface.
+ */
+export interface PresignTransportClient {
+  presign(input: {
+    companyUid: string;
+    op?: PresignOp;
+    expiresIn?: number;
+    keys: PresignKeyInput[];
+  }): Promise<{ results: PresignResultRow[]; expiresAt: string }>;
+  listFiles(
+    companyUid: string,
+    prefix?: string,
+    cursor?: string,
+  ): Promise<{
+    objects: VaultListedObject[];
+    cursor: string | null;
+    truncated: boolean;
+  }>;
+}
+
+// ---------------------------------------------------------------------------
+// Wire-primitive shapes
+// ---------------------------------------------------------------------------
+
+export interface PutObjectInput {
+  key: string;
+  body: Buffer;
+  contentType: string;
+  /** S3 user metadata (x-amz-meta-*). Lowercased keys by convention. */
+  metadata?: Record<string, string>;
+}
+
+export interface GetObjectResult {
+  body: Buffer;
+  /** S3 user metadata (keys lowercased by S3). */
+  metadata?: Record<string, string>;
+}
+
+export interface ListObjectsInput {
+  prefix?: string;
+  continuationToken?: string;
+}
+
+export interface ListedRemoteObject {
+  key: string;
+  size: number;
+  lastModified: Date;
+  etag: string;
+}
+
+export interface ListObjectsResult {
+  objects: ListedRemoteObject[];
+  /** Opaque cursor for the next page; undefined when the listing is exhausted. */
+  nextContinuationToken?: string;
+}
+
+export interface HeadObjectResult {
+  lastModified: Date;
+  etag: string;
+  size: number;
+  metadata?: Record<string, string>;
+}
+
+/**
+ * The minimal byte/metadata transport s3.ts needs. Deliberately narrow — no
+ * symlink, mode, or created-at concepts leak in here; those live one layer up
+ * in s3.ts and compose on top of these five primitives.
+ */
+export interface ObjectIO {
+  putObject(input: PutObjectInput): Promise<{ etag: string }>;
+  getObject(key: string): Promise<GetObjectResult>;
+  listObjects(input: ListObjectsInput): Promise<ListObjectsResult>;
+  deleteObject(key: string): Promise<void>;
+  /** Null when the key does not exist (404 / 403-as-absent). */
+  headObject(key: string): Promise<HeadObjectResult | null>;
+  /**
+   * Optional batch pre-mint. Warms an internal URL cache for `keys` under `op`
+   * so subsequent per-key get/head (and, when primed, put/delete) calls reuse a
+   * pre-signed URL instead of issuing one presign request each. This is what
+   * turns an N-file sync from N presign calls into ceil(N/chunk) — the
+   * difference between staying under and blowing past the 100-req/hr limit on a
+   * bulk pull. The S3 SDK transport has no presign step and omits this (the
+   * per-call cost there is the SDK request itself, not a separate presign).
+   * Best-effort: a failed chunk or per-key denial simply leaves those keys
+   * uncached, and the per-key call falls back to a single presign.
+   */
+  prime?(op: PresignOp, keys: PresignKeyInput[]): Promise<void>;
+  /**
+   * True if a live primed PUT URL exists for `key`. Lets uploadFile/uploadSymlink
+   * skip recomputing metadata + the created-at HEAD when a `prime("put", …)`
+   * pre-pass already signed the metadata into the cached URL (the upload just
+   * sends the body, replaying the cached headers). Absent (undefined) on the S3
+   * SDK transport → callers take their normal compute-metadata path.
+   */
+  hasPrimedPut?(key: string): boolean;
+}
+
+// ---------------------------------------------------------------------------
+// S3 SDK transport (default)
+// ---------------------------------------------------------------------------
+
+function stripQuotes(etag: string | undefined): string {
+  return etag ? etag.replace(/^"|"$/g, "") : "";
+}
+
+async function drainToBuffer(
+  body: AsyncIterable<Uint8Array> | undefined,
+): Promise<Buffer> {
+  if (!body) return Buffer.alloc(0);
+  const chunks: Buffer[] = [];
+  for await (const chunk of body) {
+    chunks.push(Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks);
+}
+
+/**
+ * Direct-to-S3 transport over STS-vended credentials. A fresh client per
+ * instance so the latest credentials from the EntityContext are always used
+ * (caching/refresh is the caller's concern, at the EntityContext level — see
+ * the original buildClient note in s3.ts).
+ */
+export class S3SdkObjectIO implements ObjectIO {
+  private readonly client: S3Client;
+  private readonly bucket: string;
+
+  constructor(ctx: EntityContext) {
+    this.bucket = ctx.bucketName;
+    this.client = new S3Client({
+      region: ctx.region,
+      credentials: {
+        accessKeyId: ctx.credentials.accessKeyId,
+        secretAccessKey: ctx.credentials.secretAccessKey,
+        sessionToken: ctx.credentials.sessionToken,
+      },
+    });
+  }
+
+  async putObject(input: PutObjectInput): Promise<{ etag: string }> {
+    const res = await this.client.send(
+      new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: input.key,
+        Body: input.body,
+        ContentType: input.contentType,
+        ...(input.metadata && Object.keys(input.metadata).length > 0
+          ? { Metadata: input.metadata }
+          : {}),
+      }),
+    );
+    return { etag: res.ETag || "" };
+  }
+
+  async getObject(key: string): Promise<GetObjectResult> {
+    const res = await this.client.send(
+      new GetObjectCommand({ Bucket: this.bucket, Key: key }),
+    );
+    if (!res.Body) {
+      throw new Error(`Empty response for ${key}`);
+    }
+    const body = await drainToBuffer(res.Body as AsyncIterable<Uint8Array>);
+    return { body, metadata: res.Metadata };
+  }
+
+  async listObjects(input: ListObjectsInput): Promise<ListObjectsResult> {
+    const res = await this.client.send(
+      new ListObjectsV2Command({
+        Bucket: this.bucket,
+        Prefix: input.prefix,
+        ContinuationToken: input.continuationToken,
+      }),
+    );
+    const objects: ListedRemoteObject[] = [];
+    for (const obj of res.Contents || []) {
+      if (!obj.Key) continue;
+      objects.push({
+        key: obj.Key,
+        size: obj.Size ?? 0,
+        lastModified: obj.LastModified || new Date(),
+        etag: obj.ETag || "",
+      });
+    }
+    return {
+      objects,
+      nextContinuationToken: res.NextContinuationToken,
+    };
+  }
+
+  async deleteObject(key: string): Promise<void> {
+    await this.client.send(
+      new DeleteObjectCommand({ Bucket: this.bucket, Key: key }),
+    );
+  }
+
+  async headObject(key: string): Promise<HeadObjectResult | null> {
+    try {
+      const res = await this.client.send(
+        new HeadObjectCommand({ Bucket: this.bucket, Key: key }),
+      );
+      return {
+        lastModified: res.LastModified || new Date(),
+        etag: res.ETag || "",
+        size: res.ContentLength || 0,
+        metadata: res.Metadata,
+      };
+    } catch (err: unknown) {
+      if (
+        err &&
+        typeof err === "object" &&
+        "name" in err &&
+        (err as { name?: string }).name === "NotFound"
+      ) {
+        return null;
+      }
+      throw err;
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Presigned-URL transport
+// ---------------------------------------------------------------------------
+
+/**
+ * Pull every `x-amz-meta-*` response header into a plain metadata record with
+ * the prefix stripped. S3 surfaces user metadata this way on GET/HEAD; fetch
+ * lowercases header names, matching how the SDK lowercases `Metadata` keys, so
+ * the read path in s3.ts is identical across both transports.
+ */
+function metaFromHeaders(headers: Headers): Record<string, string> {
+  const meta: Record<string, string> = {};
+  headers.forEach((value, name) => {
+    if (name.startsWith("x-amz-meta-")) {
+      meta[name.slice("x-amz-meta-".length)] = value;
+    }
+  });
+  return meta;
+}
+
+function firstRowOrThrow(
+  results: PresignResultRow[],
+  key: string,
+  op: string,
+): PresignResultRow {
+  const row = results[0];
+  if (!row) {
+    throw new Error(`presign ${op} returned no row for ${key}`);
+  }
+  if (row.error || !row.url) {
+    throw new Error(
+      `presign ${op} denied for ${key}: ${row.error ?? "no url"}${
+        row.code ? ` (${row.code})` : ""
+      }`,
+    );
+  }
+  return row;
+}
+
+/**
+ * An error shaped like the AWS SDK's NoSuchKey/NotFound so existing catch
+ * sites in s3.ts (which test `err.name === "NotFound"`) treat a presigned
+ * 404 the same as an SDK 404.
+ */
+function notFoundError(key: string): Error {
+  return Object.assign(new Error(`Not found: ${key}`), { name: "NotFound" });
+}
+
+/**
+ * Max keys per presign request when priming — the server's hard batch cap
+ * (hq-pro files-presign MAX_BATCH_KEYS = 1000). One presign call costs ONE
+ * audit row toward the 100/hr limit regardless of how many keys it carries, so
+ * filling the batch is strictly better: a 5.5k-file pull is 6 presign calls at
+ * 1000/chunk vs 55 at 100. (The sibling list page size is fixed at 1000 by
+ * AWS's ListObjectsV2 MaxKeys cap — that one we can't raise.)
+ */
+const PRIME_CHUNK = 1000;
+/**
+ * Lifetime requested for primed URLs. Generous (30 min) so a whole sync batch
+ * completes within one prime, but well inside the presign Lambda role's
+ * credential lifetime (a presigned URL cannot outlive the creds that signed
+ * it). A batch that somehow outruns this falls back to per-key single presign.
+ */
+const PRIME_EXPIRES_IN_SECONDS = 1800;
+/** Concurrent prime chunks in flight (each is one presign HTTP call). */
+const PRIME_CONCURRENCY = 4;
+/** Treat a cached URL within this window of expiry as a miss (re-presign). */
+const CACHE_SAFETY_MS = 60_000;
+
+interface CacheEntry {
+  url: string;
+  headers?: Record<string, string>;
+  expiresAtMs: number;
+}
+
+/**
+ * Thrown when a presign mint is skipped because the per-user 100/hr vault rate
+ * budget is exhausted. Distinct name so callers can tell "deferred, retry next
+ * sync" apart from a real transfer failure. The key was NOT synced.
+ */
+export class RateLimitedError extends Error {
+  constructor(
+    readonly key: string,
+    readonly op: PresignOp,
+    options?: { cause?: unknown },
+  ) {
+    super(`rate limited (100/hr) — ${op} ${key} deferred to next sync`);
+    this.name = "RateLimited";
+    if (options?.cause !== undefined) {
+      (this as { cause?: unknown }).cause = options.cause;
+    }
+  }
+}
+
+/**
+ * One-way circuit breaker shared across a run's per-company transports. The
+ * first 429 (vault rate budget exhausted) trips it; thereafter every UNCACHED
+ * presign fails fast with {@link RateLimitedError} instead of hitting the wire.
+ *
+ * Without this, an exhausted budget spirals: prime chunks 429 → keys uncached
+ * → per-file presign → each 429s (after VaultClient's own 3 retries +
+ * backoff) → an 86-minute storm of ~10k doomed calls (observed live). Tripping
+ * once and short-circuiting turns that into a clean fast finish: primed URLs
+ * still work, un-primed keys are deferred, and the run reports them so the next
+ * sync (after the rolling hour recovers) picks them up.
+ */
+export class RateLimitBreaker {
+  private tripped = false;
+  isTripped(): boolean {
+    return this.tripped;
+  }
+  trip(): void {
+    this.tripped = true;
+  }
+}
+
+/** A VaultClient 429 (rate budget exhausted) after its own retries. */
+function isRateLimit(err: unknown): boolean {
+  return err instanceof VaultClientError && err.statusCode === 429;
+}
+
+/**
+ * Transport that moves bytes over short-lived presigned URLs minted by the
+ * vault-service. Holds no AWS credentials. `companyUid` is the EntityContext's
+ * `uid` — the server resolves the per-company bucket from it, so cross-company
+ * reach is structurally impossible (same authority model as the list/presign
+ * handlers).
+ *
+ * URL cache: {@link prime} batch-mints URLs into `urlCache` (keyed by op+key)
+ * so the per-file get/head calls during a sync reuse them instead of issuing a
+ * presign request each. A single instance is shared across all s3.ts calls for
+ * one company within a run (see {@link presignObjectIOFactory} memoization), so
+ * a prime before the transfer loop warms the cache the loop then drains.
+ */
+export class PresignObjectIO implements ObjectIO {
+  private readonly urlCache = new Map<string, CacheEntry>();
+
+  constructor(
+    private readonly vault: PresignTransportClient,
+    private readonly companyUid: string,
+    // Shared across a run's per-company instances by the factory; a directly
+    // constructed instance gets its own (fine for tests / one-offs).
+    private readonly breaker: RateLimitBreaker = new RateLimitBreaker(),
+  ) {}
+
+  private cacheKey(op: PresignOp, key: string): string {
+    return `${op}${key}`;
+  }
+
+  hasPrimedPut(key: string): boolean {
+    return this.cached("put", key) !== undefined;
+  }
+
+  /** A live (non-expiring) cached URL for op+key, or undefined. */
+  private cached(op: PresignOp, key: string): CacheEntry | undefined {
+    const hit = this.urlCache.get(this.cacheKey(op, key));
+    if (!hit) return undefined;
+    if (Date.now() >= hit.expiresAtMs - CACHE_SAFETY_MS) {
+      this.urlCache.delete(this.cacheKey(op, key));
+      return undefined;
+    }
+    return hit;
+  }
+
+  /**
+   * Resolve a presigned URL (+ replay headers) for op+key: cache hit if primed,
+   * else a single presign. Throws on per-key denial (matches the SDK path's
+   * access error). `extra` carries PUT contentType/metadata on the miss path.
+   */
+  private async resolveUrl(
+    op: PresignOp,
+    key: string,
+    extra?: { contentType?: string; metadata?: Record<string, string> },
+  ): Promise<{ url: string; headers?: Record<string, string> }> {
+    const hit = this.cached(op, key);
+    if (hit) return { url: hit.url, headers: hit.headers };
+    // Primed URLs still serve once the breaker trips (no wire needed); only an
+    // uncached key on an exhausted budget fails fast.
+    if (this.breaker.isTripped()) throw new RateLimitedError(key, op);
+    let results;
+    try {
+      ({ results } = await this.vault.presign({
+        companyUid: this.companyUid,
+        op,
+        keys: [{ key, op, ...extra }],
+      }));
+    } catch (err) {
+      if (isRateLimit(err)) {
+        this.breaker.trip();
+        throw new RateLimitedError(key, op, { cause: err });
+      }
+      throw err;
+    }
+    const row = firstRowOrThrow(results, key, op);
+    return { url: row.url!, headers: row.headers };
+  }
+
+  async prime(op: PresignOp, keys: PresignKeyInput[]): Promise<void> {
+    if (keys.length === 0) return;
+    const chunks: PresignKeyInput[][] = [];
+    for (let i = 0; i < keys.length; i += PRIME_CHUNK) {
+      chunks.push(keys.slice(i, i + PRIME_CHUNK));
+    }
+    let next = 0;
+    const worker = async (): Promise<void> => {
+      while (next < chunks.length) {
+        // Budget already exhausted — stop priming; remaining keys defer.
+        if (this.breaker.isTripped()) return;
+        const chunk = chunks[next++];
+        let resp;
+        try {
+          resp = await this.vault.presign({
+            companyUid: this.companyUid,
+            op,
+            expiresIn: PRIME_EXPIRES_IN_SECONDS,
+            keys: chunk.map((k) => ({ ...k, op })),
+          });
+        } catch (err) {
+          if (isRateLimit(err)) {
+            // Budget exhausted mid-prime: trip the breaker and stop. Priming
+            // on means every remaining chunk + per-file fallback would 429 —
+            // the spiral. Tripping makes the transfer loop fail fast instead.
+            this.breaker.trip();
+            return;
+          }
+          // A non-429 chunk failure just means those keys aren't cached — the
+          // per-key call will single-presign. Never let priming fail the sync.
+          continue;
+        }
+        const now = Date.now();
+        for (const row of resp.results) {
+          if (row.error || !row.url) continue;
+          this.urlCache.set(this.cacheKey(op, row.key), {
+            url: row.url,
+            headers: row.headers,
+            expiresAtMs: now + (row.expiresIn ?? PRIME_EXPIRES_IN_SECONDS) * 1000,
+          });
+        }
+      }
+    };
+    await Promise.all(
+      Array.from({ length: Math.min(PRIME_CONCURRENCY, chunks.length) }, worker),
+    );
+  }
+
+  async putObject(input: PutObjectInput): Promise<{ etag: string }> {
+    const row = await this.resolveUrl("put", input.key, {
+      contentType: input.contentType,
+      ...(input.metadata && Object.keys(input.metadata).length > 0
+        ? { metadata: input.metadata }
+        : {}),
+    });
+    // The server signs Content-Type, SSE-KMS, and every x-amz-meta-* into the
+    // signature and returns them in `headers`; they MUST be replayed verbatim
+    // or SigV4 rejects the PUT.
+    const res = await fetchWithRetry(
+      row.url,
+      { method: "PUT", body: input.body, headers: row.headers ?? {} },
+      `presigned PUT ${input.key}`,
+    );
+    if (!res.ok) {
+      const detail = await safeText(res);
+      throw new Error(
+        `presigned PUT failed for ${input.key}: ${res.status} ${detail}`,
+      );
+    }
+    return { etag: stripQuotes(res.headers.get("etag") ?? undefined) };
+  }
+
+  async getObject(key: string): Promise<GetObjectResult> {
+    const row = await this.resolveUrl("get", key);
+    const res = await fetchWithRetry(row.url, { method: "GET" }, `presigned GET ${key}`);
+    if (res.status === 404) {
+      await cancelBody(res);
+      throw notFoundError(key);
+    }
+    if (!res.ok) {
+      const detail = await safeText(res);
+      throw new Error(`presigned GET failed for ${key}: ${res.status} ${detail}`);
+    }
+    const body = Buffer.from(await res.arrayBuffer());
+    return { body, metadata: metaFromHeaders(res.headers) };
+  }
+
+  async listObjects(input: ListObjectsInput): Promise<ListObjectsResult> {
+    const { objects, cursor } = await this.vault.listFiles(
+      this.companyUid,
+      input.prefix,
+      input.continuationToken,
+    );
+    return {
+      objects: objects.map((o) => ({
+        key: o.key,
+        size: o.size,
+        lastModified: o.lastModified ? new Date(o.lastModified) : new Date(),
+        etag: o.etag ?? "",
+      })),
+      nextContinuationToken: cursor ?? undefined,
+    };
+  }
+
+  async deleteObject(key: string): Promise<void> {
+    const row = await this.resolveUrl("delete", key);
+    const res = await fetchWithRetry(row.url, { method: "DELETE" }, `presigned DELETE ${key}`);
+    // S3 DELETE is idempotent — a 204 (deleted) and a 404 (already gone) are
+    // both success for the sync engine's purposes.
+    if (!res.ok && res.status !== 404) {
+      const detail = await safeText(res);
+      throw new Error(
+        `presigned DELETE failed for ${key}: ${res.status} ${detail}`,
+      );
+    }
+  }
+
+  async headObject(key: string): Promise<HeadObjectResult | null> {
+    // The presign endpoint has no HEAD op (get/put/delete only). A presigned
+    // GET signs the GET method, so we issue a real GET and read only the
+    // response headers, cancelling the body stream before it downloads — the
+    // headers (etag, content-length, last-modified, x-amz-meta-*) are all we
+    // need and arrive before the body. Cheap for the created-at-preservation
+    // and conflict-detection call sites that use headObject. Reuses the GET
+    // cache: a prime("get", …) before a pull warms these HEADs for free.
+    let url: string;
+    const hit = this.cached("get", key);
+    if (hit) {
+      url = hit.url;
+    } else {
+      if (this.breaker.isTripped()) throw new RateLimitedError(key, "get");
+      let results;
+      try {
+        ({ results } = await this.vault.presign({
+          companyUid: this.companyUid,
+          op: "get",
+          keys: [{ key, op: "get" }],
+        }));
+      } catch (err) {
+        if (isRateLimit(err)) {
+          this.breaker.trip();
+          throw new RateLimitedError(key, "get", { cause: err });
+        }
+        throw err;
+      }
+      const row = results[0];
+      if (!row || row.error || !row.url) {
+        // A per-key denial here means the caller can't read the key — treat as
+        // absent for HEAD semantics (the SDK path would 403, which callers map
+        // to "no usable head"); they all tolerate null.
+        return null;
+      }
+      url = row.url;
+    }
+    const res = await fetchWithRetry(url, { method: "GET" }, `presigned HEAD ${key}`);
+    if (res.status === 404 || res.status === 403) {
+      await cancelBody(res);
+      return null;
+    }
+    if (!res.ok) {
+      await cancelBody(res);
+      const detail = await safeText(res);
+      throw new Error(`presigned HEAD failed for ${key}: ${res.status} ${detail}`);
+    }
+    const result: HeadObjectResult = {
+      lastModified: parseLastModified(res.headers.get("last-modified")),
+      etag: stripQuotes(res.headers.get("etag") ?? undefined),
+      size: Number(res.headers.get("content-length") ?? "0"),
+      metadata: metaFromHeaders(res.headers),
+    };
+    await cancelBody(res);
+    return result;
+  }
+}
+
+async function safeText(res: Response): Promise<string> {
+  try {
+    return (await res.text()).slice(0, 200);
+  } catch {
+    return "";
+  }
+}
+
+async function cancelBody(res: Response): Promise<void> {
+  try {
+    await res.body?.cancel();
+  } catch {
+    // Best-effort — the socket is released either way once GC'd.
+  }
+}
+
+function parseLastModified(value: string | null): Date {
+  if (!value) return new Date();
+  const d = new Date(value);
+  return Number.isNaN(d.getTime()) ? new Date() : d;
+}
+
+// ---------------------------------------------------------------------------
+// Transient-failure retry for the presigned-URL fetches
+// ---------------------------------------------------------------------------
+//
+// The AWS S3 SDK retries transient errors (5xx, throttling, dropped sockets)
+// automatically with backoff. Moving the byte transfer to `fetch` over a
+// presigned URL dropped that resilience — and at sync scale (thousands of
+// objects per pull) transient S3 5xx (notably 503 SlowDown) are routine, so
+// without retry a large sync sporadically loses files. This restores
+// SDK-parity: retry network errors and transient 5xx with exponential backoff
+// + jitter; 4xx (404/403) are definitive and pass straight through.
+
+const FETCH_MAX_RETRIES = 3;
+const FETCH_BASE_DELAY_MS = 400;
+
+function isTransientStatus(status: number): boolean {
+  return status === 500 || status === 502 || status === 503 || status === 504;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * `fetch` with bounded retry on network errors + transient 5xx. The presigned
+ * URL is reusable until expiry and the bodies are in-memory Buffers, so a
+ * retry simply re-issues the same request. Jitter avoids a thundering-herd
+ * re-retry when the whole transfer pool hits a 503 SlowDown at once. After
+ * exhausting retries on a 5xx it returns the final (failing) Response so the
+ * caller's normal status handling reports it; network errors throw.
+ */
+async function fetchWithRetry(
+  url: string,
+  init: RequestInit,
+  what: string,
+): Promise<Response> {
+  let lastError: unknown;
+  for (let attempt = 0; attempt <= FETCH_MAX_RETRIES; attempt++) {
+    if (attempt > 0) {
+      const backoff = FETCH_BASE_DELAY_MS * 2 ** (attempt - 1);
+      const jitter = Math.floor(Math.random() * FETCH_BASE_DELAY_MS);
+      await sleep(backoff + jitter);
+    }
+    let res: Response;
+    try {
+      res = await fetch(url, init);
+    } catch (err) {
+      lastError = err; // socket reset / DNS / TLS — retry
+      continue;
+    }
+    if (isTransientStatus(res.status) && attempt < FETCH_MAX_RETRIES) {
+      await cancelBody(res); // free the socket before backoff
+      lastError = new Error(`${what}: transient ${res.status}`);
+      continue;
+    }
+    return res; // success, a non-transient status, or the last 5xx attempt
+  }
+  throw lastError instanceof Error
+    ? lastError
+    : new Error(`${what}: failed after ${FETCH_MAX_RETRIES} retries`);
+}
+
+// ---------------------------------------------------------------------------
+// Factory registry — selected once per session by runRunner
+// ---------------------------------------------------------------------------
+
+export type ObjectIOFactory = (ctx: EntityContext) => ObjectIO;
+
+const DEFAULT_FACTORY: ObjectIOFactory = (ctx) => new S3SdkObjectIO(ctx);
+
+let activeFactory: ObjectIOFactory = DEFAULT_FACTORY;
+
+/**
+ * Install the transport factory for the current process. Passing `null`
+ * resets to the default S3 SDK transport. Called once by `runRunner` after it
+ * resolves the caller's identity + feature-flag gate; every subsequent s3.ts
+ * call resolves its transport through this.
+ */
+export function setObjectIOFactory(factory: ObjectIOFactory | null): void {
+  activeFactory = factory ?? DEFAULT_FACTORY;
+}
+
+/** Resolve the transport for an EntityContext using the active factory. */
+export function resolveObjectIO(ctx: EntityContext): ObjectIO {
+  return activeFactory(ctx);
+}
+
+/**
+ * Build a factory that routes every EntityContext through the presigned-URL
+ * transport, reusing the one already-authenticated VaultClient and deriving
+ * the per-company authority from `ctx.uid`.
+ */
+export function presignObjectIOFactory(
+  vault: PresignTransportClient,
+): ObjectIOFactory {
+  // Memoize one PresignObjectIO per company for the run, so a prime() and the
+  // transfer loop that drains its URL cache share the SAME instance. (Safe to
+  // memoize: PresignObjectIO holds only the vault client — whose token is a
+  // live getter — and the stable companyUid; it captures no rotating
+  // credentials, unlike S3SdkObjectIO, which is why the default factory is
+  // intentionally NOT memoized.)
+  // One breaker per run, shared across companies: the 100/hr budget is
+  // per-user, so a 429 in one company means the whole run should stop minting.
+  const breaker = new RateLimitBreaker();
+  const byCompany = new Map<string, PresignObjectIO>();
+  return (ctx) => {
+    // Personal vaults are a PERSON entity (prs_*) accessed via the membership-
+    // less vend-self model; the list/presign endpoints are membership-gated and
+    // 403 ("no active membership in company prs_…") for them. Personal vaults
+    // also have no ACL-scale problem (single owner), so they don't need
+    // presign — keep them on the S3 SDK (STS) transport. Presign is for company
+    // vaults (cmp_*), which is where the unbounded-grants problem lives.
+    if (!ctx.uid.startsWith("cmp_")) {
+      return new S3SdkObjectIO(ctx);
+    }
+    let io = byCompany.get(ctx.uid);
+    if (!io) {
+      io = new PresignObjectIO(vault, ctx.uid, breaker);
+      byCompany.set(ctx.uid, io);
+    }
+    return io;
+  };
+}