@farthershore/backend 0.19.0 → 0.20.0

package/dist/testing/index.js

@@ -1,8 +1,13 @@
 import { createRequire as __createRequire } from "node:module";const require=__createRequire(import.meta.url);
 var __defProp = Object.defineProperty;
 var __getOwnPropNames = Object.getOwnPropertyNames;
-var __esm = (fn, res) => function __init() {
-  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+var __esm = (fn, res, err) => function __init() {
+  if (err) throw err[0];
+  try {
+    return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+  } catch (e) {
+    throw err = [e], e;
+  }
 };
 var __export = (target, all) => {
   for (var name in all)
@@ -452,16 +457,106 @@ function statusForCode(code) {
   return 401;
 }
 
+// src/core/deadline.ts
+var DEADLINE_MS = {
+  /** Boot-blocking; generous because it runs once and gates startup. */
+  bootstrap: 1e4,
+  /** On the inbound verification path — must not hold a request open. */
+  jwks: 5e3,
+  /** Background economic report, retried by the caller. */
+  metering: 1e4,
+  /** Background attested usage callback. */
+  postStreamUsage: 1e4,
+  /** Best-effort heartbeat; never blocks anything. */
+  health: 5e3,
+  /** Boot-time route drift report; fail-open at the caller. */
+  report: 1e4
+};
+var MAX_RESPONSE_BYTES = 1048576;
+var ResponseTooLargeError = class extends Error {
+  constructor(limit) {
+    super(`response body exceeded ${limit} bytes and was cancelled`);
+    this.name = "ResponseTooLargeError";
+  }
+};
+var DeadlineExceededError = class extends Error {
+  operation;
+  constructor(operation, timeoutMs) {
+    super(`${operation} exceeded its ${timeoutMs}ms deadline`);
+    this.name = "TimeoutError";
+    this.operation = operation;
+  }
+};
+async function fetchWithDeadline(fetchImpl, input, init, operation, options = {}) {
+  const timeoutMs = options.timeoutMs ?? DEADLINE_MS[operation];
+  const timeout = AbortSignal.timeout(timeoutMs);
+  const signal = options.callerSignal ? AbortSignal.any([options.callerSignal, timeout]) : timeout;
+  try {
+    return await fetchImpl(input, { ...init, signal });
+  } catch (cause) {
+    if (options.callerSignal?.aborted) throw cause;
+    if (timeout.aborted) throw new DeadlineExceededError(operation, timeoutMs);
+    throw cause;
+  }
+}
+async function readBoundedText(response, limit = MAX_RESPONSE_BYTES) {
+  const body = response.body;
+  if (!body) {
+    const text = await response.text();
+    if (byteLength(text) > limit) throw new ResponseTooLargeError(limit);
+    return text;
+  }
+  const reader = body.getReader();
+  const chunks = [];
+  let total = 0;
+  try {
+    for (; ; ) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (!value) continue;
+      total += value.byteLength;
+      if (total > limit) {
+        await reader.cancel();
+        throw new ResponseTooLargeError(limit);
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  const joined = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    joined.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return new TextDecoder().decode(joined);
+}
+async function readBoundedJson(response, limit = MAX_RESPONSE_BYTES) {
+  return JSON.parse(await readBoundedText(response, limit));
+}
+function byteLength(text) {
+  return new TextEncoder().encode(text).byteLength;
+}
+
 // src/core/jwks.ts
 var DEFAULT_CACHE_TTL_MS = 5 * 6e4;
+var DEFAULT_HARD_STALE_MS = 15 * 6e4;
 var DEFAULT_NEGATIVE_CACHE_MS = 3e4;
+function finiteMsOr(value, fallback) {
+  if (value === void 0) return fallback;
+  if (!Number.isFinite(value) || value < 0) return fallback;
+  return value;
+}
 var MAX_NEGATIVE_KIDS = 1e3;
 var JwksClient = class {
   jwksUrl;
   fetchImpl;
   cacheTtlMs;
+  hardStaleMs;
   negativeCacheMs;
   now;
+  onObservation;
   keysByKid = /* @__PURE__ */ new Map();
   fetchedAt = 0;
   hasFetchedOnce = false;
@@ -470,21 +565,34 @@ var JwksClient = class {
   constructor(options) {
     this.jwksUrl = options.jwksUrl;
     this.fetchImpl = options.fetchImpl ?? globalThis.fetch;
-    this.cacheTtlMs = options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS;
+    this.cacheTtlMs = finiteMsOr(options.cacheTtlMs, DEFAULT_CACHE_TTL_MS);
+    this.hardStaleMs = Math.max(
+      finiteMsOr(options.hardStaleMs, DEFAULT_HARD_STALE_MS),
+      this.cacheTtlMs
+    );
     this.negativeCacheMs = options.negativeCacheMs ?? DEFAULT_NEGATIVE_CACHE_MS;
     this.now = options.now ?? (() => Date.now());
+    this.onObservation = options.onObservation;
   }
   /**
    * Resolve a public JWK for `kid`, fail-closed. Throws FartherShoreError with
-   * `jwks_unavailable` (cold cache + fetch failed) or `unknown_key_id`.
+   * `jwks_unavailable` (cold cache, or a hard-stale cache whose refresh is
+   * failing) or `unknown_key_id`.
    */
   async getKey(kid) {
     const cached = this.keysByKid.get(kid);
-    if (cached && !this.isStale()) return cached;
+    if (cached && !this.isStale()) {
+      this.observe("fresh", kid);
+      return cached;
+    }
     const negAt = this.negativeKids.get(kid);
     if (negAt !== void 0 && this.now() - negAt < this.negativeCacheMs) {
       const warm = this.keysByKid.get(kid);
-      if (warm) return warm;
+      if (warm) {
+        this.assertWithinHardStale();
+        this.observe(this.cacheState(), kid);
+        return warm;
+      }
       throw new FartherShoreError(
         "unknown_key_id",
         `signing key '${kid}' is not present in the JWKS`
@@ -494,6 +602,7 @@ var JwksClient = class {
     const key2 = this.keysByKid.get(kid);
     if (key2) {
       this.negativeKids.delete(kid);
+      this.observe(this.cacheState(), kid);
       return key2;
     }
     this.rememberMissingKid(kid);
@@ -513,8 +622,37 @@ var JwksClient = class {
     }
     this.negativeKids.set(kid, this.now());
   }
+  ageMs() {
+    return this.now() - this.fetchedAt;
+  }
   isStale() {
-    return this.now() - this.fetchedAt >= this.cacheTtlMs;
+    return this.ageMs() >= this.cacheTtlMs;
+  }
+  isHardStale() {
+    return this.ageMs() >= this.hardStaleMs;
+  }
+  /** Current freshness of the cached key set. */
+  cacheState() {
+    if (!this.hasFetchedOnce) return "cold";
+    if (this.isHardStale()) return "hard_stale";
+    if (this.isStale()) return "soft_stale";
+    return "fresh";
+  }
+  observe(state, kid) {
+    this.onObservation?.({
+      state,
+      ageMs: this.hasFetchedOnce ? this.ageMs() : 0,
+      ...kid !== void 0 ? { kid } : {}
+    });
+  }
+  /** Fail closed when the cached key set is past the hard-stale ceiling. */
+  assertWithinHardStale() {
+    if (!this.isHardStale()) return;
+    this.observe("hard_stale");
+    throw new FartherShoreError(
+      "jwks_unavailable",
+      `JWKS key set is ${Math.round(this.ageMs() / 1e3)}s old, past the ${Math.round(this.hardStaleMs / 1e3)}s hard-stale limit, and cannot be refreshed; refusing to vouch for keys that may have been revoked`
+    );
   }
   /** Single-flight refresh: concurrent callers share one fetch. */
   async refresh() {
@@ -527,24 +665,27 @@ var JwksClient = class {
   async doFetch() {
     let response;
     try {
-      response = await this.fetchImpl(this.jwksUrl, {
-        headers: { accept: "application/json" }
-      });
+      response = await fetchWithDeadline(
+        this.fetchImpl,
+        this.jwksUrl,
+        { headers: { accept: "application/json" } },
+        "jwks"
+      );
     } catch (cause) {
-      this.failOnColdCache(cause);
+      this.handleRefreshFailure(cause);
       return;
     }
     if (!response.ok) {
-      this.failOnColdCache(
+      this.handleRefreshFailure(
         new Error(`JWKS endpoint returned HTTP ${response.status}`)
       );
       return;
     }
     let doc;
     try {
-      doc = await response.json();
+      doc = await readBoundedJson(response);
     } catch (cause) {
-      this.failOnColdCache(cause);
+      this.handleRefreshFailure(cause);
       return;
     }
     const next = /* @__PURE__ */ new Map();
@@ -557,15 +698,27 @@ var JwksClient = class {
     this.negativeKids.clear();
   }
   /**
-   * Stale-while-revalidate: with a warm cache, swallow the refresh failure and
-   * keep serving the last-known keys. With a COLD cache, fail closed.
+   * BOUNDED stale-while-revalidate. A COLD cache fails closed. A warm cache
+   * inside the soft window swallows the failure and keeps serving. Past the
+   * hard-stale ceiling it fails closed too — availability is worth a bounded
+   * window of degraded trust, not an unbounded one.
    */
-  failOnColdCache(cause) {
-    if (this.hasFetchedOnce) return;
-    throw new FartherShoreError(
-      "jwks_unavailable",
-      `JWKS unavailable on a cold cache: ${stringifyCause(cause)}`
-    );
+  handleRefreshFailure(cause) {
+    if (!this.hasFetchedOnce) {
+      this.observe("cold");
+      throw new FartherShoreError(
+        "jwks_unavailable",
+        `JWKS unavailable on a cold cache: ${stringifyCause(cause)}`
+      );
+    }
+    if (this.isHardStale()) {
+      this.observe("hard_stale");
+      throw new FartherShoreError(
+        "jwks_unavailable",
+        `JWKS refresh failed and the cached key set is ${Math.round(this.ageMs() / 1e3)}s old, past the ${Math.round(this.hardStaleMs / 1e3)}s hard-stale limit: ${stringifyCause(cause)}`
+      );
+    }
+    this.observe("soft_stale");
   }
 };
 function stringifyCause(cause) {
@@ -968,6 +1121,7 @@ import { dirname as dirname2 } from "node:path";
 // src/core/bootstrap.ts
 var BOOTSTRAP_PATH = "/v1/runtime/bootstrap";
 var DEFAULT_MIN_REFRESH_SECONDS = 30;
+var DEFAULT_MAX_STALE_SECONDS = 300;
 var BootstrapClient = class {
   runtimeToken;
   endpoint;
@@ -975,6 +1129,7 @@ var BootstrapClient = class {
   fetchImpl;
   now;
   minRefreshSeconds;
+  maxStaleMs;
   cached = null;
   fetchedAt = 0;
   refreshAfterMs = 0;
@@ -998,6 +1153,7 @@ var BootstrapClient = class {
     this.fetchImpl = options.fetchImpl ?? globalThis.fetch;
     this.now = options.now ?? (() => Date.now());
     this.minRefreshSeconds = options.minRefreshSeconds ?? DEFAULT_MIN_REFRESH_SECONDS;
+    this.maxStaleMs = (options.maxStaleSeconds ?? DEFAULT_MAX_STALE_SECONDS) * 1e3;
   }
   /** Cached config when fresh; otherwise refreshes. */
   async get() {
@@ -1019,20 +1175,37 @@ var BootstrapClient = class {
   isStale() {
     return this.now() - this.fetchedAt >= this.refreshAfterMs;
   }
+  isHardStale() {
+    return this.now() - this.fetchedAt >= this.maxStaleMs;
+  }
+  cachedOrThrowOnHardStale(reason) {
+    if (this.cached && !this.isHardStale()) return this.cached;
+    throw new FartherShoreError(
+      "jwks_unavailable",
+      `bootstrap refresh failed with stale cached authorization metadata: ${reason}`
+    );
+  }
   async doBootstrap() {
     let response;
     try {
-      response = await this.fetchImpl(this.endpoint, {
-        method: "POST",
-        headers: {
-          authorization: `Bearer ${this.runtimeToken}`,
-          "content-type": "application/json",
-          accept: "application/json"
+      response = await fetchWithDeadline(
+        this.fetchImpl,
+        this.endpoint,
+        {
+          method: "POST",
+          headers: {
+            authorization: `Bearer ${this.runtimeToken}`,
+            "content-type": "application/json",
+            accept: "application/json"
+          },
+          body: JSON.stringify(this.request)
         },
-        body: JSON.stringify(this.request)
-      });
+        "bootstrap"
+      );
     } catch (cause) {
-      if (this.cached) return this.cached;
+      if (this.cached) {
+        return this.cachedOrThrowOnHardStale(stringify(cause));
+      }
       throw new FartherShoreError(
         "jwks_unavailable",
         `bootstrap request failed: ${stringify(cause)}`
@@ -1045,13 +1218,15 @@ var BootstrapClient = class {
       );
     }
     if (!response.ok) {
-      if (this.cached) return this.cached;
+      if (this.cached) {
+        return this.cachedOrThrowOnHardStale(`HTTP ${response.status}`);
+      }
       throw new FartherShoreError(
         "jwks_unavailable",
         `bootstrap returned HTTP ${response.status}`
       );
     }
-    const body = await response.json();
+    const body = await readBoundedJson(response);
     this.cached = body;
     this.fetchedAt = this.now();
     const refreshSeconds = Math.max(
@@ -1084,17 +1259,22 @@ async function reportHealth(options) {
   const fetchImpl = options.fetchImpl ?? globalThis.fetch;
   const endpoint = `${options.coreUrl.replace(/\/+$/, "")}${HEALTH_PATH}`;
   try {
-    const response = await fetchImpl(endpoint, {
-      method: "POST",
-      headers: {
-        authorization: `Bearer ${options.runtimeToken}`,
-        "content-type": "application/json"
+    const response = await fetchWithDeadline(
+      fetchImpl,
+      endpoint,
+      {
+        method: "POST",
+        headers: {
+          authorization: `Bearer ${options.runtimeToken}`,
+          "content-type": "application/json"
+        },
+        body: JSON.stringify({
+          status: options.status,
+          ...options.instanceId ? { instanceId: options.instanceId } : {}
+        })
       },
-      body: JSON.stringify({
-        status: options.status,
-        ...options.instanceId ? { instanceId: options.instanceId } : {}
-      })
-    });
+      "health"
+    );
     return response.ok;
   } catch {
     return false;
@@ -1243,15 +1423,20 @@ var MeteringClient = class {
     for (let attempt = 0; attempt < this.maxRetries; attempt += 1) {
       let retryAfter = null;
       try {
-        const response = await this.fetchImpl(this.endpoint, {
-          method: "POST",
-          headers: {
-            authorization: `Bearer ${this.config.credential}`,
-            "content-type": "application/json",
-            accept: "application/json"
+        const response = await fetchWithDeadline(
+          this.fetchImpl,
+          this.endpoint,
+          {
+            method: "POST",
+            headers: {
+              authorization: `Bearer ${this.config.credential}`,
+              "content-type": "application/json",
+              accept: "application/json"
+            },
+            body: JSON.stringify(event)
           },
-          body: JSON.stringify(event)
-        });
+          "metering"
+        );
         if (response.ok) return true;
         if (!isTransientStatus(response.status)) return false;
         retryAfter = retryAfterMs(response.headers);
@@ -1319,6 +1504,7 @@ var PostStreamUsageClient = class {
   logger;
   sleep;
   retryDelaysMs;
+  maxRetryDelayMs;
   constructor(options) {
     this.config = options.config;
     this.endpoint = resolveEndpoint2(options.config.endpoint, options.coreUrl);
@@ -1327,6 +1513,7 @@ var PostStreamUsageClient = class {
     this.logger = options.logger ?? ((message) => console.warn(message));
     this.sleep = options.sleep ?? sleep;
     this.retryDelaysMs = options.retryDelaysMs ?? [100, 250, 500];
+    this.maxRetryDelayMs = options.maxRetryDelayMs ?? 1e4;
   }
   async reportUsage(input) {
     try {
@@ -1355,19 +1542,36 @@ var PostStreamUsageClient = class {
       const event = { ...unsigned, signature };
       const body = JSON.stringify(event);
       for (let attempt = 0; ; attempt += 1) {
-        const response = await this.fetchImpl(this.endpoint, {
-          method: "POST",
-          headers: {
-            authorization: `Bearer ${this.config.credential}`,
-            "content-type": "application/json",
-            accept: "application/json"
-          },
-          body
-        });
+        let response;
+        try {
+          response = await fetchWithDeadline(
+            this.fetchImpl,
+            this.endpoint,
+            {
+              method: "POST",
+              headers: {
+                authorization: `Bearer ${this.config.credential}`,
+                "content-type": "application/json",
+                accept: "application/json"
+              },
+              body
+            },
+            "postStreamUsage"
+          );
+        } catch (cause) {
+          const delayMs2 = this.retryDelayForAttempt(attempt, null);
+          if (delayMs2 === null) throw cause;
+          await this.sleep(delayMs2);
+          continue;
+        }
         if (response.ok) return { ok: true };
         const requestNotFound = await isPostStreamRequestNotFound(response);
-        const delayMs = this.retryDelaysMs[attempt];
-        if (!requestNotFound || delayMs === void 0) {
+        const retryable = requestNotFound || isRetryableStatus(response.status);
+        const delayMs = this.retryDelayForAttempt(
+          attempt,
+          retryAfterMs2(response.headers)
+        );
+        if (!retryable || delayMs === null) {
           throw new Error(`metering endpoint returned ${response.status}`);
         }
         await this.sleep(delayMs);
@@ -1378,6 +1582,12 @@ var PostStreamUsageClient = class {
       return { ok: false, reason };
     }
   }
+  retryDelayForAttempt(attempt, retryAfterMs3) {
+    const fallback = this.retryDelaysMs[attempt];
+    if (fallback === void 0) return null;
+    if (retryAfterMs3 === null) return fallback;
+    return Math.min(retryAfterMs3, this.maxRetryDelayMs);
+  }
 };
 async function isPostStreamRequestNotFound(response) {
   if (response.status !== 422) return false;
@@ -1388,6 +1598,18 @@ async function isPostStreamRequestNotFound(response) {
     return false;
   }
 }
+function isRetryableStatus(status) {
+  return status === 429 || status >= 500 && status <= 599;
+}
+function retryAfterMs2(headers) {
+  const raw = headers.get("retry-after");
+  if (!raw) return null;
+  const seconds = Number(raw);
+  if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1e3;
+  const dateMs = Date.parse(raw);
+  if (!Number.isFinite(dateMs)) return null;
+  return Math.max(0, dateMs - Date.now());
+}
 function sleep(delayMs) {
   return new Promise((resolve) => setTimeout(resolve, delayMs));
 }
@@ -1463,6 +1685,37 @@ var NonceCache = class {
   }
 };
 
+// src/core/replay-protection.ts
+function resolveReplayProtection(input = {}) {
+  if (input.nonceStore) {
+    return {
+      // An opted-in shared store that is DOWN must not degrade to "no replay
+      // check" — that would make knocking it over a way to switch the
+      // protection off entirely.
+      store: failClosed(input.nonceStore),
+      diagnostic: { mode: "shared", crossReplica: true }
+    };
+  }
+  return {
+    store: new NonceCache(),
+    diagnostic: { mode: "single-instance", crossReplica: false }
+  };
+}
+function failClosed(store) {
+  return {
+    async checkAndRemember(id) {
+      try {
+        return await store.checkAndRemember(id);
+      } catch (cause) {
+        throw new FartherShoreError(
+          "replayed_nonce",
+          `replay store is unavailable, refusing the request rather than skipping one-time-use enforcement: ${cause instanceof Error ? cause.message : String(cause)}`
+        );
+      }
+    }
+  };
+}
+
 // src/core/shutdown.ts
 var ShutdownManager = class {
   hooks = [];
@@ -2156,7 +2409,7 @@ function headerGetter(headers) {
 
 // src/core/runtime.ts
 var DEFAULT_CORE_URL = "https://core.farthershore.com";
-var SDK_VERSION = "0.19.0".length > 0 ? "0.19.0" : "0.0.0-dev";
+var SDK_VERSION = "0.20.0".length > 0 ? "0.20.0" : "0.0.0-dev";
 var FartherShore = class {
   bootstrapClient;
   fetchImpl;
@@ -2169,6 +2422,7 @@ var FartherShore = class {
   /** OPTIONAL HS256 secret(s) — defense-in-depth over the cv=2 X-Fs-Context. */
   contextSecrets;
   nonceCache;
+  replayProtectionDiagnostic;
   shutdownManager = new ShutdownManager();
   jwks = null;
   meteringClient = null;
@@ -2186,7 +2440,9 @@ var FartherShore = class {
     this.meteringEnabledOverride = options.metering?.enabled ?? true;
     this.tunnelOptions = options.tunnel ?? {};
     this.instanceId = options.instanceId;
-    this.nonceCache = options.nonceStore ?? new NonceCache();
+    const replay = resolveReplayProtection({ nonceStore: options.nonceStore });
+    this.nonceCache = replay.store;
+    this.replayProtectionDiagnostic = replay.diagnostic;
     this.contextSecrets = options.contextSecrets ?? parseContextSecrets(env.FS_CONTEXT_SECRETS);
     this.bootstrapClient = new BootstrapClient({
       runtimeToken,
@@ -2276,14 +2532,19 @@ var FartherShore = class {
   buildReportSink() {
     const post = async (path, body) => {
       const base = this.coreUrl.replace(/\/$/, "");
-      const res = await this.fetchImpl(`${base}${path}`, {
-        method: "POST",
-        headers: {
-          "content-type": "application/json",
-          authorization: `Bearer ${this.runtimeToken}`
+      const res = await fetchWithDeadline(
+        this.fetchImpl,
+        `${base}${path}`,
+        {
+          method: "POST",
+          headers: {
+            "content-type": "application/json",
+            authorization: `Bearer ${this.runtimeToken}`
+          },
+          body: JSON.stringify(body)
         },
-        body: JSON.stringify(body)
-      });
+        "report"
+      );
       if (!res.ok) throw new Error(`runtime report ${path} -> ${res.status}`);
     };
     return {
@@ -2410,6 +2671,15 @@ var FartherShore = class {
       return { ok: false, reason };
     }
   }
+  /**
+   * How far replay protection actually reaches — `"shared"` (enforced across
+   * every replica) or `"single-instance"` (this process only). Deployment
+   * diagnostic: log it at boot, or assert on it in a smoke test, so the mode is
+   * never a surprise. See {@link FartherShoreInitOptions.nonceStore}.
+   */
+  replayProtection() {
+    return this.replayProtectionDiagnostic;
+  }
   /** Current local health report. */
   health() {
     const config = this.bootstrapClient.peek();

package/dist/types/core/bootstrap.d.ts

@@ -12,6 +12,11 @@ export type BootstrapClientOptions = {
     now?: () => number;
     /** Minimum seconds between refreshes regardless of server hint. */
     minRefreshSeconds?: number;
+    /**
+     * Maximum age for cached bootstrap authorization metadata during transient
+     * refresh failures. Defaults to 5 minutes.
+     */
+    maxStaleSeconds?: number;
 };
 /**
  * Caches the bootstrap response and refreshes it lazily. `get()` returns the
@@ -24,6 +29,7 @@ export declare class BootstrapClient {
     private readonly fetchImpl;
     private readonly now;
     private readonly minRefreshSeconds;
+    private readonly maxStaleMs;
     private cached;
     private fetchedAt;
     private refreshAfterMs;
@@ -36,5 +42,7 @@ export declare class BootstrapClient {
     /** Last cached value without triggering a refresh (null until bootstrapped). */
     peek(): RuntimeBootstrapResponse | null;
     private isStale;
+    private isHardStale;
+    private cachedOrThrowOnHardStale;
     private doBootstrap;
 }