@gkoos/caracal 0.1.0

package/src/adapters/fetch/retry-after.ts

@@ -0,0 +1,111 @@
+import type { RetryContext } from "../../core/retry.js"
+
+/**
+ * Opt-in retry pacing for `@gkoos/caracal/fetch` that honours the HTTP
+ * `Retry-After` header. It is protocol-specific, so it lives with the
+ * adapter rather than in the protocol-agnostic core.
+ */
+
+const DEFAULTS = {
+  baseMs: 100,
+  factor: 2,
+  maxDelayMs: 30_000,
+  jitterRatio: 0.1,
+} as const
+
+const deltaSecondsPattern = /^\d+$/
+
+/**
+ * Structurally extracts a `Headers`-like object. `instanceof Response` is
+ * unreliable across realms and custom fetch implementations.
+ */
+function headersOf(value: unknown): Headers | undefined {
+  if (typeof value !== "object" || value === null) return undefined
+  const headers = (value as { headers?: unknown }).headers
+  if (typeof headers !== "object" || headers === null) return undefined
+  const get = (headers as { get?: unknown }).get
+  return typeof get === "function" ? (headers as Headers) : undefined
+}
+
+/**
+ * Parses `Retry-After` (delta-seconds or HTTP-date) from a settled fetch
+ * outcome. Reads the response from `context.result`, or from
+ * `context.error` when a custom fetch implementation throws a
+ * response-bearing error.
+ *
+ * Returns `undefined` when no usable header is present. Malformed values
+ * are ignored rather than thrown.
+ */
+export function retryAfterMs(
+  context: RetryContext,
+  now: number = Date.now(),
+): number | undefined {
+  const headers = headersOf(context.result) ?? headersOf(context.error)
+  if (headers === undefined) return undefined
+
+  const value = headers.get("retry-after")?.trim()
+  if (!value) return undefined
+
+  if (deltaSecondsPattern.test(value)) {
+    const seconds = Number(value)
+    return Number.isSafeInteger(seconds) ? seconds * 1000 : undefined
+  }
+
+  const timestamp = Date.parse(value)
+  return Number.isNaN(timestamp) ? undefined : Math.max(0, timestamp - now)
+}
+
+/** Options for `createRetryAfterDelay`. */
+export interface RetryAfterDelayOptions {
+  /** Base of the exponential backoff in ms. Default: 100. */
+  readonly baseMs?: number
+  /** Exponential growth factor (>= 1). Default: 2. */
+  readonly factor?: number
+  /** Ceiling applied to the deterministic wait in ms. Default: 30 000. */
+  readonly maxDelayMs?: number
+  /** Additive jitter as a fraction of the wait, within [0, 1]. Default: 0.1. */
+  readonly jitterRatio?: number
+}
+
+/** A `RetryDelay` that reads `Retry-After` from the settled outcome. */
+export type RetryAfterDelay = (attempt: number, context: RetryContext) => number
+
+/**
+ * Builds a `RetryDelay` that waits for the longer of exponential backoff
+ * and the `Retry-After` the server sent, then adds additive jitter.
+ *
+ * Jitter only ever lengthens the wait, so a server-provided minimum is
+ * never retried early.
+ */
+export function createRetryAfterDelay(
+  options: RetryAfterDelayOptions = {},
+): RetryAfterDelay {
+  const baseMs = options.baseMs ?? DEFAULTS.baseMs
+  const factor = options.factor ?? DEFAULTS.factor
+  const maxDelayMs = options.maxDelayMs ?? DEFAULTS.maxDelayMs
+  const jitterRatio = options.jitterRatio ?? DEFAULTS.jitterRatio
+
+  if (!Number.isFinite(baseMs) || baseMs < 0)
+    throw new RangeError("retryAfterDelay baseMs must be finite and >= 0")
+  if (!Number.isFinite(factor) || factor < 1)
+    throw new RangeError("retryAfterDelay factor must be finite and >= 1")
+  if (!Number.isFinite(maxDelayMs) || maxDelayMs < 0)
+    throw new RangeError("retryAfterDelay maxDelayMs must be finite and >= 0")
+  if (!Number.isFinite(jitterRatio) || jitterRatio < 0 || jitterRatio > 1)
+    throw new RangeError("retryAfterDelay jitterRatio must be within [0, 1]")
+
+  return (attempt, context) => {
+    const backoff = Math.min(baseMs * factor ** (attempt - 1), maxDelayMs)
+    const base = Math.min(
+      Math.max(backoff, retryAfterMs(context) ?? 0),
+      maxDelayMs,
+    )
+    return base + Math.random() * base * jitterRatio
+  }
+}
+
+/**
+ * Ready-to-use default: `retry({ maxAttempts: 3, delay: retryAfterDelay })`.
+ * Use `createRetryAfterDelay()` to change the pacing.
+ */
+export const retryAfterDelay: RetryAfterDelay = createRetryAfterDelay()

package/src/adapters/postgres/adapter.ts

@@ -0,0 +1,102 @@
+import type { QueryResult } from "pg"
+
+import type {
+  Adapter,
+  Classification,
+  OperationCapabilities,
+  Outcome,
+} from "../../core/types.js"
+
+export type PostgresReplay =
+  | OperationCapabilities["replay"]
+  | ((args: PostgresQueryArgs) => OperationCapabilities["replay"])
+
+export type PostgresQueryArgs = Readonly<{
+  sql: string
+  values?: readonly unknown[]
+  /** Application-declared per-query replay trait; never inferred from SQL. */
+  replay?: OperationCapabilities["replay"]
+}>
+
+export interface PostgresQueryable {
+  query(query: {
+    text: string
+    values?: readonly unknown[]
+  }): Promise<QueryResult>
+}
+
+export interface PostgresAdapterOptions {
+  readonly replay?: PostgresReplay
+  readonly classifyError?: (error: unknown) => Classification
+}
+
+const retryableSqlStates = new Set([
+  "40001", // serialization_failure
+  "40P01", // deadlock_detected
+  "55P03", // lock_not_available
+  "53300", // too_many_connections
+  "57P01", // admin_shutdown
+  "57P02", // crash_shutdown
+  "57P03", // cannot_connect_now
+])
+
+function sqlState(error: unknown): string | undefined {
+  if (typeof error !== "object" || error === null || !("code" in error)) {
+    return undefined
+  }
+
+  return typeof error.code === "string" ? error.code : undefined
+}
+
+function defaultErrorClassification(error: unknown): Classification {
+  const state = sqlState(error)
+  return state?.startsWith("08") === true ||
+    (state !== undefined && retryableSqlStates.has(state))
+    ? "retryable"
+    : "failure"
+}
+
+function replayFor(
+  args: PostgresQueryArgs,
+  configured: PostgresReplay | undefined,
+): OperationCapabilities["replay"] {
+  if (args.replay !== undefined) {
+    return args.replay
+  }
+
+  return typeof configured === "function"
+    ? configured(args)
+    : (configured ?? "unknown")
+}
+
+/**
+ * Adapter for node-postgres (`pg`) 8.x query clients/pools. The standard
+ * `query()` contract does not provide a portable AbortSignal cancellation path,
+ * so this adapter accurately declares abort as unsupported.
+ */
+export function postgresAdapter(
+  client: PostgresQueryable,
+  options: PostgresAdapterOptions = {},
+): Adapter<PostgresQueryArgs, QueryResult> {
+  return Object.freeze({
+    capabilities(args: PostgresQueryArgs): OperationCapabilities {
+      return { abort: "unsupported", replay: replayFor(args, options.replay) }
+    },
+    execute(args: PostgresQueryArgs): Promise<QueryResult> {
+      return client.query({
+        text: args.sql,
+        values: args.values === undefined ? undefined : [...args.values],
+      })
+    },
+    classify(outcome: Outcome<QueryResult>): Classification {
+      if (outcome.status === "success") {
+        return "success"
+      }
+
+      return (
+        options.classifyError?.(outcome.error) ??
+        defaultErrorClassification(outcome.error)
+      )
+    },
+  })
+}

package/src/adapters/postgres/index.ts

@@ -0,0 +1,7 @@
+export type {
+  PostgresAdapterOptions,
+  PostgresQueryArgs,
+  PostgresQueryable,
+  PostgresReplay,
+} from "./adapter.js"
+export { postgresAdapter } from "./adapter.js"

package/src/coordination/redis/bulkhead.ts

@@ -0,0 +1,61 @@
+import type { BulkheadCoordinator } from "../../core/bulkhead.js"
+import { CoordinatorUnavailableError } from "./client.js"
+import { evalScript } from "./eval-script.js"
+import { coordinationKey } from "./keys.js"
+import type { RedisScriptClient } from "./leases.js"
+import { bulkheadLeaseV1 } from "./scripts.js"
+export function redisCoordinator(
+  client: RedisScriptClient,
+  options: { namespace: string },
+): BulkheadCoordinator {
+  const namespace = options.namespace
+  coordinationKey(namespace, "bulkhead", "validate", "validate")
+  const coordinator: BulkheadCoordinator = {
+    async command(identity, action, token, leaseMs, limit) {
+      if (
+        !["acquire", "renew", "release"].includes(action) ||
+        typeof token !== "string" ||
+        !token ||
+        token.length > 256
+      )
+        throw new TypeError("Invalid bulkhead lease action or token")
+      if (
+        ![leaseMs, limit].every(
+          (value) => Number.isSafeInteger(value) && value > 0,
+        ) ||
+        leaseMs > 86400000
+      )
+        throw new RangeError("Invalid bulkhead lease duration or limit")
+      const key = coordinationKey(
+        namespace,
+        `bulkhead:${identity.name}`,
+        identity.operation,
+        identity.scope,
+      )
+      try {
+        const result = await evalScript(
+          client,
+          bulkheadLeaseV1,
+          1,
+          key,
+          action,
+          token,
+          leaseMs,
+          limit,
+        )
+        if (
+          !Array.isArray(result) ||
+          result.length !== 2 ||
+          ![0, 1].includes(result[0]) ||
+          !Number.isSafeInteger(result[1]) ||
+          result[1] < 0
+        )
+          throw new Error("Invalid bulkhead reply")
+        return { allowed: result[0] === 1, occupancy: result[1] as number }
+      } catch (error) {
+        throw new CoordinatorUnavailableError(error)
+      }
+    },
+  }
+  return Object.freeze(coordinator)
+}

package/src/coordination/redis/circuit-breaker.ts

@@ -0,0 +1,270 @@
+import type {
+  AdmitProbeResult,
+  BreakerCoordinator,
+  BreakerIdentity,
+  BreakerState,
+  ObserveResult,
+  SettleProbeResult,
+} from "../../core/circuit-breaker.js"
+import { CoordinatorUnavailableError } from "./client.js"
+import { evalScript } from "./eval-script.js"
+import { coordinationKey } from "./keys.js"
+import type { RedisScriptClient } from "./leases.js"
+import {
+  breakerAdmitProbeV1,
+  breakerObserveV1,
+  breakerSettleProbeV1,
+} from "./scripts.js"
+
+// ---------------------------------------------------------------------------
+// Key helpers
+// ---------------------------------------------------------------------------
+
+function breakerKeys(
+  namespace: string,
+  identity: BreakerIdentity,
+): [hashKey: string, obsKey: string, probeKey: string] {
+  const policy = `breaker:${identity.name}`
+  const hash = coordinationKey(
+    namespace,
+    policy,
+    identity.operation,
+    identity.scope,
+    "breaker",
+  )
+  const obs = coordinationKey(
+    namespace,
+    policy,
+    identity.operation,
+    identity.scope,
+    "observations",
+  )
+  const prob = coordinationKey(
+    namespace,
+    policy,
+    identity.operation,
+    identity.scope,
+    "probes",
+  )
+  return [hash, obs, prob]
+}
+
+// ---------------------------------------------------------------------------
+// Reply validation helpers
+// ---------------------------------------------------------------------------
+
+function assertArray(reply: unknown, minLen: number, label: string): unknown[] {
+  if (
+    !Array.isArray(reply) ||
+    reply.length < minLen ||
+    reply.some((v) => typeof v !== "number" || !Number.isSafeInteger(v))
+  ) {
+    throw new Error(
+      `Invalid ${label} reply from Redis: ${JSON.stringify(reply)}`,
+    )
+  }
+  return reply as unknown[]
+}
+
+const STATE_CODES: BreakerState[] = ["closed", "open", "half-open"]
+
+function decodeState(code: number): BreakerState {
+  const s = STATE_CODES[code]
+  if (!s) throw new Error(`Unknown breaker state code: ${code}`)
+  return s
+}
+
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+
+/**
+ * Policy-specific Redis coordinator for `circuitBreaker.distributed()`.
+ *
+ * The caller owns `client` and must connect/disconnect it independently.
+ * `namespace` is prepended to every key; use a per-service, per-environment
+ * value to avoid cross-deployment state collisions.
+ */
+export function redisCircuitBreakerCoordinator(
+  client: RedisScriptClient,
+  options: { readonly namespace: string },
+): BreakerCoordinator {
+  const { namespace } = options
+  // Validate namespace eagerly (coordinationKey throws on bad input).
+  coordinationKey(
+    namespace,
+    "breaker:validate",
+    "validate",
+    "validate",
+    "breaker",
+  )
+
+  const coordinator: BreakerCoordinator = {
+    async readState(identity) {
+      const [hashKey] = breakerKeys(namespace, identity)
+      try {
+        const fields = await client.hmget(hashKey, "state", "generation")
+        const rawState = fields[0]
+        if (!rawState) return null
+        const state = rawState as BreakerState
+        if (!STATE_CODES.includes(state))
+          throw new Error(`Unknown breaker state: ${rawState}`)
+        const generation = parseInt(fields[1] ?? "0", 10)
+        return { state, generation }
+      } catch (error) {
+        throw new CoordinatorUnavailableError(error)
+      }
+    },
+
+    async observe(identity, params) {
+      const [hashKey, obsKey] = breakerKeys(namespace, identity)
+      const {
+        generation,
+        outcome,
+        uuid,
+        windowTtlMs,
+        minimumThroughput,
+        failureThresholdNumerator,
+        windowSize,
+        openMs,
+      } = params
+      try {
+        const reply = await evalScript(
+          client,
+          breakerObserveV1,
+          2,
+          hashKey,
+          obsKey,
+          outcome,
+          generation,
+          windowTtlMs,
+          minimumThroughput,
+          failureThresholdNumerator,
+          windowSize,
+          openMs,
+          uuid,
+        )
+        const r = assertArray(reply, 5, "breakerObserveV1") as number[]
+        const [status, , newGen, windowTotal, windowFailures] = r
+        if (status === 0) {
+          return {
+            type: "stale",
+            currentGeneration: newGen,
+          } satisfies ObserveResult
+        }
+        if (status === 2) {
+          return {
+            type: "opened",
+            newGeneration: newGen,
+            windowTotal,
+            windowFailures,
+          } satisfies ObserveResult
+        }
+        return {
+          type: "observed",
+          generation: newGen,
+          windowTotal,
+          windowFailures,
+        } satisfies ObserveResult
+      } catch (error) {
+        throw new CoordinatorUnavailableError(error)
+      }
+    },
+
+    async admitProbe(identity, params) {
+      const [hashKey, , probeKey] = breakerKeys(namespace, identity)
+      const { probeToken, openMs, halfOpenProbes, probeLeaseTtlMs } = params
+      try {
+        const reply = await evalScript(
+          client,
+          breakerAdmitProbeV1,
+          2,
+          hashKey,
+          probeKey,
+          probeToken,
+          openMs,
+          halfOpenProbes,
+          probeLeaseTtlMs,
+        )
+        const r = assertArray(reply, 5, "breakerAdmitProbeV1") as number[]
+        const [status, stateCode, gen, probeCount, transitioned] = r
+        if (status === 0) {
+          const reason =
+            stateCode === 0
+              ? "closed"
+              : stateCode === 1
+                ? "open"
+                : "probe-limit"
+          return {
+            type: "rejected",
+            reason,
+            generation: gen,
+          } satisfies AdmitProbeResult
+        }
+        return {
+          type: "admitted",
+          generation: gen,
+          probeCount,
+          stateChanged: transitioned === 1,
+        } satisfies AdmitProbeResult
+      } catch (error) {
+        throw new CoordinatorUnavailableError(error)
+      }
+    },
+
+    async settleProbe(identity, params) {
+      const [hashKey, , probeKey] = breakerKeys(namespace, identity)
+      const {
+        probeToken,
+        outcome,
+        generation,
+        halfOpenSuccesses,
+        openMs,
+        windowTtlMs,
+      } = params
+      try {
+        const reply = await evalScript(
+          client,
+          breakerSettleProbeV1,
+          2,
+          hashKey,
+          probeKey,
+          probeToken,
+          outcome,
+          generation,
+          halfOpenSuccesses,
+          openMs,
+          // Floor for the CLOSED cleanup TTL; older callers that do not pass it
+          // keep the historical openMs x 2 behaviour.
+          windowTtlMs ?? openMs * 2,
+        )
+        const r = assertArray(reply, 3, "breakerSettleProbeV1") as number[]
+        const [status, stateCode, newGen] = r
+        if (status === 0) {
+          return {
+            type: "stale",
+            generation: newGen,
+          } satisfies SettleProbeResult
+        }
+        const state = decodeState(stateCode)
+        if (status === 2) {
+          return {
+            type: "transitioned",
+            newState: state,
+            newGeneration: newGen,
+            previousState: "half-open",
+          } satisfies SettleProbeResult
+        }
+        return {
+          type: "settled",
+          state,
+          generation: newGen,
+        } satisfies SettleProbeResult
+      } catch (error) {
+        throw new CoordinatorUnavailableError(error)
+      }
+    },
+  }
+
+  return Object.freeze(coordinator)
+}

package/src/coordination/redis/client.ts

@@ -0,0 +1,78 @@
+import { Cluster, Redis } from "ioredis"
+
+export class CoordinatorUnavailableError extends Error {
+  readonly coordination = "distributed"
+  constructor(cause: unknown) {
+    super("Redis coordination unavailable; command outcome may be unknown", {
+      cause,
+    })
+    this.name = "CoordinatorUnavailableError"
+  }
+}
+
+function attachErrorSink(emitter: {
+  on(event: "error", listener: () => void): void
+}): void {
+  emitter.on("error", () => {
+    /* Commands surface errors; never crash on an unhandled EventEmitter error. */
+  })
+}
+
+/** Standalone Redis client. Caller owns it and must disconnect at shutdown. */
+export function createCoordinationClient(
+  url: string,
+  commandTimeout = 1000,
+): Redis {
+  if (!Number.isSafeInteger(commandTimeout) || commandTimeout < 1)
+    throw new RangeError("commandTimeout must be a positive integer")
+  const client = new Redis(url, {
+    lazyConnect: true,
+    enableOfflineQueue: false,
+    maxRetriesPerRequest: 0,
+    autoResendUnfulfilledCommands: false,
+    commandTimeout,
+    connectTimeout: commandTimeout,
+    retryStrategy: (attempt) => Math.min(attempt * 50, 1000),
+  })
+  attachErrorSink(client)
+  return client
+}
+
+export interface ClusterNode {
+  readonly host: string
+  readonly port: number
+}
+
+/**
+ * Redis Cluster client. Caller owns it and must disconnect at shutdown.
+ *
+ * All coordination keys use a hash-tag so that every key for a given
+ * policy+operation+scope lands on the same cluster slot. Multi-key Lua
+ * scripts (circuit breaker) are therefore cluster-safe without cross-slot
+ * concerns.
+ *
+ * Call `connect()` before use and `disconnect()` at shutdown, matching the
+ * standalone client contract.
+ */
+export function createCoordinationClusterClient(
+  nodes: ReadonlyArray<ClusterNode>,
+  commandTimeout = 1000,
+): Cluster {
+  if (!Number.isSafeInteger(commandTimeout) || commandTimeout < 1)
+    throw new RangeError("commandTimeout must be a positive integer")
+  if (!Array.isArray(nodes) || nodes.length === 0)
+    throw new RangeError("nodes must be a non-empty array of { host, port }")
+  const cluster = new Cluster([...nodes], {
+    lazyConnect: true,
+    enableOfflineQueue: false,
+    clusterRetryStrategy: (attempt) => Math.min(attempt * 50, 1000),
+    redisOptions: {
+      commandTimeout,
+      connectTimeout: commandTimeout,
+      maxRetriesPerRequest: 0,
+      autoResendUnfulfilledCommands: false,
+    },
+  })
+  attachErrorSink(cluster)
+  return cluster
+}

package/src/coordination/redis/eval-script.ts

@@ -0,0 +1,71 @@
+import { createHash } from "node:crypto"
+
+/**
+ * Minimal Lua-script capability required by the Redis coordinators.
+ *
+ * `eval` is always required.  `evalsha` is optional so structural test doubles
+ * and clients that only expose `eval` keep working; when it is missing the
+ * script body is sent with `eval`.
+ */
+export interface ScriptClient {
+  eval(
+    script: string,
+    numberOfKeys: number,
+    ...args: (string | number)[]
+  ): Promise<unknown>
+  evalsha?(
+    sha: string,
+    numberOfKeys: number,
+    ...args: (string | number)[]
+  ): Promise<unknown>
+}
+
+const scriptHashes = new Map<string, string>()
+
+/** SHA1 of a script body.  Redis keys its script cache by exactly this value. */
+export function scriptSha(script: string): string {
+  const cached = scriptHashes.get(script)
+  if (cached !== undefined) return cached
+  const sha = createHash("sha1").update(script).digest("hex")
+  scriptHashes.set(script, sha)
+  return sha
+}
+
+function errorMessage(error: unknown): string {
+  if (typeof error === "string") return error
+  if (error instanceof Error) return error.message
+  return ""
+}
+
+/**
+ * Executes a Lua script, sending the body only when it has to.
+ *
+ * The script cache is keyed by SHA1, so the steady state is `EVALSHA` - a
+ * 40-byte hash instead of kilobytes of Lua on every call.  The full body goes
+ * out with `EVAL` only when the client cannot send `EVALSHA`, or when the
+ * server answers `NOSCRIPT`: the script cache lost the script to a restart, a
+ * `SCRIPT FLUSH`, or (Redis 7.4 and later) LRU eviction under memory pressure.
+ *
+ * Retrying through `EVAL` is safe because Redis rejects an unknown SHA1 before
+ * executing anything, so the failed command is known not to have run.  This is
+ * deliberately narrower than the coordinator's command-timeout rule, where the
+ * outcome is unknown and a replay is never allowed.  A timeout error is not a
+ * `NOSCRIPT` error and propagates unchanged.
+ */
+export async function evalScript(
+  client: ScriptClient,
+  script: string,
+  numberOfKeys: number,
+  ...args: (string | number)[]
+): Promise<unknown> {
+  const evalsha = client.evalsha
+  if (typeof evalsha !== "function") {
+    return client.eval(script, numberOfKeys, ...args)
+  }
+  try {
+    return await evalsha.call(client, scriptSha(script), numberOfKeys, ...args)
+  } catch (error) {
+    if (!errorMessage(error).includes("NOSCRIPT")) throw error
+    return client.eval(script, numberOfKeys, ...args)
+  }
+}