@procedures/client 0.2.2 → 0.3.0

package/src/client/middleware/circuit-breaker.ts

@@ -12,6 +12,11 @@
 
 import type { ClientMiddleware, ClientRunner, ClientContext, TypedClientMiddleware } from "../types.js";
 import type { CircuitBreakerContext } from "./contexts.js";
+import { optionalMs, type Duration } from "../../duration.js";
+import { isAvailabilityFailure } from "./items.js";
+import { realClock, type Cancel, type Clock } from "../../clock.js";
+import { CIRCUIT_BREAKER_MACHINE, type MachineTransition } from "../../machines.js";
+import { TIMER_DEFAULTS } from "../../timers.js";
 
 /**
  * Circuit breaker state.
@@ -29,16 +34,16 @@ export interface CircuitBreakerOptions {
   failureThreshold?: number;
 
   /**
-   * Time window for counting failures (in milliseconds).
-   * @default 10000 (10 seconds)
+   * Time window for counting failures.
+   * @default 10 seconds
    */
-  failureWindow?: number;
+  failureWindow?: Duration;
 
   /**
-   * Time to wait before attempting to close circuit (in milliseconds).
-   * @default 30000 (30 seconds)
+   * Time to wait before attempting to close circuit.
+   * @default 30 seconds
    */
-  resetTimeout?: number;
+  resetTimeout?: Duration;
 
   /**
    * Number of successful requests required to close circuit from HALF_OPEN.
@@ -55,8 +60,11 @@ export interface CircuitBreakerOptions {
 
   /**
    * Custom error predicate.
-   * Return true if error should count as failure.
-   * @default All errors count as failures
+   * Return true if error should count as failure. An error that is not a failure counts as
+   * nothing: not a failure and not a success.
+   * @default `isAvailabilityFailure`: only UNAVAILABLE, TIMEOUT, a stopped server, a gateway
+   * error and network errors count. An abort or a validation error does not (roadmap R12: before,
+   * every error counted, so a caller that aborted could open the circuit).
    */
   isFailure?: (error: Error) => boolean;
 
@@ -64,12 +72,30 @@ export interface CircuitBreakerOptions {
    * Callback when circuit state changes.
    */
   onStateChange?: (oldState: CircuitState, newState: CircuitState) => void;
+
+  /**
+   * The clock of the window and of the reset (roadmap R35). The default is the clock of the
+   * platform. A test gives a fake clock.
+   */
+  clock?: Clock;
+
+  /**
+   * This function gets each change of the state, with its event, as a transition of
+   * `CIRCUIT_BREAKER_MACHINE` (roadmap R35). A test replays them against the document.
+   */
+  onTransition?: (transition: MachineTransition) => void;
+
+  /** The name of the breaker in its transitions. The default is "circuit-breaker". */
+  name?: string;
 }
 
 /**
  * Failure record for tracking errors.
  */
 interface FailureRecord {
+  /** The time of the monotonic clock: the window uses it */
+  at: number;
+  /** The wall-clock time, for the statistics */
   timestamp: number;
   error: Error;
 }
@@ -114,8 +140,8 @@ export class CircuitBreakerError extends Error {
  * ```typescript
  * client.use(createCircuitBreakerMiddleware({
  *   failureThreshold: 5,      // Open after 5 failures
- *   failureWindow: 10000,      // Within 10 seconds
- *   resetTimeout: 30000,       // Try again after 30 seconds
+ *   failureWindow: seconds(10),  // Within 10 seconds
+ *   resetTimeout: seconds(30),   // Try again after 30 seconds
  *   successThreshold: 2        // Close after 2 successes
  * }));
  * ```
@@ -125,13 +151,13 @@ export function createCircuitBreakerMiddleware(
 ): TypedClientMiddleware<CircuitBreakerContext, {}> {
   const {
     failureThreshold = 5,
-    failureWindow = 10000,
-    resetTimeout = 30000,
     successThreshold = 2,
-    isFailure = () => true,
+    isFailure = isAvailabilityFailure,
     onStateChange,
   } = options;
   const halfOpenMaxRequests = options.halfOpenMaxRequests ?? successThreshold;
+  const failureWindow = optionalMs(options.failureWindow, "failureWindow") ?? TIMER_DEFAULTS.circuitBreaker.failureWindowMs;
+  const resetTimeout = optionalMs(options.resetTimeout, "resetTimeout") ?? TIMER_DEFAULTS.circuitBreaker.resetTimeoutMs;
 
   let state: CircuitState = "CLOSED";
   let failures: FailureRecord[] = [];
@@ -139,23 +165,26 @@ export function createCircuitBreakerMiddleware(
   let totalRequests = 0;
   let probesInFlight = 0;
   let lastStateChange = Date.now();
-  let resetTimer: ReturnType<typeof setTimeout> | null = null;
+  let resetTimer: Cancel | null = null;
+  const clock = options.clock ?? realClock;
+  const instance = options.name ?? "circuit-breaker";
 
   /**
-   * Change circuit state.
+   * Change circuit state. `on` is the event of `CIRCUIT_BREAKER_MACHINE`.
    */
-  function setState(newState: CircuitState): void {
+  function setState(newState: CircuitState, on: string): void {
     const oldState = state;
     state = newState;
     if (oldState !== newState) lastStateChange = Date.now();
 
-    if (oldState !== newState && onStateChange) {
-      onStateChange(oldState, newState);
+    if (oldState !== newState) {
+      options.onTransition?.({ machine: CIRCUIT_BREAKER_MACHINE.id, instance, from: oldState, on, to: newState, at: clock.now() });
+      onStateChange?.(oldState, newState);
     }
 
     // Clear reset timer when moving to CLOSED
     if (newState === "CLOSED" && resetTimer) {
-      clearTimeout(resetTimer);
+      resetTimer();
       resetTimer = null;
     }
   }
@@ -164,22 +193,23 @@ export function createCircuitBreakerMiddleware(
    * Record a failure.
    */
   function recordFailure(error: Error): void {
-    const now = Date.now();
+    // The window uses a monotonic clock: a change of the wall clock does not move it (R12)
+    const now = clock.now();
 
     // Add failure
-    failures.push({ timestamp: now, error });
+    failures.push({ at: now, timestamp: Date.now(), error });
 
     // Remove old failures outside window
-    failures = failures.filter((f) => now - f.timestamp < failureWindow);
+    failures = failures.filter((f) => now - f.at < failureWindow);
 
     if (state === "HALF_OPEN") {
       // Any failure while testing reopens the circuit immediately — don't wait for the full
       // failureThreshold. See documentation/BUGS-2026-07.md (H11).
-      setState("OPEN");
+      setState("OPEN", "probe-failure");
       successes = 0;
       scheduleReset();
     } else if (state === "CLOSED" && failures.length >= failureThreshold) {
-      setState("OPEN");
+      setState("OPEN", "failure-threshold");
       scheduleReset();
     }
   }
@@ -192,14 +222,14 @@ export function createCircuitBreakerMiddleware(
       successes++;
       if (successes >= successThreshold) {
         // Enough successes - close circuit
-        setState("CLOSED");
+        setState("CLOSED", "probe-successes");
         failures = [];
         successes = 0;
       }
     } else if (state === "CLOSED") {
       // Clear old failures on success
-      const now = Date.now();
-      failures = failures.filter((f) => now - f.timestamp < failureWindow);
+      const now = clock.now();
+      failures = failures.filter((f) => now - f.at < failureWindow);
     }
   }
 
@@ -208,15 +238,15 @@ export function createCircuitBreakerMiddleware(
    */
   function scheduleReset(): void {
     if (resetTimer) {
-      clearTimeout(resetTimer);
+      resetTimer();
     }
 
-    resetTimer = setTimeout(() => {
+    resetTimer = clock.after(resetTimeout, () => {
       if (state === "OPEN") {
-        setState("HALF_OPEN");
+        setState("HALF_OPEN", "reset-timeout");
         successes = 0;
       }
-    }, resetTimeout);
+    });
     // Don't keep the event loop alive just for the reset timer. See BUGS-2026-07.md (M4).
     resetTimer.unref?.();
   }
@@ -239,10 +269,10 @@ export function createCircuitBreakerMiddleware(
   }
 
   function getStats(): CircuitBreakerStats {
-    const now = Date.now();
+    const now = clock.now();
     return {
       state,
-      failures: failures.filter((f) => now - f.timestamp < failureWindow).length,
+      failures: failures.filter((f) => now - f.at < failureWindow).length,
       successes,
       totalRequests,
       lastFailureTime: failures.length > 0 ? failures[failures.length - 1]!.timestamp : null,
@@ -257,10 +287,10 @@ export function createCircuitBreakerMiddleware(
       totalRequests++;
       // The state for the other middlewares (CircuitBreakerContext)
       const writeState = (): void => {
-        const now = Date.now();
+        const now = clock.now();
         context.message.metadata["circuitBreaker"] = {
           state,
-          failures: failures.filter((f) => now - f.timestamp < failureWindow).length,
+          failures: failures.filter((f) => now - f.at < failureWindow).length,
           successes,
         };
       };

package/src/client/middleware/contexts.ts

@@ -15,6 +15,8 @@
  * ```
  */
 
+import type { Duration } from "../../duration.js";
+
 // =============================================================================
 // Retry Middleware Context
 // =============================================================================
@@ -29,12 +31,17 @@ export interface RetryContext {
     attempt?: number;
     /** Maximum number of attempts: an override of a call, then the value that the middleware uses */
     maxAttempts?: number;
-    /** An override of the base delay, in milliseconds */
-    delay?: number;
+    /** An override of the base delay */
+    delay?: Duration;
     /** An override of the factor of the delay from one attempt to the next */
     backoffMultiplier?: number;
-    /** An override of the longest delay, in milliseconds */
-    maxDelay?: number;
+    /** An override of the longest delay */
+    maxDelay?: Duration;
+    /**
+     * True: the call can go again after a try with an indeterminate outcome, because a second
+     * run changes nothing more (roadmap R1). It overrides the `idempotent` option of the middleware.
+     */
+    idempotent?: boolean;
   };
 }
 
@@ -51,8 +58,8 @@ export interface RetryContext {
  */
 export interface CacheContext {
   cache: {
-    /** The time-to-live of the entry that this call stores, in milliseconds */
-    ttl?: number;
+    /** The time-to-live of the entry that this call stores */
+    ttl?: Duration;
     /** True: the call skips the cache (no read, no write) */
     bypass?: boolean;
     /** True: the call fetches and replaces the stored response */
@@ -72,10 +79,10 @@ export interface CacheContext {
  */
 export interface TimeoutContext {
   timeout: {
-    /** Overall timeout in milliseconds (if configured) */
-    overall?: number;
-    /** Per-attempt timeout in milliseconds (if configured) */
-    perAttempt?: number;
+    /** The overall timeout (if configured) */
+    overall?: Duration;
+    /** The timeout of each attempt (if configured) */
+    perAttempt?: Duration;
   };
 }
 

package/src/client/middleware/items.ts

@@ -8,6 +8,7 @@
  */
 
 import type { ResponseItem } from "../types.js";
+import { realClock, type Clock } from "../../clock.js";
 
 /** This function makes an error item. */
 export function errorItem<TRes>(id: string, code: string, message: string, retryable: boolean): ResponseItem<TRes> {
@@ -19,6 +20,72 @@ export function abortedItem<TRes>(id: string): ResponseItem<TRes> {
   return errorItem<TRes>(id, "ABORTED", "Request was aborted", false);
 }
 
+/**
+ * What a failed try tells about its request, in the metadata of the error item (`delivery`).
+ * - `not-sent`: the request did not leave the client. A retry is safe.
+ * - `indeterminate`: the request left, and no answer came. The server can have run it, so only
+ *   an idempotent call can go again (the three outcomes of `@mark1russell7/service`).
+ * An error item without the field is an answer of the server: its `retryable` flag decides.
+ */
+export type Delivery = "not-sent" | "indeterminate";
+
+/** The metadata key of the delivery of a failed try. */
+export const DELIVERY_METADATA_KEY = "delivery";
+
+/** This function marks an error item with the delivery of its request. */
+export function withDelivery<TRes>(item: ResponseItem<TRes>, delivery: Delivery): ResponseItem<TRes> {
+  return { ...item, metadata: { ...item.metadata, [DELIVERY_METADATA_KEY]: delivery } };
+}
+
+/** The metadata key of the time that the server asks a client to wait before a retry, in milliseconds. */
+export const RETRY_AFTER_METADATA_KEY = "retry-after";
+
+/** The time that an error item asks a client to wait before a retry, in milliseconds, or undefined. */
+export function retryAfterOf(item: ResponseItem<unknown>): number | undefined {
+  const value = item.metadata?.[RETRY_AFTER_METADATA_KEY];
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
+}
+
+/**
+ * The error codes that say that the service is not available: the server is down, busy, slow or
+ * stopping, or the network failed. A circuit breaker counts only these (roadmap R12). An abort,
+ * a validation error, a refusal or an error of the handler says nothing about the health of the
+ * service.
+ */
+const AVAILABILITY_CODES = new Set([
+  "UNAVAILABLE",
+  "TIMEOUT",
+  "SERVER_STOPPED",
+  "SERVICE_UNAVAILABLE",
+  "BAD_GATEWAY",
+  "GATEWAY_TIMEOUT",
+  "NETWORK_ERROR",
+  "CONNECTION_REFUSED",
+  "STREAM_ENDED",
+  // The connection to the other side closed before the call left (roadmap R30: before, the list
+  // did not have this code of R24)
+  "CONNECTION_CLOSED",
+]);
+
+/** True when an error says that the service is not available (see AVAILABILITY_CODES). */
+export function isAvailabilityFailure(error: unknown): boolean {
+  const code = (error as { code?: unknown } | undefined)?.code;
+  if (typeof code === "string" && AVAILABILITY_CODES.has(code)) return true;
+  return isNetworkError(error);
+}
+
+/** The time of a monotonic clock in milliseconds: a change of the wall clock does not move it. */
+export function monotonicNow(): number {
+  const performance = (globalThis as { performance?: { now(): number } }).performance;
+  return performance ? performance.now() : Date.now();
+}
+
+/** The delivery of a failed try, or undefined for an answer of the server. */
+export function deliveryOf(item: ResponseItem<unknown>): Delivery | undefined {
+  const delivery = item.metadata?.[DELIVERY_METADATA_KEY];
+  return delivery === "not-sent" || delivery === "indeterminate" ? delivery : undefined;
+}
+
 const NETWORK_CODES = new Set([
   "ECONNREFUSED",
   "ECONNRESET",
@@ -52,35 +119,63 @@ export function isNetworkError(error: unknown): boolean {
   return /WebSocket (connection closed|disconnected)|Connection timeout/.test(error.message);
 }
 
+/** The network codes that tell that the connection never opened, so no request left. */
+const NOT_SENT_CODES = new Set([
+  "ECONNREFUSED",
+  "ENETUNREACH",
+  "EHOSTUNREACH",
+  "ENOTFOUND",
+  "EAI_AGAIN",
+  "UND_ERR_CONNECT_TIMEOUT",
+  "CONNECTION_REFUSED",
+]);
+
+/**
+ * This function tells if a thrown error proves that the request did not leave the client: the
+ * connection was refused, the name did not resolve, or the WebSocket did not open in time.
+ * A reset or a close during the request does not prove it (the request can have arrived).
+ */
+export function isNotSentError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+  const code = (error as { code?: unknown }).code;
+  if (typeof code === "string" && NOT_SENT_CODES.has(code)) return true;
+  const cause = (error as { cause?: unknown }).cause;
+  if (cause !== undefined && cause !== error && isNotSentError(cause)) return true;
+  // The WebSocket transport fails a request that waited for a connection that did not come
+  return /^(WebSocket disconnected|Connection timeout)$/.test(error.message);
+}
+
 /**
  * The item that stands for a thrown error, so that `shouldRetry` can decide about it. It is
- * retryable only when the error is a network failure.
+ * retryable only when the error is a network failure. Its delivery tells if the request left.
  */
 export function thrownItem<TRes>(id: string, error: unknown): ResponseItem<TRes> {
   const code = (error as { code?: unknown } | undefined)?.code;
-  return errorItem<TRes>(
+  const item = errorItem<TRes>(
     id,
     typeof code === "string" ? code : "THROWN",
     error instanceof Error ? error.message : String(error),
     isNetworkError(error),
   );
+  return withDelivery(item, isNotSentError(error) ? "not-sent" : "indeterminate");
 }
 
 /**
  * This function waits `ms` milliseconds. It returns false at once when the signal aborts (deep
  * dive TRN-8: before, the backoff of the retry middleware ignored the signal).
  */
-export function sleep(ms: number, signal?: AbortSignal): Promise<boolean> {
+export function sleep(ms: number, signal?: AbortSignal, clock: Clock = realClock): Promise<boolean> {
   if (signal?.aborted) return Promise.resolve(false);
   return new Promise((resolve) => {
     const onAbort = (): void => {
-      clearTimeout(timer);
+      cancel();
       resolve(false);
     };
-    const timer = setTimeout(() => {
+    // The clock of the caller (roadmap R35): a test gives a fake clock
+    const cancel = clock.after(Math.max(0, ms), () => {
       signal?.removeEventListener("abort", onAbort);
       resolve(true);
-    }, Math.max(0, ms));
+    });
     signal?.addEventListener("abort", onAbort, { once: true });
   });
 }

package/src/client/middleware/lru.ts

@@ -63,4 +63,16 @@ export class LruMap<K, V> {
   clear(): void {
     this.entries.clear();
   }
+
+  /** Delete each key whose value matches. This does not count as a use. The number deleted. */
+  deleteWhere(matches: (value: V, key: K) => boolean): number {
+    let deleted = 0;
+    for (const [key, value] of this.entries) {
+      if (matches(value, key)) {
+        this.entries.delete(key);
+        deleted++;
+      }
+    }
+    return deleted;
+  }
 }

package/src/client/middleware/rate-limit.ts

@@ -10,6 +10,8 @@
 import type { ClientMiddleware, ClientRunner, ClientContext, TypedClientMiddleware } from "../types.js";
 import type { RateLimitContext } from "./contexts.js";
 import { abortedItem } from "./items.js";
+import { optionalMs, type Duration } from "../../duration.js";
+import { TIMER_DEFAULTS } from "../../timers.js";
 
 /**
  * Rate limiting options.
@@ -25,7 +27,7 @@ export interface RateLimitOptions {
    * Time window in milliseconds.
    * @default 60000 (1 minute)
    */
-  window?: number;
+  window?: Duration;
 
   /**
    * Behavior when rate limit is exceeded.
@@ -96,14 +98,14 @@ interface QueuedRequest {
  * // Reject strategy (default)
  * client.use(createRateLimitMiddleware({
  *   maxRequests: 100,    // 100 requests
- *   window: 60000,       // per minute
+ *   window: minutes(1),
  *   strategy: "reject"   // Throw error when exceeded
  * }));
  *
  * // Queue strategy
  * client.use(createRateLimitMiddleware({
  *   maxRequests: 10,
- *   window: 1000,
+ *   window: seconds(1),
  *   strategy: "queue",   // Queue requests when exceeded
  *   maxQueueSize: 50
  * }));
@@ -114,12 +116,12 @@ export function createRateLimitMiddleware(
 ): TypedClientMiddleware<RateLimitContext, {}> {
   const {
     maxRequests = 100,
-    window = 60000,
     strategy = "reject",
     maxQueueSize = 100,
     message = "Rate limit exceeded",
     onRateLimitExceeded,
   } = options;
+  const window = optionalMs(options.window, "window") ?? TIMER_DEFAULTS.rateLimit.windowMs;
 
   // Token bucket state
   let tokens = maxRequests;
@@ -273,9 +275,9 @@ export function createRateLimitMiddleware(
  * @example
  * ```typescript
  * client.use(createPerServiceRateLimiter({
- *   users: { maxRequests: 100, window: 60000 },
- *   orders: { maxRequests: 50, window: 60000 },
- *   default: { maxRequests: 200, window: 60000 }
+ *   users: { maxRequests: 100, window: minutes(1) },
+ *   orders: { maxRequests: 50, window: minutes(1) },
+ *   default: { maxRequests: 200, window: minutes(1) }
  * }));
  * ```
  */

package/src/client/middleware/resilience.ts

@@ -0,0 +1,81 @@
+/**
+ * The resilience preset (roadmap R10): the timeouts, the cache, the retry, the circuit breaker and
+ * the rate limit of a client, in the one order that works.
+ *
+ * The first layer is the outermost one:
+ *
+ * 1. The overall timeout: it covers the whole call, retries included.
+ * 2. The cache: a hit makes no request, so no try and no slot of the rate limit.
+ * 3. The retry: it starts each try.
+ * 4. The circuit breaker: it counts each try, and an open breaker stops a retry loop.
+ * 5. The rate limit: each try takes a slot.
+ * 6. The timeout of one try: inside the retry, so the retry can start the next try.
+ *
+ * Each order pitfall of a hand-made stack is impossible here: an overall timeout inside the
+ * retry, a per-try timeout outside it, or a rate limit outside the retry. The deadline of the
+ * call goes to the server as `procedure-timeout` with each try (`client/deadline.ts`).
+ */
+
+import type { ClientMiddleware, ClientRunner, TypedClientMiddleware } from "../types.js";
+import type { CacheContext, CircuitBreakerContext, RateLimitContext, RetryContext, TimeoutContext } from "./contexts.js";
+import { createCacheMiddleware, type CacheOptions } from "./cache.js";
+import { createCircuitBreakerMiddleware, type CircuitBreakerOptions } from "./circuit-breaker.js";
+import { createRateLimitMiddleware, type RateLimitOptions } from "./rate-limit.js";
+import { createRetryMiddleware, type RetryOptions } from "./retry.js";
+import { createOverallTimeoutMiddleware, createTimeoutMiddleware } from "./timeout.js";
+import type { Duration } from "../../duration.js";
+
+/** The parts of the preset. An absent part is not in the stack. */
+export interface ResilienceOptions {
+  /** The time of the whole call, retries included. */
+  overall?: Duration | undefined;
+  /** The time of one try. It is at most the remaining time of the call. */
+  perAttempt?: Duration | undefined;
+  /** The retry. Its `idempotent` option decides which calls go again after an indeterminate try. */
+  retry?: RetryOptions | undefined;
+  /** The circuit breaker. */
+  breaker?: CircuitBreakerOptions | undefined;
+  /** The cache of the reads that opt in. */
+  cache?: CacheOptions | undefined;
+  /** The rate limit of the tries. */
+  rateLimit?: RateLimitOptions | undefined;
+  /** The message of a timeout error. */
+  message?: string | undefined;
+}
+
+/** The context that the preset gives: the context of each part that it can have. */
+export type ResilienceContext = TimeoutContext & Partial<RetryContext & CacheContext & CircuitBreakerContext & RateLimitContext>;
+
+/**
+ * This function gives one middleware with the parts of `options` in the correct order (see the
+ * file comment). Add it with one `use()`.
+ *
+ * @example
+ * ```typescript
+ * const client = new Client(transport).use(
+ *   resilience({
+ *     overall: seconds(10),
+ *     perAttempt: seconds(3),
+ *     retry: { maxRetries: 2, idempotent: idempotentIn(registry) },
+ *     breaker: { failureThreshold: 5 },
+ *   }),
+ * );
+ * ```
+ */
+export function resilience(options: ResilienceOptions): TypedClientMiddleware<ResilienceContext, {}> {
+  const layers: ClientMiddleware[] = [];
+  const message = options.message;
+  if (options.overall) {
+    layers.push(createOverallTimeoutMiddleware({ overall: options.overall, ...(message ? { message: `${message} (overall)` } : {}) }) as ClientMiddleware);
+  }
+  if (options.cache) layers.push(createCacheMiddleware(options.cache) as ClientMiddleware);
+  if (options.retry) layers.push(createRetryMiddleware(options.retry) as ClientMiddleware);
+  if (options.breaker) layers.push(createCircuitBreakerMiddleware(options.breaker) as ClientMiddleware);
+  if (options.rateLimit) layers.push(createRateLimitMiddleware(options.rateLimit) as ClientMiddleware);
+  if (options.perAttempt) {
+    layers.push(createTimeoutMiddleware({ perAttempt: options.perAttempt, ...(message ? { message: `${message} (per-attempt)` } : {}) }) as ClientMiddleware);
+  }
+  const composed = <TReq, TRes>(next: ClientRunner<TReq, TRes>): ClientRunner<TReq, TRes> =>
+    layers.reduceRight<ClientRunner<TReq, TRes>>((inner, layer) => (layer as ClientMiddleware<TReq, TRes>)(inner), next);
+  return composed as TypedClientMiddleware<ResilienceContext, {}>;
+}