@vereda/http 1.0.0

package/src/queue/circuit-breaker.ts

@@ -0,0 +1,373 @@
+import type { AppError } from "../core/errors.ts";
+import {
+	type CircuitBreakerConfig,
+	DEFAULT_FAILURE_THRESHOLD,
+	DEFAULT_HALF_OPEN_MAX_ATTEMPTS,
+	DEFAULT_RESET_TIMEOUT_MS,
+	type PartitionConfig,
+} from "../core/types.ts";
+import { DEFAULT_PARTITION_TTL_MS } from "./bulkhead.ts";
+import { RETRIABLE_KINDS } from "./policy.ts";
+
+type CircuitState = "closed" | "open" | "half-open";
+
+/** Error kinds that mean the attempt never left the process — no request was
+ *  ever dispatched, so the host's health is untouched. Currently only a body
+ *  factory (`RequestOptions.body` as a function) that threw before `fetch`
+ *  was called (`src/queue/executor.ts`). A raw-`ReadableStream` body is
+ *  rejected even earlier, before the breaker is asked for a permit at all
+ *  (`client.ts` validates it ahead of `tryAcquire()`), so it never reaches
+ *  this classification in the first place. */
+const NEVER_SENT_KINDS: ReadonlySet<AppError["kind"]> = new Set(["configuration"]);
+
+// ---------------------------------------------------------------------------
+// Rolling window — fixed-size time buckets for the failure-percentage mode
+// ---------------------------------------------------------------------------
+
+interface Bucket {
+	/** Bucket start time (ms since epoch), used to determine expiry. */
+	start: number;
+	total: number;
+	failures: number;
+}
+
+const DEFAULT_BUCKET_COUNT = 10;
+
+class RollingWindow {
+	private readonly sizeMs: number;
+	private readonly bucketMs: number;
+	private buckets: Bucket[] = [];
+
+	constructor(sizeMs: number, bucketCount: number = DEFAULT_BUCKET_COUNT) {
+		this.sizeMs = sizeMs;
+		this.bucketMs = Math.max(1, Math.floor(sizeMs / bucketCount));
+	}
+
+	recordSuccess(now: number = Date.now()): void {
+		const bucket = this.currentBucket(now);
+		bucket.total++;
+	}
+
+	recordFailure(now: number = Date.now()): void {
+		const bucket = this.currentBucket(now);
+		bucket.total++;
+		bucket.failures++;
+	}
+
+	totalRequests(now: number = Date.now()): number {
+		this.prune(now);
+		return this.buckets.reduce((sum, b) => sum + b.total, 0);
+	}
+
+	failureRate(now: number = Date.now()): number {
+		this.prune(now);
+		const total = this.buckets.reduce((sum, b) => sum + b.total, 0);
+		if (total === 0) return 0;
+		const failures = this.buckets.reduce((sum, b) => sum + b.failures, 0);
+		return (failures / total) * 100;
+	}
+
+	private currentBucket(now: number): Bucket {
+		this.prune(now);
+		const bucketStart = Math.floor(now / this.bucketMs) * this.bucketMs;
+		const last = this.buckets[this.buckets.length - 1];
+		if (last && last.start === bucketStart) {
+			return last;
+		}
+		const bucket: Bucket = { start: bucketStart, total: 0, failures: 0 };
+		this.buckets.push(bucket);
+		return bucket;
+	}
+
+	private prune(now: number): void {
+		const cutoff = now - this.sizeMs;
+		this.buckets = this.buckets.filter((b) => b.start >= cutoff);
+	}
+}
+
+// ---------------------------------------------------------------------------
+// CircuitBreaker — one per partition
+// ---------------------------------------------------------------------------
+
+/** One admission through the breaker, returned by `tryAcquire()`. Exactly one
+ *  of its methods takes effect (later calls are no-ops), so a caller can
+ *  report the outcome and still `release()` unconditionally in a `finally`.
+ *  In half-open, the permit owns a trial slot; every exit path — success,
+ *  failure, a non-failure error, cancellation, a veto — must hand it back,
+ *  or the breaker is wedged half-open with no trial slots left (B1). */
+export interface CircuitPermit {
+	/** The attempt succeeded. */
+	success(): void;
+	/** The attempt failed with `error`. Errors the breaker doesn't classify as
+	 *  failures (e.g. a 404) count as successes — the host answered. An error
+	 *  that shows the attempt never reached the host (e.g. a body factory that
+	 *  threw) is ignored instead — same effect as calling `release()` — since
+	 *  it carries no information about the host's health. */
+	failure(error: AppError): void;
+	/** The permit was not used to reach the server (cancelled, vetoed, queue
+	 *  full…). Frees a half-open trial slot without recording an outcome. */
+	release(): void;
+}
+
+export class CircuitBreaker {
+	private readonly partition: string;
+	private readonly config: CircuitBreakerConfig;
+	private readonly onStateChange?: (partition: string, state: "open" | "closed") => void;
+
+	private state: CircuitState = "closed";
+	private consecutiveFailures = 0;
+	private openedAt = 0;
+	private halfOpenInFlight = 0;
+	/** Bumped on every open -> half-open transition, so a permit can tell
+	 *  whether the trial slot it reserved still belongs to the current
+	 *  half-open episode or to one that has since ended. */
+	private halfOpenGeneration = 0;
+	private readonly window?: RollingWindow;
+
+	constructor(
+		partition: string,
+		config: CircuitBreakerConfig = {},
+		onStateChange?: (partition: string, state: "open" | "closed") => void,
+	) {
+		this.partition = partition;
+		this.config = config;
+		this.onStateChange = onStateChange;
+		if (config.window) {
+			this.window = new RollingWindow(config.window.sizeMs);
+		}
+	}
+
+	/** Whether a request may proceed against this partition right now.
+	 *  Always true when the breaker is disabled. Handles the open -> half-open
+	 *  transition on `resetTimeoutMs` elapse, and reserves a half-open trial
+	 *  slot atomically so concurrent callers don't over-admit trials. */
+	canRequest(): boolean {
+		if (!this.config.enabled) return true;
+
+		if (this.state === "open") {
+			const resetTimeoutMs = this.config.resetTimeoutMs ?? DEFAULT_RESET_TIMEOUT_MS;
+			if (Date.now() - this.openedAt >= resetTimeoutMs) {
+				this.state = "half-open";
+				this.halfOpenInFlight = 0;
+				this.halfOpenGeneration++;
+			} else {
+				return false;
+			}
+		}
+
+		if (this.state === "half-open") {
+			const maxAttempts = this.config.halfOpenMaxAttempts ?? DEFAULT_HALF_OPEN_MAX_ATTEMPTS;
+			if (this.halfOpenInFlight >= maxAttempts) {
+				return false;
+			}
+			// Reserve the slot immediately (side-effecting) so two callers
+			// arriving before either trial resolves can't both be admitted.
+			this.halfOpenInFlight++;
+			return true;
+		}
+
+		// closed
+		return true;
+	}
+
+	/** Admit a request, or return `null` when the circuit rejects it. Prefer
+	 *  this over the raw `canRequest()`/`record*()` pair: the returned permit
+	 *  guarantees a reserved half-open trial slot is always given back. */
+	tryAcquire(): CircuitPermit | null {
+		if (!this.canRequest()) return null;
+
+		// null = admitted while closed (or disabled) — no trial slot held.
+		const trialGeneration = this.config.enabled && this.state === "half-open" ? this.halfOpenGeneration : null;
+		const isCurrentTrial = () => trialGeneration !== null && trialGeneration === this.halfOpenGeneration;
+		// A request admitted while closed that completes during a later
+		// half-open episode must not decide that episode: only its own trials do.
+		const mayRecord = () => this.state !== "half-open" || isCurrentTrial();
+		const freeTrialSlot = () => {
+			if (this.state === "half-open" && isCurrentTrial()) {
+				this.halfOpenInFlight = Math.max(0, this.halfOpenInFlight - 1);
+			}
+		};
+
+		let settled = false;
+		const settle = (fn: () => void) => {
+			if (settled) return;
+			settled = true;
+			fn();
+		};
+
+		return {
+			success: () =>
+				settle(() => {
+					if (mayRecord()) this.recordSuccess();
+				}),
+			failure: (error) =>
+				settle(() => {
+					if (!mayRecord()) return;
+					// Never-sent outcomes are ignored outright — not even isFailure is
+					// consulted, since there is no host-health signal to classify.
+					if (NEVER_SENT_KINDS.has(error.kind)) {
+						freeTrialSlot();
+						return;
+					}
+					// Classify once here and record the already-classified failure
+					// directly — `recordFailure()` would consult isFailure a second time.
+					if (this.isFailure(error)) {
+						this.recordClassifiedFailure();
+					} else {
+						this.recordNeutral();
+						freeTrialSlot();
+					}
+				}),
+			release: () => settle(freeTrialSlot),
+		};
+	}
+
+	/** Record a successful attempt outcome. No-op when disabled. */
+	recordSuccess(): void {
+		if (!this.config.enabled) return;
+
+		if (this.state === "half-open") {
+			this.halfOpenInFlight = Math.max(0, this.halfOpenInFlight - 1);
+			this.state = "closed";
+			this.resetCounters();
+			this.onStateChange?.(this.partition, "closed");
+			return;
+		}
+
+		if (this.state === "closed") {
+			this.consecutiveFailures = 0;
+			this.window?.recordSuccess();
+		}
+	}
+
+	/** Record a failed attempt outcome. No-op when disabled or when the error
+	 *  is not classified as a failure. */
+	recordFailure(error: AppError): void {
+		if (!this.config.enabled) return;
+		if (!this.isFailure(error)) return;
+		this.recordClassifiedFailure();
+	}
+
+	/** Record a failure the caller has already classified via `isFailure`, so a
+	 *  user-supplied classifier runs exactly once per outcome. No-op when disabled. */
+	private recordClassifiedFailure(): void {
+		if (!this.config.enabled) return;
+
+		if (this.state === "half-open") {
+			this.halfOpenInFlight = Math.max(0, this.halfOpenInFlight - 1);
+			this.state = "open";
+			this.openedAt = Date.now();
+			this.onStateChange?.(this.partition, "open");
+			return;
+		}
+
+		if (this.state === "closed") {
+			this.consecutiveFailures++;
+			this.window?.recordFailure();
+			if (this.evaluateTripCondition()) {
+				this.state = "open";
+				this.openedAt = Date.now();
+				this.onStateChange?.(this.partition, "open");
+			}
+		}
+	}
+
+	/** Not consulted for a `NEVER_SENT_KINDS` error (`tryAcquire()` short-circuits
+	 *  before calling this) — such an error never reached the host, so there is
+	 *  nothing for a caller-supplied classifier to legitimately weigh in on. */
+	private isFailure(error: AppError): boolean {
+		return this.config.isFailure ? this.config.isFailure(error) : RETRIABLE_KINDS.has(error.kind);
+	}
+
+	/** An attempt reached the server but ended in an error the breaker does
+	 *  not count as a failure — e.g. a 404, or a 200 whose body failed `parse`.
+	 *  The breaker measures availability, and the host answered, so this is
+	 *  recorded as a success in every state (the Resilience4j default for
+	 *  unrecorded errors): closed resets the consecutive-failure run and counts
+	 *  as a non-failure in the rolling window; a half-open trial closes the
+	 *  circuit. Classify it as a failure via `isFailure` to change that. */
+	private recordNeutral(): void {
+		this.recordSuccess();
+	}
+
+	private resetCounters(): void {
+		this.consecutiveFailures = 0;
+		this.halfOpenInFlight = 0;
+	}
+
+	private evaluateTripCondition(): boolean {
+		const windowConfig = this.config.window;
+		if (this.window && windowConfig) {
+			return (
+				this.window.totalRequests() >= windowConfig.minimumRequests &&
+				this.window.failureRate() > windowConfig.failureRatePercent
+			);
+		}
+		return this.consecutiveFailures >= (this.config.failureThreshold ?? DEFAULT_FAILURE_THRESHOLD);
+	}
+}
+
+// ---------------------------------------------------------------------------
+// CircuitBreakerRegistry — one breaker per partition, mirrors BulkheadRegistry
+// ---------------------------------------------------------------------------
+
+type CircuitBreakerEntry = [CircuitBreaker, number];
+
+export class CircuitBreakerRegistry {
+	private readonly breakers = new Map<string, CircuitBreakerEntry>();
+	private readonly ttlMs: number;
+	private readonly globalConfig: CircuitBreakerConfig;
+	private readonly partitionConfigs: Record<string, PartitionConfig>;
+	private readonly sweepInterval: number;
+	private readonly onStateChange?: (partition: string, state: "open" | "closed") => void;
+	private callCounter = 0;
+
+	constructor(
+		globalConfig: CircuitBreakerConfig = {},
+		partitionConfigs: Record<string, PartitionConfig> = {},
+		ttlMs: number = DEFAULT_PARTITION_TTL_MS,
+		onStateChange?: (partition: string, state: "open" | "closed") => void,
+	) {
+		this.ttlMs = ttlMs;
+		this.globalConfig = globalConfig;
+		this.partitionConfigs = partitionConfigs;
+		this.sweepInterval = 10;
+		this.onStateChange = onStateChange;
+	}
+
+	get(partitionName: string): CircuitBreaker {
+		this.callCounter++;
+
+		if (!this.breakers.has(partitionName)) {
+			const partitionConfig = this.partitionConfigs[partitionName]?.circuitBreaker ?? {};
+			const merged: CircuitBreakerConfig = {
+				...this.globalConfig,
+				...partitionConfig,
+			};
+			this.breakers.set(partitionName, [new CircuitBreaker(partitionName, merged, this.onStateChange), Date.now()]);
+		}
+		const [cb] = this.breakers.get(partitionName)!;
+		// Update last accessed time
+		this.breakers.set(partitionName, [cb, Date.now()]);
+
+		// Sweep stale entries periodically
+		if (this.callCounter % this.sweepInterval === 0 || this.breakers.size > 100) {
+			this.prune();
+		}
+
+		return cb;
+	}
+
+	prune(): void {
+		const now = Date.now();
+		for (const [key, [, lastAccessed]] of this.breakers) {
+			if (now - lastAccessed > this.ttlMs) {
+				this.breakers.delete(key);
+			}
+		}
+	}
+
+	delete(partitionName: string): void {
+		this.breakers.delete(partitionName);
+	}
+}