@vereda/http 1.0.0

package/src/queue/retry.ts

@@ -0,0 +1,380 @@
+import { buildBackoffFn, DEFAULT_MAX_DELAY_MS } from "../core/backoff.ts";
+import type { AppError } from "../core/errors.ts";
+import {
+	CancelledError,
+	CircuitOpenError,
+	DeadlineExceededError,
+	MaxRetriesExceededError,
+	NO_TIMEOUT_CONFIGURED,
+	QueueFullError,
+	RetryableStatusError,
+	TimeoutError,
+} from "../core/errors.ts";
+import {
+	type BackoffOptions,
+	DEFAULT_MAX_RETRIES,
+	isBoundedMs,
+	type RequestOptions,
+	type RetryConfig,
+	type TimeoutConfig,
+} from "../core/types.ts";
+import type { Ticket, TicketController } from "../ticket/ticket.ts";
+import type { Bulkhead } from "./bulkhead.ts";
+import type { CircuitBreaker } from "./circuit-breaker.ts";
+import { executeRequest, type MiddlewareFn } from "./executor.ts";
+import { defaultRetryPolicy, type RetryPolicyContext, shouldRetry } from "./policy.ts";
+import type { Semaphore } from "./semaphore.ts";
+
+export interface RetryJobOptions {
+	/** Fully resolved request URL (baseUrl applied) — what each retry fetches. */
+	url: string;
+	/** `url` as it may appear in errors/logs (query-redacted unless disabled). */
+	displayUrl: string;
+	requestOptions: RequestOptions<unknown>;
+	timeoutConfig: TimeoutConfig;
+	retryConfig: RetryConfig;
+	/** Absolute `Date.now()` timestamp of the ticket's whole-ticket deadline
+	 *  (`startTime + timeoutConfig.totalMs`), or undefined when `totalMs`
+	 *  isn't bounded. Forwarded to `executeRequest` so a handed-off Response
+	 *  body (unread success or `HttpError`) is bounded by it — see
+	 *  `ExecuteRequest.deadlineAt`. */
+	deadlineAt?: number;
+	ticket: Ticket<unknown>;
+	controller: TicketController<unknown>;
+	middleware: MiddlewareFn[];
+	/** Per-attempt bulkhead for retry scheduling. */
+	bulkhead: Bulkhead;
+	/** Global concurrency semaphore acquired after the partition slot (D4). */
+	semaphore?: Semaphore;
+	/** Per-partition circuit breaker. Inert unless configured/enabled. */
+	circuitBreaker: CircuitBreaker;
+	/** Partition name, used to construct CircuitOpenError when the breaker is open. */
+	partition: string;
+	/** The error from the first attempt (fired client-side before queuing). */
+	firstError: AppError;
+	/** Ms the first attempt spent waiting for a bulkhead/semaphore permit,
+	 *  carried over so the final queuedMs reported to the client covers the
+	 *  whole ticket lifetime, not just the retries run inside this loop. */
+	initialQueuedMs: number;
+	onRetry?: (attempt: number, delayMs: number, error: AppError) => void;
+	/** Called on success so the client can emit the success event with the
+	 *  response status code, the total attempt count, and the cumulative ms
+	 *  spent waiting for a bulkhead/semaphore permit across all attempts. */
+	onSuccess?: (statusCode: number, attempts: number, queuedMs: number) => void;
+	/** Called on failure (exhausted retries or policy veto inside the loop)
+	 *  so the client can emit the failure event. `queuedMs` is cumulative
+	 *  across all attempts made so far. */
+	onFailure?: (error: AppError, attempts: number, queuedMs: number) => void;
+	/** Called on cancellation (user cancel, deadline, or external signal)
+	 *  so the client can emit the cancelled event. `queuedMs` is cumulative
+	 *  across all attempts made so far. */
+	onCancelled?: (attempts: number, queuedMs: number) => void;
+	/** Called before markDone to clean up external resources (e.g. signal listeners). */
+	onCleanup?: () => void;
+	/** Custom fetch function. Falls back to globalThis.fetch. */
+	fetch?: typeof globalThis.fetch;
+}
+
+export async function runRetryLoop(job: RetryJobOptions): Promise<void> {
+	const {
+		url,
+		displayUrl,
+		requestOptions,
+		timeoutConfig,
+		retryConfig,
+		deadlineAt,
+		ticket,
+		controller,
+		middleware,
+		bulkhead,
+		semaphore,
+		circuitBreaker,
+		partition,
+		firstError,
+		initialQueuedMs,
+		onRetry,
+		onSuccess,
+		onFailure,
+		onCancelled,
+		onCleanup,
+		fetch: customFetch,
+	} = job;
+
+	const maxRetries = retryConfig.maxRetries ?? DEFAULT_MAX_RETRIES;
+	const backoffFn = buildBackoffFn(retryConfig.backoff);
+	const backoffCap =
+		retryConfig.backoff && typeof retryConfig.backoff === "object"
+			? ((retryConfig.backoff as BackoffOptions).maxDelayMs ?? DEFAULT_MAX_DELAY_MS)
+			: DEFAULT_MAX_DELAY_MS;
+
+	let lastError: AppError = firstError;
+	let totalAttempts = 1; // first attempt already fired client-side
+	let totalQueuedMs = initialQueuedMs; // cumulative bulkhead/semaphore wait across all attempts
+
+	// Attempt 0's error was already vetted (default policy + retryWhen) by
+	// client.ts's vetoed() before this loop was ever entered — that's what
+	// decided to queue the ticket for retry in the first place. `ctx` is
+	// constant across the loop (derived only from requestOptions/retryConfig),
+	// so it's built once here and reused both by the loop's gate below and by
+	// the final-attempt check after the loop.
+	const ctx: RetryPolicyContext = {
+		method: requestOptions.method ?? "GET",
+		headers: requestOptions.headers,
+		idempotent: retryConfig.idempotent,
+	};
+
+	for (let attempt = 0; attempt < maxRetries; attempt++) {
+		if (ticket.isCancelled) {
+			onCleanup?.();
+			onCancelled?.(totalAttempts, totalQueuedMs);
+			controller.markDone({
+				success: false,
+				error: new CancelledError(),
+			} as never);
+			return;
+		}
+
+		const permit = circuitBreaker.tryAcquire();
+		if (!permit) {
+			onCleanup?.();
+			const error = new CircuitOpenError(partition);
+			onFailure?.(error, totalAttempts, totalQueuedMs);
+			controller.markDone({ success: false, error } as never);
+			return;
+		}
+
+		// Every exit from this iteration settles the permit: an outcome below,
+		// or release() for the veto/cancel/deadline/queue-full paths (B1).
+		try {
+			// Consult the default policy + retryWhen for every retry iteration
+			// *except the first* (attempt 0): that error is always `firstError`,
+			// already vetted by client.ts's vetoed() before this loop was ever
+			// entered. Re-checking it here would call a user-provided retryWhen
+			// twice for the same failed attempt. From attempt 1 on, `lastError`
+			// is a fresh error produced by a retry this loop just ran, so the
+			// gate must run — it's what lets retryWhen limit retries to N+1
+			// attempts total.
+			if (attempt > 0 && !shouldRetry(lastError, attempt, ctx, retryConfig.retryWhen)) {
+				onCleanup?.();
+				onFailure?.(lastError, totalAttempts, totalQueuedMs);
+				controller.markDone({ success: false, error: lastError } as never);
+				return;
+			}
+
+			// All retries get backoff. Every retry is gated by the policy check above.
+			const delayMs = resolveRetryDelay(lastError, backoffFn, attempt, backoffCap);
+			onRetry?.(attempt, delayMs, lastError);
+			controller.markRetrying(attempt, delayMs);
+
+			try {
+				await sleep(delayMs, ticket.signal);
+			} catch {
+				// The deadline timer aborts the ticket signal. Distinguish deadline from
+				// user cancellation: deadline only aborts the signal (abortSignal()),
+				// while user cancellation sets _cancelled = true via cancel().
+				onCleanup?.();
+				if (!ticket.isCancelled && isBoundedMs(timeoutConfig.totalMs)) {
+					const error = new DeadlineExceededError(displayUrl, timeoutConfig.totalMs);
+					onFailure?.(error, totalAttempts, totalQueuedMs);
+					controller.markDone({
+						success: false,
+						error,
+					} as never);
+				} else {
+					onCancelled?.(totalAttempts, totalQueuedMs);
+					controller.markDone({
+						success: false,
+						error: new CancelledError(),
+					} as never);
+				}
+				return;
+			}
+
+			if (ticket.isCancelled) {
+				onCleanup?.();
+				onCancelled?.(totalAttempts, totalQueuedMs);
+				controller.markDone({
+					success: false,
+					error: new CancelledError(),
+				} as never);
+				return;
+			}
+
+			totalAttempts++;
+
+			// Per-attempt bulkhead scheduling: each retry acquires its own slot,
+			// releases it after execution, so other tickets aren't blocked (#5, D4).
+			// The global semaphore is acquired after the partition slot (D4).
+			let result: Awaited<ReturnType<typeof executeRequest>>;
+			try {
+				result = await bulkhead.run(
+					() =>
+						executeRequest(
+							{
+								url,
+								options: requestOptions,
+								timeoutConfig,
+								retryConfig,
+								deadlineAt,
+								displayUrl,
+								signal: ticket.signal,
+								attempt: attempt + 1,
+								ticketId: ticket.id,
+								partition: bulkhead.name,
+								fetch: customFetch,
+							},
+							middleware,
+						),
+					semaphore,
+					(queuedMs) => {
+						totalQueuedMs += queuedMs;
+					},
+					ticket.signal,
+				);
+			} catch (err) {
+				if (err instanceof CancelledError) {
+					// Cancelled (or deadline-aborted) while still queued for a slot or
+					// permit (B9): nothing was dispatched, so back out the optimistic
+					// totalAttempts++ and let the "cancelled" case settle it.
+					totalAttempts--;
+					result = { kind: "cancelled" };
+				} else if (err instanceof QueueFullError) {
+					// Queue is at capacity — bulkhead.run() rejected before task()
+					// ever ran, so this iteration's optimistic totalAttempts++ above
+					// must be backed out; it reports attempts actually dispatched.
+					const attemptsMade = totalAttempts - 1;
+					onCleanup?.();
+					if (ticket.isCancelled) {
+						// cancel() may have resolved the ticket directly (ticket.ts),
+						// bypassing markDone, while this bulkhead/semaphore acquisition
+						// was still pending. Match every other cancellation checkpoint
+						// in this loop: notify via onCancelled, not onFailure, so the
+						// lifecycle-event stream agrees with the ticket's actual outcome.
+						onCancelled?.(attemptsMade, totalQueuedMs);
+					} else {
+						// Emit failure here (not in the client's outer .catch) because
+						// totalQueuedMs — the wait accumulated by retries that already
+						// ran — only exists in this closure; the outer catch only has
+						// the first attempt's queuedMs.
+						onFailure?.(err, attemptsMade, totalQueuedMs);
+					}
+					controller.markDone({ success: false, error: err } as never);
+					throw err;
+				} else {
+					throw err;
+				}
+			}
+
+			switch (result.kind) {
+				case "success": {
+					permit.success();
+					onCleanup?.();
+					if (result.result.success) {
+						onSuccess?.(result.result.raw.status, totalAttempts, totalQueuedMs);
+					}
+					controller.markDone(result.result);
+					return;
+				}
+
+				case "cancelled":
+					onCleanup?.();
+					if (!ticket.isCancelled && isBoundedMs(timeoutConfig.totalMs)) {
+						const error = new DeadlineExceededError(displayUrl, timeoutConfig.totalMs);
+						onFailure?.(error, totalAttempts, totalQueuedMs);
+						controller.markDone({
+							success: false,
+							error,
+						} as never);
+					} else {
+						onCancelled?.(totalAttempts, totalQueuedMs);
+						controller.markDone({
+							success: false,
+							error: new CancelledError(),
+						} as never);
+					}
+					return;
+
+				case "timeout":
+					lastError = new TimeoutError(displayUrl, timeoutConfig.attemptMs ?? NO_TIMEOUT_CONFIGURED);
+					permit.failure(lastError);
+					break;
+
+				case "error":
+					lastError = result.error;
+					permit.failure(lastError);
+					break;
+			}
+		} finally {
+			permit.release();
+		}
+	}
+
+	if (maxRetries === 0) {
+		// Zero retries configured/executed — surface the underlying error raw,
+		// never wrapped in MaxRetriesExceededError.
+		onCleanup?.();
+		onFailure?.(lastError, totalAttempts, totalQueuedMs);
+		controller.markDone({ success: false, error: lastError } as never);
+		return;
+	}
+
+	// All retries exhausted — total attempts = 1 (first) + maxRetries (loop)
+	totalAttempts = 1 + maxRetries;
+	onCleanup?.();
+
+	// The final attempt's own error was never run through the gate above —
+	// the loop only checks the *previous* error at the top of each iteration,
+	// and there is no next iteration for this one. If it wouldn't have been
+	// retried anyway (e.g. a non-retryable HttpError, or a ValidationError
+	// from a failed parse), it's terminal on its own merits: surface it raw,
+	// same as a mid-loop veto, instead of wrapping it in
+	// MaxRetriesExceededError. Only genuinely transient exhaustion — the
+	// final error was itself retryable, retries just ran out — gets wrapped.
+	// This intentionally checks the default policy only, not retryWhen:
+	// retryWhen is never consulted for an attempt that has no retries left.
+	if (!defaultRetryPolicy(lastError, maxRetries, ctx)) {
+		onFailure?.(lastError, totalAttempts, totalQueuedMs);
+		controller.markDone({ success: false, error: lastError } as never);
+		return;
+	}
+
+	const exhaustedError = new MaxRetriesExceededError(totalAttempts, lastError);
+	onFailure?.(exhaustedError, totalAttempts, totalQueuedMs);
+	controller.markDone({
+		success: false,
+		error: exhaustedError,
+	} as never);
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+	return new Promise((resolve, reject) => {
+		if (signal?.aborted) {
+			reject(signal.reason ?? new DOMException("The operation was aborted", "AbortError"));
+			return;
+		}
+		const timer = setTimeout(() => {
+			signal?.removeEventListener("abort", onAbort);
+			resolve();
+		}, ms);
+		const onAbort = () => {
+			clearTimeout(timer);
+			reject(signal?.reason ?? new DOMException("The operation was aborted", "AbortError"));
+		};
+		signal?.addEventListener("abort", onAbort, { once: true });
+	});
+}
+
+/** Resolve the delay before a retry (decision D3): a Retry-After-derived
+ *  `retryAfterMs` wins when present (capped at `cap`, no jitter), otherwise
+ *  the configured backoff drives the delay. */
+function resolveRetryDelay(
+	lastError: AppError,
+	backoffFn: (attempt: number) => number,
+	attempt: number,
+	cap: number,
+): number {
+	if (lastError instanceof RetryableStatusError && lastError.retryAfterMs !== undefined) {
+		return Math.min(lastError.retryAfterMs, cap);
+	}
+	return backoffFn(attempt);
+}

package/src/queue/semaphore.ts

@@ -0,0 +1,91 @@
+import { CancelledError, QueueFullError } from "../core/errors.ts";
+import { DEFAULT_GLOBAL_QUEUE_SIZE } from "../core/types.ts";
+
+/**
+ * A counting semaphore that limits total concurrent executions across all
+ * partitions (the global concurrency cap — decision D1, see
+ * `ClientConfig.concurrency` / `DEFAULT_GLOBAL_CONCURRENCY` for the default).
+ *
+ * Each `acquire()` returns a `release` callback. When no permit is available
+ * the caller is queued (up to `maxQueueSize`, see `ClientConfig.maxQueueSize` /
+ * `DEFAULT_GLOBAL_QUEUE_SIZE`) and resolved when a permit is released.
+ * Exceeding the queue limit rejects immediately with QueueFullError.
+ */
+export class Semaphore {
+	private available: number;
+	private readonly waitQueue: Array<() => void> = [];
+	private readonly maxQueueSize: number;
+
+	constructor(permits: number, maxQueueSize = DEFAULT_GLOBAL_QUEUE_SIZE) {
+		this.available = permits;
+		this.maxQueueSize = maxQueueSize;
+	}
+
+	/** Number of callers currently waiting for a permit. */
+	get queueLength(): number {
+		return this.waitQueue.length;
+	}
+
+	/** Number of permits currently free (not held by an in-flight request). */
+	get availablePermits(): number {
+		return this.available;
+	}
+
+	/** Acquire a permit. Resolves to a `release` function when granted;
+	 *  calling `release` more than once is a no-op.
+	 *
+	 *  If `signal` aborts while the caller is still queued, the waiter is
+	 *  removed from the queue — freeing its place for live callers — and the
+	 *  promise rejects with `CancelledError` (B9). An already-aborted signal
+	 *  rejects immediately without taking a permit. */
+	acquire(signal?: AbortSignal): Promise<() => void> {
+		if (signal?.aborted) {
+			return Promise.reject(new CancelledError());
+		}
+
+		if (this.available > 0) {
+			this.available--;
+			return Promise.resolve(this.releaseOnce());
+		}
+
+		if (this.waitQueue.length >= this.maxQueueSize) {
+			return Promise.reject(new QueueFullError("global", this.waitQueue.length, this.maxQueueSize));
+		}
+
+		return new Promise<() => void>((resolve, reject) => {
+			const onAbort = () => {
+				const idx = this.waitQueue.indexOf(grant);
+				if (idx !== -1) this.waitQueue.splice(idx, 1);
+				reject(new CancelledError());
+			};
+			const grant = () => {
+				signal?.removeEventListener("abort", onAbort);
+				this.available--;
+				resolve(this.releaseOnce());
+			};
+			signal?.addEventListener("abort", onAbort, { once: true });
+			this.waitQueue.push(grant);
+		});
+	}
+
+	private releaseOnce(): () => void {
+		let released = false;
+		return () => {
+			if (released) return;
+			released = true;
+			this.release();
+		};
+	}
+
+	private release(): void {
+		this.available++;
+		this._drain();
+	}
+
+	private _drain(): void {
+		while (this.available > 0 && this.waitQueue.length > 0) {
+			const next = this.waitQueue.shift()!;
+			next();
+		}
+	}
+}