@adaptic/utils 0.0.990 → 0.0.991

package/dist/index.cjs

@@ -6044,6 +6044,202 @@ async function createLimitOrder(auth, params = {
 }
 
 // Utility function for debug logging
+/** Sliding-window size (number of recent outcomes tracked per host). */
+const CIRCUIT_WINDOW_SIZE = 20;
+/** Maximum age (ms) of an outcome before it is dropped from the window. */
+const CIRCUIT_WINDOW_TTL_MS = 30_000;
+/**
+ * Failure ratio (0..1) at which the breaker trips when the window
+ * is full (i.e. CIRCUIT_WINDOW_SIZE outcomes recorded).
+ *
+ * Tuned to be permissive enough that a few transient failures during
+ * normal operation do not trip the breaker, but tight enough that a
+ * persistent upstream outage (where >= half of recent calls are
+ * failing) is detected quickly. With WINDOW=20 and threshold=0.6 we
+ * need 12 failures within the most recent 20 attempts to trip.
+ */
+const CIRCUIT_TRIP_FAILURE_RATIO = 0.6;
+/**
+ * Minimum number of recent outcomes required before the failure-ratio
+ * gate is even evaluated. Without this guard, a single failure on a
+ * cold host would give a 100 % ratio and trip the breaker on the
+ * second attempt — which would be useless protection while also being
+ * highly destructive.
+ */
+const CIRCUIT_MIN_SAMPLES = 8;
+/**
+ * How long the breaker stays open before transitioning to half-open
+ * (i.e. allowing the next request through as a probe). The next
+ * outcome — success or failure — fully recloses or re-opens the
+ * breaker. 5 s aligns with typical upstream recovery times and is
+ * short enough that the next caller drives the probe.
+ */
+const CIRCUIT_OPEN_COOLDOWN_MS = 5_000;
+/**
+ * Fail-fast latency. We sleep a short, deterministic amount before
+ * rejecting so the upstream caller's outer timeout (commonly 10–25 s)
+ * sees a clean rejection rather than a synchronous reject-storm that
+ * could starve other event-loop work. 100 ms aligns with the user's
+ * G1 analysis target: "cap per-symbol fallback latency at 100 ms
+ * during outages — eliminating the 25 s × N-concurrent-symbols
+ * event-loop hog."
+ */
+const CIRCUIT_FAIL_FAST_LATENCY_MS = 100;
+const circuitStates = new Map();
+function getCircuitState(host) {
+    let state = circuitStates.get(host);
+    if (!state) {
+        state = {
+            host,
+            recent: [],
+            openedAt: 0,
+            lastTripFailureRatio: 0,
+        };
+        circuitStates.set(host, state);
+    }
+    return state;
+}
+function pruneStaleSamples(state, now) {
+    const cutoff = now - CIRCUIT_WINDOW_TTL_MS;
+    // Remove entries older than the TTL. Window is small so a simple
+    // filter is fine — no need for a deque structure.
+    if (state.recent.length === 0)
+        return;
+    let firstFreshIndex = 0;
+    while (firstFreshIndex < state.recent.length &&
+        state.recent[firstFreshIndex].atMs < cutoff) {
+        firstFreshIndex += 1;
+    }
+    if (firstFreshIndex > 0) {
+        state.recent.splice(0, firstFreshIndex);
+    }
+}
+function recordOutcome(host, ok) {
+    const now = Date.now();
+    const state = getCircuitState(host);
+    pruneStaleSamples(state, now);
+    state.recent.push({ atMs: now, ok });
+    if (state.recent.length > CIRCUIT_WINDOW_SIZE) {
+        state.recent.shift();
+    }
+    // If we are in the open or half-open window and just received an
+    // outcome, decide whether to close or re-open.
+    if (state.openedAt > 0) {
+        if (now - state.openedAt >= CIRCUIT_OPEN_COOLDOWN_MS) {
+            // Half-open probe outcome arrived.
+            if (ok) {
+                // Recovered. Close the breaker.
+                state.openedAt = 0;
+                state.lastTripFailureRatio = 0;
+                getLogger().info(`Circuit breaker for ${host} closed — upstream recovered`, { host });
+            }
+            else {
+                // Probe failed. Re-open the breaker for another cooldown.
+                state.openedAt = now;
+            }
+            return;
+        }
+        // Still inside the cooldown window — outcomes are recorded for
+        // statistics but the breaker stays open regardless.
+        return;
+    }
+    // Closed breaker — evaluate whether the failure ratio has tripped.
+    if (state.recent.length < CIRCUIT_MIN_SAMPLES)
+        return;
+    const failures = state.recent.reduce((acc, entry) => acc + (entry.ok ? 0 : 1), 0);
+    const ratio = failures / state.recent.length;
+    if (ratio >= CIRCUIT_TRIP_FAILURE_RATIO) {
+        state.openedAt = now;
+        state.lastTripFailureRatio = ratio;
+        getLogger().warn(`Circuit breaker for ${host} opened — ${failures}/${state.recent.length} recent attempts failed (${(ratio * 100).toFixed(0)}%); fail-fast for ${CIRCUIT_OPEN_COOLDOWN_MS}ms`, {
+            host,
+            recentFailures: failures,
+            recentSamples: state.recent.length,
+            failureRatio: ratio,
+            cooldownMs: CIRCUIT_OPEN_COOLDOWN_MS,
+        });
+    }
+}
+function isCircuitOpen(host, now) {
+    const state = circuitStates.get(host);
+    if (!state || state.openedAt === 0)
+        return false;
+    pruneStaleSamples(state, now);
+    return now - state.openedAt < CIRCUIT_OPEN_COOLDOWN_MS;
+}
+/**
+ * Error thrown by {@link fetchWithRetry} when the per-host circuit
+ * breaker is open and fail-fast suppression is in effect.
+ *
+ * Carries the host, the failure ratio that tripped the breaker, and
+ * the remaining cooldown so callers can render an actionable log.
+ */
+class CircuitOpenError extends Error {
+    code = "MASSIVE_CIRCUIT_OPEN";
+    host;
+    tripFailureRatio;
+    cooldownRemainingMs;
+    constructor(host, tripFailureRatio, cooldownRemainingMs) {
+        super(`Circuit open for ${host} — fail-fast (recent failure ratio ${(tripFailureRatio * 100).toFixed(0)}%, retry in ${cooldownRemainingMs}ms)`);
+        this.name = "CircuitOpenError";
+        this.host = host;
+        this.tripFailureRatio = tripFailureRatio;
+        this.cooldownRemainingMs = cooldownRemainingMs;
+    }
+}
+/**
+ * Force-close the breaker for a given host. Exposed for tests and
+ * operator-runbook scripts so a stuck-open breaker can be reset
+ * without bouncing the process. Not intended for hot-path use.
+ *
+ * @param host The hostname whose breaker should be reset.
+ */
+function resetCircuitBreaker(host) {
+    const state = circuitStates.get(host);
+    if (!state)
+        return;
+    state.openedAt = 0;
+    state.lastTripFailureRatio = 0;
+    state.recent = [];
+}
+/**
+ * Snapshot of all known per-host circuit-breaker states. Intended for
+ * an operational-truth / status endpoint to surface upstream health.
+ *
+ * @returns Map of host → {open, openedAt, recentSamples, failureRatio,
+ *          lastTripFailureRatio}.
+ */
+function getCircuitBreakerSnapshot() {
+    const now = Date.now();
+    const snapshot = {};
+    for (const [host, state] of circuitStates) {
+        pruneStaleSamples(state, now);
+        const failures = state.recent.reduce((acc, entry) => acc + (entry.ok ? 0 : 1), 0);
+        const ratio = state.recent.length > 0 ? failures / state.recent.length : 0;
+        snapshot[host] = {
+            open: state.openedAt > 0 && now - state.openedAt < CIRCUIT_OPEN_COOLDOWN_MS,
+            openedAt: state.openedAt,
+            cooldownRemainingMs: state.openedAt > 0
+                ? Math.max(0, CIRCUIT_OPEN_COOLDOWN_MS - (now - state.openedAt))
+                : 0,
+            recentSamples: state.recent.length,
+            failureRatio: ratio,
+            lastTripFailureRatio: state.lastTripFailureRatio,
+        };
+    }
+    return snapshot;
+}
+function hostnameFromUrl(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return null;
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /**
  * Debug logging utility that respects environment debug flags.
  * Logs messages through the configured structured logger when LUMIC_DEBUG
@@ -6155,9 +6351,46 @@ function hideApiKeyFromurl(url) {
  * @throws Will throw an error if the fetch fails after the specified number of retries.
  */
 async function fetchWithRetry(url, options = {}, retries = 3, initialBackoff = 1000) {
+    // Per-host circuit-breaker check. When upstream is failing
+    // pervasively (e.g. Massive REST in an outage), short-circuit to a
+    // 100 ms fail-fast rejection so the caller's outer timeout (25 s,
+    // 10 s, etc.) does not pile concurrent symbol fetches against a
+    // dead host. Without this, an upstream outage during a screener tick
+    // can lock up the event loop for up to (timeout × N-concurrent-symbols).
+    const host = hostnameFromUrl(url);
+    if (host) {
+        const now = Date.now();
+        if (isCircuitOpen(host, now)) {
+            const state = getCircuitState(host);
+            const cooldownRemainingMs = Math.max(0, CIRCUIT_OPEN_COOLDOWN_MS - (now - state.openedAt));
+            await sleep(CIRCUIT_FAIL_FAST_LATENCY_MS);
+            throw new CircuitOpenError(host, state.lastTripFailureRatio, cooldownRemainingMs);
+        }
+    }
     return withRetry(async () => {
-        const response = await fetch(url, options);
+        let response;
+        try {
+            response = await fetch(url, options);
+        }
+        catch (networkError) {
+            // Network failure (e.g. DNS, connection refused, TLS abort).
+            // Record against the breaker so a sustained outage trips it
+            // for subsequent calls; then re-throw so withRetry handles
+            // its own retry policy.
+            if (host)
+                recordOutcome(host, false);
+            throw networkError;
+        }
         if (!response.ok) {
+            // Classify the outcome for the circuit breaker. 5xx and 429
+            // are upstream-health signals (record as failure). 4xx client
+            // errors are caller-side and should NOT affect the breaker —
+            // a request with a bad API key shouldn't trip the host's
+            // breaker for everyone else.
+            const upstreamUnhealthy = response.status === 429 ||
+                (response.status >= 500 && response.status < 600);
+            if (host)
+                recordOutcome(host, !upstreamUnhealthy);
             // Enhanced HTTP error handling with specific error types
             if (response.status === 429) {
                 // Check for Retry-After header
@@ -6189,6 +6422,10 @@ async function fetchWithRetry(url, options = {}, retries = 3, initialBackoff = 1
             error.response = response;
             throw error;
         }
+        // Success — record against the breaker so a healthy upstream
+        // closes any half-open state cleanly.
+        if (host)
+            recordOutcome(host, true);
         return response;
     }, {
         maxRetries: retries,
@@ -69379,6 +69616,17 @@ const adaptic = {
         logIfDebug: logIfDebug,
         fetchWithRetry: fetchWithRetry,
         validateMassiveApiKey: validateMassiveApiKey,
+        /**
+         * Force-close a stuck-open per-host circuit breaker. Operator
+         * runbook utility — see {@link misc.resetCircuitBreaker}.
+         */
+        resetCircuitBreaker: resetCircuitBreaker,
+        /**
+         * Read-only snapshot of all per-host circuit-breaker states for
+         * use in operational-truth endpoints. See
+         * {@link misc.getCircuitBreakerSnapshot}.
+         */
+        getCircuitBreakerSnapshot: getCircuitBreakerSnapshot,
     },
     rateLimiter: {
         TokenBucketRateLimiter,
@@ -69417,6 +69665,7 @@ exports.AssetAllocationEngine = AssetAllocationEngine;
 exports.AuthenticationError = AuthenticationError;
 exports.BTC_PAIRS = BTC_PAIRS;
 exports.BarError = BarError;
+exports.CircuitOpenError = CircuitOpenError;
 exports.CryptoDataError = CryptoDataError;
 exports.CryptoOrderError = CryptoOrderError;
 exports.DEFAULT_CACHE_OPTIONS = DEFAULT_CACHE_OPTIONS;