@adaptic/utils 0.0.990 → 0.0.992

package/dist/index.cjs

@@ -2691,6 +2691,48 @@ class AlpacaMarketDataAPI extends require$$0$1.EventEmitter {
     };
     reconnectAttempts = {};
     reconnectTimers = {};
+    /**
+     * Wall-clock timestamp of the most recent Alpaca app-level error code
+     * 406 ("connection limit exceeded") received on each stream. Used by
+     * {@link scheduleReconnect} to apply a long backoff with jitter rather
+     * than the normal sub-second exponential ramp — without this, hitting
+     * Alpaca's account-wide concurrent-connection cap (typical for blue/green
+     * deploy rollovers where the old pod's WS slots haven't been released
+     * yet) produced a 10-attempt retry storm that compounded the slot
+     * pressure and consumed the per-account connection quota across the
+     * organisation.
+     *
+     * Cleared once the long-backoff retry is scheduled so that subsequent
+     * normal failures fall back to the standard sub-second exponential.
+     *
+     * @see CONNECTION_LIMIT_BACKOFF_MS / CONNECTION_LIMIT_BACKOFF_JITTER_MS
+     */
+    lastConnectionLimitAt = {};
+    /**
+     * Five-minute base backoff after Alpaca's app-level 406. Long enough
+     * for Alpaca's server-side cleanup to release stale slots in typical
+     * rollover scenarios; short enough that an operator doesn't need to
+     * intervene. Mirrors the equivalent MassiveClient
+     * `MAX_CONNECTIONS_RETRY_DELAY_MS` (engine v1.0.59) so both providers
+     * behave identically under the same failure mode.
+     */
+    CONNECTION_LIMIT_BACKOFF_MS = 5 * 60_000;
+    /**
+     * ±30 s of uniform jitter on the connection-limit backoff. Prevents
+     * a thundering-herd retry when all three streams (stock / option /
+     * crypto) hit 406 simultaneously during a deploy rollover — without
+     * jitter they'd all retry at the same wall-clock instant and could
+     * re-trip the account cap together.
+     */
+    CONNECTION_LIMIT_BACKOFF_JITTER_MS = 30_000;
+    /**
+     * Recency window within which a 406 is considered "still applicable"
+     * to a subsequent reconnect-schedule call. The 406 message handler
+     * stamps {@link lastConnectionLimitAt} and the `close` handler fires
+     * shortly afterwards (sub-second typically) — the window is wide
+     * enough to absorb scheduling delays without false-positives.
+     */
+    CONNECTION_LIMIT_RECENCY_MS = 30_000;
     setMode(mode = "production") {
         if (mode === "sandbox") {
             // sandbox mode
@@ -2822,6 +2864,18 @@ class AlpacaMarketDataAPI extends require$$0$1.EventEmitter {
                 }
                 else if (message.T === "error") {
                     log$l(`${streamType} stream error: ${message.msg} (code: ${message.code}, raw: ${JSON.stringify(message)})`, { type: "error" });
+                    // Alpaca code 406: "connection limit exceeded" — account-wide
+                    // concurrent-WS cap reached. The Alpaca server will close the
+                    // socket immediately after this frame, which would normally
+                    // trigger our standard sub-second exponential reconnect chain
+                    // (1 s, 2 s, 4 s, 8 s, 16 s, 30 s × 5) — exactly the wrong
+                    // behaviour against a rate-limit response. Stamp the recency
+                    // marker so {@link scheduleReconnect} switches to the
+                    // 5-minute jittered backoff instead.
+                    if (typeof message.code === "number" &&
+                        message.code === 406) {
+                        this.lastConnectionLimitAt[streamType] = Date.now();
+                    }
                 }
                 else if (message.S) {
                     super.emit(`${streamType}-${message.T}`, message);
@@ -2855,6 +2909,37 @@ class AlpacaMarketDataAPI extends require$$0$1.EventEmitter {
         });
     }
     scheduleReconnect(streamType) {
+        // 406-recovery fast path. When the most recent close was preceded
+        // by an Alpaca app-level 406 ("connection limit exceeded"), the
+        // standard sub-second exponential ramp is exactly wrong — it
+        // hammers the rate-limit endpoint and prolongs the slot pressure.
+        // Use a 5-minute jittered backoff instead and reset the normal
+        // attempt counter so we don't fall off the end of maxAttempts
+        // and permanently give up on a transient rollover blip.
+        const connectionLimitAt = this.lastConnectionLimitAt[streamType];
+        const isRecentConnectionLimit = typeof connectionLimitAt === "number" &&
+            Date.now() - connectionLimitAt <= this.CONNECTION_LIMIT_RECENCY_MS;
+        if (isRecentConnectionLimit) {
+            const jitter = Math.floor((Math.random() - 0.5) *
+                2 *
+                this.CONNECTION_LIMIT_BACKOFF_JITTER_MS);
+            const delayMs = this.CONNECTION_LIMIT_BACKOFF_MS + jitter;
+            // Reset normal attempt counter so the next 406 retry doesn't
+            // inherit a stale exponential cap.
+            this.reconnectAttempts[streamType] = 0;
+            // Consume the recency marker — subsequent reconnects fall back
+            // to the standard exponential path unless a new 406 arrives.
+            delete this.lastConnectionLimitAt[streamType];
+            log$l(`${streamType} stream: Alpaca 406 connection-limit recovery — backing off ${Math.round(delayMs / 1000)}s before retry to allow account-wide slot release`, { type: "warn" });
+            if (this.reconnectTimers[streamType]) {
+                clearTimeout(this.reconnectTimers[streamType]);
+            }
+            this.reconnectTimers[streamType] = setTimeout(() => {
+                log$l(`${streamType} stream: attempting reconnect after 406-recovery backoff`, { type: "info" });
+                this.connect(streamType);
+            }, delayMs);
+            return;
+        }
         const attempts = this.reconnectAttempts[streamType] ?? 0;
         const maxAttempts = 10;
         if (attempts >= maxAttempts) {
@@ -6044,6 +6129,202 @@ async function createLimitOrder(auth, params = {
 }
 
 // Utility function for debug logging
+/** Sliding-window size (number of recent outcomes tracked per host). */
+const CIRCUIT_WINDOW_SIZE = 20;
+/** Maximum age (ms) of an outcome before it is dropped from the window. */
+const CIRCUIT_WINDOW_TTL_MS = 30_000;
+/**
+ * Failure ratio (0..1) at which the breaker trips when the window
+ * is full (i.e. CIRCUIT_WINDOW_SIZE outcomes recorded).
+ *
+ * Tuned to be permissive enough that a few transient failures during
+ * normal operation do not trip the breaker, but tight enough that a
+ * persistent upstream outage (where >= half of recent calls are
+ * failing) is detected quickly. With WINDOW=20 and threshold=0.6 we
+ * need 12 failures within the most recent 20 attempts to trip.
+ */
+const CIRCUIT_TRIP_FAILURE_RATIO = 0.6;
+/**
+ * Minimum number of recent outcomes required before the failure-ratio
+ * gate is even evaluated. Without this guard, a single failure on a
+ * cold host would give a 100 % ratio and trip the breaker on the
+ * second attempt — which would be useless protection while also being
+ * highly destructive.
+ */
+const CIRCUIT_MIN_SAMPLES = 8;
+/**
+ * How long the breaker stays open before transitioning to half-open
+ * (i.e. allowing the next request through as a probe). The next
+ * outcome — success or failure — fully recloses or re-opens the
+ * breaker. 5 s aligns with typical upstream recovery times and is
+ * short enough that the next caller drives the probe.
+ */
+const CIRCUIT_OPEN_COOLDOWN_MS = 5_000;
+/**
+ * Fail-fast latency. We sleep a short, deterministic amount before
+ * rejecting so the upstream caller's outer timeout (commonly 10–25 s)
+ * sees a clean rejection rather than a synchronous reject-storm that
+ * could starve other event-loop work. 100 ms aligns with the user's
+ * G1 analysis target: "cap per-symbol fallback latency at 100 ms
+ * during outages — eliminating the 25 s × N-concurrent-symbols
+ * event-loop hog."
+ */
+const CIRCUIT_FAIL_FAST_LATENCY_MS = 100;
+const circuitStates = new Map();
+function getCircuitState(host) {
+    let state = circuitStates.get(host);
+    if (!state) {
+        state = {
+            host,
+            recent: [],
+            openedAt: 0,
+            lastTripFailureRatio: 0,
+        };
+        circuitStates.set(host, state);
+    }
+    return state;
+}
+function pruneStaleSamples(state, now) {
+    const cutoff = now - CIRCUIT_WINDOW_TTL_MS;
+    // Remove entries older than the TTL. Window is small so a simple
+    // filter is fine — no need for a deque structure.
+    if (state.recent.length === 0)
+        return;
+    let firstFreshIndex = 0;
+    while (firstFreshIndex < state.recent.length &&
+        state.recent[firstFreshIndex].atMs < cutoff) {
+        firstFreshIndex += 1;
+    }
+    if (firstFreshIndex > 0) {
+        state.recent.splice(0, firstFreshIndex);
+    }
+}
+function recordOutcome(host, ok) {
+    const now = Date.now();
+    const state = getCircuitState(host);
+    pruneStaleSamples(state, now);
+    state.recent.push({ atMs: now, ok });
+    if (state.recent.length > CIRCUIT_WINDOW_SIZE) {
+        state.recent.shift();
+    }
+    // If we are in the open or half-open window and just received an
+    // outcome, decide whether to close or re-open.
+    if (state.openedAt > 0) {
+        if (now - state.openedAt >= CIRCUIT_OPEN_COOLDOWN_MS) {
+            // Half-open probe outcome arrived.
+            if (ok) {
+                // Recovered. Close the breaker.
+                state.openedAt = 0;
+                state.lastTripFailureRatio = 0;
+                getLogger().info(`Circuit breaker for ${host} closed — upstream recovered`, { host });
+            }
+            else {
+                // Probe failed. Re-open the breaker for another cooldown.
+                state.openedAt = now;
+            }
+            return;
+        }
+        // Still inside the cooldown window — outcomes are recorded for
+        // statistics but the breaker stays open regardless.
+        return;
+    }
+    // Closed breaker — evaluate whether the failure ratio has tripped.
+    if (state.recent.length < CIRCUIT_MIN_SAMPLES)
+        return;
+    const failures = state.recent.reduce((acc, entry) => acc + (entry.ok ? 0 : 1), 0);
+    const ratio = failures / state.recent.length;
+    if (ratio >= CIRCUIT_TRIP_FAILURE_RATIO) {
+        state.openedAt = now;
+        state.lastTripFailureRatio = ratio;
+        getLogger().warn(`Circuit breaker for ${host} opened — ${failures}/${state.recent.length} recent attempts failed (${(ratio * 100).toFixed(0)}%); fail-fast for ${CIRCUIT_OPEN_COOLDOWN_MS}ms`, {
+            host,
+            recentFailures: failures,
+            recentSamples: state.recent.length,
+            failureRatio: ratio,
+            cooldownMs: CIRCUIT_OPEN_COOLDOWN_MS,
+        });
+    }
+}
+function isCircuitOpen(host, now) {
+    const state = circuitStates.get(host);
+    if (!state || state.openedAt === 0)
+        return false;
+    pruneStaleSamples(state, now);
+    return now - state.openedAt < CIRCUIT_OPEN_COOLDOWN_MS;
+}
+/**
+ * Error thrown by {@link fetchWithRetry} when the per-host circuit
+ * breaker is open and fail-fast suppression is in effect.
+ *
+ * Carries the host, the failure ratio that tripped the breaker, and
+ * the remaining cooldown so callers can render an actionable log.
+ */
+class CircuitOpenError extends Error {
+    code = "MASSIVE_CIRCUIT_OPEN";
+    host;
+    tripFailureRatio;
+    cooldownRemainingMs;
+    constructor(host, tripFailureRatio, cooldownRemainingMs) {
+        super(`Circuit open for ${host} — fail-fast (recent failure ratio ${(tripFailureRatio * 100).toFixed(0)}%, retry in ${cooldownRemainingMs}ms)`);
+        this.name = "CircuitOpenError";
+        this.host = host;
+        this.tripFailureRatio = tripFailureRatio;
+        this.cooldownRemainingMs = cooldownRemainingMs;
+    }
+}
+/**
+ * Force-close the breaker for a given host. Exposed for tests and
+ * operator-runbook scripts so a stuck-open breaker can be reset
+ * without bouncing the process. Not intended for hot-path use.
+ *
+ * @param host The hostname whose breaker should be reset.
+ */
+function resetCircuitBreaker(host) {
+    const state = circuitStates.get(host);
+    if (!state)
+        return;
+    state.openedAt = 0;
+    state.lastTripFailureRatio = 0;
+    state.recent = [];
+}
+/**
+ * Snapshot of all known per-host circuit-breaker states. Intended for
+ * an operational-truth / status endpoint to surface upstream health.
+ *
+ * @returns Map of host → {open, openedAt, recentSamples, failureRatio,
+ *          lastTripFailureRatio}.
+ */
+function getCircuitBreakerSnapshot() {
+    const now = Date.now();
+    const snapshot = {};
+    for (const [host, state] of circuitStates) {
+        pruneStaleSamples(state, now);
+        const failures = state.recent.reduce((acc, entry) => acc + (entry.ok ? 0 : 1), 0);
+        const ratio = state.recent.length > 0 ? failures / state.recent.length : 0;
+        snapshot[host] = {
+            open: state.openedAt > 0 && now - state.openedAt < CIRCUIT_OPEN_COOLDOWN_MS,
+            openedAt: state.openedAt,
+            cooldownRemainingMs: state.openedAt > 0
+                ? Math.max(0, CIRCUIT_OPEN_COOLDOWN_MS - (now - state.openedAt))
+                : 0,
+            recentSamples: state.recent.length,
+            failureRatio: ratio,
+            lastTripFailureRatio: state.lastTripFailureRatio,
+        };
+    }
+    return snapshot;
+}
+function hostnameFromUrl(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return null;
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /**
  * Debug logging utility that respects environment debug flags.
  * Logs messages through the configured structured logger when LUMIC_DEBUG
@@ -6155,9 +6436,46 @@ function hideApiKeyFromurl(url) {
  * @throws Will throw an error if the fetch fails after the specified number of retries.
  */
 async function fetchWithRetry(url, options = {}, retries = 3, initialBackoff = 1000) {
+    // Per-host circuit-breaker check. When upstream is failing
+    // pervasively (e.g. Massive REST in an outage), short-circuit to a
+    // 100 ms fail-fast rejection so the caller's outer timeout (25 s,
+    // 10 s, etc.) does not pile concurrent symbol fetches against a
+    // dead host. Without this, an upstream outage during a screener tick
+    // can lock up the event loop for up to (timeout × N-concurrent-symbols).
+    const host = hostnameFromUrl(url);
+    if (host) {
+        const now = Date.now();
+        if (isCircuitOpen(host, now)) {
+            const state = getCircuitState(host);
+            const cooldownRemainingMs = Math.max(0, CIRCUIT_OPEN_COOLDOWN_MS - (now - state.openedAt));
+            await sleep(CIRCUIT_FAIL_FAST_LATENCY_MS);
+            throw new CircuitOpenError(host, state.lastTripFailureRatio, cooldownRemainingMs);
+        }
+    }
     return withRetry(async () => {
-        const response = await fetch(url, options);
+        let response;
+        try {
+            response = await fetch(url, options);
+        }
+        catch (networkError) {
+            // Network failure (e.g. DNS, connection refused, TLS abort).
+            // Record against the breaker so a sustained outage trips it
+            // for subsequent calls; then re-throw so withRetry handles
+            // its own retry policy.
+            if (host)
+                recordOutcome(host, false);
+            throw networkError;
+        }
         if (!response.ok) {
+            // Classify the outcome for the circuit breaker. 5xx and 429
+            // are upstream-health signals (record as failure). 4xx client
+            // errors are caller-side and should NOT affect the breaker —
+            // a request with a bad API key shouldn't trip the host's
+            // breaker for everyone else.
+            const upstreamUnhealthy = response.status === 429 ||
+                (response.status >= 500 && response.status < 600);
+            if (host)
+                recordOutcome(host, !upstreamUnhealthy);
             // Enhanced HTTP error handling with specific error types
             if (response.status === 429) {
                 // Check for Retry-After header
@@ -6189,6 +6507,10 @@ async function fetchWithRetry(url, options = {}, retries = 3, initialBackoff = 1
             error.response = response;
             throw error;
         }
+        // Success — record against the breaker so a healthy upstream
+        // closes any half-open state cleanly.
+        if (host)
+            recordOutcome(host, true);
         return response;
     }, {
         maxRetries: retries,
@@ -69379,6 +69701,17 @@ const adaptic = {
         logIfDebug: logIfDebug,
         fetchWithRetry: fetchWithRetry,
         validateMassiveApiKey: validateMassiveApiKey,
+        /**
+         * Force-close a stuck-open per-host circuit breaker. Operator
+         * runbook utility — see {@link misc.resetCircuitBreaker}.
+         */
+        resetCircuitBreaker: resetCircuitBreaker,
+        /**
+         * Read-only snapshot of all per-host circuit-breaker states for
+         * use in operational-truth endpoints. See
+         * {@link misc.getCircuitBreakerSnapshot}.
+         */
+        getCircuitBreakerSnapshot: getCircuitBreakerSnapshot,
     },
     rateLimiter: {
         TokenBucketRateLimiter,
@@ -69417,6 +69750,7 @@ exports.AssetAllocationEngine = AssetAllocationEngine;
 exports.AuthenticationError = AuthenticationError;
 exports.BTC_PAIRS = BTC_PAIRS;
 exports.BarError = BarError;
+exports.CircuitOpenError = CircuitOpenError;
 exports.CryptoDataError = CryptoDataError;
 exports.CryptoOrderError = CryptoOrderError;
 exports.DEFAULT_CACHE_OPTIONS = DEFAULT_CACHE_OPTIONS;