@code-yeongyu/senpi 2026.9.27-4 → 2026.9.28-2

package/dist/core/retry-fallback/circuit.d.ts

@@ -0,0 +1,101 @@
+import type { FallbackLogger } from "./log.ts";
+/**
+ * Fallback-chain circuit breaker. An entry that fails out of a configured chain
+ * opens a circuit keyed by its base selector (`provider/id`); while it is open,
+ * chain resolution skips the entry instead of spending its retry budget again.
+ *
+ * One breaker is shared by every session in the process that uses the same agent
+ * directory - the config that defines the providers and selectors - so a session
+ * started by `/new`, `/resume`, or `/fork` (each gets a fresh model runtime) and
+ * an in-process subagent do not re-discover a dead provider.
+ * The class is clock-free: every call receives `now`, keeping sessions' injected
+ * clocks authoritative and tests deterministic.
+ */
+export declare const DEFAULT_CIRCUIT_COOLDOWN_MS = 60000;
+export declare const DEFAULT_CIRCUIT_MAX_COOLDOWN_MS: number;
+export interface FallbackCircuitSettings {
+    /** First cooldown after an entry fails out of a chain; doubles per consecutive open. 0 disables the breaker. */
+    circuitCooldownMs?: number;
+    circuitMaxCooldownMs?: number;
+}
+export interface ResolvedFallbackCircuitSettings {
+    cooldownMs: number;
+    maxCooldownMs: number;
+}
+export interface CircuitOpenRequest {
+    now: number;
+    cooldownMs: number;
+    maxCooldownMs: number;
+    retryAfterMs?: number;
+}
+export interface ProbeToken {
+    readonly selector: string;
+    readonly owner: string;
+    readonly generation: number;
+}
+export type CircuitAdmission = {
+    kind: "closed";
+} | {
+    kind: "open";
+} | {
+    kind: "probe";
+    token: ProbeToken;
+};
+export declare function resolveFallbackCircuitSettings(settings: FallbackCircuitSettings | undefined): ResolvedFallbackCircuitSettings;
+export declare class FallbackCircuitBreaker {
+    private readonly circuits;
+    private generations;
+    get size(): number;
+    open(selector: string, request: CircuitOpenRequest): number;
+    isOpen(selector: string, now: number, owner: string): boolean;
+    /**
+     * Atomic admission: a closed selector passes, a cooling one or one whose probe
+     * another requester holds is refused, and a half-open one hands `owner` its
+     * single probe under a fresh generation. The probe stays exclusive until its
+     * token is released or the circuit settles - it never expires on a timer, so a
+     * slow probe cannot overlap a second one, and a stale token never releases a
+     * later acquisition.
+     */
+    admit(selector: string, now: number, owner: string): CircuitAdmission;
+    release(token: ProbeToken): void;
+    releaseOwnersWithPrefix(prefix: string): void;
+    close(selector: string): void;
+    has(selector: string): boolean;
+    sweep(now: number): void;
+}
+/**
+ * A session's requests are admitted per lane, not per session: the foreground
+ * turn is one lane, each background probe-back its own. Two lanes of one session
+ * never share a probe, so a probe-back in flight keeps the foreground turn off
+ * the entry exactly as it keeps a sibling session off.
+ */
+export type CircuitLane = "turn" | `probe-back:${number}`;
+export interface FallbackCircuitAccess {
+    noteFailure(selector: string, failure: {
+        retryAfterMs?: number;
+        errorMessage?: string;
+    }): void;
+    /** Whether the breaker holds any state for the selector, i.e. it, not a session cooldown, governs recovery. */
+    governs(selector: string): boolean;
+    isOpen(selector: string, lane?: CircuitLane): boolean;
+    admit(selector: string, lane?: CircuitLane): CircuitAdmission;
+    release(token: ProbeToken): void;
+    releaseAll(): void;
+    close(selector: string): void;
+}
+export interface FallbackCircuitAccessDeps {
+    breaker: FallbackCircuitBreaker;
+    owner(): string;
+    now(): number;
+    settings(): ResolvedFallbackCircuitSettings;
+    logger: FallbackLogger;
+}
+export declare function createFallbackCircuitAccess(deps: FallbackCircuitAccessDeps): FallbackCircuitAccess;
+export declare function fallbackCircuitsFor(agentDir: string): FallbackCircuitBreaker;
+/** Holds the agent dir's breaker for one session; `release` is idempotent. */
+export declare function acquireFallbackCircuits(agentDir: string): {
+    breaker: FallbackCircuitBreaker;
+    release(): void;
+};
+export declare function monotonicNow(): number;
+//# sourceMappingURL=circuit.d.ts.map

package/dist/core/retry-fallback/circuit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"circuit.d.ts","sourceRoot":"","sources":["../../../src/core/retry-fallback/circuit.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/C;;;;;;;;;;;GAWG;AAEH,eAAO,MAAM,2BAA2B,QAAS,CAAC;AAClD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAE3D,MAAM,WAAW,uBAAuB;IACvC,gHAAgH;IAChH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,+BAA+B;IAC/C,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,MAAM,gBAAgB,GAAG;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,CAAC;AAe5G,wBAAgB,8BAA8B,CAC7C,QAAQ,EAAE,uBAAuB,GAAG,SAAS,GAC3C,+BAA+B,CAIjC;AAED,qBAAa,sBAAsB;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAC/D,OAAO,CAAC,WAAW,CAAK;IAExB,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG,MAAM,CAuB1D;IAED,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAK5D;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAOpE;IAED,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAM/B;IAED,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAI5C;IAED,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAE5B;IAED,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAE7B;IAED,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAIvB;CACD;AAED;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,cAAc,MAAM,EAAE,CAAC;AAE1D,MAAM,WAAW,qBAAqB;IACrC,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC/F,+GAA+G;IAC/G,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACnC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;IACtD,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,gBAAgB,CAAC;IAC9D,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACjC,UAAU,IAAI,IAAI,CAAC;IACnB,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,MAAM,WAAW,yBAAyB;IACzC,OAAO,EAAE,sBAAsB,CAAC;IAChC,KAAK,IAAI,MAAM,CAAC;IAChB,GAAG,IAAI,MAAM,CAAC;IACd,QAAQ,IAAI,+BAA+B,CAAC;IAC5C,MAAM,EAAE,cAAc,CAAC;CACvB;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,yBAAyB,GAAG,qBAAqB,CAuBlG;AAkCD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,sBAAsB,CAE5E;AAED,8EAA8E;AAC9E,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,sBAAsB,CAAC;IAAC,OAAO,IAAI,IAAI,CAAA;CAAE,CAa9G;AAED,wBAAgB,YAAY,IAAI,MAAM,CAErC"}

package/dist/core/retry-fallback/circuit.js

@@ -0,0 +1,175 @@
+import { resolve } from "node:path";
+import { parseRetryAfterMsMarker } from "@earendil-works/pi-ai/utils/retry-hint";
+/**
+ * Fallback-chain circuit breaker. An entry that fails out of a configured chain
+ * opens a circuit keyed by its base selector (`provider/id`); while it is open,
+ * chain resolution skips the entry instead of spending its retry budget again.
+ *
+ * One breaker is shared by every session in the process that uses the same agent
+ * directory - the config that defines the providers and selectors - so a session
+ * started by `/new`, `/resume`, or `/fork` (each gets a fresh model runtime) and
+ * an in-process subagent do not re-discover a dead provider.
+ * The class is clock-free: every call receives `now`, keeping sessions' injected
+ * clocks authoritative and tests deterministic.
+ */
+export const DEFAULT_CIRCUIT_COOLDOWN_MS = 60_000;
+export const DEFAULT_CIRCUIT_MAX_COOLDOWN_MS = 30 * 60_000;
+function nonNegativeMs(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : fallback;
+}
+export function resolveFallbackCircuitSettings(settings) {
+    const cooldownMs = nonNegativeMs(settings?.circuitCooldownMs, DEFAULT_CIRCUIT_COOLDOWN_MS);
+    const maxCooldownMs = nonNegativeMs(settings?.circuitMaxCooldownMs, DEFAULT_CIRCUIT_MAX_COOLDOWN_MS);
+    return { cooldownMs, maxCooldownMs: Math.max(cooldownMs, maxCooldownMs) };
+}
+export class FallbackCircuitBreaker {
+    constructor() {
+        this.circuits = new Map();
+        this.generations = 0;
+    }
+    get size() {
+        return this.circuits.size;
+    }
+    open(selector, request) {
+        const previous = this.circuits.get(selector);
+        const consecutiveOpens = (previous?.consecutiveOpens ?? 0) + 1;
+        const cooldownMs = Math.min(request.cooldownMs * 2 ** (consecutiveOpens - 1), request.maxCooldownMs);
+        const retryAfterMs = request.retryAfterMs;
+        const hinted = retryAfterMs !== undefined && Number.isFinite(retryAfterMs) && retryAfterMs > 0
+            ? request.now + retryAfterMs
+            : 0;
+        const retryFloorUntil = Math.max(hinted, previous && previous.retryFloorUntil > request.now ? previous.retryFloorUntil : 0);
+        const openUntil = Math.max(request.now + cooldownMs, retryFloorUntil);
+        this.circuits.set(selector, {
+            openUntil,
+            retryFloorUntil,
+            consecutiveOpens,
+            cooldownMs,
+            maxCooldownMs: request.maxCooldownMs,
+            probe: undefined,
+        });
+        return openUntil;
+    }
+    isOpen(selector, now, owner) {
+        const circuit = this.circuits.get(selector);
+        if (!circuit)
+            return false;
+        if (now < circuit.openUntil)
+            return true;
+        return circuit.probe !== undefined && circuit.probe.owner !== owner;
+    }
+    /**
+     * Atomic admission: a closed selector passes, a cooling one or one whose probe
+     * another requester holds is refused, and a half-open one hands `owner` its
+     * single probe under a fresh generation. The probe stays exclusive until its
+     * token is released or the circuit settles - it never expires on a timer, so a
+     * slow probe cannot overlap a second one, and a stale token never releases a
+     * later acquisition.
+     */
+    admit(selector, now, owner) {
+        const circuit = this.circuits.get(selector);
+        if (!circuit)
+            return { kind: "closed" };
+        if (now < circuit.openUntil)
+            return { kind: "open" };
+        if (circuit.probe && circuit.probe.owner !== owner)
+            return { kind: "open" };
+        circuit.probe ??= { owner, generation: ++this.generations };
+        return { kind: "probe", token: { selector, owner, generation: circuit.probe.generation } };
+    }
+    release(token) {
+        const probe = this.circuits.get(token.selector)?.probe;
+        if (probe && probe.owner === token.owner && probe.generation === token.generation) {
+            const circuit = this.circuits.get(token.selector);
+            if (circuit)
+                circuit.probe = undefined;
+        }
+    }
+    releaseOwnersWithPrefix(prefix) {
+        for (const circuit of this.circuits.values()) {
+            if (circuit.probe?.owner.startsWith(prefix))
+                circuit.probe = undefined;
+        }
+    }
+    close(selector) {
+        this.circuits.delete(selector);
+    }
+    has(selector) {
+        return this.circuits.has(selector);
+    }
+    sweep(now) {
+        for (const [selector, circuit] of this.circuits) {
+            if (!circuit.probe && now >= circuit.openUntil + circuit.maxCooldownMs)
+                this.circuits.delete(selector);
+        }
+    }
+}
+export function createFallbackCircuitAccess(deps) {
+    const enabled = () => deps.settings().cooldownMs > 0;
+    return {
+        noteFailure(selector, failure) {
+            if (!enabled())
+                return;
+            const { cooldownMs, maxCooldownMs } = deps.settings();
+            const now = deps.now();
+            const retryAfterMs = failure.retryAfterMs ??
+                (failure.errorMessage === undefined ? undefined : parseRetryAfterMsMarker(failure.errorMessage));
+            deps.breaker.sweep(now);
+            const openUntil = deps.breaker.open(selector, { now, cooldownMs, maxCooldownMs, retryAfterMs });
+            deps.logger.info("circuit_opened", { selector, durationMs: openUntil - now, retryAfterMs });
+        },
+        governs: (selector) => enabled() && deps.breaker.has(selector),
+        isOpen: (selector, lane = "turn") => enabled() && deps.breaker.isOpen(selector, deps.now(), laneOwner(deps.owner(), lane)),
+        admit: (selector, lane = "turn") => enabled() ? deps.breaker.admit(selector, deps.now(), laneOwner(deps.owner(), lane)) : { kind: "closed" },
+        release: (token) => deps.breaker.release(token),
+        releaseAll: () => deps.breaker.releaseOwnersWithPrefix(`${deps.owner()}:`),
+        close: (selector) => deps.breaker.close(selector),
+    };
+}
+function laneOwner(session, lane) {
+    return `${session}:${lane}`;
+}
+const breakersByAgentDir = new Map();
+/**
+ * An entry leaves the registry only when no live session holds it and it has no
+ * circuit left to share; a held breaker keeps its identity however empty it is.
+ */
+function evictUnheldEmptyBreakers() {
+    for (const [key, entry] of breakersByAgentDir) {
+        if (entry.owners === 0 && entry.breaker.size === 0)
+            breakersByAgentDir.delete(key);
+    }
+}
+function entryFor(agentDir) {
+    const key = resolve(agentDir);
+    let entry = breakersByAgentDir.get(key);
+    if (!entry) {
+        evictUnheldEmptyBreakers();
+        entry = { breaker: new FallbackCircuitBreaker(), owners: 0 };
+        breakersByAgentDir.set(key, entry);
+    }
+    return entry;
+}
+export function fallbackCircuitsFor(agentDir) {
+    return entryFor(agentDir).breaker;
+}
+/** Holds the agent dir's breaker for one session; `release` is idempotent. */
+export function acquireFallbackCircuits(agentDir) {
+    const entry = entryFor(agentDir);
+    entry.owners++;
+    let released = false;
+    return {
+        breaker: entry.breaker,
+        release() {
+            if (released)
+                return;
+            released = true;
+            entry.owners--;
+            evictUnheldEmptyBreakers();
+        },
+    };
+}
+export function monotonicNow() {
+    return performance.timeOrigin + performance.now();
+}
+//# sourceMappingURL=circuit.js.map

package/dist/core/retry-fallback/circuit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"circuit.js","sourceRoot":"","sources":["../../../src/core/retry-fallback/circuit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAC;AAGjF;;;;;;;;;;;GAWG;AAEH,MAAM,CAAC,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAClD,MAAM,CAAC,MAAM,+BAA+B,GAAG,EAAE,GAAG,MAAM,CAAC;AAqC3D,SAAS,aAAa,CAAC,KAAc,EAAE,QAAgB;IACtD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC7C,QAA6C;IAE7C,MAAM,UAAU,GAAG,aAAa,CAAC,QAAQ,EAAE,iBAAiB,EAAE,2BAA2B,CAAC,CAAC;IAC3F,MAAM,aAAa,GAAG,aAAa,CAAC,QAAQ,EAAE,oBAAoB,EAAE,+BAA+B,CAAC,CAAC;IACrG,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,aAAa,CAAC,EAAE,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,sBAAsB;IAAnC;QACkB,aAAQ,GAAG,IAAI,GAAG,EAA2B,CAAC;QACvD,gBAAW,GAAG,CAAC,CAAC;IAkFzB,CAAC;IAhFA,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC,QAAgB,EAAE,OAA2B;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QACrG,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QAC1C,MAAM,MAAM,GACX,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC;YAC9E,CAAC,CAAC,OAAO,CAAC,GAAG,GAAG,YAAY;YAC5B,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAC/B,MAAM,EACN,QAAQ,IAAI,QAAQ,CAAC,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CACjF,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,GAAG,UAAU,EAAE,eAAe,CAAC,CAAC;QACtE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC3B,SAAS;YACT,eAAe;YACf,gBAAgB;YAChB,UAAU;YACV,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,KAAK,EAAE,SAAS;SAChB,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,CAAC,QAAgB,EAAE,GAAW,EAAE,KAAa;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,IAAI,GAAG,GAAG,OAAO,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QACzC,OAAO,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC;IACrE,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,QAAgB,EAAE,GAAW,EAAE,KAAa;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACxC,IAAI,GAAG,GAAG,OAAO,CAAC,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACrD,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC5E,OAAO,CAAC,KAAK,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;QAC5D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC;IAC5F,CAAC;IAED,OAAO,CAAC,KAAiB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC;QACvD,IAAI,KAAK,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACnF,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAClD,IAAI,OAAO;gBAAE,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC;QACxC,CAAC;IACF,CAAC;IAED,uBAAuB,CAAC,MAAc;QACrC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,IAAI,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;gBAAE,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC;QACxE,CAAC;IACF,CAAC;IAED,KAAK,CAAC,QAAgB;QACrB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,GAAG,CAAC,QAAgB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,GAAW;QAChB,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjD,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,GAAG,IAAI,OAAO,CAAC,SAAS,GAAG,OAAO,CAAC,aAAa;gBAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACxG,CAAC;IACF,CAAC;CACD;AA6BD,MAAM,UAAU,2BAA2B,CAAC,IAA+B;IAC1E,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,UAAU,GAAG,CAAC,CAAC;IACrD,OAAO;QACN,WAAW,CAAC,QAAQ,EAAE,OAAO;YAC5B,IAAI,CAAC,OAAO,EAAE;gBAAE,OAAO;YACvB,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,YAAY,GACjB,OAAO,CAAC,YAAY;gBACpB,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;YAClG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACxB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,CAAC;YAChG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,GAAG,GAAG,EAAE,YAAY,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC9D,MAAM,EAAE,CAAC,QAAQ,EAAE,IAAI,GAAG,MAAM,EAAE,EAAE,CACnC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;QACtF,KAAK,EAAE,CAAC,QAAQ,EAAE,IAAI,GAAG,MAAM,EAAE,EAAE,CAClC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;QACzG,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC;QAC/C,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC;QAC1E,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjD,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,OAAe,EAAE,IAAiB;IACpD,OAAO,GAAG,OAAO,IAAI,IAAI,EAAE,CAAC;AAC7B,CAAC;AAOD,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAwB,CAAC;AAE3D;;;GAGG;AACH,SAAS,wBAAwB;IAChC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,kBAAkB,EAAE,CAAC;QAC/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC;YAAE,kBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACpF,CAAC;AACF,CAAC;AAED,SAAS,QAAQ,CAAC,QAAgB;IACjC,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,IAAI,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,wBAAwB,EAAE,CAAC;QAC3B,KAAK,GAAG,EAAE,OAAO,EAAE,IAAI,sBAAsB,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAC7D,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IACnD,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC;AACnC,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,uBAAuB,CAAC,QAAgB;IACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACjC,KAAK,CAAC,MAAM,EAAE,CAAC;IACf,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,OAAO;QACN,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO;YACN,IAAI,QAAQ;gBAAE,OAAO;YACrB,QAAQ,GAAG,IAAI,CAAC;YAChB,KAAK,CAAC,MAAM,EAAE,CAAC;YACf,wBAAwB,EAAE,CAAC;QAC5B,CAAC;KACD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY;IAC3B,OAAO,WAAW,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;AACnD,CAAC","sourcesContent":["import { resolve } from \"node:path\";\nimport { parseRetryAfterMsMarker } from \"@earendil-works/pi-ai/utils/retry-hint\";\nimport type { FallbackLogger } from \"./log.ts\";\n\n/**\n * Fallback-chain circuit breaker. An entry that fails out of a configured chain\n * opens a circuit keyed by its base selector (`provider/id`); while it is open,\n * chain resolution skips the entry instead of spending its retry budget again.\n *\n * One breaker is shared by every session in the process that uses the same agent\n * directory - the config that defines the providers and selectors - so a session\n * started by `/new`, `/resume`, or `/fork` (each gets a fresh model runtime) and\n * an in-process subagent do not re-discover a dead provider.\n * The class is clock-free: every call receives `now`, keeping sessions' injected\n * clocks authoritative and tests deterministic.\n */\n\nexport const DEFAULT_CIRCUIT_COOLDOWN_MS = 60_000;\nexport const DEFAULT_CIRCUIT_MAX_COOLDOWN_MS = 30 * 60_000;\n\nexport interface FallbackCircuitSettings {\n\t/** First cooldown after an entry fails out of a chain; doubles per consecutive open. 0 disables the breaker. */\n\tcircuitCooldownMs?: number;\n\tcircuitMaxCooldownMs?: number;\n}\n\nexport interface ResolvedFallbackCircuitSettings {\n\tcooldownMs: number;\n\tmaxCooldownMs: number;\n}\n\nexport interface CircuitOpenRequest {\n\tnow: number;\n\tcooldownMs: number;\n\tmaxCooldownMs: number;\n\tretryAfterMs?: number;\n}\n\nexport interface ProbeToken {\n\treadonly selector: string;\n\treadonly owner: string;\n\treadonly generation: number;\n}\n\nexport type CircuitAdmission = { kind: \"closed\" } | { kind: \"open\" } | { kind: \"probe\"; token: ProbeToken };\n\ninterface SelectorCircuit {\n\topenUntil: number;\n\tretryFloorUntil: number;\n\tconsecutiveOpens: number;\n\tcooldownMs: number;\n\tmaxCooldownMs: number;\n\tprobe: { owner: string; generation: number } | undefined;\n}\n\nfunction nonNegativeMs(value: unknown, fallback: number): number {\n\treturn typeof value === \"number\" && Number.isFinite(value) && value >= 0 ? value : fallback;\n}\n\nexport function resolveFallbackCircuitSettings(\n\tsettings: FallbackCircuitSettings | undefined,\n): ResolvedFallbackCircuitSettings {\n\tconst cooldownMs = nonNegativeMs(settings?.circuitCooldownMs, DEFAULT_CIRCUIT_COOLDOWN_MS);\n\tconst maxCooldownMs = nonNegativeMs(settings?.circuitMaxCooldownMs, DEFAULT_CIRCUIT_MAX_COOLDOWN_MS);\n\treturn { cooldownMs, maxCooldownMs: Math.max(cooldownMs, maxCooldownMs) };\n}\n\nexport class FallbackCircuitBreaker {\n\tprivate readonly circuits = new Map<string, SelectorCircuit>();\n\tprivate generations = 0;\n\n\tget size(): number {\n\t\treturn this.circuits.size;\n\t}\n\n\topen(selector: string, request: CircuitOpenRequest): number {\n\t\tconst previous = this.circuits.get(selector);\n\t\tconst consecutiveOpens = (previous?.consecutiveOpens ?? 0) + 1;\n\t\tconst cooldownMs = Math.min(request.cooldownMs * 2 ** (consecutiveOpens - 1), request.maxCooldownMs);\n\t\tconst retryAfterMs = request.retryAfterMs;\n\t\tconst hinted =\n\t\t\tretryAfterMs !== undefined && Number.isFinite(retryAfterMs) && retryAfterMs > 0\n\t\t\t\t? request.now + retryAfterMs\n\t\t\t\t: 0;\n\t\tconst retryFloorUntil = Math.max(\n\t\t\thinted,\n\t\t\tprevious && previous.retryFloorUntil > request.now ? previous.retryFloorUntil : 0,\n\t\t);\n\t\tconst openUntil = Math.max(request.now + cooldownMs, retryFloorUntil);\n\t\tthis.circuits.set(selector, {\n\t\t\topenUntil,\n\t\t\tretryFloorUntil,\n\t\t\tconsecutiveOpens,\n\t\t\tcooldownMs,\n\t\t\tmaxCooldownMs: request.maxCooldownMs,\n\t\t\tprobe: undefined,\n\t\t});\n\t\treturn openUntil;\n\t}\n\n\tisOpen(selector: string, now: number, owner: string): boolean {\n\t\tconst circuit = this.circuits.get(selector);\n\t\tif (!circuit) return false;\n\t\tif (now < circuit.openUntil) return true;\n\t\treturn circuit.probe !== undefined && circuit.probe.owner !== owner;\n\t}\n\n\t/**\n\t * Atomic admission: a closed selector passes, a cooling one or one whose probe\n\t * another requester holds is refused, and a half-open one hands `owner` its\n\t * single probe under a fresh generation. The probe stays exclusive until its\n\t * token is released or the circuit settles - it never expires on a timer, so a\n\t * slow probe cannot overlap a second one, and a stale token never releases a\n\t * later acquisition.\n\t */\n\tadmit(selector: string, now: number, owner: string): CircuitAdmission {\n\t\tconst circuit = this.circuits.get(selector);\n\t\tif (!circuit) return { kind: \"closed\" };\n\t\tif (now < circuit.openUntil) return { kind: \"open\" };\n\t\tif (circuit.probe && circuit.probe.owner !== owner) return { kind: \"open\" };\n\t\tcircuit.probe ??= { owner, generation: ++this.generations };\n\t\treturn { kind: \"probe\", token: { selector, owner, generation: circuit.probe.generation } };\n\t}\n\n\trelease(token: ProbeToken): void {\n\t\tconst probe = this.circuits.get(token.selector)?.probe;\n\t\tif (probe && probe.owner === token.owner && probe.generation === token.generation) {\n\t\t\tconst circuit = this.circuits.get(token.selector);\n\t\t\tif (circuit) circuit.probe = undefined;\n\t\t}\n\t}\n\n\treleaseOwnersWithPrefix(prefix: string): void {\n\t\tfor (const circuit of this.circuits.values()) {\n\t\t\tif (circuit.probe?.owner.startsWith(prefix)) circuit.probe = undefined;\n\t\t}\n\t}\n\n\tclose(selector: string): void {\n\t\tthis.circuits.delete(selector);\n\t}\n\n\thas(selector: string): boolean {\n\t\treturn this.circuits.has(selector);\n\t}\n\n\tsweep(now: number): void {\n\t\tfor (const [selector, circuit] of this.circuits) {\n\t\t\tif (!circuit.probe && now >= circuit.openUntil + circuit.maxCooldownMs) this.circuits.delete(selector);\n\t\t}\n\t}\n}\n\n/**\n * A session's requests are admitted per lane, not per session: the foreground\n * turn is one lane, each background probe-back its own. Two lanes of one session\n * never share a probe, so a probe-back in flight keeps the foreground turn off\n * the entry exactly as it keeps a sibling session off.\n */\nexport type CircuitLane = \"turn\" | `probe-back:${number}`;\n\nexport interface FallbackCircuitAccess {\n\tnoteFailure(selector: string, failure: { retryAfterMs?: number; errorMessage?: string }): void;\n\t/** Whether the breaker holds any state for the selector, i.e. it, not a session cooldown, governs recovery. */\n\tgoverns(selector: string): boolean;\n\tisOpen(selector: string, lane?: CircuitLane): boolean;\n\tadmit(selector: string, lane?: CircuitLane): CircuitAdmission;\n\trelease(token: ProbeToken): void;\n\treleaseAll(): void;\n\tclose(selector: string): void;\n}\n\nexport interface FallbackCircuitAccessDeps {\n\tbreaker: FallbackCircuitBreaker;\n\towner(): string;\n\tnow(): number;\n\tsettings(): ResolvedFallbackCircuitSettings;\n\tlogger: FallbackLogger;\n}\n\nexport function createFallbackCircuitAccess(deps: FallbackCircuitAccessDeps): FallbackCircuitAccess {\n\tconst enabled = () => deps.settings().cooldownMs > 0;\n\treturn {\n\t\tnoteFailure(selector, failure) {\n\t\t\tif (!enabled()) return;\n\t\t\tconst { cooldownMs, maxCooldownMs } = deps.settings();\n\t\t\tconst now = deps.now();\n\t\t\tconst retryAfterMs =\n\t\t\t\tfailure.retryAfterMs ??\n\t\t\t\t(failure.errorMessage === undefined ? undefined : parseRetryAfterMsMarker(failure.errorMessage));\n\t\t\tdeps.breaker.sweep(now);\n\t\t\tconst openUntil = deps.breaker.open(selector, { now, cooldownMs, maxCooldownMs, retryAfterMs });\n\t\t\tdeps.logger.info(\"circuit_opened\", { selector, durationMs: openUntil - now, retryAfterMs });\n\t\t},\n\t\tgoverns: (selector) => enabled() && deps.breaker.has(selector),\n\t\tisOpen: (selector, lane = \"turn\") =>\n\t\t\tenabled() && deps.breaker.isOpen(selector, deps.now(), laneOwner(deps.owner(), lane)),\n\t\tadmit: (selector, lane = \"turn\") =>\n\t\t\tenabled() ? deps.breaker.admit(selector, deps.now(), laneOwner(deps.owner(), lane)) : { kind: \"closed\" },\n\t\trelease: (token) => deps.breaker.release(token),\n\t\treleaseAll: () => deps.breaker.releaseOwnersWithPrefix(`${deps.owner()}:`),\n\t\tclose: (selector) => deps.breaker.close(selector),\n\t};\n}\n\nfunction laneOwner(session: string, lane: CircuitLane): string {\n\treturn `${session}:${lane}`;\n}\n\ninterface BreakerEntry {\n\treadonly breaker: FallbackCircuitBreaker;\n\towners: number;\n}\n\nconst breakersByAgentDir = new Map<string, BreakerEntry>();\n\n/**\n * An entry leaves the registry only when no live session holds it and it has no\n * circuit left to share; a held breaker keeps its identity however empty it is.\n */\nfunction evictUnheldEmptyBreakers(): void {\n\tfor (const [key, entry] of breakersByAgentDir) {\n\t\tif (entry.owners === 0 && entry.breaker.size === 0) breakersByAgentDir.delete(key);\n\t}\n}\n\nfunction entryFor(agentDir: string): BreakerEntry {\n\tconst key = resolve(agentDir);\n\tlet entry = breakersByAgentDir.get(key);\n\tif (!entry) {\n\t\tevictUnheldEmptyBreakers();\n\t\tentry = { breaker: new FallbackCircuitBreaker(), owners: 0 };\n\t\tbreakersByAgentDir.set(key, entry);\n\t}\n\treturn entry;\n}\n\nexport function fallbackCircuitsFor(agentDir: string): FallbackCircuitBreaker {\n\treturn entryFor(agentDir).breaker;\n}\n\n/** Holds the agent dir's breaker for one session; `release` is idempotent. */\nexport function acquireFallbackCircuits(agentDir: string): { breaker: FallbackCircuitBreaker; release(): void } {\n\tconst entry = entryFor(agentDir);\n\tentry.owners++;\n\tlet released = false;\n\treturn {\n\t\tbreaker: entry.breaker,\n\t\trelease() {\n\t\t\tif (released) return;\n\t\t\treleased = true;\n\t\t\tentry.owners--;\n\t\t\tevictUnheldEmptyBreakers();\n\t\t},\n\t};\n}\n\nexport function monotonicNow(): number {\n\treturn performance.timeOrigin + performance.now();\n}\n"]}

package/dist/core/retry-fallback/controller-types.d.ts

@@ -0,0 +1,58 @@
+import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
+import type { Api, Model } from "@earendil-works/pi-ai";
+import type { FallbackCircuitAccess } from "./circuit.ts";
+import type { SelectorCooldowns } from "./cooldown.ts";
+import type { FallbackLogger } from "./log.ts";
+export interface ActiveFallbackState {
+    chainKey: string;
+    originalSelector: string;
+    originalThinkingLevel?: ThinkingLevel;
+    lastAppliedThinkingLevel?: ThinkingLevel;
+    /** Pin provenance: refusal contributions release on compaction, billing never. */
+    pinnedByRefusal: boolean;
+    pinnedByBilling: boolean;
+    /** Derived OR of the two provenance flags - the only field consumers read. */
+    pinned: boolean;
+}
+export type FallbackReason = "transient" | "refusal" | "hard-error" | "billing";
+export type CircuitFailure = {
+    errorMessage?: string;
+    retryAfterMs?: number;
+};
+export interface FallbackSettings {
+    modelFallback: boolean;
+    chains: Readonly<Record<string, readonly string[]>>;
+}
+export interface RetryFallbackControllerDeps {
+    getSettings(): FallbackSettings;
+    registry: {
+        find(provider: string, id: string): Model<Api> | undefined;
+        getAll(): Model<Api>[];
+        /** Ranks bare-selector expansion: OAuth-credential providers come first. */
+        isUsingOAuth?(model: Model<Api>): boolean;
+        /** Filters bare-selector expansion: a definitive `false` keeps a lane that can never serve out of the chain. */
+        isFallbackEligible?(model: Model<Api>): boolean;
+    };
+    cooldowns: SelectorCooldowns;
+    /** Chain-entry circuits shared with every session in the process on the same agent dir. */
+    circuits?: FallbackCircuitAccess;
+    logger: FallbackLogger;
+    switchModel(model: Model<Api>, thinking: ThinkingLevel, reason: "fallback" | "fallback-revert"): Promise<void>;
+    emit(event: {
+        type: "retry_fallback_applied";
+        from: string;
+        to: string;
+        chainKey: string;
+        reason: FallbackReason;
+    } | {
+        type: "retry_fallback_reverted";
+        from: string;
+        to: string;
+    }): void;
+    getCurrentSelector(): {
+        model: Model<Api>;
+        thinkingLevel?: ThinkingLevel;
+    } | undefined;
+    isAuthAvailable(provider: string): boolean;
+}
+//# sourceMappingURL=controller-types.d.ts.map

package/dist/core/retry-fallback/controller-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"controller-types.d.ts","sourceRoot":"","sources":["../../../src/core/retry-fallback/controller-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/C,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,aAAa,CAAC;IACtC,wBAAwB,CAAC,EAAE,aAAa,CAAC;IACzC,kFAAkF;IAClF,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,8EAA8E;IAC9E,MAAM,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,SAAS,GAAG,YAAY,GAAG,SAAS,CAAC;AAChF,MAAM,MAAM,cAAc,GAAG;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9E,MAAM,WAAW,gBAAgB;IAChC,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC;CACpD;AAED,MAAM,WAAW,2BAA2B;IAC3C,WAAW,IAAI,gBAAgB,CAAC;IAChC,QAAQ,EAAE;QACT,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,4EAA4E;QAC5E,YAAY,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;QAC1C,gHAAgH;QAChH,kBAAkB,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;KAChD,CAAC;IACF,SAAS,EAAE,iBAAiB,CAAC;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,MAAM,EAAE,cAAc,CAAC;IACvB,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,GAAG,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/G,IAAI,CACH,KAAK,EACF;QACA,IAAI,EAAE,wBAAwB,CAAC;QAC/B,IAAI,EAAE,MAAM,CAAC;QACb,EAAE,EAAE,MAAM,CAAC;QACX,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,cAAc,CAAC;KACtB,GACD;QAAE,IAAI,EAAE,yBAAyB,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,GAC9D,IAAI,CAAC;IACR,kBAAkB,IAAI;QAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAAC,aAAa,CAAC,EAAE,aAAa,CAAA;KAAE,GAAG,SAAS,CAAC;IACvF,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3C"}

package/dist/core/retry-fallback/controller-types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=controller-types.js.map

package/dist/core/retry-fallback/controller-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"controller-types.js","sourceRoot":"","sources":["../../../src/core/retry-fallback/controller-types.ts"],"names":[],"mappings":"","sourcesContent":["import type { ThinkingLevel } from \"@earendil-works/pi-agent-core\";\nimport type { Api, Model } from \"@earendil-works/pi-ai\";\nimport type { FallbackCircuitAccess } from \"./circuit.ts\";\nimport type { SelectorCooldowns } from \"./cooldown.ts\";\nimport type { FallbackLogger } from \"./log.ts\";\n\nexport interface ActiveFallbackState {\n\tchainKey: string;\n\toriginalSelector: string;\n\toriginalThinkingLevel?: ThinkingLevel;\n\tlastAppliedThinkingLevel?: ThinkingLevel;\n\t/** Pin provenance: refusal contributions release on compaction, billing never. */\n\tpinnedByRefusal: boolean;\n\tpinnedByBilling: boolean;\n\t/** Derived OR of the two provenance flags - the only field consumers read. */\n\tpinned: boolean;\n}\n\nexport type FallbackReason = \"transient\" | \"refusal\" | \"hard-error\" | \"billing\";\nexport type CircuitFailure = { errorMessage?: string; retryAfterMs?: number };\n\nexport interface FallbackSettings {\n\tmodelFallback: boolean;\n\tchains: Readonly<Record<string, readonly string[]>>;\n}\n\nexport interface RetryFallbackControllerDeps {\n\tgetSettings(): FallbackSettings;\n\tregistry: {\n\t\tfind(provider: string, id: string): Model<Api> | undefined;\n\t\tgetAll(): Model<Api>[];\n\t\t/** Ranks bare-selector expansion: OAuth-credential providers come first. */\n\t\tisUsingOAuth?(model: Model<Api>): boolean;\n\t\t/** Filters bare-selector expansion: a definitive `false` keeps a lane that can never serve out of the chain. */\n\t\tisFallbackEligible?(model: Model<Api>): boolean;\n\t};\n\tcooldowns: SelectorCooldowns;\n\t/** Chain-entry circuits shared with every session in the process on the same agent dir. */\n\tcircuits?: FallbackCircuitAccess;\n\tlogger: FallbackLogger;\n\tswitchModel(model: Model<Api>, thinking: ThinkingLevel, reason: \"fallback\" | \"fallback-revert\"): Promise<void>;\n\temit(\n\t\tevent:\n\t\t\t| {\n\t\t\t\t\ttype: \"retry_fallback_applied\";\n\t\t\t\t\tfrom: string;\n\t\t\t\t\tto: string;\n\t\t\t\t\tchainKey: string;\n\t\t\t\t\treason: FallbackReason;\n\t\t\t  }\n\t\t\t| { type: \"retry_fallback_reverted\"; from: string; to: string },\n\t): void;\n\tgetCurrentSelector(): { model: Model<Api>; thinkingLevel?: ThinkingLevel } | undefined;\n\tisAuthAvailable(provider: string): boolean;\n}\n"]}

package/dist/core/retry-fallback/controller.d.ts

@@ -1,58 +1,14 @@
 import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
 import { type Api, type Model } from "@earendil-works/pi-ai";
-import type { SelectorCooldowns } from "./cooldown.ts";
-import type { FallbackLogger } from "./log.ts";
-export interface ActiveFallbackState {
-    chainKey: string;
-    originalSelector: string;
-    originalThinkingLevel?: ThinkingLevel;
-    lastAppliedThinkingLevel?: ThinkingLevel;
-    /** Pin provenance: refusal contributions release on compaction, billing never. */
-    pinnedByRefusal: boolean;
-    pinnedByBilling: boolean;
-    /** Derived OR of the two provenance flags - the only field consumers read. */
-    pinned: boolean;
-}
-type FallbackReason = "transient" | "refusal" | "hard-error" | "billing";
-interface FallbackSettings {
-    modelFallback: boolean;
-    chains: Readonly<Record<string, readonly string[]>>;
-}
-export interface RetryFallbackControllerDeps {
-    getSettings(): FallbackSettings;
-    registry: {
-        find(provider: string, id: string): Model<Api> | undefined;
-        getAll(): Model<Api>[];
-        /** Ranks bare-selector expansion: OAuth-credential providers come first. */
-        isUsingOAuth?(model: Model<Api>): boolean;
-        /** Filters bare-selector expansion: a definitive `false` keeps a lane that can never serve out of the chain. */
-        isFallbackEligible?(model: Model<Api>): boolean;
-    };
-    cooldowns: SelectorCooldowns;
-    logger: FallbackLogger;
-    switchModel(model: Model<Api>, thinking: ThinkingLevel, reason: "fallback" | "fallback-revert"): Promise<void>;
-    emit(event: {
-        type: "retry_fallback_applied";
-        from: string;
-        to: string;
-        chainKey: string;
-        reason: FallbackReason;
-    } | {
-        type: "retry_fallback_reverted";
-        from: string;
-        to: string;
-    }): void;
-    getCurrentSelector(): {
-        model: Model<Api>;
-        thinkingLevel?: ThinkingLevel;
-    } | undefined;
-    isAuthAvailable(provider: string): boolean;
-}
+import { CircuitProbes } from "./circuit-probes.ts";
+import type { ActiveFallbackState, CircuitFailure, FallbackReason, RetryFallbackControllerDeps } from "./controller-types.ts";
+export type { ActiveFallbackState, RetryFallbackControllerDeps } from "./controller-types.ts";
 export declare class RetryFallbackController {
     private readonly deps;
     private readonly triedSelectors;
     private state;
     private lastExhaustedChainKey;
+    readonly probes: CircuitProbes;
     private canonicalCache;
     constructor(deps: RetryFallbackControllerDeps);
     get activeState(): Readonly<ActiveFallbackState> | undefined;
@@ -92,13 +48,29 @@ export declare class RetryFallbackController {
     notifyCompactionApplied(): boolean;
     /** A user-driven model change abandons the fallback window entirely. */
     clearForManualModelChange(model: Model<Api>): void;
+    /**
+     * Records a failure that makes the entry unusable for every session - billing
+     * or quota exhaustion, or a transient failure that spent its budget or failed a
+     * half-open probe - independent of whether a fallback candidate remains.
+     */
+    noteHealthFailure(model: Model<Api>, thinkingLevel: ThinkingLevel | undefined, failure: CircuitFailure): void;
+    /**
+     * Turn-boundary skip: when the current entry's circuit was opened by this or a
+     * sibling session, move to the next closed entry without spending a request on
+     * it. With no closed entry left the current one stays and probes, so the chain
+     * never refuses a turn; a half-open current entry is claimed for this session.
+     */
+    rerouteAroundOpenCircuit(): Promise<boolean>;
     tryFallback(reason: FallbackReason, failure: {
         errorMessage?: string;
         retryAfterMs?: number;
     }): Promise<boolean>;
+    private applyCandidate;
+    /** The chain governing `current`, when fallback is enabled; a model's own chain wins over an active one. */
+    private managedChainKey;
     private nextCandidate;
+    private switchOrRelease;
     private selectThinking;
     private skip;
 }
-export {};
 //# sourceMappingURL=controller.d.ts.map

package/dist/core/retry-fallback/controller.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"controller.d.ts","sourceRoot":"","sources":["../../../src/core/retry-fallback/controller.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,KAAK,GAAG,EAAsB,KAAK,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAWjF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/C,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,aAAa,CAAC;IACtC,wBAAwB,CAAC,EAAE,aAAa,CAAC;IACzC,kFAAkF;IAClF,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,8EAA8E;IAC9E,MAAM,EAAE,OAAO,CAAC;CAChB;AAED,KAAK,cAAc,GAAG,WAAW,GAAG,SAAS,GAAG,YAAY,GAAG,SAAS,CAAC;AAEzE,UAAU,gBAAgB;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC;CACpD;AAED,MAAM,WAAW,2BAA2B;IAC3C,WAAW,IAAI,gBAAgB,CAAC;IAChC,QAAQ,EAAE;QACT,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,4EAA4E;QAC5E,YAAY,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;QAC1C,gHAAgH;QAChH,kBAAkB,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;KAChD,CAAC;IACF,SAAS,EAAE,iBAAiB,CAAC;IAC7B,MAAM,EAAE,cAAc,CAAC;IACvB,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,GAAG,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/G,IAAI,CACH,KAAK,EACF;QACA,IAAI,EAAE,wBAAwB,CAAC;QAC/B,IAAI,EAAE,MAAM,CAAC;QACb,EAAE,EAAE,MAAM,CAAC;QACX,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,cAAc,CAAC;KACtB,GACD;QAAE,IAAI,EAAE,yBAAyB,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,GAC9D,IAAI,CAAC;IACR,kBAAkB,IAAI;QAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAAC,aAAa,CAAC,EAAE,aAAa,CAAA;KAAE,GAAG,SAAS,CAAC;IACvF,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3C;AAED,qBAAa,uBAAuB;IACnC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA8B;IACnD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,qBAAqB,CAAqB;IAQlD,OAAO,CAAC,cAAc,CAAsD;IAE5E,YAAY,IAAI,EAAE,2BAA2B,EAE5C;IAED,IAAI,WAAW,IAAI,QAAQ,CAAC,mBAAmB,CAAC,GAAG,SAAS,CAE3D;IAED,IAAI,iBAAiB,IAAI,MAAM,GAAG,SAAS,CAE1C;IAED,SAAS,IAAI,IAAI,CAGhB;IAED,KAAK,IAAI,IAAI,CAIZ;IAED,OAAO,CAAC,eAAe;IASvB,cAAc,IAAI,OAAO,CAExB;IAED;;;;;OAKG;IACH,kBAAkB,IAAI,OAAO,CAK5B;IAED;;;;;OAKG;IACG,mBAAmB,CAAC,YAAY,EAAE,iBAAiB,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAsBrF;IAED;;;OAGG;IACH,uBAAuB,IAAI,IAAI,CAE9B;IAED;;;;;;;;OAQG;IACH,uBAAuB,IAAI,OAAO,CAajC;IAED,wEAAwE;IACxE,yBAAyB,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAMjD;IAEK,WAAW,CAChB,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC,OAAO,CAAC,CA8BlB;IAED,OAAO,CAAC,aAAa;IAoDrB,OAAO,CAAC,cAAc;IAYtB,OAAO,CAAC,IAAI;CAGZ"}
+{"version":3,"file":"controller.d.ts","sourceRoot":"","sources":["../../../src/core/retry-fallback/controller.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,KAAK,GAAG,EAAsB,KAAK,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAWjF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EACX,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,2BAA2B,EAC3B,MAAM,uBAAuB,CAAC;AAE/B,YAAY,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AAE9F,qBAAa,uBAAuB;IACnC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA8B;IACnD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,qBAAqB,CAAqB;IAClD,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAQ/B,OAAO,CAAC,cAAc,CAAsD;IAE5E,YAAY,IAAI,EAAE,2BAA2B,EAG5C;IAED,IAAI,WAAW,IAAI,QAAQ,CAAC,mBAAmB,CAAC,GAAG,SAAS,CAE3D;IAED,IAAI,iBAAiB,IAAI,MAAM,GAAG,SAAS,CAE1C;IAED,SAAS,IAAI,IAAI,CAGhB;IAED,KAAK,IAAI,IAAI,CAIZ;IAED,OAAO,CAAC,eAAe;IASvB,cAAc,IAAI,OAAO,CAExB;IAED;;;;;OAKG;IACH,kBAAkB,IAAI,OAAO,CAK5B;IAED;;;;;OAKG;IACG,mBAAmB,CAAC,YAAY,EAAE,iBAAiB,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CA4BrF;IAED;;;OAGG;IACH,uBAAuB,IAAI,IAAI,CAE9B;IAED;;;;;;;;OAQG;IACH,uBAAuB,IAAI,OAAO,CAajC;IAED,wEAAwE;IACxE,yBAAyB,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAQjD;IAED;;;;OAIG;IACH,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,aAAa,EAAE,aAAa,GAAG,SAAS,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAG5G;IAED;;;;;OAKG;IACG,wBAAwB,IAAI,OAAO,CAAC,OAAO,CAAC,CAUjD;IAEK,WAAW,CAChB,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC,OAAO,CAAC,CAelB;YAEa,cAAc;IA2B5B,4GAA4G;IAC5G,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,aAAa;YA2BP,eAAe;IAa7B,OAAO,CAAC,cAAc;IAYtB,OAAO,CAAC,IAAI;CAGZ"}