@gamaze/hicortex 0.19.5 → 0.20.0

package/dist/llm-flight.js

@@ -0,0 +1,190 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_STALE_MS = void 0;
+exports.llmFlightLockPath = llmFlightLockPath;
+exports.acquireLlmFlight = acquireLlmFlight;
+/**
+ * Single-flight guard for LLM endpoint calls (#355).
+ *
+ * Local single-user model servers (one big-context model on a personal
+ * machine) have a standing failure mode: TWO concurrent large-context
+ * requests stall or OOM the server — and take the whole machine with it.
+ * One Hicortex server is several LLM callers at once: the daemon distills
+ * every inbound /distill concurrently, and the nightly's consolidation is a
+ * separate OS process on its own timers. This guard makes "never two
+ * in-flight requests to the same endpoint" STRUCTURAL instead of a hope
+ * that timers do not overlap.
+ *
+ * Same idiom as capture.ts's capture.lock (A5): O_EXCL create, dead-pid or
+ * TTL staleness with a TOCTOU re-race, and FAIL-OPEN — a filesystem refusal
+ * logs a loud warning and lets the call proceed unserialized. The guard must
+ * never be the reason recall or distillation stops on a healthy endpoint.
+ *
+ * The lock file lives in the hicortex home, one file per endpoint
+ * (sha1 of `provider@baseUrl`), so an install with multiple endpoints
+ * serializes within each endpoint, not across them.
+ */
+const node_crypto_1 = require("node:crypto");
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+/**
+ * Default staleness floor: a lock older than this is stale REGARDLESS of the
+ * recorded pid (guards the recycled-pid case, capture A5 fix 2). The caller
+ * passes `staleMs = max(this default, 2× llmTimeoutMs)` so an operator who
+ * raises the timeout ceiling can never have a LIVE call's lock reclaimed
+ * mid-flight (CR #355 finding 3 — a reclaim-while-running is exactly the
+ * two-concurrent-calls crash class this guard exists to prevent).
+ */
+exports.DEFAULT_STALE_MS = 30 * 60 * 1000;
+/**
+ * A lock file younger than this whose holder is unreadable (empty/invalid
+ * JSON) is treated as LIVE: its creator is between the O_EXCL create and the
+ * writeSync — stealing in that window is the same crash class. Only an
+ * unreadable file OLDER than the grace period is junk to reclaim
+ * (CR #355 finding 4).
+ */
+const GRACE_MS = 2_000;
+/** LLM-scale polling: waits are seconds-to-minutes, not capture-scale. */
+const POLL_MS = 100;
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Warn at most once per process that the guard failed open (#355 fail-open). */
+let warnedOpen = false;
+/** Lock path for an endpoint key — exported for tests and diagnostics. */
+function llmFlightLockPath(homeDir, endpointKey) {
+    const hash = (0, node_crypto_1.createHash)("sha1").update(endpointKey).digest("hex").slice(0, 16);
+    return (0, node_path_1.join)(homeDir, `llm-flight-${hash}.lock`);
+}
+/**
+ * Acquire the single-flight lock for `endpointKey`, waiting up to `waitMs`.
+ * Never throws: every filesystem surprise degrades to `{ kind: "open" }`.
+ */
+async function acquireLlmFlight(homeDir, endpointKey, waitMs, staleMs = exports.DEFAULT_STALE_MS) {
+    const lockPath = llmFlightLockPath(homeDir, endpointKey);
+    try {
+        (0, node_fs_1.mkdirSync)(homeDir, { recursive: true });
+    }
+    catch {
+        /* best effort — the create below surfaces real problems */
+    }
+    const deadline = Date.now() + waitMs;
+    for (;;) {
+        const attempt = tryAcquireOnce(lockPath, endpointKey, staleMs);
+        if (attempt.kind !== "busy")
+            return attempt;
+        if (Date.now() >= deadline)
+            return { kind: "timeout" };
+        await sleep(Math.max(1, Math.min(POLL_MS, deadline - Date.now())));
+    }
+}
+/** One acquire attempt: create-if-free, else reclaim-if-stale. */
+function tryAcquireOnce(lockPath, endpointKey, staleMs) {
+    const release = () => {
+        try {
+            (0, node_fs_1.unlinkSync)(lockPath);
+        }
+        catch {
+            /* already gone */
+        }
+    };
+    const create = () => {
+        try {
+            const fd = (0, node_fs_1.openSync)(lockPath, "wx"); // O_CREAT | O_EXCL
+            // The lease records how long THIS holder may legitimately run
+            // (now + staleMs) — a waiter with a smaller budget judges staleness
+            // by the recorded lease, never by its own parameter (2nd-review
+            // finding 3: a short-timeout waiter must not TTL-reclaim a live
+            // long-timeout holder's lock).
+            (0, node_fs_1.writeSync)(fd, JSON.stringify({
+                pid: process.pid,
+                endpoint: endpointKey,
+                leaseUntil: Date.now() + staleMs,
+            }));
+            (0, node_fs_1.closeSync)(fd);
+            return true;
+        }
+        catch (err) {
+            if (err.code === "EEXIST")
+                return false;
+            throw err;
+        }
+    };
+    try {
+        if (create())
+            return { kind: "acquired", release };
+        const holder = readHolder(lockPath);
+        if (!isStale(lockPath, holder, staleMs))
+            return { kind: "busy" };
+        // Stale. Re-verify the holder has not changed (another reclaimer may
+        // have taken it), unlink, re-race the O_EXCL create (capture fix 12).
+        // Compare by VALUE (pid/endpoint) — readHolder returns a fresh object
+        // per call, so reference equality would always differ.
+        const recheck = readHolder(lockPath);
+        if (recheck?.pid !== holder?.pid ||
+            recheck?.endpoint !== holder?.endpoint) {
+            return { kind: "busy" };
+        }
+        try {
+            (0, node_fs_1.unlinkSync)(lockPath);
+        }
+        catch {
+            /* raced with another reclaimer */
+        }
+        return create() ? { kind: "acquired", release } : { kind: "busy" };
+    }
+    catch {
+        // Filesystem refused the lock op entirely — do not wedge LLM traffic on
+        // the guard; proceed unserialized (the behaviour before #355).
+        if (!warnedOpen) {
+            warnedOpen = true;
+            console.warn(`[hicortex] LLM single-flight lock unavailable (${lockPath}) — ` +
+                `proceeding WITHOUT serialization. This is safe for recall but ` +
+                `concurrent LLM calls can stall a single-user local model server.`);
+        }
+        return { kind: "open" };
+    }
+}
+function readHolder(lockPath) {
+    try {
+        const parsed = JSON.parse((0, node_fs_1.readFileSync)(lockPath, "utf-8"));
+        if (typeof parsed.pid === "number" && Number.isFinite(parsed.pid))
+            return parsed;
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Stale = dead pid, OR past the HOLDER's recorded lease, OR (lease-less
+ *  lock files) older than the waiter's `staleMs`. An unreadable holder
+ *  (mid-write or corrupt) is live within the grace period, junk after it. */
+function isStale(lockPath, holder, staleMs) {
+    if (!holder) {
+        try {
+            return Date.now() - (0, node_fs_1.statSync)(lockPath).mtimeMs > GRACE_MS;
+        }
+        catch {
+            return true;
+        }
+    }
+    if (!isProcessAlive(holder.pid))
+        return true;
+    if (typeof holder.leaseUntil === "number" && Number.isFinite(holder.leaseUntil)) {
+        return Date.now() > holder.leaseUntil;
+    }
+    try {
+        return Date.now() - (0, node_fs_1.statSync)(lockPath).mtimeMs > staleMs;
+    }
+    catch {
+        return true;
+    }
+}
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        // ESRCH = no such process; EPERM = exists but not ours (still alive).
+        return err.code === "EPERM";
+    }
+}

package/dist/llm.d.ts

@@ -32,6 +32,31 @@ export interface LlmConfig {
     ollamaFlushEvery?: number;
     /** Ms to wait after an ollama flush for the runner to release. */
     ollamaFlushWaitMs?: number;
+    /** ONE per-call timeout ceiling for every phase (#337). Default 900000 — the
+     *  AbortSignal.timeout value passed by all four phase wrappers (the old
+     *  600000 scoring special-case is gone). See HicortexConfig.llmTimeoutMs. */
+    timeoutMs?: number;
+    /** Consecutive ladder-exhausted total failures before the circuit breaker
+     *  opens (#337). Default 3; 0 disables. See HicortexConfig.llmBreakerThreshold. */
+    breakerThreshold?: number;
+    /** How long an OPEN breaker stays open before the next call becomes a trial
+     *  (#337). Default 600000. See HicortexConfig.llmBreakerCooldownMs. */
+    breakerCooldownMs?: number;
+    /** Timeout for the readiness probe's single generation attempt (#337).
+     *  Default 60000. See HicortexConfig.llmProbeTimeoutMs. */
+    probeTimeoutMs?: number;
+    /** TTL the daemon caches a probe outcome for (#337). Default 300000.
+     *  See HicortexConfig.llmProbeTtlMs. */
+    probeTtlMs?: number;
+    /** Single-flight serialization: at most ONE in-flight LLM request per
+     *  endpoint, ever (#355). Default true — a correctness property for local
+     *  single-user model servers (two concurrent large-context calls stall/OOM
+     *  the server and the machine under it). See HicortexConfig.llmSingleFlight. */
+    singleFlight?: boolean;
+    /** How long a queued call waits for the in-flight call before failing as
+     *  endpoint-down (#355). Default 900000 — the same ceiling as llmTimeoutMs.
+     *  See HicortexConfig.llmSingleFlightWaitMs. */
+    singleFlightWaitMs?: number;
 }
 /**
  * Resolve LLM configuration from explicit config-file overrides or
@@ -135,6 +160,18 @@ export interface LlmResult {
     text: string;
     usage?: LlmUsage;
 }
+/** Thrown when the per-endpoint circuit breaker is open (#337) — no network I/O happened. */
+export declare class LlmCircuitOpenError extends Error {
+    constructor(endpoint: string, cooldownRemainingMs: number);
+}
+/**
+ * The TOTAL-failure class the retry ladder matches (#337, unchanged strings —
+ * this is the same matcher the ladder has always used, now also the breaker's
+ * definition of "endpoint may be down"). A fast HTTP 500, a parse error, or a
+ * rate limit is NOT in this class: those prove the endpoint ANSWERS.
+ */
+declare function isTotalFailure(message: string): boolean;
+export { isTotalFailure };
 export declare class LlmClient {
     private config;
     private ollamaCallCount;
@@ -144,6 +181,18 @@ export declare class LlmClient {
     private get rateLimitedUntil();
     /** Check if we're currently rate limited */
     get isRateLimited(): boolean;
+    /**
+     * True once the endpoint's circuit breaker has tripped and no success has
+     * reset it since (#337). Note this stays true past the cooldown until a
+     * trial succeeds — "past cooldown" means calls are LET THROUGH (one trial),
+     * not "healthy". The nightly reads this after runConsolidation to override
+     * a fail-soft "completed" report with "endpoint_down".
+     */
+    get breakerOpen(): boolean;
+    /** Record one ladder-exhausted TOTAL failure; open the breaker at threshold. */
+    private recordBreakerFailure;
+    /** Any success resets the endpoint's breaker (closed + counter zeroed). */
+    private resetBreaker;
     private handleRateLimit;
     /**
      * Fast-tier completion (importance scoring, simple tasks). One model serves
@@ -168,8 +217,34 @@ export declare class LlmClient {
      * serves all phases (#231) — thin wrapper kept for call-site readability.
      */
     completeClassify(prompt: string, maxTokens?: number): Promise<LlmResult>;
+    /**
+     * Readiness probe (#337): ONE minimal generation request (max output 1
+     * token) through the normal provider dispatch. Asks the question liveness
+     * checks CANNOT: "can this endpoint GENERATE right now?" — the incident
+     * gateway kept answering /v1/models for hours while every completion hung.
+     * Single attempt: no retry ladder (a dead endpoint must cost one fast
+     * failure, not a 3.5-min ladder), and it never accrues to the circuit
+     * breaker (probing is diagnosis, not traffic). Catch-all → false; the
+     * callers translate that into "endpoint_down" / a 503, never an exception.
+     */
+    probe(timeoutMs?: number): Promise<boolean>;
     private complete;
     private completeOnce;
+    /** The single-flight wait budget for a call with ceiling `timeoutMs`.
+     *  An explicit `llmSingleFlightWaitMs` wins; otherwise the default is
+     *  max(900 s, llmTimeoutMs) so a waiter never gives up before a legitimate
+     *  in-flight call's own (possibly raised) ceiling expires (2nd-review
+     *  finding 2 — a hardcoded 900 s made a raised-timeout install treat a
+     *  healthy-busy endpoint as down). */
+    private flightWaitMs;
+    /** Resolve the flight guard for this call, or undefined when disabled.
+     *  `staleMs` is derived from THIS call's timeout ceiling (≥ the 30-min
+     *  floor, 2× timeout) so a raised `llmTimeoutMs` can never get a live
+     *  call's lock reclaimed mid-flight (CR #355 finding 3). The lock file
+     *  records its own lease, so reclaim is judged by the HOLDER's lease, not
+     *  this waiter's parameter (2nd-review finding 3). */
+    private acquireFlightGuard;
+    private dispatchOnce;
     /**
      * Claude CLI: shell out to `claude -p` for subscription users.
      * No API key needed — uses CC's authenticated session.

package/dist/llm.js

@@ -18,14 +18,29 @@
  * OpenAI, Anthropic, Google, Ollama, OpenRouter, and Claude CLI.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LlmClient = exports.RateLimitError = exports.resolveLlmConfigForCC = void 0;
+exports.LlmClient = exports.LlmCircuitOpenError = exports.RateLimitError = exports.resolveLlmConfigForCC = void 0;
 exports.resolveExplicitLlmConfig = resolveExplicitLlmConfig;
 exports.applyTierTuningOverlay = applyTierTuningOverlay;
 exports.resolveSavedLlmConfig = resolveSavedLlmConfig;
 exports.findClaudeBinary = findClaudeBinary;
 exports.claudeCliConfig = claudeCliConfig;
 exports.probeOllama = probeOllama;
+exports.isTotalFailure = isTotalFailure;
 const config_read_js_1 = require("./config-read.js");
+// #355: single-flight guard + canonical home (where the per-endpoint flight
+// lock files live — the daemon and the nightly share the home, so the lock
+// serializes them across processes).
+const llm_flight_js_1 = require("./llm-flight.js");
+const paths_js_1 = require("./paths.js");
+// #337: the openai-compat + anthropic request paths fetch through undici's OWN
+// fetch with an explicit dispatcher (below). Node's global fetch is also undici
+// under the hood, but with a hidden 5-minute response-HEADER timer that fires
+// FIRST on non-streaming completions — a completion only sends its headers after
+// generation finishes, so a legitimate >5-min generation is abandoned client-side
+// while the server keeps generating for the dead client (the 2026-08-23/24
+// incident's amplification mechanism). The ollama path already streams to dodge
+// this; these paths get the dispatcher instead.
+const undici_1 = require("undici");
 /**
  * Resolve LLM configuration from explicit config-file overrides or
  * Hicortex-specific env vars only. Returns null when nothing explicit is set.
@@ -104,6 +119,36 @@ function applyTierTuningOverlay(llmConfig, savedConfig) {
     if (savedConfig.ollamaFlushWaitMs !== undefined) {
         llmConfig.ollamaFlushWaitMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "ollamaFlushWaitMs", 180000);
     }
+    // #337 resilience knobs. Same boundary discipline as the keys above: absent =
+    // call-site defaults (timeout 900 s, threshold 3, cooldown 10 min, probe
+    // timeout 60 s, probe TTL 5 min), wrong-typed values warn and fall back.
+    // breakerThreshold uses readNonNegativeConfig because 0 is a VALID value
+    // ("disable the breaker") — the same reason ollamaFlushEvery uses it.
+    if (savedConfig.llmTimeoutMs !== undefined) {
+        llmConfig.timeoutMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "llmTimeoutMs", 900000);
+    }
+    if (savedConfig.llmBreakerThreshold !== undefined) {
+        llmConfig.breakerThreshold = (0, config_read_js_1.readNonNegativeConfig)(savedConfig, "llmBreakerThreshold", 3);
+    }
+    if (savedConfig.llmBreakerCooldownMs !== undefined) {
+        llmConfig.breakerCooldownMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "llmBreakerCooldownMs", 600000);
+    }
+    if (savedConfig.llmProbeTimeoutMs !== undefined) {
+        llmConfig.probeTimeoutMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "llmProbeTimeoutMs", 60000);
+    }
+    if (savedConfig.llmProbeTtlMs !== undefined) {
+        llmConfig.probeTtlMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "llmProbeTtlMs", 300000);
+    }
+    // #355 single-flight. Default ON (a correctness property, not a tuning
+    // option); the wait budget defaults to the timeout ceiling so a queued call
+    // never gives up before the in-flight call's own ceiling expires.
+    const singleFlight = (0, config_read_js_1.readStrictBoolean)(savedConfig, "llmSingleFlight");
+    if (singleFlight !== undefined) {
+        llmConfig.singleFlight = singleFlight;
+    }
+    if (savedConfig.llmSingleFlightWaitMs !== undefined) {
+        llmConfig.singleFlightWaitMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "llmSingleFlightWaitMs", 900000);
+    }
 }
 /**
  * Resolve an LlmConfig from a saved ~/.hicortex/config.json object.
@@ -234,6 +279,18 @@ async function probeOllama(baseUrl = "http://localhost:11434") {
 // ---------------------------------------------------------------------------
 // LLM Client class
 // ---------------------------------------------------------------------------
+// #337: one timeout ceiling. undici (Node's fetch implementation) silently
+// enforces a ~5-minute response-header timer and a body timer on every request.
+// On a NON-STREAMING completion the headers only arrive when generation
+// finishes, so that hidden timer — not our AbortSignal — was the real ceiling,
+// and when it fired the server kept generating for the abandoned client. This
+// module-scoped Agent disables both timers for the LLM request paths that opt
+// in (completeAnthropic + completeOpenAiCompat), making `llmTimeoutMs` the only
+// ceiling. Module scope = one connection pool shared by every LlmClient in the
+// process (the ollama path keeps global fetch — it already streams, so the
+// header timer can't fire there; claude-cli is a subprocess with its own
+// timeout).
+const llmDispatcher = new undici_1.Agent({ headersTimeout: 0, bodyTimeout: 0 });
 const DEFAULT_RATE_LIMIT_RETRY_MS = 5 * 60 * 60 * 1000 + 60_000; // 5h01m safety margin
 class RateLimitError extends Error {
     retryAfterMs;
@@ -251,6 +308,28 @@ exports.RateLimitError = RateLimitError;
 // (#231), so in practice there is a single client per process today; the
 // module-level map keeps the state shared correctly if that ever changes.
 const rateLimitedUntilByEndpoint = new Map();
+const breakerByEndpoint = new Map();
+/** Thrown when the per-endpoint circuit breaker is open (#337) — no network I/O happened. */
+class LlmCircuitOpenError extends Error {
+    constructor(endpoint, cooldownRemainingMs) {
+        super(`LLM circuit breaker open for ${endpoint} — failing fast ` +
+            `(endpoint deemed down; next trial in ~${Math.round(cooldownRemainingMs / 1000)}s)`);
+        this.name = "LlmCircuitOpenError";
+    }
+}
+exports.LlmCircuitOpenError = LlmCircuitOpenError;
+/**
+ * The TOTAL-failure class the retry ladder matches (#337, unchanged strings —
+ * this is the same matcher the ladder has always used, now also the breaker's
+ * definition of "endpoint may be down"). A fast HTTP 500, a parse error, or a
+ * rate limit is NOT in this class: those prove the endpoint ANSWERS.
+ */
+function isTotalFailure(message) {
+    return (message.includes("fetch failed") ||
+        message.includes("ECONNREFUSED") ||
+        message.includes("timeout") ||
+        message.includes("Headers Timeout"));
+}
 class LlmClient {
     config;
     ollamaCallCount = 0;
@@ -268,6 +347,43 @@ class LlmClient {
     get isRateLimited() {
         return Date.now() < this.rateLimitedUntil;
     }
+    /**
+     * True once the endpoint's circuit breaker has tripped and no success has
+     * reset it since (#337). Note this stays true past the cooldown until a
+     * trial succeeds — "past cooldown" means calls are LET THROUGH (one trial),
+     * not "healthy". The nightly reads this after runConsolidation to override
+     * a fail-soft "completed" report with "endpoint_down".
+     */
+    get breakerOpen() {
+        const st = breakerByEndpoint.get(this.endpointKey);
+        return st !== undefined && st.openedAt !== null;
+    }
+    /** Record one ladder-exhausted TOTAL failure; open the breaker at threshold. */
+    recordBreakerFailure() {
+        const threshold = this.config.breakerThreshold ?? 3;
+        if (threshold <= 0)
+            return; // 0 disables — never open, never fast-fail
+        const st = breakerByEndpoint.get(this.endpointKey) ?? { failures: 0, openedAt: null };
+        st.failures += 1;
+        if (st.failures >= threshold) {
+            // (Re)open. Re-opening (a failed trial past cooldown) restarts the
+            // cooldown window from NOW — the endpoint just proved itself still dead.
+            st.openedAt = Date.now();
+            // One structured line per opening — the runbook's grep target. Same
+            // key=value style as event=budget_exhausted (consolidate.ts).
+            console.warn(`[hicortex] event=circuit_open endpoint=${this.endpointKey} ` +
+                `failures=${st.failures} threshold=${threshold} ` +
+                `cooldown_ms=${this.config.breakerCooldownMs ?? 600_000}`);
+        }
+        breakerByEndpoint.set(this.endpointKey, st);
+    }
+    /** Any success resets the endpoint's breaker (closed + counter zeroed). */
+    resetBreaker() {
+        const st = breakerByEndpoint.get(this.endpointKey);
+        if (st && (st.failures !== 0 || st.openedAt !== null)) {
+            breakerByEndpoint.set(this.endpointKey, { failures: 0, openedAt: null });
+        }
+    }
     handleRateLimit(resp) {
         // Parse Retry-After header if present (seconds)
         const retryAfter = resp.headers.get("retry-after");
@@ -289,7 +405,11 @@ class LlmClient {
      */
     async completeFast(prompt, maxTokens) {
         const tokens = maxTokens ?? this.config.maxTokens ?? 8192;
-        const result = await this.complete(this.config.model, prompt, tokens, 600_000);
+        // #337: ONE ceiling for every phase (llmTimeoutMs, default 900 s). The old
+        // 600 s scoring special-case assumed fast-tier calls are short — but the
+        // ceiling only ever mattered when the endpoint was wedged, and a wedged
+        // endpoint wedges scoring too. One knob, one place.
+        const result = await this.complete(this.config.model, prompt, tokens, this.config.timeoutMs ?? 900_000);
         const flushEvery = this.config.ollamaFlushEvery ?? 0;
         if (this.config.provider === "ollama" && flushEvery > 0) {
             this.ollamaCallCount++;
@@ -306,7 +426,7 @@ class LlmClient {
      */
     async completeReflect(prompt, maxTokens) {
         const tokens = maxTokens ?? this.config.maxTokens ?? 8192;
-        return this.complete(this.config.model, prompt, tokens, 900_000);
+        return this.complete(this.config.model, prompt, tokens, this.config.timeoutMs ?? 900_000);
     }
     /**
      * Distillation-tier completion (session knowledge extraction). One model
@@ -314,7 +434,7 @@ class LlmClient {
      */
     async completeDistill(prompt, maxTokens) {
         const tokens = maxTokens ?? this.config.maxTokens ?? 8192;
-        return this.complete(this.config.model, prompt, tokens, 900_000);
+        return this.complete(this.config.model, prompt, tokens, this.config.timeoutMs ?? 900_000);
     }
     /**
      * Classification-tier completion (memory tag classification). One model
@@ -322,9 +442,40 @@ class LlmClient {
      */
     async completeClassify(prompt, maxTokens) {
         const tokens = maxTokens ?? this.config.maxTokens ?? 8192;
-        return this.complete(this.config.model, prompt, tokens, 900_000);
+        return this.complete(this.config.model, prompt, tokens, this.config.timeoutMs ?? 900_000);
+    }
+    /**
+     * Readiness probe (#337): ONE minimal generation request (max output 1
+     * token) through the normal provider dispatch. Asks the question liveness
+     * checks CANNOT: "can this endpoint GENERATE right now?" — the incident
+     * gateway kept answering /v1/models for hours while every completion hung.
+     * Single attempt: no retry ladder (a dead endpoint must cost one fast
+     * failure, not a 3.5-min ladder), and it never accrues to the circuit
+     * breaker (probing is diagnosis, not traffic). Catch-all → false; the
+     * callers translate that into "endpoint_down" / a 503, never an exception.
+     */
+    async probe(timeoutMs) {
+        try {
+            const budget = timeoutMs ?? this.config.probeTimeoutMs ?? 60_000;
+            await this.completeOnce(this.config.model, "Reply with OK.", 1, budget, budget);
+            return true;
+        }
+        catch {
+            return false;
+        }
     }
     async complete(model, prompt, maxTokens, timeoutMs) {
+        // Breaker BEFORE anything else (#337) — an open breaker must cost zero
+        // network I/O and zero ladder time. Past the cooldown we fall through:
+        // this call IS the trial.
+        const breakerSt = breakerByEndpoint.get(this.endpointKey);
+        if (breakerSt !== undefined && breakerSt.openedAt !== null) {
+            const elapsed = Date.now() - breakerSt.openedAt;
+            const cooldownMs = this.config.breakerCooldownMs ?? 600_000;
+            if (elapsed < cooldownMs) {
+                throw new LlmCircuitOpenError(this.endpointKey, cooldownMs - elapsed);
+            }
+        }
         if (this.isRateLimited) {
             throw new RateLimitError(this.rateLimitedUntil - Date.now());
         }
@@ -332,24 +483,83 @@ class LlmClient {
         let lastErr;
         for (let attempt = 0; attempt <= retryDelays.length; attempt++) {
             try {
-                return await this.completeOnce(model, prompt, maxTokens, timeoutMs);
+                const result = await this.completeOnce(model, prompt, maxTokens, timeoutMs);
+                this.resetBreaker(); // any success closes the endpoint's breaker
+                return result;
             }
             catch (err) {
                 lastErr = err instanceof Error ? err : new Error(String(err));
-                const msg = lastErr.message;
-                if (attempt < retryDelays.length && (msg.includes("fetch failed") || msg.includes("ECONNREFUSED") || msg.includes("timeout") || msg.includes("Headers Timeout"))) {
+                if (attempt < retryDelays.length && isTotalFailure(lastErr.message)) {
                     const delay = retryDelays[attempt];
-                    console.log(`[hicortex] LLM call failed (${msg.slice(0, 60)}), retry ${attempt + 1}/${retryDelays.length} in ${delay / 1000}s...`);
+                    console.log(`[hicortex] LLM call failed (${lastErr.message.slice(0, 60)}), retry ${attempt + 1}/${retryDelays.length} in ${delay / 1000}s...`);
                     await new Promise(r => setTimeout(r, delay));
                 }
-                else {
+                else if (!isTotalFailure(lastErr.message)) {
+                    // Non-retryable (HTTP error with a response, parse error,
+                    // RateLimitError): fail this call now — and it NEVER accrues to the
+                    // breaker; an endpoint that answers is not breaker-down.
                     throw lastErr;
                 }
+                // else: total-class failure on the LAST attempt — the ladder is
+                // exhausted; fall out of the loop to count + throw below.
             }
         }
+        // Ladder exhausted on total-class errors — the only path that accrues to
+        // the breaker (#337).
+        this.recordBreakerFailure();
         throw lastErr;
     }
-    async completeOnce(model, prompt, maxTokens, timeoutMs) {
+    async completeOnce(model, prompt, maxTokens, timeoutMs, 
+    /** Optional cap on the single-flight WAIT budget (the probe passes its
+     *  own timeout so a busy endpoint costs one bounded failure, not the full
+     *  900 s queue budget — the #337 "one fast failure" contract). */
+    maxFlightWaitMs) {
+        // #355 single-flight: at most ONE in-flight request per endpoint, ever —
+        // wrapping the per-attempt dispatch (not the ladder) so the lock is
+        // released between retries and a crashed attempt cannot outlive its call.
+        // The probe serializes too: a probe racing a real call would be exactly
+        // the two-concurrent-callers pattern this guard exists to prevent.
+        // Fail-open by construction: acquireLlmFlight never throws; a filesystem
+        // refusal degrades to unserialized dispatch.
+        const waitMs = Math.min(this.flightWaitMs(timeoutMs), maxFlightWaitMs ?? Infinity);
+        const guard = await this.acquireFlightGuard(timeoutMs, waitMs);
+        if (guard?.kind === "timeout") {
+            // Message deliberately contains "timeout" so isTotalFailure() matches:
+            // the ladder retries it and the breaker accrues on exhaustion — a
+            // persistently contended endpoint is indistinguishable from a slow one.
+            throw new Error(`single-flight wait timeout after ${waitMs}ms for ${this.endpointKey} — ` +
+                `another LLM call holds the flight lock (treated as endpoint-down)`);
+        }
+        try {
+            return await this.dispatchOnce(model, prompt, maxTokens, timeoutMs);
+        }
+        finally {
+            if (guard?.kind === "acquired")
+                guard.release();
+        }
+    }
+    /** The single-flight wait budget for a call with ceiling `timeoutMs`.
+     *  An explicit `llmSingleFlightWaitMs` wins; otherwise the default is
+     *  max(900 s, llmTimeoutMs) so a waiter never gives up before a legitimate
+     *  in-flight call's own (possibly raised) ceiling expires (2nd-review
+     *  finding 2 — a hardcoded 900 s made a raised-timeout install treat a
+     *  healthy-busy endpoint as down). */
+    flightWaitMs(timeoutMs) {
+        return this.config.singleFlightWaitMs ?? Math.max(900_000, timeoutMs);
+    }
+    /** Resolve the flight guard for this call, or undefined when disabled.
+     *  `staleMs` is derived from THIS call's timeout ceiling (≥ the 30-min
+     *  floor, 2× timeout) so a raised `llmTimeoutMs` can never get a live
+     *  call's lock reclaimed mid-flight (CR #355 finding 3). The lock file
+     *  records its own lease, so reclaim is judged by the HOLDER's lease, not
+     *  this waiter's parameter (2nd-review finding 3). */
+    async acquireFlightGuard(timeoutMs, waitMs) {
+        if (this.config.singleFlight === false)
+            return undefined; // kill switch
+        const staleMs = Math.max(llm_flight_js_1.DEFAULT_STALE_MS, 2 * timeoutMs);
+        return (0, llm_flight_js_1.acquireLlmFlight)((0, paths_js_1.hicortexHome)(), this.endpointKey, waitMs, staleMs);
+    }
+    async dispatchOnce(model, prompt, maxTokens, timeoutMs) {
         if (this.config.provider === "claude-cli") {
             return this.completeClaude(model, prompt, timeoutMs);
         }
@@ -512,7 +722,7 @@ class LlmClient {
         const baseUrl = this.config.baseUrl.replace(/\/$/, "");
         const hasVersion = /\/v\d+\/?$/.test(baseUrl);
         const url = hasVersion ? `${baseUrl}/messages` : `${baseUrl}/v1/messages`;
-        const resp = await fetch(url, {
+        const resp = await (0, undici_1.fetch)(url, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
@@ -525,6 +735,8 @@ class LlmClient {
                 max_tokens: maxTokens,
             }),
             signal: AbortSignal.timeout(timeoutMs),
+            // #337: disable undici's hidden header/body timers (see llmDispatcher).
+            dispatcher: llmDispatcher,
         });
         if (resp.status === 429)
             this.handleRateLimit(resp);
@@ -584,11 +796,13 @@ class LlmClient {
         if (thinking !== undefined) {
             body.chat_template_kwargs = { enable_thinking: thinking };
         }
-        const resp = await fetch(url, {
+        const resp = await (0, undici_1.fetch)(url, {
             method: "POST",
             headers,
             body: JSON.stringify(body),
             signal: AbortSignal.timeout(timeoutMs),
+            // #337: disable undici's hidden header/body timers (see llmDispatcher).
+            dispatcher: llmDispatcher,
         });
         if (resp.status === 429)
             this.handleRateLimit(resp);

package/dist/mcp-server.d.ts

@@ -12,6 +12,22 @@
  */
 import express from "express";
 import type { MemorySearchResult } from "./types.js";
+/**
+ * Resolve the /distill probe gate (#337): true when the endpoint recently
+ * proved it can GENERATE (cached outcome inside its TTL, or a fresh probe),
+ * false when the probe failed — the caller answers 503 so the capture client
+ * holds its cursor and retries next run (nothing lost, dup-over-loss).
+ * Structural `llm` parameter (anything with probe()) so wiring tests drive
+ * the real cache + TTL discipline with a counting stub.
+ */
+export declare function resolveDistillProbeGate(llm: {
+    probe(timeoutMs?: number): Promise<boolean>;
+}, llmConfig: {
+    provider: string;
+    model: string;
+    baseUrl: string;
+    probeTtlMs?: number;
+}): Promise<boolean>;
 /**
  * Resolve the request body-size limit in MB (#7, #328 item 2b). Pure —
  * exported for tests. Precedence: HICORTEX_DISTILL_BODY_LIMIT_MB env >