@gamaze/hicortex 0.19.5 → 0.19.6

package/README.md

@@ -234,6 +234,11 @@ Config at `~/.hicortex/config.json`. Created by `init`. Key options:
 | `maxTokens` | Max output tokens for all phases (default 8192). A ceiling, not a target — the model stops early when done. |
 | `ollamaFlushEvery` | Flush ollama's accumulated memory every N scoring calls. **Off by default (0)** — opt-in only for an **ollama** install whose runner RSS growth (~171 MB/call) swap-thrashes long consolidations on a RAM-constrained box; N=15 caps a cycle at ~2.5 GB. Gated on the provider being ollama (local **or** remote) — no effect for non-ollama providers. Only you can judge whether your ollama endpoint actually suffers the growth (a managed/cloud ollama host may not), so it stays off until you set it. |
 | `ollamaFlushWaitMs` | Milliseconds to wait after an ollama flush for the runner to exit + release memory (default 180000 = 3 min). |
+| `llmTimeoutMs` | The ONE timeout ceiling on every LLM call in every phase (default 900000 = 15 min). The LLM request paths disable the HTTP client's hidden 5-minute response-header timer, so this knob is the only bound — one place to tune when the endpoint is slow, no per-phase special cases. |
+| `llmBreakerThreshold` | Consecutive fully-failed LLM calls (after their built-in retry ladder) that open the per-endpoint circuit breaker (default 3; **0 disables the breaker**). While open, calls fail fast with no network I/O; an HTTP error with a response body, a malformed-reply parse error, or a rate-limit 429 never counts — only the endpoint being unreachable/hung does. |
+| `llmBreakerCooldownMs` | How long the breaker stays open before one half-open trial call goes out (default 600000 = 10 min). A failing trial re-opens it; a succeeding one resets the counter. |
+| `llmProbeTimeoutMs` | Patience of the readiness probe — one minimal 1-token generation request the nightly sends before consolidating and the daemon sends before distilling (default 60000 = 1 min). Catches a gateway that answers health/model-list queries while generation is dead; a failed probe skips consolidation (`endpoint_down`, retried next run) and answers `/distill` with a 503 so capture holds its cursor. |
+| `llmProbeTtlMs` | How long the daemon caches a `/distill` probe outcome (default 300000 = 5 min). A healthy capture cadence pays at most one probe per window; a dead endpoint turns into fast cached 503s instead of every request paying the probe timeout. |
 | `authToken` | Bearer token for endpoint auth. Generated on first `init` in server mode. Find the active token with `hicortex status` or in `~/.hicortex/config.json`. |
 | `corsAllowedOrigins` | Browser origins allowed to read cross-origin responses, e.g. `["https://ui.example.com"]`. **Empty by default** — the server sends no `Access-Control-Allow-Origin` and never `Allow-Credentials`, so no external web page can read its data. The bundled `/viz` and `/identity/ui` pages are same-origin and need no entry. |
 | `licenseKey` | Commercial license key (optional; for display in `hicortex status`) |

package/dist/llm.d.ts

@@ -32,6 +32,22 @@ export interface LlmConfig {
     ollamaFlushEvery?: number;
     /** Ms to wait after an ollama flush for the runner to release. */
     ollamaFlushWaitMs?: number;
+    /** ONE per-call timeout ceiling for every phase (#337). Default 900000 — the
+     *  AbortSignal.timeout value passed by all four phase wrappers (the old
+     *  600000 scoring special-case is gone). See HicortexConfig.llmTimeoutMs. */
+    timeoutMs?: number;
+    /** Consecutive ladder-exhausted total failures before the circuit breaker
+     *  opens (#337). Default 3; 0 disables. See HicortexConfig.llmBreakerThreshold. */
+    breakerThreshold?: number;
+    /** How long an OPEN breaker stays open before the next call becomes a trial
+     *  (#337). Default 600000. See HicortexConfig.llmBreakerCooldownMs. */
+    breakerCooldownMs?: number;
+    /** Timeout for the readiness probe's single generation attempt (#337).
+     *  Default 60000. See HicortexConfig.llmProbeTimeoutMs. */
+    probeTimeoutMs?: number;
+    /** TTL the daemon caches a probe outcome for (#337). Default 300000.
+     *  See HicortexConfig.llmProbeTtlMs. */
+    probeTtlMs?: number;
 }
 /**
  * Resolve LLM configuration from explicit config-file overrides or
@@ -135,6 +151,10 @@ export interface LlmResult {
     text: string;
     usage?: LlmUsage;
 }
+/** Thrown when the per-endpoint circuit breaker is open (#337) — no network I/O happened. */
+export declare class LlmCircuitOpenError extends Error {
+    constructor(endpoint: string, cooldownRemainingMs: number);
+}
 export declare class LlmClient {
     private config;
     private ollamaCallCount;
@@ -144,6 +164,18 @@ export declare class LlmClient {
     private get rateLimitedUntil();
     /** Check if we're currently rate limited */
     get isRateLimited(): boolean;
+    /**
+     * True once the endpoint's circuit breaker has tripped and no success has
+     * reset it since (#337). Note this stays true past the cooldown until a
+     * trial succeeds — "past cooldown" means calls are LET THROUGH (one trial),
+     * not "healthy". The nightly reads this after runConsolidation to override
+     * a fail-soft "completed" report with "endpoint_down".
+     */
+    get breakerOpen(): boolean;
+    /** Record one ladder-exhausted TOTAL failure; open the breaker at threshold. */
+    private recordBreakerFailure;
+    /** Any success resets the endpoint's breaker (closed + counter zeroed). */
+    private resetBreaker;
     private handleRateLimit;
     /**
      * Fast-tier completion (importance scoring, simple tasks). One model serves
@@ -168,6 +200,17 @@ export declare class LlmClient {
      * serves all phases (#231) — thin wrapper kept for call-site readability.
      */
     completeClassify(prompt: string, maxTokens?: number): Promise<LlmResult>;
+    /**
+     * Readiness probe (#337): ONE minimal generation request (max output 1
+     * token) through the normal provider dispatch. Asks the question liveness
+     * checks CANNOT: "can this endpoint GENERATE right now?" — the incident
+     * gateway kept answering /v1/models for hours while every completion hung.
+     * Single attempt: no retry ladder (a dead endpoint must cost one fast
+     * failure, not a 3.5-min ladder), and it never accrues to the circuit
+     * breaker (probing is diagnosis, not traffic). Catch-all → false; the
+     * callers translate that into "endpoint_down" / a 503, never an exception.
+     */
+    probe(timeoutMs?: number): Promise<boolean>;
     private complete;
     private completeOnce;
     /**

package/dist/llm.js

@@ -18,7 +18,7 @@
  * OpenAI, Anthropic, Google, Ollama, OpenRouter, and Claude CLI.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LlmClient = exports.RateLimitError = exports.resolveLlmConfigForCC = void 0;
+exports.LlmClient = exports.LlmCircuitOpenError = exports.RateLimitError = exports.resolveLlmConfigForCC = void 0;
 exports.resolveExplicitLlmConfig = resolveExplicitLlmConfig;
 exports.applyTierTuningOverlay = applyTierTuningOverlay;
 exports.resolveSavedLlmConfig = resolveSavedLlmConfig;
@@ -26,6 +26,15 @@ exports.findClaudeBinary = findClaudeBinary;
 exports.claudeCliConfig = claudeCliConfig;
 exports.probeOllama = probeOllama;
 const config_read_js_1 = require("./config-read.js");
+// #337: the openai-compat + anthropic request paths fetch through undici's OWN
+// fetch with an explicit dispatcher (below). Node's global fetch is also undici
+// under the hood, but with a hidden 5-minute response-HEADER timer that fires
+// FIRST on non-streaming completions — a completion only sends its headers after
+// generation finishes, so a legitimate >5-min generation is abandoned client-side
+// while the server keeps generating for the dead client (the 2026-08-23/24
+// incident's amplification mechanism). The ollama path already streams to dodge
+// this; these paths get the dispatcher instead.
+const undici_1 = require("undici");
 /**
  * Resolve LLM configuration from explicit config-file overrides or
  * Hicortex-specific env vars only. Returns null when nothing explicit is set.
@@ -104,6 +113,26 @@ function applyTierTuningOverlay(llmConfig, savedConfig) {
     if (savedConfig.ollamaFlushWaitMs !== undefined) {
         llmConfig.ollamaFlushWaitMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "ollamaFlushWaitMs", 180000);
     }
+    // #337 resilience knobs. Same boundary discipline as the keys above: absent =
+    // call-site defaults (timeout 900 s, threshold 3, cooldown 10 min, probe
+    // timeout 60 s, probe TTL 5 min), wrong-typed values warn and fall back.
+    // breakerThreshold uses readNonNegativeConfig because 0 is a VALID value
+    // ("disable the breaker") — the same reason ollamaFlushEvery uses it.
+    if (savedConfig.llmTimeoutMs !== undefined) {
+        llmConfig.timeoutMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "llmTimeoutMs", 900000);
+    }
+    if (savedConfig.llmBreakerThreshold !== undefined) {
+        llmConfig.breakerThreshold = (0, config_read_js_1.readNonNegativeConfig)(savedConfig, "llmBreakerThreshold", 3);
+    }
+    if (savedConfig.llmBreakerCooldownMs !== undefined) {
+        llmConfig.breakerCooldownMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "llmBreakerCooldownMs", 600000);
+    }
+    if (savedConfig.llmProbeTimeoutMs !== undefined) {
+        llmConfig.probeTimeoutMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "llmProbeTimeoutMs", 60000);
+    }
+    if (savedConfig.llmProbeTtlMs !== undefined) {
+        llmConfig.probeTtlMs = (0, config_read_js_1.readPositiveConfig)(savedConfig, "llmProbeTtlMs", 300000);
+    }
 }
 /**
  * Resolve an LlmConfig from a saved ~/.hicortex/config.json object.
@@ -234,6 +263,18 @@ async function probeOllama(baseUrl = "http://localhost:11434") {
 // ---------------------------------------------------------------------------
 // LLM Client class
 // ---------------------------------------------------------------------------
+// #337: one timeout ceiling. undici (Node's fetch implementation) silently
+// enforces a ~5-minute response-header timer and a body timer on every request.
+// On a NON-STREAMING completion the headers only arrive when generation
+// finishes, so that hidden timer — not our AbortSignal — was the real ceiling,
+// and when it fired the server kept generating for the abandoned client. This
+// module-scoped Agent disables both timers for the LLM request paths that opt
+// in (completeAnthropic + completeOpenAiCompat), making `llmTimeoutMs` the only
+// ceiling. Module scope = one connection pool shared by every LlmClient in the
+// process (the ollama path keeps global fetch — it already streams, so the
+// header timer can't fire there; claude-cli is a subprocess with its own
+// timeout).
+const llmDispatcher = new undici_1.Agent({ headersTimeout: 0, bodyTimeout: 0 });
 const DEFAULT_RATE_LIMIT_RETRY_MS = 5 * 60 * 60 * 1000 + 60_000; // 5h01m safety margin
 class RateLimitError extends Error {
     retryAfterMs;
@@ -251,6 +292,28 @@ exports.RateLimitError = RateLimitError;
 // (#231), so in practice there is a single client per process today; the
 // module-level map keeps the state shared correctly if that ever changes.
 const rateLimitedUntilByEndpoint = new Map();
+const breakerByEndpoint = new Map();
+/** Thrown when the per-endpoint circuit breaker is open (#337) — no network I/O happened. */
+class LlmCircuitOpenError extends Error {
+    constructor(endpoint, cooldownRemainingMs) {
+        super(`LLM circuit breaker open for ${endpoint} — failing fast ` +
+            `(endpoint deemed down; next trial in ~${Math.round(cooldownRemainingMs / 1000)}s)`);
+        this.name = "LlmCircuitOpenError";
+    }
+}
+exports.LlmCircuitOpenError = LlmCircuitOpenError;
+/**
+ * The TOTAL-failure class the retry ladder matches (#337, unchanged strings —
+ * this is the same matcher the ladder has always used, now also the breaker's
+ * definition of "endpoint may be down"). A fast HTTP 500, a parse error, or a
+ * rate limit is NOT in this class: those prove the endpoint ANSWERS.
+ */
+function isTotalFailure(message) {
+    return (message.includes("fetch failed") ||
+        message.includes("ECONNREFUSED") ||
+        message.includes("timeout") ||
+        message.includes("Headers Timeout"));
+}
 class LlmClient {
     config;
     ollamaCallCount = 0;
@@ -268,6 +331,43 @@ class LlmClient {
     get isRateLimited() {
         return Date.now() < this.rateLimitedUntil;
     }
+    /**
+     * True once the endpoint's circuit breaker has tripped and no success has
+     * reset it since (#337). Note this stays true past the cooldown until a
+     * trial succeeds — "past cooldown" means calls are LET THROUGH (one trial),
+     * not "healthy". The nightly reads this after runConsolidation to override
+     * a fail-soft "completed" report with "endpoint_down".
+     */
+    get breakerOpen() {
+        const st = breakerByEndpoint.get(this.endpointKey);
+        return st !== undefined && st.openedAt !== null;
+    }
+    /** Record one ladder-exhausted TOTAL failure; open the breaker at threshold. */
+    recordBreakerFailure() {
+        const threshold = this.config.breakerThreshold ?? 3;
+        if (threshold <= 0)
+            return; // 0 disables — never open, never fast-fail
+        const st = breakerByEndpoint.get(this.endpointKey) ?? { failures: 0, openedAt: null };
+        st.failures += 1;
+        if (st.failures >= threshold) {
+            // (Re)open. Re-opening (a failed trial past cooldown) restarts the
+            // cooldown window from NOW — the endpoint just proved itself still dead.
+            st.openedAt = Date.now();
+            // One structured line per opening — the runbook's grep target. Same
+            // key=value style as event=budget_exhausted (consolidate.ts).
+            console.warn(`[hicortex] event=circuit_open endpoint=${this.endpointKey} ` +
+                `failures=${st.failures} threshold=${threshold} ` +
+                `cooldown_ms=${this.config.breakerCooldownMs ?? 600_000}`);
+        }
+        breakerByEndpoint.set(this.endpointKey, st);
+    }
+    /** Any success resets the endpoint's breaker (closed + counter zeroed). */
+    resetBreaker() {
+        const st = breakerByEndpoint.get(this.endpointKey);
+        if (st && (st.failures !== 0 || st.openedAt !== null)) {
+            breakerByEndpoint.set(this.endpointKey, { failures: 0, openedAt: null });
+        }
+    }
     handleRateLimit(resp) {
         // Parse Retry-After header if present (seconds)
         const retryAfter = resp.headers.get("retry-after");
@@ -289,7 +389,11 @@ class LlmClient {
      */
     async completeFast(prompt, maxTokens) {
         const tokens = maxTokens ?? this.config.maxTokens ?? 8192;
-        const result = await this.complete(this.config.model, prompt, tokens, 600_000);
+        // #337: ONE ceiling for every phase (llmTimeoutMs, default 900 s). The old
+        // 600 s scoring special-case assumed fast-tier calls are short — but the
+        // ceiling only ever mattered when the endpoint was wedged, and a wedged
+        // endpoint wedges scoring too. One knob, one place.
+        const result = await this.complete(this.config.model, prompt, tokens, this.config.timeoutMs ?? 900_000);
         const flushEvery = this.config.ollamaFlushEvery ?? 0;
         if (this.config.provider === "ollama" && flushEvery > 0) {
             this.ollamaCallCount++;
@@ -306,7 +410,7 @@ class LlmClient {
      */
     async completeReflect(prompt, maxTokens) {
         const tokens = maxTokens ?? this.config.maxTokens ?? 8192;
-        return this.complete(this.config.model, prompt, tokens, 900_000);
+        return this.complete(this.config.model, prompt, tokens, this.config.timeoutMs ?? 900_000);
     }
     /**
      * Distillation-tier completion (session knowledge extraction). One model
@@ -314,7 +418,7 @@ class LlmClient {
      */
     async completeDistill(prompt, maxTokens) {
         const tokens = maxTokens ?? this.config.maxTokens ?? 8192;
-        return this.complete(this.config.model, prompt, tokens, 900_000);
+        return this.complete(this.config.model, prompt, tokens, this.config.timeoutMs ?? 900_000);
     }
     /**
      * Classification-tier completion (memory tag classification). One model
@@ -322,9 +426,39 @@ class LlmClient {
      */
     async completeClassify(prompt, maxTokens) {
         const tokens = maxTokens ?? this.config.maxTokens ?? 8192;
-        return this.complete(this.config.model, prompt, tokens, 900_000);
+        return this.complete(this.config.model, prompt, tokens, this.config.timeoutMs ?? 900_000);
+    }
+    /**
+     * Readiness probe (#337): ONE minimal generation request (max output 1
+     * token) through the normal provider dispatch. Asks the question liveness
+     * checks CANNOT: "can this endpoint GENERATE right now?" — the incident
+     * gateway kept answering /v1/models for hours while every completion hung.
+     * Single attempt: no retry ladder (a dead endpoint must cost one fast
+     * failure, not a 3.5-min ladder), and it never accrues to the circuit
+     * breaker (probing is diagnosis, not traffic). Catch-all → false; the
+     * callers translate that into "endpoint_down" / a 503, never an exception.
+     */
+    async probe(timeoutMs) {
+        try {
+            await this.completeOnce(this.config.model, "Reply with OK.", 1, timeoutMs ?? this.config.probeTimeoutMs ?? 60_000);
+            return true;
+        }
+        catch {
+            return false;
+        }
     }
     async complete(model, prompt, maxTokens, timeoutMs) {
+        // Breaker BEFORE anything else (#337) — an open breaker must cost zero
+        // network I/O and zero ladder time. Past the cooldown we fall through:
+        // this call IS the trial.
+        const breakerSt = breakerByEndpoint.get(this.endpointKey);
+        if (breakerSt !== undefined && breakerSt.openedAt !== null) {
+            const elapsed = Date.now() - breakerSt.openedAt;
+            const cooldownMs = this.config.breakerCooldownMs ?? 600_000;
+            if (elapsed < cooldownMs) {
+                throw new LlmCircuitOpenError(this.endpointKey, cooldownMs - elapsed);
+            }
+        }
         if (this.isRateLimited) {
             throw new RateLimitError(this.rateLimitedUntil - Date.now());
         }
@@ -332,21 +466,30 @@ class LlmClient {
         let lastErr;
         for (let attempt = 0; attempt <= retryDelays.length; attempt++) {
             try {
-                return await this.completeOnce(model, prompt, maxTokens, timeoutMs);
+                const result = await this.completeOnce(model, prompt, maxTokens, timeoutMs);
+                this.resetBreaker(); // any success closes the endpoint's breaker
+                return result;
             }
             catch (err) {
                 lastErr = err instanceof Error ? err : new Error(String(err));
-                const msg = lastErr.message;
-                if (attempt < retryDelays.length && (msg.includes("fetch failed") || msg.includes("ECONNREFUSED") || msg.includes("timeout") || msg.includes("Headers Timeout"))) {
+                if (attempt < retryDelays.length && isTotalFailure(lastErr.message)) {
                     const delay = retryDelays[attempt];
-                    console.log(`[hicortex] LLM call failed (${msg.slice(0, 60)}), retry ${attempt + 1}/${retryDelays.length} in ${delay / 1000}s...`);
+                    console.log(`[hicortex] LLM call failed (${lastErr.message.slice(0, 60)}), retry ${attempt + 1}/${retryDelays.length} in ${delay / 1000}s...`);
                     await new Promise(r => setTimeout(r, delay));
                 }
-                else {
+                else if (!isTotalFailure(lastErr.message)) {
+                    // Non-retryable (HTTP error with a response, parse error,
+                    // RateLimitError): fail this call now — and it NEVER accrues to the
+                    // breaker; an endpoint that answers is not breaker-down.
                     throw lastErr;
                 }
+                // else: total-class failure on the LAST attempt — the ladder is
+                // exhausted; fall out of the loop to count + throw below.
             }
         }
+        // Ladder exhausted on total-class errors — the only path that accrues to
+        // the breaker (#337).
+        this.recordBreakerFailure();
         throw lastErr;
     }
     async completeOnce(model, prompt, maxTokens, timeoutMs) {
@@ -512,7 +655,7 @@ class LlmClient {
         const baseUrl = this.config.baseUrl.replace(/\/$/, "");
         const hasVersion = /\/v\d+\/?$/.test(baseUrl);
         const url = hasVersion ? `${baseUrl}/messages` : `${baseUrl}/v1/messages`;
-        const resp = await fetch(url, {
+        const resp = await (0, undici_1.fetch)(url, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
@@ -525,6 +668,8 @@ class LlmClient {
                 max_tokens: maxTokens,
             }),
             signal: AbortSignal.timeout(timeoutMs),
+            // #337: disable undici's hidden header/body timers (see llmDispatcher).
+            dispatcher: llmDispatcher,
         });
         if (resp.status === 429)
             this.handleRateLimit(resp);
@@ -584,11 +729,13 @@ class LlmClient {
         if (thinking !== undefined) {
             body.chat_template_kwargs = { enable_thinking: thinking };
         }
-        const resp = await fetch(url, {
+        const resp = await (0, undici_1.fetch)(url, {
             method: "POST",
             headers,
             body: JSON.stringify(body),
             signal: AbortSignal.timeout(timeoutMs),
+            // #337: disable undici's hidden header/body timers (see llmDispatcher).
+            dispatcher: llmDispatcher,
         });
         if (resp.status === 429)
             this.handleRateLimit(resp);

package/dist/mcp-server.d.ts

@@ -12,6 +12,22 @@
  */
 import express from "express";
 import type { MemorySearchResult } from "./types.js";
+/**
+ * Resolve the /distill probe gate (#337): true when the endpoint recently
+ * proved it can GENERATE (cached outcome inside its TTL, or a fresh probe),
+ * false when the probe failed — the caller answers 503 so the capture client
+ * holds its cursor and retries next run (nothing lost, dup-over-loss).
+ * Structural `llm` parameter (anything with probe()) so wiring tests drive
+ * the real cache + TTL discipline with a counting stub.
+ */
+export declare function resolveDistillProbeGate(llm: {
+    probe(timeoutMs?: number): Promise<boolean>;
+}, llmConfig: {
+    provider: string;
+    model: string;
+    baseUrl: string;
+    probeTtlMs?: number;
+}): Promise<boolean>;
 /**
  * Resolve the request body-size limit in MB (#7, #328 item 2b). Pure —
  * exported for tests. Precedence: HICORTEX_DISTILL_BODY_LIMIT_MB env >

package/dist/mcp-server.js

@@ -48,6 +48,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveDistillProbeGate = resolveDistillProbeGate;
 exports.resolveBodyLimitMb = resolveBodyLimitMb;
 exports.makeBodyLimitErrorHandler = makeBodyLimitErrorHandler;
 exports.makeContentLengthGate = makeContentLengthGate;
@@ -126,6 +127,33 @@ let memoryInstructionsEnabled = true;
 // Cache detectChunkSize results keyed by "<provider>/<model>@<baseUrl>" so we
 // probe each endpoint once per server boot rather than once per /distill request.
 const chunkSizeCache = new Map();
+// #337: /distill readiness-probe outcomes, keyed like chunkSizeCache. The
+// daemon probes the GENERATION path (one 1-token completion) before
+// distilling — liveness-style endpoint checks cannot catch the incident
+// signature (a wedged gateway that keeps answering /v1/models while every
+// completion hangs). Outcomes are cached for llmProbeTtlMs (default 5 min),
+// so a healthy capture cadence pays at most one probe per window and a DEAD
+// endpoint turns into fast cached 503s instead of every request paying the
+// probe timeout. Module-scoped like chunkSizeCache (state spans requests).
+const distillProbeCache = new Map();
+/**
+ * Resolve the /distill probe gate (#337): true when the endpoint recently
+ * proved it can GENERATE (cached outcome inside its TTL, or a fresh probe),
+ * false when the probe failed — the caller answers 503 so the capture client
+ * holds its cursor and retries next run (nothing lost, dup-over-loss).
+ * Structural `llm` parameter (anything with probe()) so wiring tests drive
+ * the real cache + TTL discipline with a counting stub.
+ */
+async function resolveDistillProbeGate(llm, llmConfig) {
+    const key = `${llmConfig.provider}/${llmConfig.model}@${llmConfig.baseUrl}`;
+    const ttlMs = llmConfig.probeTtlMs ?? 300_000;
+    const cached = distillProbeCache.get(key);
+    if (cached && Date.now() - cached.at < ttlMs)
+        return cached.ok;
+    const ok = await llm.probe();
+    distillProbeCache.set(key, { ok, at: Date.now() });
+    return ok;
+}
 let VERSION = "0.3.x";
 try {
     const pkg = JSON.parse(require("node:fs").readFileSync(require("node:path").join(__dirname, "..", "package.json"), "utf-8"));
@@ -1165,6 +1193,16 @@ async function startServer(options = {}) {
                 return;
             }
         }
+        // #337: readiness gate — cached minimal generation probe BEFORE
+        // detectChunkSize + distillSession, so a dead endpoint never even pays the
+        // chunk-size probe. Placed AFTER the dedup short-circuits (a duplicate
+        // costs nothing and must not trip the gate). On failure: 503 with the
+        // diagnosis — the capture client treats non-201/200 as transient and holds
+        // its cursor (capture.ts), so the segment is retried next run, never lost.
+        if (!(await resolveDistillProbeGate(llm, llmConfig))) {
+            res.status(503).json({ error: "LLM endpoint not generating — session will be retried" });
+            return;
+        }
         // Cache detectChunkSize per endpoint so we probe at most once per server boot.
         // numCtx is passed so chunk size derives from the request's ACTUAL context
         // window (#231, #228) — the chunker and the request agree by construction.

package/dist/nightly.js

@@ -555,6 +555,10 @@ async function runNightly(options = {}) {
         // nothing-to-do short-circuit (zero LLM calls), NOT a failure.
         // "throttled" (#246) = the llmTokensPerMonth fair-use cap was projected to
         // be exceeded, so consolidation was skipped before any LLM call.
+        // "endpoint_down" (#337) = the pre-consolidation readiness probe failed, or
+        // the LLM circuit breaker was open after the run — transient (retried next
+        // run), and NEVER "completed": the stages fail soft, so without this
+        // override a dead-endpoint run would report clean.
         let consolidationStatus;
         // #246: total consolidation tokens consumed this run (hoisted for telemetry
         // + the dashboard snapshot). Undefined when consolidation didn't run at all
@@ -620,95 +624,119 @@ async function runNightly(options = {}) {
                     }
                 }
                 if (consolidationStatus !== "throttled") {
-                    // One model serves all phases (#231) — there is no separate endpoint to
-                    // pre-flight. If the model doesn't answer, `complete()` already retries at
-                    // 30s/60s/120s (~3.5 min); anything still failing after that is an outage,
-                    // not a blip. A failed phase costs latency, not data: capture cursors hold
-                    // on failure (dup-over-loss), and consolidation has resumable cursors
-                    // (domainCursor, supersessionCursor). The nightly runs 2-4×/day, so the
-                    // wait is hours — no polling, no new config. (Issue #231.)
-                    const cfgDomains = (0, domain_classify_js_1.parseConfigDomains)(savedConfig);
-                    console.log(`[hicortex] Running consolidation...`);
-                    const report = await (0, consolidate_js_1.runConsolidation)(db, llm, embedder_js_1.embed, dryRun, false, undefined, {
-                        domains: cfgDomains,
-                        contentDomainsReady: true,
-                        weakPrimaryFloor: (0, nofit_js_1.resolveWeakPrimaryFloor)(savedConfig),
-                    }, {
-                        minSimilarity: savedConfig?.supersessionMinSimilarity,
-                        maxCalls: savedConfig?.supersessionMaxCalls,
-                    }, 
-                    // #241: config-driven total LLM-call ceiling (default 5000, was 200).
-                    (0, config_read_js_1.readPositiveConfig)(savedConfig ?? {}, "consolidateMaxLlmCalls", consolidate_js_1.CONSOLIDATE_MAX_LLM_CALLS), 
-                    // #245: soft cap on the corpus (default 10000; 0 disables eviction).
-                    memorySoftCapResolved);
-                    console.log(`[hicortex] Consolidation ${report.status} in ${report.elapsed_seconds}s` +
-                        (report.stages.reflection ? ` (${report.stages.reflection.lessons_generated} lessons)` : ""));
-                    consolidationStatus = report.status;
-                    // #245: capture the eviction count for the dashboard snapshot. The
-                    // stage always returns `evicted` (0 when under cap / disabled); report
-                    // it as 0 (a real value), not undefined, when the stage ran.
-                    evictedCount = report.stages.memory_cap?.evicted ?? 0;
-                    // Only set when reflection actually RAN (not skipped). A skipped stage
-                    // (e.g. endpoint offline, #232 fail-soft) must NOT collapse to 0 — that
-                    // would make "endpoint down" indistinguishable from "prompt too tight"
-                    // in the fleet aggregate. Leave undefined so the optional field is
-                    // omitted and the aggregate buckets skipped runs separately.
-                    const refl = report.stages.reflection;
-                    if (refl && !refl.skipped)
-                        lessonsGenerated = refl.lessons_generated;
-                    // #246: surface token totals for telemetry + dashboard snapshot. Both
-                    // fields stay undefined on a skipped/failed run (no metered calls →
-                    // nothing to report; the optional fields are omitted from the ping).
-                    const tokensTotal = report.budget?.tokens_total;
-                    if (tokensTotal && tokensTotal.total > 0) {
-                        tokensThisRun = tokensTotal.total;
-                        tokensByStage = report.budget?.tokens_by_stage;
+                    // #337: readiness probe — ONE minimal generation request before any
+                    // consolidation phase. This REPLACES the #231 no-preflight decision
+                    // (its premise "a failed phase costs latency, not data" was falsified
+                    // by the 2026-08-23/24 incident: the gateway answered /v1/models —
+                    // liveness — while generation was dead, and the nightly retried into
+                    // it for ~5 h, making the wedge monotonically worse). A failed probe
+                    // skips consolidation entirely with the diagnosis "LLM endpoint not
+                    // generating" (not-generating, not slow) and status endpoint_down —
+                    // a transient outcome: consolidation has resumable cursors, the
+                    // nightly re-runs 2-4×/day, and capture is unaffected (the daemon's
+                    // /distill path has its own probe). Zero LLM phases run when the
+                    // probe fails — one fast failure is the whole cost.
+                    const probeOk = await llm.probe();
+                    if (!probeOk) {
+                        console.error("[hicortex] LLM endpoint not generating — consolidation skipped " +
+                            "(endpoint_down, will retry next run). See the ops runbook's " +
+                            "known failure signatures; llmProbeTimeoutMs tunes the probe's patience.");
+                        consolidationStatus = "endpoint_down";
                     }
-                    // #255: budget exhaustion — always populated when consolidation ran
-                    // (report.budget.exhausted is a boolean). The dashboard + telemetry
-                    // treat true as a quality-degradation health signal. The
-                    // ran-vs-didn't-run distinction is carried by `budgetCallsUsed`/
-                    // `budgetMaxCalls` (forwarded whenever consolidation ran), NOT by a
-                    // false `budget_exhausted` flag — the snapshot forwards
-                    // `budget_exhausted` only on exhaustion (alert state), so the
-                    // aggregate reads: calls_used present + budget_exhausted undefined
-                    // = "ran and didn't exhaust"; calls_used undefined = "didn't run".
-                    budgetExhausted = report.budget?.exhausted;
-                    budgetDeferredByStage = report.budget?.deferred_by_stage;
-                    budgetCallsUsed = report.budget?.calls_used;
-                    budgetMaxCalls = report.budget?.max_calls;
-                    // #246: accrue to state.json (monthly reset + last-run estimate for
-                    // the next throttle check). Written even on a failed run — a partial
-                    // run that made metered calls before the failure still spent tokens,
-                    // and the next run's estimate should reflect that.
-                    if (!dryRun) {
-                        (0, state_js_1.updateState)((s) => {
-                            const now = new Date();
-                            const cur = s.llmTokensThisPeriod;
-                            let periodStart = cur?.periodStart ?? now.toISOString();
-                            let prompt = cur?.prompt ?? 0;
-                            let completion = cur?.completion ?? 0;
-                            let total = cur?.total ?? 0;
-                            // Monthly reset: if periodStart is in a previous calendar month,
-                            // zero the accrual before adding this run's contribution.
-                            const startD = new Date(periodStart);
-                            if (startD.getUTCFullYear() !== now.getUTCFullYear() ||
-                                startD.getUTCMonth() !== now.getUTCMonth()) {
-                                periodStart = now.toISOString();
-                                prompt = 0;
-                                completion = 0;
-                                total = 0;
-                            }
-                            if (tokensTotal) {
-                                prompt += tokensTotal.prompt;
-                                completion += tokensTotal.completion;
-                                total += tokensTotal.total;
-                            }
-                            s.llmTokensThisPeriod = {
-                                prompt, completion, total, periodStart,
-                            };
-                            s.llmTokensLastRun = tokensTotal?.total ?? 0;
-                        }, stateDir);
+                    else {
+                        const cfgDomains = (0, domain_classify_js_1.parseConfigDomains)(savedConfig);
+                        console.log(`[hicortex] Running consolidation...`);
+                        const report = await (0, consolidate_js_1.runConsolidation)(db, llm, embedder_js_1.embed, dryRun, false, undefined, {
+                            domains: cfgDomains,
+                            contentDomainsReady: true,
+                            weakPrimaryFloor: (0, nofit_js_1.resolveWeakPrimaryFloor)(savedConfig),
+                        }, {
+                            minSimilarity: savedConfig?.supersessionMinSimilarity,
+                            maxCalls: savedConfig?.supersessionMaxCalls,
+                        }, 
+                        // #241: config-driven total LLM-call ceiling (default 5000, was 200).
+                        (0, config_read_js_1.readPositiveConfig)(savedConfig ?? {}, "consolidateMaxLlmCalls", consolidate_js_1.CONSOLIDATE_MAX_LLM_CALLS), 
+                        // #245: soft cap on the corpus (default 10000; 0 disables eviction).
+                        memorySoftCapResolved);
+                        console.log(`[hicortex] Consolidation ${report.status} in ${report.elapsed_seconds}s` +
+                            (report.stages.reflection ? ` (${report.stages.reflection.lessons_generated} lessons)` : ""));
+                        consolidationStatus = report.status;
+                        // #337: the stages fail soft, so a run against an endpoint that died
+                        // MID-run would otherwise report "completed". An open breaker is the
+                        // honest signal — override to endpoint_down (lastConsolidated still
+                        // only advances on a clean "completed", so the work is re-run).
+                        if (llm.breakerOpen) {
+                            console.error(`[hicortex] LLM circuit breaker OPEN after consolidation — ` +
+                                `overriding "${report.status}" to endpoint_down (stages failed ` +
+                                `soft against a down endpoint; will retry next run).`);
+                            consolidationStatus = "endpoint_down";
+                        }
+                        // #245: capture the eviction count for the dashboard snapshot. The
+                        // stage always returns `evicted` (0 when under cap / disabled); report
+                        // it as 0 (a real value), not undefined, when the stage ran.
+                        evictedCount = report.stages.memory_cap?.evicted ?? 0;
+                        // Only set when reflection actually RAN (not skipped). A skipped stage
+                        // (e.g. endpoint offline, #232 fail-soft) must NOT collapse to 0 — that
+                        // would make "endpoint down" indistinguishable from "prompt too tight"
+                        // in the fleet aggregate. Leave undefined so the optional field is
+                        // omitted and the aggregate buckets skipped runs separately.
+                        const refl = report.stages.reflection;
+                        if (refl && !refl.skipped)
+                            lessonsGenerated = refl.lessons_generated;
+                        // #246: surface token totals for telemetry + dashboard snapshot. Both
+                        // fields stay undefined on a skipped/failed run (no metered calls →
+                        // nothing to report; the optional fields are omitted from the ping).
+                        const tokensTotal = report.budget?.tokens_total;
+                        if (tokensTotal && tokensTotal.total > 0) {
+                            tokensThisRun = tokensTotal.total;
+                            tokensByStage = report.budget?.tokens_by_stage;
+                        }
+                        // #255: budget exhaustion — always populated when consolidation ran
+                        // (report.budget.exhausted is a boolean). The dashboard + telemetry
+                        // treat true as a quality-degradation health signal. The
+                        // ran-vs-didn't-run distinction is carried by `budgetCallsUsed`/
+                        // `budgetMaxCalls` (forwarded whenever consolidation ran), NOT by a
+                        // false `budget_exhausted` flag — the snapshot forwards
+                        // `budget_exhausted` only on exhaustion (alert state), so the
+                        // aggregate reads: calls_used present + budget_exhausted undefined
+                        // = "ran and didn't exhaust"; calls_used undefined = "didn't run".
+                        budgetExhausted = report.budget?.exhausted;
+                        budgetDeferredByStage = report.budget?.deferred_by_stage;
+                        budgetCallsUsed = report.budget?.calls_used;
+                        budgetMaxCalls = report.budget?.max_calls;
+                        // #246: accrue to state.json (monthly reset + last-run estimate for
+                        // the next throttle check). Written even on a failed run — a partial
+                        // run that made metered calls before the failure still spent tokens,
+                        // and the next run's estimate should reflect that.
+                        if (!dryRun) {
+                            (0, state_js_1.updateState)((s) => {
+                                const now = new Date();
+                                const cur = s.llmTokensThisPeriod;
+                                let periodStart = cur?.periodStart ?? now.toISOString();
+                                let prompt = cur?.prompt ?? 0;
+                                let completion = cur?.completion ?? 0;
+                                let total = cur?.total ?? 0;
+                                // Monthly reset: if periodStart is in a previous calendar month,
+                                // zero the accrual before adding this run's contribution.
+                                const startD = new Date(periodStart);
+                                if (startD.getUTCFullYear() !== now.getUTCFullYear() ||
+                                    startD.getUTCMonth() !== now.getUTCMonth()) {
+                                    periodStart = now.toISOString();
+                                    prompt = 0;
+                                    completion = 0;
+                                    total = 0;
+                                }
+                                if (tokensTotal) {
+                                    prompt += tokensTotal.prompt;
+                                    completion += tokensTotal.completion;
+                                    total += tokensTotal.total;
+                                }
+                                s.llmTokensThisPeriod = {
+                                    prompt, completion, total, periodStart,
+                                };
+                                s.llmTokensLastRun = tokensTotal?.total ?? 0;
+                            }, stateDir);
+                        }
                     }
                 }
             }

package/dist/telemetry.d.ts

@@ -89,15 +89,19 @@ export interface TelemetryPayload {
      * Consolidation outcome for THIS full nightly (server mode only —
      * capture-only runs send no nightly ping, so the field is absent there).
      * `runConsolidation`'s status: "completed" | "skipped" | "failed", plus
-     * "no_llm" when consolidation was skipped because no LLM was configured, and
+     * "no_llm" when consolidation was skipped because no LLM was configured,
      * "throttled" (#246) when the run was skipped because the
-     * `llmTokensPerMonth` fair-use cap was projected to be exceeded.
+     * `llmTokensPerMonth` fair-use cap was projected to be exceeded, and
+     * "endpoint_down" (#337) when the pre-consolidation readiness probe failed
+     * or the LLM circuit breaker was open after the run — a TRANSIENT state
+     * (retried next run), never reported as "completed" even though the stages
+     * fail soft.
      * "skipped" = the built-in nothing-to-do short-circuit (no new + no unscored
      * memories → zero LLM calls), NOT a failure. Lets the fleet aggregate tell a
      * real consolidation run from a no-op without repurposing `ok` (which is the
      * capture-health signal). 0.17+.
      */
-    consolidation?: "completed" | "skipped" | "failed" | "no_llm" | "throttled";
+    consolidation?: "completed" | "skipped" | "failed" | "no_llm" | "throttled" | "endpoint_down";
     /**
      * Total LLM tokens consumed by THIS nightly's consolidation (#246) — the
      * BudgetTracker total. Server-mode only (capture-only + client runs make no

package/dist/types.d.ts

@@ -392,6 +392,48 @@ export interface HicortexConfig {
      * doubled for margin). Only relevant when `ollamaFlushEvery` > 0.
      */
     ollamaFlushWaitMs?: number;
+    /**
+     * ONE per-attempt timeout ceiling (ms) for every LLM phase — distill,
+     * reflect, classify, and scoring alike (#337). Default 900000 (15 min). The
+     * openai-compat and anthropic requests fetch through an undici dispatcher
+     * with undici's hidden 5-minute header/body timers disabled, so this knob is
+     * the ONLY ceiling: a legitimate long generation is no longer abandoned
+     * client-side at 5 min while the server keeps generating for the dead
+     * client (the 2026-08-23/24 incident's amplification mechanism). Before
+     * #337, scoring used a 600 s ceiling and the other phases 900 s; one knob
+     * now covers all four. No effect on the ollama path (already streams) or
+     * claude-cli (subprocess timeout).
+     */
+    llmTimeoutMs?: number;
+    /**
+     * Consecutive ladder-exhausted TOTAL failures (fetch-failed / ECONNREFUSED /
+     * timeout / "Headers Timeout" class) after which the per-endpoint circuit
+     * breaker opens (#337). Default 3; `0` disables. While open, calls throw
+     * `LlmCircuitOpenError` immediately with NO network I/O. HTTP error statuses
+     * with a response, parse errors, and rate limits never count (they throw
+     * before the retry ladder can be exhausted). Any success resets the counter.
+     */
+    llmBreakerThreshold?: number;
+    /**
+     * How long (ms) an open circuit breaker stays open before the next call
+     * becomes a half-open trial (#337). Default 600000 (10 min). A trial failure
+     * re-opens the breaker; a trial success resets it.
+     */
+    llmBreakerCooldownMs?: number;
+    /**
+     * Timeout (ms) for the readiness probe's single 1-token generation attempt
+     * (#337). Default 60000. The probe asks "can this endpoint GENERATE", which
+     * /health-style liveness checks cannot answer (a wedged gateway keeps
+     * answering /v1/models). Read by the nightly before consolidation and by the
+     * daemon before distilling.
+     */
+    llmProbeTimeoutMs?: number;
+    /**
+     * How long (ms) the daemon caches a /distill probe outcome before probing
+     * again (#337). Default 300000 — a healthy capture cadence pays at most one
+     * probe per window. Nightly runs are single-shot and never cache.
+     */
+    llmProbeTtlMs?: number;
     /**
      * Max lessons injected into an agent's session-start context (default 10).
      * Lessons are ranked per-session by project/domain affinity + recency +

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gamaze/hicortex",
-  "version": "0.19.5",
+  "version": "0.19.6",
   "description": "Persistent agent identity for AI agents \u2014 a hand-edited identity layer, nightly-distilled experience, and lessons injected every session, shared across your whole fleet. Works with Hermes, OpenClaw, Claude Code, and Pi.",
   "main": "dist/index.js",
   "bin": {
@@ -71,6 +71,7 @@
     "better-sqlite3": "^12.11.1",
     "express": "^4.21.0",
     "sqlite-vec": "^0.1.7",
-    "tar-stream": "^2.2.0"
+    "tar-stream": "^2.2.0",
+    "undici": "^8.10.0"
   }
 }