circle-ir-ai 2.43.0 → 2.45.2

package/dist/llm/ax-client.js

@@ -104,6 +104,86 @@ export const crossFileTaintSignature = AxSignature.create(`
  * Maximum consecutive LLM failures before skipping further calls
  */
 const MAX_CONSECUTIVE_FAILURES = 5;
+/**
+ * cognium-ai#217 — project-level dead-endpoint breaker threshold.
+ *
+ * The per-file breaker (`MAX_CONSECUTIVE_FAILURES`) is reset between files
+ * (`resetCircuitBreaker`) so one bad file doesn't disable LLM for the whole
+ * project. But when the *endpoint itself* is dead (empty-body 60s hangs,
+ * timeouts, connection failures on every call — the #217 Gemma-vLLM stall),
+ * that per-file reset means each new file re-pays ~5×60s of dead time before
+ * re-tripping, consuming the entire per-repo wall-clock budget (16/72 Tier-1
+ * repos hit the 60-min cap with zero output).
+ *
+ * This counts *endpoint-dead* failures (empty/timeout/connection) that are
+ * NOT interrupted by a success, and — unlike the per-file breaker — is NOT
+ * cleared by `resetCircuitBreaker`. Once it trips, LLM stays off for the rest
+ * of the scan and the run finishes static-only. Set to 4× the per-file
+ * threshold so a single unlucky file can't project-kill, while a genuinely
+ * dead endpoint trips after ~4 files (~2 min wall-clock at concurrency 10,
+ * vs the 60-min cap). A single success resets the counter, so a healthy
+ * endpoint with scattered blips never trips.
+ */
+const PROJECT_DEAD_THRESHOLD = 20;
+// ---------------------------------------------------------------------------
+// cognium-ai#228 — MODULE-LEVEL circuit-breaker state.
+//
+// `getAxLLMClient()` returns a FRESH `AxLLMClient` per request (intentional —
+// the per-call `lastErrorCategory`/`lastUsage`/`lastAttemptCount` fields would
+// race under concurrency if a client were shared). But that means an
+// *instance*-level breaker never accumulates across the throwaway clients —
+// discovery/enrichment/verification each spin up their own client, so the
+// per-file breaker (#16) and the project-dead breaker (#217) were both no-ops
+// in the real scan path (confirmed on apache/camel: 613 empty-response calls,
+// breaker never tripped). The cap (#160) worked only because it lives in
+// call-limiter.ts at module scope. So the breaker state lives here, shared
+// across every client; the instance methods below are thin accessors.
+let breakerConsecutiveFailures = 0;
+let breakerLLMDisabled = false;
+let breakerConnectionWarned = false;
+let breakerProjectDeadCalls = 0;
+let breakerLLMProjectDead = false;
+let breakerProjectDeadWarned = false;
+/**
+ * cognium-ai#228 — reset ALL breaker state for a fresh scan. Called at the
+ * true top-level scan entry (scanner's file loop, single-file workflow) so a
+ * long-lived process (circle-pack) doesn't carry a tripped breaker across
+ * repos. The per-file `resetCircuitBreaker()` deliberately does NOT clear the
+ * project-dead state (#217); this does.
+ */
+export function resetLLMBreakerForScan() {
+    breakerConsecutiveFailures = 0;
+    breakerLLMDisabled = false;
+    breakerConnectionWarned = false;
+    breakerProjectDeadCalls = 0;
+    breakerLLMProjectDead = false;
+    breakerProjectDeadWarned = false;
+}
+/**
+ * cognium-ai#217 — reset only the PER-FILE breaker (consecutive-failure count
+ * + the soft disable) between files within a scan, so one bad file doesn't
+ * disable LLM for the whole run. Deliberately does NOT clear the project-dead
+ * state — that's the cross-file backstop that must survive per-file resets.
+ */
+export function resetLLMPerFileBreaker() {
+    breakerConsecutiveFailures = 0;
+    breakerLLMDisabled = false;
+    breakerConnectionWarned = false;
+}
+/** cognium-ai#217 — true once the project-dead breaker has tripped. Module
+ *  accessor so callers (CLI degraded-status) don't need a client handle. */
+export function isLLMEndpointDead() {
+    return breakerLLMProjectDead;
+}
+/** cognium-ai#228 — test/inspection accessor for the shared breaker state. */
+export function getLLMBreakerState() {
+    return {
+        consecutiveFailures: breakerConsecutiveFailures,
+        llmDisabled: breakerLLMDisabled,
+        projectDeadCalls: breakerProjectDeadCalls,
+        llmProjectDead: breakerLLMProjectDead,
+    };
+}
 /**
  * Maximum code context length to send to LLM (in characters).
  * Prevents 400 errors from exceeding model context limits AND keeps
@@ -214,6 +294,53 @@ const TRANSIENT_5XX_STATUSES = new Set([500, 502, 503, 504]);
  * proxy doesn't burn the rate-limit budget.
  */
 const MAX_TRANSIENT_5XX_RETRIES = 3;
+/**
+ * cognium-ai#222 — Maximum retries for an *opaque* HTTP 400. Some
+ * OpenAI-compatible providers (observed: Novita.ai `qwen/qwen3-coder-next`)
+ * intermittently return `HTTP 400 invalid_request_error` for requests that
+ * succeed on a later attempt — a transient server-side rejection dressed as
+ * a client error. Small budget, independent of the 5xx/429 budgets.
+ */
+const MAX_TRANSIENT_400_RETRIES = 2;
+/**
+ * cognium-ai#222 — decide whether an HTTP 400 body looks *transient*
+ * (retryable) rather than a deterministic request defect.
+ *
+ * Deterministic 400s carry an actionable, request-shape message we must NOT
+ * retry — OpenAI: `"Unsupported parameter: 'temperature'"`,
+ * `"Use 'max_completion_tokens' instead of 'max_tokens'"`, unknown model,
+ * context-length-exceeded, content-policy. Retrying those just loops.
+ *
+ * Opaque 400s carry only a generic `invalid_request_error` + a `trace_id`
+ * and no actionable parameter name — empirically these are transient
+ * provider hiccups and recover on retry. Default to NOT retrying when the
+ * body is missing/ambiguous (conservative — preserves prior behavior).
+ */
+export function isTransient400(body) {
+    if (!body)
+        return false; // no body → can't prove transient → terminal
+    const b = body.toLowerCase();
+    // Deterministic client-fix signals → never retry.
+    if (b.includes('unsupported parameter') ||
+        b.includes('max_completion_tokens') ||
+        b.includes('unsupported value') ||
+        b.includes('context length') ||
+        b.includes('context_length') ||
+        b.includes('maximum context') ||
+        b.includes('too many tokens') ||
+        b.includes('model_not_found') ||
+        b.includes('does not exist') ||
+        b.includes('content management policy') ||
+        b.includes('content_policy') ||
+        b.includes('content filter')) {
+        return false;
+    }
+    // Opaque provider rejection → treat as transient. Novita emits
+    // `{"message":"invalid request error trace_id: …","type":"invalid_request_error"}`.
+    return (b.includes('invalid_request_error') ||
+        b.includes('invalid request error') ||
+        b.includes('trace_id'));
+}
 /**
  * Strip `<think>...</think>` reasoning blocks from LLM output.
  *
@@ -298,9 +425,9 @@ export class AxLLMClient {
     config;
     ai;
     generators = new Map();
-    consecutiveFailures = 0;
-    llmDisabled = false;
-    connectionFailureWarned = false;
+    // cognium-ai#228 — circuit-breaker state is module-level (see top of file),
+    // NOT per-instance, because `getAxLLMClient()` hands out a fresh client per
+    // request. The breaker methods below read/write the shared `breaker*` vars.
     /**
      * Set by `_chatJSONImpl` on failure so the outer `chatJSON` wrapper
      * can write a categorical reason to the JSONL log without
@@ -386,7 +513,7 @@ export class AxLLMClient {
      * Check if LLM is currently available (not disabled due to failures)
      */
     isAvailable() {
-        return !this.llmDisabled;
+        return !breakerLLMDisabled;
     }
     /**
      * Reset the circuit breaker so LLM calls are re-enabled.
@@ -394,23 +521,55 @@ export class AxLLMClient {
      * LLM for the entire project analysis.
      */
     resetCircuitBreaker() {
-        this.consecutiveFailures = 0;
-        this.llmDisabled = false;
+        resetLLMPerFileBreaker();
+    }
+    /**
+     * cognium-ai#217 — true once the project-level dead-endpoint breaker has
+     * tripped (the LLM endpoint returned empty/timeout/connection failures on
+     * ~4 files' worth of calls with no intervening success). The scan then
+     * runs static-only; callers can surface a `degraded: llm-unavailable`
+     * status instead of reporting a plain size timeout.
+     */
+    isLLMProjectDead() {
+        return breakerLLMProjectDead;
     }
     /**
-     * Reset failure counter (call when LLM succeeds)
+     * Reset the per-file failure counter. Called on any HTTP 200 — note an
+     * HTTP-200-but-empty body is NOT a genuine success (it's the #217
+     * dead-endpoint signature), so the project-dead counter is reset elsewhere
+     * (on genuine content success in `chatJSON`), not here.
      */
     resetFailures() {
-        this.consecutiveFailures = 0;
+        breakerConsecutiveFailures = 0;
     }
     /**
      * Track a failure and potentially disable LLM
      */
     trackFailure() {
-        this.consecutiveFailures++;
-        if (this.consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+        breakerConsecutiveFailures++;
+        if (breakerConsecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
             console.warn(`LLM disabled after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`);
-            this.llmDisabled = true;
+            breakerLLMDisabled = true;
+        }
+    }
+    /**
+     * cognium-ai#217 — track an *endpoint-dead* failure (empty response,
+     * timeout, or connection error). Unlike {@link trackFailure}, this counter
+     * survives the per-file `resetCircuitBreaker()`, so a wholesale endpoint
+     * stall trips a project-level breaker and the scan finishes static-only
+     * rather than re-paying dead time on every file to the wall-clock cap.
+     */
+    trackDeadCall() {
+        breakerProjectDeadCalls++;
+        if (!breakerLLMProjectDead &&
+            breakerProjectDeadCalls >= PROJECT_DEAD_THRESHOLD) {
+            breakerLLMProjectDead = true;
+            if (!breakerProjectDeadWarned) {
+                breakerProjectDeadWarned = true;
+                console.warn(`\nLLM endpoint appears dead: ${breakerProjectDeadCalls} empty/timeout/connection ` +
+                    `failures with no success (${this.config.baseUrl}). Disabling LLM for the rest ` +
+                    `of this scan and continuing static-only. Use --no-llm to silence.\n`);
+            }
         }
     }
     /**
@@ -450,14 +609,15 @@ export class AxLLMClient {
      * default LLM endpoint isn't reachable).
      */
     tripBreakerOnConnectionFailure(err) {
-        if (!this.connectionFailureWarned) {
-            this.connectionFailureWarned = true;
+        if (!breakerConnectionWarned) {
+            breakerConnectionWarned = true;
             const msg = err instanceof Error ? err.message : String(err);
             console.warn(`\nLLM endpoint unreachable (${this.config.baseUrl}): ${msg}`);
             console.warn('Falling back to static analysis. Use --no-llm to silence, or set LLM_BASE_URL/LLM_API_KEY.\n');
         }
-        this.consecutiveFailures = MAX_CONSECUTIVE_FAILURES;
-        this.llmDisabled = true;
+        breakerConsecutiveFailures = MAX_CONSECUTIVE_FAILURES;
+        breakerLLMDisabled = true;
+        this.trackDeadCall(); // #217: also counts toward the project-dead breaker
     }
     /**
      * Get or create a generator for a signature.
@@ -783,6 +943,14 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
         catch (e) {
             errorMsg = e instanceof Error ? e.message : String(e);
         }
+        // cognium-ai#217 — a genuine content success proves the endpoint is
+        // alive: reset the project-dead counter so scattered failures on a
+        // healthy endpoint never accumulate to the trip. (An HTTP-200-but-empty
+        // body returns null here, so it does NOT reset — that's the dead
+        // signature the breaker is counting.)
+        if (result !== null && errorMsg === undefined) {
+            breakerProjectDeadCalls = 0;
+        }
         // Derive the category: thrown exceptions take precedence; otherwise
         // use whatever `_chatJSONImpl` recorded. A null result with no
         // category is the historical "parse_error" path (no closer cause).
@@ -851,8 +1019,9 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
      * consecutive-failure circuit breaker (rate limits are transient).
      */
     async _chatJSONImpl(systemPrompt, userPrompt, phase = 'enrichment', retryCount = 0, rateLimitRetryCount = 0) {
-        // Skip if LLM has been disabled due to repeated failures
-        if (this.llmDisabled) {
+        // Skip if LLM has been disabled due to repeated failures (per-file
+        // breaker) or the endpoint is dead across the whole scan (#217).
+        if (breakerLLMDisabled || breakerLLMProjectDead) {
             return null;
         }
         // Use the correct phase config
@@ -864,13 +1033,16 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
         const maxRetries = 1; // parse / timeout / network retries
         const maxRateLimitRetries = MAX_RATE_LIMIT_RETRIES;
         const maxTransient5xxRetries = MAX_TRANSIENT_5XX_RETRIES;
+        const maxTransient400Retries = MAX_TRANSIENT_400_RETRIES;
         let attempt = retryCount; // 0 = initial; >0 = reduced-token retry
         let rateLimitAttempt = rateLimitRetryCount;
         let transient5xxAttempt = 0; // independent budget for 5xx
+        let transient400Attempt = 0; // cognium-ai#222: budget for opaque 400
         // Per-attempt logging (cognium-ai#87). One JSONL `kind:'attempt'` entry per
         // loop iteration when LLM_LOG_JSONL or LLM_DEEP is set. Upper-bound on
         // total possible iterations covers all three independent retry budgets.
-        const maxAttempts = maxRetries + 1 + maxRateLimitRetries + maxTransient5xxRetries;
+        const maxAttempts = maxRetries + 1 + maxRateLimitRetries + maxTransient5xxRetries +
+            maxTransient400Retries;
         let attemptIdx = 0;
         const promptChars = systemPrompt.length + userPrompt.length;
         const emitAttempt = (outcome, attemptStartMs, extras = {}) => {
@@ -905,13 +1077,13 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
         // error, so retrying is just burning the failure budget. One clear
         // diagnostic on stderr, then static-only for the rest of the run.
         const stripped = phaseConfig.model.replace(/^[a-z]+\//, '');
-        if (/-pro$/i.test(stripped) && !this.llmDisabled) {
+        if (/-pro$/i.test(stripped) && !breakerLLMDisabled) {
             console.error(`LLM model '${phaseConfig.model}' is not a chat-completions model ` +
                 `(OpenAI rejects */chat/completions for *-pro variants). ` +
                 `Use a chat-capable model — gpt-5-nano, gpt-4o-mini, gpt-4o, or claude-*-latest. ` +
                 `Falling back to static analysis. Use --no-llm to silence.`);
             this.lastErrorCategory = 'http_error';
-            this.llmDisabled = true;
+            breakerLLMDisabled = true;
             return null;
         }
         // All LLM requests go through the shared throttled queue.
@@ -927,10 +1099,13 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
                 while (true) {
                     attemptIdx++;
                     const attemptStart = Date.now();
-                    // On retry, request fewer tokens to reduce truncation risk
-                    const maxTokens = attempt > 0
-                        ? Math.min(phaseConfig.maxTokens, 4000)
-                        : phaseConfig.maxTokens;
+                    // cognium-ai#223: `max_tokens` is the OUTPUT cap — lowering it makes
+                    // the model truncate *sooner*. The old heuristic shrank it to 4000
+                    // on retry "to reduce truncation risk", which is backwards when the
+                    // failure IS output truncation (the dominant discovery parse-error:
+                    // a verdict array cut off mid-`reasoning`). Keep the full budget on
+                    // retry so a re-attempt has room to finish the JSON.
+                    const maxTokens = phaseConfig.maxTokens;
                     const controller = new AbortController();
                     const timeoutMs = phaseConfig.timeout || 60000;
                     const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
@@ -997,6 +1172,7 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
                             this.lastErrorCategory = 'connection_error';
                         }
                         this.trackFailure();
+                        this.trackDeadCall(); // #217: timeout / network error = endpoint-dead
                         return null;
                     }
                     if (!response.ok) {
@@ -1034,10 +1210,14 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
                             transient5xxAttempt++;
                             continue;
                         }
-                        // #43: log the response body too, not just the status code.
+                        // #43: read the response body once, not just the status code.
                         // OpenAI's 400s contain useful messages like "Unsupported
                         // parameter: 'temperature'" or "Use 'max_completion_tokens'
-                        // instead of 'max_tokens'" that would otherwise be invisible.
+                        // instead of 'max_tokens'" that would otherwise be invisible —
+                        // and cognium-ai#222 uses the body to classify 400s as
+                        // transient-vs-terminal. `response.text()` can only be consumed
+                        // once, so this single read feeds both the retry decision and the
+                        // terminal log below.
                         let bodySnippet = '';
                         try {
                             bodySnippet = (await response.text()).slice(0, 300);
@@ -1045,6 +1225,25 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
                         catch {
                             // ignore — body may have been consumed or stream errored
                         }
+                        // cognium-ai#222: opaque 400 (e.g. Novita `invalid_request_error`)
+                        // is a transient provider rejection — retry with backoff, like a
+                        // 5xx. Deterministic parameter-error 400s (`isTransient400` →
+                        // false) fall through to the terminal path unchanged.
+                        if (response.status === 400 &&
+                            transient400Attempt < maxTransient400Retries &&
+                            isTransient400(bodySnippet)) {
+                            const sleepMs = parseRetryAfterMs(response.headers.get('Retry-After'), transient400Attempt);
+                            console.error(`LLM opaque 400 (invalid_request_error); sleeping ${sleepMs}ms ` +
+                                `before retry ${transient400Attempt + 1}/${maxTransient400Retries}`);
+                            emitAttempt('transient_client_error', attemptStart, {
+                                httpStatus: 400,
+                                backoffMs: sleepMs,
+                                responseSnippet: bodySnippet || undefined,
+                            });
+                            await sleep(sleepMs);
+                            transient400Attempt++;
+                            continue;
+                        }
                         console.error(`LLM call failed: HTTP ${response.status}${bodySnippet ? ' — ' + bodySnippet : ''}`);
                         let terminalCategory;
                         if (response.status === 429) {
@@ -1100,6 +1299,7 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
                         this.lastErrorCategory = 'empty_response';
                         emitAttempt('empty_response', attemptStart, { httpStatus: response.status });
                         this.trackFailure();
+                        this.trackDeadCall(); // #217: empty body = endpoint-dead (the 60s stall)
                         return null;
                     }
                     // Parse JSON from response (handle markdown code blocks)
@@ -1198,9 +1398,7 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
      * Handles cases where LLM output was cut off mid-response
      */
     tryRecoverTruncatedJSON(jsonStr) {
-        // Count open brackets
-        let braces = 0;
-        let brackets = 0;
+        // First pass: detect a mid-string truncation (unbalanced quotes).
         let inString = false;
         let escaped = false;
         for (const char of jsonStr) {
@@ -1216,18 +1414,8 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
                 inString = !inString;
                 continue;
             }
-            if (inString)
-                continue;
-            if (char === '{')
-                braces++;
-            if (char === '}')
-                braces--;
-            if (char === '[')
-                brackets++;
-            if (char === ']')
-                brackets--;
         }
-        // If we're truncated mid-string, close it
+        // If we're truncated mid-string, close it.
         if (inString) {
             jsonStr = jsonStr + '"';
         }
@@ -1237,14 +1425,44 @@ INFORMATION EXPOSURE (CWE-200/209) SPECIFIC:
         if (lastValidEnd > 0 && lastValidEnd < jsonStr.length - 1) {
             jsonStr = jsonStr.substring(0, lastValidEnd);
         }
-        // Add missing brackets
-        while (brackets > 0) {
-            jsonStr += ']';
-            brackets--;
+        // cognium-ai#223: when the truncation point is a comma,
+        // `findLastCompleteValue` keeps it, so appending closers yields `…,}]`
+        // (invalid trailing comma). Strip a dangling `"key":` (no value) and any
+        // trailing comma. A *complete* trailing `"key":"value"` ends in `"`, so
+        // neither pattern touches it.
+        jsonStr = jsonStr
+            .replace(/,\s*"[^"]*"\s*:\s*$/, '') // drop `, "key":` with no value
+            .replace(/,\s*$/, ''); // drop trailing comma
+        // cognium-ai#223: close the OPEN delimiters in reverse nesting order.
+        // The previous code closed all `]` then all `}`, which produced invalid
+        // output like `…"path_traversal"]}` for a truncated `[{…` (array closed
+        // before the object it contains). Rebuild the open-delimiter stack on
+        // the final string and append the matching closers innermost-first.
+        const stack = [];
+        inString = false;
+        escaped = false;
+        for (const char of jsonStr) {
+            if (escaped) {
+                escaped = false;
+                continue;
+            }
+            if (char === '\\') {
+                escaped = true;
+                continue;
+            }
+            if (char === '"') {
+                inString = !inString;
+                continue;
+            }
+            if (inString)
+                continue;
+            if (char === '{' || char === '[')
+                stack.push(char);
+            else if (char === '}' || char === ']')
+                stack.pop();
         }
-        while (braces > 0) {
-            jsonStr += '}';
-            braces--;
+        for (let i = stack.length - 1; i >= 0; i--) {
+            jsonStr += stack[i] === '{' ? '}' : ']';
         }
         // Quick validation - should start with { or [
         const trimmed = jsonStr.trim();