circle-ir-ai 2.35.0 → 2.37.0

package/dist/llm/llm-error-counter.js

@@ -0,0 +1,109 @@
+/**
+ * LLM error counter (cognium-ai#191)
+ *
+ * Process-wide singleton that tracks aggregate LLM discovery/verification
+ * failures by categorical `kind`. Populated by callers whose outer catch
+ * blocks previously swallowed transport errors into a "not vulnerable"
+ * result shape (e.g. `discovery.ts:524-531`).
+ *
+ * Two consumers:
+ *
+ *   1. Benchmark runners (`benchmarks/runners/run-cwe-bench-java.ts` +
+ *      cousins) read `snapshot()` at end-of-run to decide whether to emit
+ *      a "high LLM failure rate" banner. Without this, a total-outage run
+ *      is indistinguishable from a real static-only baseline in the score
+ *      output — which was the root cause of the identical 65/120 seen on
+ *      `zai-org/glm-5.2` and `minimax/minimax-m3` in the 2026-07-02/03
+ *      Novita sweeps.
+ *
+ *   2. The first `increment()` call also mirrors a single stderr warning
+ *      so operators running a plain `cognium-ai scan --llm` notice the
+ *      failure without needing to open the JSONL log. Subsequent calls
+ *      are silent (avoids stderr flooding on a persistent outage).
+ *
+ * Categorization reuses `LLMErrorCategory` from `call-logger.ts`. Callers
+ * that don't have a categorical reason pass `'unknown'`.
+ *
+ * Related:
+ *  - cognium-ai#189 (circle-ir-ai 2.36.0) — cache Round 3 model-in-path.
+ *    Complementary: #189 makes cache-collision regressions loud; this
+ *    module makes LLM-call failures loud.
+ *  - cognium-ai#87 — per-attempt JSONL logging. Attempt-level failures
+ *    still land in the log; this counter is the aggregate signal.
+ *  - The existing JSONL `errorCategory` field on `kind:'call'` rows is
+ *    the source-of-truth for offline analysis. This counter is a live
+ *    process-local mirror sized for immediate benchmark-runner use.
+ */
+const ALL_KINDS = [
+    'empty_response',
+    'parse_error',
+    'timeout',
+    'http_error',
+    'rate_limited',
+    'connection_error',
+    'transient_server_error',
+    'unknown',
+];
+class LLMErrorCounter {
+    counts = new Map();
+    warned = false;
+    /**
+     * Record one LLM-call failure. First call also mirrors a single
+     * stderr line so operators notice a broken model config on a bare
+     * `cognium-ai scan --llm` invocation. Subsequent failures are silent.
+     */
+    increment(kind, sampleMessage) {
+        this.counts.set(kind, (this.counts.get(kind) ?? 0) + 1);
+        if (!this.warned) {
+            this.warned = true;
+            const suffix = sampleMessage ? `: ${trim(sampleMessage)}` : '';
+            try {
+                process.stderr.write(`[llm] first discovery-stage LLM error (${kind})${suffix}\n` +
+                    `[llm] subsequent errors are aggregated silently; see LLM_LOG_JSONL for per-call detail\n`);
+            }
+            catch {
+                // swallow — logging failure must not affect analysis
+            }
+        }
+    }
+    /**
+     * Return a stable point-in-time snapshot keyed by every known kind so
+     * downstream printers can format a fixed-width breakdown without
+     * probing for undefined keys. Zero-value kinds are included.
+     */
+    snapshot() {
+        const out = {};
+        for (const k of ALL_KINDS) {
+            out[k] = this.counts.get(k) ?? 0;
+        }
+        return out;
+    }
+    /** Total across all kinds. Cheap; recomputed on each call. */
+    total() {
+        let sum = 0;
+        for (const n of this.counts.values())
+            sum += n;
+        return sum;
+    }
+    /**
+     * Test-only: clear counts + reset the first-warning latch. Also called
+     * by benchmark harnesses that want per-repo counters rather than a
+     * process-wide roll-up.
+     */
+    reset() {
+        this.counts.clear();
+        this.warned = false;
+    }
+}
+/**
+ * Process-wide singleton. Callers use this directly rather than passing
+ * the counter through discovery/verification APIs — every LLM call path
+ * currently shares one client anyway, and a live singleton matches the
+ * existing `llmCallLogger` pattern.
+ */
+export const llmErrorCounter = new LLMErrorCounter();
+/** Trim a message for the first-error stderr line. */
+function trim(s, max = 200) {
+    return s.length > max ? s.slice(0, max) + '…' : s;
+}
+//# sourceMappingURL=llm-error-counter.js.map

package/dist/llm/llm-error-counter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-error-counter.js","sourceRoot":"","sources":["../../src/llm/llm-error-counter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAMH,MAAM,SAAS,GAAkC;IAC/C,gBAAgB;IAChB,aAAa;IACb,SAAS;IACT,YAAY;IACZ,cAAc;IACd,kBAAkB;IAClB,wBAAwB;IACxB,SAAS;CACV,CAAC;AAEF,MAAM,eAAe;IACX,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAC;IAC/C,MAAM,GAAG,KAAK,CAAC;IAEvB;;;;OAIG;IACH,SAAS,CAAC,IAAwB,EAAE,aAAsB;QACxD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,0CAA0C,IAAI,IAAI,MAAM,IAAI;oBAC5D,0FAA0F,CAC3F,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,qDAAqD;YACvD,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,MAAM,GAAG,GAAG,EAAwC,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,8DAA8D;IAC9D,KAAK;QACH,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YAAE,GAAG,IAAI,CAAC,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;AAErD,sDAAsD;AACtD,SAAS,IAAI,CAAC,CAAS,EAAE,GAAG,GAAG,GAAG;IAChC,OAAO,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACpD,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "circle-ir-ai",
-  "version": "2.35.0",
+  "version": "2.37.0",
   "description": "LLM-enhanced SAST analysis built on circle-ir",
   "main": "dist/index.js",
   "module": "dist/index.js",