@remnic/core 10.0.37 → 10.0.38

package/dist/who-knows.js

@@ -9,8 +9,8 @@ import {
   loadWhoKnowsEntities,
   tokenizeTopic,
   validateWhoKnowsInput
-} from "./chunk-DK5VMJL3.js";
-import "./chunk-R6TCM47S.js";
+} from "./chunk-VSSQIBVU.js";
+import "./chunk-WCBLPKG3.js";
 import "./chunk-UHVXE24Y.js";
 import "./chunk-DM5QZ3YU.js";
 import "./chunk-VY6BD5KB.js";
@@ -37,7 +37,7 @@ import "./chunk-S2HLHAZ4.js";
 import "./chunk-DM2T26WE.js";
 import "./chunk-LDXUBPMO.js";
 import "./chunk-FF6N7BXC.js";
-import "./chunk-5EILJ6W2.js";
+import "./chunk-WXPBDEMW.js";
 import "./chunk-AAD2ZS6P.js";
 import "./chunk-YQ6SLFNZ.js";
 import "./chunk-ZIKENA2C.js";
@@ -64,7 +64,7 @@ import "./chunk-OR42ZI54.js";
 import "./chunk-23G5P7TA.js";
 import "./chunk-C64XATK7.js";
 import "./chunk-42JKGUFJ.js";
-import "./chunk-GMMZ45C4.js";
+import "./chunk-AVVCPA7U.js";
 import "./chunk-I55KAVET.js";
 import "./chunk-L2EXJQJP.js";
 import "./chunk-45GF7Y3V.js";
@@ -77,7 +77,7 @@ import "./chunk-YCVB7X56.js";
 import "./chunk-PQVCH2AE.js";
 import "./chunk-PJKTFXE2.js";
 import "./chunk-W5BIQOAJ.js";
-import "./chunk-A6PKEU7T.js";
+import "./chunk-UFM3PJDT.js";
 import "./chunk-YBA52OKG.js";
 import "./chunk-TDCR6PQL.js";
 import "./chunk-BT7NVCML.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/core",
-  "version": "10.0.37",
+  "version": "10.0.38",
   "description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
   "type": "module",
   "main": "dist/index.js",
@@ -3347,7 +3347,7 @@
     "core"
   ],
   "peerDependencies": {
-    "@remnic/coding-graph": "^10.0.37"
+    "@remnic/coding-graph": "^10.0.38"
   },
   "peerDependenciesMeta": {
     "@remnic/coding-graph": {

package/src/config.test.ts

@@ -1060,6 +1060,22 @@ test("parseConfig localLlmTimeoutMs clamps invalid values to a positive timeout"
   assert.equal(parseConfig({ localLlmTimeoutMs: Number.NaN }).localLlmTimeoutMs, 180_000);
 });
 
+test("parseConfig extractionJudgeTimeoutMs defaults to 20000 — compact verdict measures 2.1-2.5 s on a small fast judge model", () => {
+  assert.equal(parseConfig({}).extractionJudgeTimeoutMs, 20_000);
+});
+
+test("parseConfig extractionJudgeTimeoutMs accepts numbers and CLI-style numeric strings", () => {
+  assert.equal(parseConfig({ extractionJudgeTimeoutMs: 45_000 }).extractionJudgeTimeoutMs, 45_000);
+  assert.equal(parseConfig({ extractionJudgeTimeoutMs: "90000" }).extractionJudgeTimeoutMs, 90_000);
+});
+
+test("parseConfig extractionJudgeTimeoutMs falls back to the default on unusable values and caps the top", () => {
+  for (const value of [0, -1, Number.NaN, "abc"]) {
+    assert.equal(parseConfig({ extractionJudgeTimeoutMs: value }).extractionJudgeTimeoutMs, 20_000);
+  }
+  assert.equal(parseConfig({ extractionJudgeTimeoutMs: 10_000_000 }).extractionJudgeTimeoutMs, 3_600_000);
+});
+
 test("parseConfig validates localLlmMaxContext as a usable context window", () => {
   assert.equal(parseConfig({ localLlmMaxContext: 4096 }).localLlmMaxContext, 4096);
   assert.equal(parseConfig({ localLlmMaxContext: "8192" }).localLlmMaxContext, 8192);

package/src/config.ts

@@ -2731,6 +2731,20 @@ export function parseConfig(
       typeof cfg.extractionJudgeBatchSize === "number" && Number.isFinite(cfg.extractionJudgeBatchSize)
         ? Math.max(1, Math.round(cfg.extractionJudgeBatchSize))
         : 20,
+    // Judge LLM timeout (issue #376 follow-up, Studio 2026-10-29; tightened in
+    // PR 3264): the judge requests a compact verdict with a batch-scaled
+    // completion budget (24 tokens per candidate, up to 2048). Measured
+    // 2.1-2.5 s per call on a small fast judge
+    // model; the big shared core model measured 13-22 s per compact call,
+    // 41 s verbose. The 20s default assumes the fast model, stays fail-open
+    // on timeout, and needs a raise if extractionJudgeModel points at the
+    // slow core model. Non-positive / unusable values fall back to the
+    // default rather than a 1ms timeout that would silently disable judging.
+    extractionJudgeTimeoutMs: ((): number => {
+      const coerced = coerceNumber(cfg.extractionJudgeTimeoutMs);
+      if (coerced === undefined || coerced < 1) return 20_000;
+      return Math.min(3_600_000, Math.floor(coerced));
+    })(),
     extractionJudgeShadow: cfg.extractionJudgeShadow === true,
     // Defer cap (issue #562 PR 2): max re-deferrals for the same candidate
     // text before the verdict is forcibly converted to reject.

package/src/extraction-judge-chain.test.ts

@@ -2,8 +2,11 @@ import assert from "node:assert/strict";
 import test from "node:test";
 
 import { parseConfig } from "./config.js";
-import { judgeFactDurability } from "./extraction-judge.js";
-import type { FallbackLlmClient } from "./fallback-llm.js";
+import { judgeFactDurability, JUDGE_SYSTEM_PROMPT, judgeMaxTokens, MAX_JUDGE_BATCH, MAX_JUDGE_TOKENS } from "./extraction-judge.js";
+import { FallbackLlmClient } from "./fallback-llm.js";
+import type { FallbackLlmOptions, FallbackLlmResponse } from "./fallback-llm.js";
+import { LocalLlmClient } from "./local-llm.js";
+import type { LocalLlmChatCompletionOptions, LocalLlmChatCompletionResult } from "./local-llm-helpers.js";
 
 // Regression for cursor review on #1425: the extraction judge must route gateway
 // fallback calls through the SAME precedence as ExtractionEngine.withGatewayAgent
@@ -53,3 +56,296 @@ test("judge falls back to gatewayAgentId when no taskModelChain is set", async (
   assert.equal(captured[0]?.agentId, "persona-agent");
   assert.equal(captured[0]?.modelChain, undefined);
 });
+
+type ChatOptions = LocalLlmChatCompletionOptions;
+
+/** A real FallbackLlmClient whose gateway call is replaced: records the options the judge passes. */
+class RecordingFallbackLlm extends FallbackLlmClient {
+  constructor(private readonly sink: FallbackLlmOptions[]) {
+    super();
+  }
+
+  override isAvailable(): boolean {
+    return true;
+  }
+
+  override async chatCompletion(
+    _messages: Array<{ role: "system" | "user" | "assistant"; content: string }>,
+    options: FallbackLlmOptions = {},
+  ): Promise<FallbackLlmResponse | null> {
+    this.sink.push(options);
+    return { content: JSON.stringify([{ index: 0, durable: true, reason: "ok" }]), modelUsed: "recording-fake" };
+  }
+}
+
+/** A real LocalLlmClient whose network call is replaced: records the options the judge passes. */
+class RecordingLocalLlm extends LocalLlmClient {
+  constructor(
+    private readonly sink: ChatOptions[],
+    private readonly reply: string,
+  ) {
+    super(parseConfig({}));
+  }
+
+  override async chatCompletion(
+    _messages: Array<{ role: string; content: string }>,
+    options: ChatOptions = {},
+  ): Promise<LocalLlmChatCompletionResult | null> {
+    this.sink.push(options);
+    return { content: this.reply };
+  }
+}
+
+interface ExampleVerdict {
+  index?: number;
+  kind?: string;
+  reason?: string;
+  durable?: boolean;
+}
+
+// Regression (Studio 2026-10-29): the judge hardcoded timeoutMs: 1500 while the
+// deployed local model needs tens of seconds — 94 straight aborts, 0 completions.
+// The timeout must come from config so operators can set one the model can meet.
+
+test("judge forwards the configured timeout to the local LLM call", async () => {
+  const config = parseConfig({ extractionJudgeTimeoutMs: 45_000 });
+  const captured: ChatOptions[] = [];
+  const localLlm = new RecordingLocalLlm(captured, JSON.stringify([{ index: 0, durable: true, reason: "ok" }]));
+
+  await judgeFactDurability([candidate], config, localLlm, null, new Map(), new Map());
+
+  assert.equal(captured.length, 1, "judge should call the local LLM once");
+  assert.equal(captured[0]?.timeoutMs, 45_000);
+});
+
+// ---------------------------------------------------------------------------
+// Compact verdict request (PR 3264 follow-up): the judge asks for a minimal
+// `[{"index":N,"kind":"accept|reject|defer"}]` reply with a batch-scaled
+// completion budget (24 tokens per candidate, up to 2048) instead of a flat 2048. The big budget let
+// a verbose 59-305s reply hide inside the old 150s timeout; the compact one
+// measures 2.1-2.5 s per call on a small fast judge model (13-22 s, 41 s
+// verbose on the big shared core model), inside the 20s default.
+// ---------------------------------------------------------------------------
+
+function captureLocal(captured: ChatOptions[], content?: string): LocalLlmClient {
+  return new RecordingLocalLlm(captured, content ?? JSON.stringify([{ index: 0, kind: "reject" }]));
+}
+
+const EIGHT_CANDIDATES = Array.from({ length: 8 }, (_, i) => ({
+  text: `sample candidate fact ${i}`,
+  category: "preference",
+  confidence: 0.5,
+}));
+
+test("judge uses a compact completion budget scaled to the batch size", async () => {
+  const config = parseConfig({});
+  const captured: ChatOptions[] = [];
+
+  await judgeFactDurability(EIGHT_CANDIDATES, config, captureLocal(captured), null, new Map(), new Map());
+
+  assert.equal(captured.length, 1, "judge should call the local LLM once");
+  assert.equal(captured[0]?.maxTokens, judgeMaxTokens(8));
+  assert.ok(
+    (captured[0]?.maxTokens ?? Number.POSITIVE_INFINITY) <= judgeMaxTokens(8),
+    `completion budget must stay small, got ${captured[0]?.maxTokens}`,
+  );
+});
+
+test("judgeMaxTokens leaves room for one verdict per candidate, up to a cap", () => {
+  assert.equal(judgeMaxTokens(1), 64);
+  assert.equal(judgeMaxTokens(8), 192);
+  assert.ok(judgeMaxTokens(50) >= 50 * 16, "a 50-candidate batch needs a verdict array that fits");
+  assert.equal(judgeMaxTokens(500), 2048, "cap");
+});
+
+test("judge bounds the whole local call, retries included, with one deadline signal", async () => {
+  const config = parseConfig({ extractionJudgeTimeoutMs: 45_000 });
+  const captured: ChatOptions[] = [];
+
+  await judgeFactDurability([candidate], config, captureLocal(captured), null, new Map(), new Map());
+
+  assert.ok(captured[0]?.signal instanceof AbortSignal, "judge passes an AbortSignal so retries share one deadline");
+  assert.equal(captured[0]?.signal?.aborted, false);
+});
+
+
+test("system prompt requests the compact verdict shape only", () => {
+  const lines = JUDGE_SYSTEM_PROMPT.split("\n");
+  const exampleIdx = lines.findIndex((l) => l.trim() === "Example output:");
+  assert.ok(exampleIdx >= 0, "prompt must contain an 'Example output:' line");
+  // SAFETY: the line is the example JSON checked into the prompt text this test pins.
+  const example = JSON.parse(lines[exampleIdx + 1].trim()) as { verdicts?: ExampleVerdict[] };
+  assert.ok(
+    Array.isArray(example.verdicts) && example.verdicts.length > 0,
+    'example must use the {"verdicts":[...]} wrapper — response_format json_object rejects a bare top-level array',
+  );
+  for (const item of example.verdicts) {
+    assert.ok(Number.isInteger(item.index), "example index must be an integer");
+    assert.ok(
+      item.kind === "accept" || item.kind === "reject" || item.kind === "defer",
+      `example kind must be a parser-accepted verdict, got ${item.kind}`,
+    );
+    assert.equal(item.reason, undefined, "compact request must not ask for reasons");
+    assert.equal(item.durable, undefined, "compact request must not ask for the durable flag");
+  }
+});
+
+test("judge still accepts the legacy verbose verdict shape", async () => {
+  const config = parseConfig({});
+  const captured: ChatOptions[] = [];
+  const llm = captureLocal(
+    captured,
+    JSON.stringify([
+      { index: 0, kind: "accept", durable: true, reason: "Stable preference" },
+      { index: 1, durable: false, reason: "Ephemeral build status" },
+      { index: 2, kind: "defer", durable: false, reason: "Ambiguous pronoun" },
+    ]),
+  );
+  const candidates = [
+    { text: "user prefers dark mode", category: "preference", confidence: 0.5 },
+    { text: "the build is running now", category: "task", confidence: 0.5 },
+    { text: "he said they'd follow up", category: "task", confidence: 0.5 },
+  ];
+
+  const result = await judgeFactDurability(candidates, config, llm, null, new Map(), new Map());
+
+  assert.deepEqual(
+    { ...result.verdicts.get(0) },
+    { durable: true, reason: "Stable preference", kind: "accept" },
+  );
+  assert.deepEqual(
+    { ...result.verdicts.get(1) },
+    { durable: false, reason: "Ephemeral build status" },
+  );
+  assert.deepEqual(
+    { ...result.verdicts.get(2) },
+    { durable: false, reason: "Ambiguous pronoun", kind: "defer" },
+  );
+});
+
+test("judge parses the {\"verdicts\":[...]} wrapper object the prompt requests", async () => {
+  const config = parseConfig({});
+  const captured: ChatOptions[] = [];
+  const llm = captureLocal(
+    captured,
+    JSON.stringify({
+      verdicts: [
+        { index: 0, kind: "accept" },
+        { index: 1, kind: "reject" },
+      ],
+    }),
+  );
+  const candidates = [
+    { text: "user prefers dark mode", category: "preference", confidence: 0.5 },
+    { text: "the build is running now", category: "task", confidence: 0.5 },
+  ];
+
+  const result = await judgeFactDurability(candidates, config, llm, null, new Map(), new Map());
+
+  assert.equal(result.judged, 2, "both verdicts must resolve from the wrapper");
+  assert.equal(result.verdicts.get(0)?.kind, "accept");
+  assert.equal(result.verdicts.get(1)?.kind, "reject");
+});
+
+test("judge still parses the compact bare-array shape", async () => {
+  const config = parseConfig({});
+  const captured: ChatOptions[] = [];
+  const llm = captureLocal(
+    captured,
+    JSON.stringify([
+      { index: 0, kind: "reject" },
+      { index: 1, kind: "accept" },
+    ]),
+  );
+  const candidates = [
+    { text: "the build is running now", category: "task", confidence: 0.5 },
+    { text: "user prefers dark mode", category: "preference", confidence: 0.5 },
+  ];
+
+  const result = await judgeFactDurability(candidates, config, llm, null, new Map(), new Map());
+
+  assert.equal(result.judged, 2, "both verdicts must resolve from the bare array");
+  assert.equal(result.verdicts.get(0)?.kind, "reject");
+  assert.equal(result.verdicts.get(1)?.kind, "accept");
+});
+
+test("judge treats a single verdict object as one verdict at its index, not the whole batch", async () => {
+  const config = parseConfig({});
+  const captured: ChatOptions[] = [];
+  const llm = captureLocal(captured, JSON.stringify({ index: 1, kind: "reject" }));
+  const candidates = [
+    { text: "user prefers dark mode", category: "preference", confidence: 0.5 },
+    { text: "the build is running now", category: "task", confidence: 0.5 },
+    { text: "he said they'd follow up", category: "task", confidence: 0.5 },
+  ];
+
+  const result = await judgeFactDurability(candidates, config, llm, null, new Map(), new Map());
+
+  assert.equal(result.judged, 1, "only the indexed candidate comes from the LLM");
+  assert.equal(result.verdicts.get(1)?.kind, "reject");
+  assert.equal(result.verdicts.get(1)?.durable, false);
+  assert.equal(result.verdicts.get(0)?.kind, undefined, "index 0 must not inherit the reject");
+  assert.equal(result.verdicts.get(0)?.durable, true, "index 0 fail-open approves");
+  assert.equal(result.verdicts.get(2)?.kind, undefined, "index 2 must not inherit the reject");
+  assert.equal(result.verdicts.get(2)?.durable, true, "index 2 fail-open approves");
+});
+
+// ---------------------------------------------------------------------------
+// Judge model override: extractionJudgeModel must reach the LLM call options
+// (options.model) on both the local and fallback paths, and stay absent when
+// unset so routing keeps the default model.
+// ---------------------------------------------------------------------------
+
+test("judge forwards extractionJudgeModel to the local LLM options", async () => {
+  const config = parseConfig({ extractionJudgeModel: "qwen3.8-27b-64k-nothink" });
+  const captured: ChatOptions[] = [];
+
+  await judgeFactDurability([candidate], config, captureLocal(captured), null, new Map(), new Map());
+
+  assert.equal(captured[0]?.model, "qwen3.8-27b-64k-nothink");
+});
+
+test("judge omits model from local LLM options when extractionJudgeModel is unset", async () => {
+  const config = parseConfig({});
+  const captured: ChatOptions[] = [];
+
+  await judgeFactDurability([candidate], config, captureLocal(captured), null, new Map(), new Map());
+
+  assert.equal(captured[0]?.model, undefined);
+});
+
+test("judge forwards extractionJudgeModel to the fallback LLM options in gateway mode", async () => {
+  const config = parseConfig({ modelSource: "gateway", extractionJudgeModel: "fast-judge" });
+  const captured: FallbackLlmOptions[] = [];
+
+  await judgeFactDurability([candidate], config, null, new RecordingFallbackLlm(captured), new Map(), new Map());
+
+  assert.equal(captured[0]?.model, "fast-judge");
+});
+
+test("judge defaults to a 20s timeout when none is configured", async () => {
+  const config = parseConfig({});
+  const captured: FallbackLlmOptions[] = [];
+
+  await judgeFactDurability([candidate], config, null, new RecordingFallbackLlm(captured), new Map(), new Map());
+
+  assert.equal(captured[0]?.timeoutMs, 20_000);
+});
+
+test("a configured batch size above what the token cap can answer is split", async () => {
+  const config = parseConfig({ extractionJudgeBatchSize: 200 });
+  const captured: ChatOptions[] = [];
+  const candidates = Array.from({ length: 200 }, (_, i) => ({
+    text: `distinct candidate fact number ${i}`,
+    category: "preference",
+    confidence: 0.5,
+  }));
+
+  await judgeFactDurability(candidates, config, captureLocal(captured), null, new Map(), new Map());
+
+  assert.equal(captured.length, 3, "200 candidates at 85 per request");
+  for (const call of captured) {
+    assert.ok((call.maxTokens ?? 0) <= MAX_JUDGE_TOKENS, "never above the cap");
+  }
+  assert.equal(MAX_JUDGE_BATCH * 24 <= MAX_JUDGE_TOKENS, true, "a full batch fits its own budget");
+});