@remnic/core 9.69.36 → 9.69.38

package/src/config.test.ts

@@ -2724,3 +2724,175 @@ test("parseConfig coerces string-typed sharedContextAllowBindingAuthority from t
     );
   }
 });
+
+test("parseConfig consumes backgroundGeneration and leaves openaiBaseUrl untouched", () => {
+  withIsolatedConnectorsDir(false, () => {
+    const previousKey = process.env.OPENAI_API_KEY;
+    const previousBase = process.env.OPENAI_BASE_URL;
+    delete process.env.OPENAI_API_KEY;
+    delete process.env.OPENAI_BASE_URL;
+    try {
+      const parsed = parseConfig({
+        backgroundGeneration: {
+          endpoint: "http://127.0.0.1:8765/v1/chat/completions",
+          token: "bridge-token-fixture",
+          timeoutSeconds: 120,
+        },
+      });
+      assert.equal(parsed.openaiBaseUrl, undefined);
+      assert.equal(parsed.openaiApiKey, undefined);
+      assert.deepEqual(parsed.backgroundGeneration, {
+        endpoint: "http://127.0.0.1:8765/v1/chat/completions",
+        token: "bridge-token-fixture",
+        timeoutSeconds: 120,
+      });
+
+      const explicitOpenAi = parseConfig({
+        backgroundGeneration: {
+          endpoint: "http://127.0.0.1:8765/v1/chat/completions",
+          token: "bridge-token-fixture",
+          timeoutSeconds: 120,
+        },
+        openaiBaseUrl: "http://127.0.0.1:9999/v1",
+        openaiApiKey: "keep-me",
+      });
+      assert.equal(explicitOpenAi.openaiBaseUrl, "http://127.0.0.1:9999/v1");
+      assert.equal(explicitOpenAi.openaiApiKey, "keep-me");
+      assert.equal(
+        explicitOpenAi.backgroundGeneration?.endpoint,
+        "http://127.0.0.1:8765/v1/chat/completions",
+      );
+    } finally {
+      if (previousKey === undefined) delete process.env.OPENAI_API_KEY;
+      else process.env.OPENAI_API_KEY = previousKey;
+      if (previousBase === undefined) delete process.env.OPENAI_BASE_URL;
+      else process.env.OPENAI_BASE_URL = previousBase;
+    }
+  });
+});
+
+test("parseConfig ignores llmBridgeClientConfigPath", () => {
+  withIsolatedConnectorsDir(false, () => {
+    const parsed = parseConfig({
+      llmBridgeClientConfigPath: "/no/such/remnic-llm-bridge-client.json",
+    });
+    assert.equal(parsed.backgroundGeneration, undefined);
+  });
+});
+
+test("parseConfig rejects backgroundGeneration without a token", () => {
+  withIsolatedConnectorsDir(false, () => {
+    assert.throws(
+      () =>
+        parseConfig({
+          backgroundGeneration: {
+            endpoint: "http://127.0.0.1:8765/v1/chat/completions",
+          },
+        }),
+      /must include a token/,
+    );
+  });
+});
+
+test("parseConfig ignores Hermes timeout_seconds on backgroundGeneration", () => {
+  withIsolatedConnectorsDir(false, () => {
+    const parsed = parseConfig({
+      backgroundGeneration: {
+        endpoint: "http://127.0.0.1:8765/v1/chat/completions",
+        token: "bridge-token-fixture",
+        timeout_seconds: 45,
+      },
+    });
+    assert.equal(parsed.backgroundGeneration?.timeoutSeconds, 120);
+  });
+});
+
+test("parseConfig keeps env openaiBaseUrl isolated from backgroundGeneration", () => {
+  withIsolatedConnectorsDir(false, () => {
+    const previousKey = process.env.OPENAI_API_KEY;
+    const previousBase = process.env.OPENAI_BASE_URL;
+    process.env.OPENAI_API_KEY = "env-openai-key";
+    process.env.OPENAI_BASE_URL = "http://127.0.0.1:9999/v1";
+    try {
+      const parsed = parseConfig({
+        backgroundGeneration: {
+          endpoint: "http://127.0.0.1:8765/v1/chat/completions",
+          token: "bridge-token-fixture",
+          timeoutSeconds: 45,
+        },
+      });
+      assert.equal(parsed.openaiBaseUrl, "http://127.0.0.1:9999/v1");
+      assert.equal(parsed.openaiApiKey, "env-openai-key");
+      assert.deepEqual(parsed.backgroundGeneration, {
+        endpoint: "http://127.0.0.1:8765/v1/chat/completions",
+        token: "bridge-token-fixture",
+        timeoutSeconds: 45,
+      });
+    } finally {
+      if (previousKey === undefined) delete process.env.OPENAI_API_KEY;
+      else process.env.OPENAI_API_KEY = previousKey;
+      if (previousBase === undefined) delete process.env.OPENAI_BASE_URL;
+      else process.env.OPENAI_BASE_URL = previousBase;
+    }
+  });
+});
+
+test("parseConfig rejects a non-object backgroundGeneration value", () => {
+  withIsolatedConnectorsDir(false, () => {
+    assert.throws(
+      () => parseConfig({ backgroundGeneration: "http://127.0.0.1:8765/v1/chat/completions" }),
+      /backgroundGeneration must be an object/,
+    );
+  });
+});
+
+test("parseConfig strips a slash-rich bridge endpoint without regex", () => {
+  withIsolatedConnectorsDir(false, () => {
+    const parsed = parseConfig({
+      backgroundGeneration: {
+        endpoint: `http://127.0.0.1:8765/v1/chat/completions${"/".repeat(10_000)}`,
+        token: "bridge-token-fixture",
+      },
+    });
+    assert.equal(
+      parsed.backgroundGeneration?.endpoint,
+      "http://127.0.0.1:8765/v1/chat/completions",
+    );
+    assert.equal(parsed.openaiBaseUrl, undefined);
+  });
+});
+
+test("parseConfig rejects plaintext non-loopback backgroundGeneration endpoints", () => {
+  withIsolatedConnectorsDir(false, () => {
+    for (const endpoint of [
+      "http://192.168.10.20:8765/v1/chat/completions",
+      "http://example.test/v1/chat/completions",
+    ]) {
+      assert.throws(
+        () =>
+          parseConfig({
+            backgroundGeneration: {
+              endpoint,
+              token: "bridge-token-fixture",
+            },
+          }),
+        /must use HTTPS unless the host is loopback/,
+      );
+    }
+
+    for (const endpoint of [
+      "http://127.0.0.1:8765/v1/chat/completions",
+      "http://localhost:8765/v1/chat/completions",
+      "http://[::1]:8765/v1/chat/completions",
+      "https://example.test/v1/chat/completions",
+    ]) {
+      const parsed = parseConfig({
+        backgroundGeneration: {
+          endpoint,
+          token: "bridge-token-fixture",
+        },
+      });
+      assert.equal(parsed.backgroundGeneration?.endpoint, endpoint);
+    }
+  });
+});

package/src/config.ts

@@ -25,6 +25,7 @@ import type {
   TriggerMode,
   TrustWeights,
 } from "./types.js";
+import { parseLocalLlmConfig } from "./local-llm-config.js";
 import { parseConvergeConfig } from "./converge-config.js";
 import { parseExternalWikiRecallGuard } from "./external-wiki-guard.js";
 import { log } from "./logger.js";
@@ -40,7 +41,7 @@ import { coerceBool, coerceBooleanLike, coerceInstallExtension, coerceNumber } f
 import { parseSubjectRuntimeConfig } from "./subject-config.js";
 import { parseRecallStateViews } from "./recall-state-view.js";
 import { parseRecallConcurrencyConfig } from "./recall-concurrency-config.js";
-import { parseExtractionLivenessConfig } from "./extraction-liveness.js";
+import { parseExtractionFields } from "./extraction-span-config.js";
 import { parseReplicaPeersConfig } from "./replica-peers-config.js";
 import { parseDependencyPropagationConfig } from "./dependency-propagation-config.js";
 import { parseProceduralMaintenanceConfig } from "./procedural/maintenance-config.js";
@@ -2332,7 +2333,7 @@ export function parseConfig(
     dreaming,
     dreamsPhases,
     procedural,
-    extractionLiveness: parseExtractionLivenessConfig(cfg),
+    ...parseExtractionFields(cfg),
     replicaPeers: parseReplicaPeersConfig(cfg),
     converge: parseConvergeConfig(cfg.converge),
     wearables,
@@ -2680,49 +2681,7 @@ export function parseConfig(
       typeof cfg.abstractionNodeStoreDir === "string" && cfg.abstractionNodeStoreDir.trim().length > 0
         ? cfg.abstractionNodeStoreDir.trim()
         : path.join(memoryDir, "state", "abstraction-nodes"),
-    // Local LLM Provider (v2.1)
-    localLlmEnabled: cfg.localLlmEnabled === true || cfg.localLlmEnabled === "true", // default: false
-    localLlmUrl:
-      typeof cfg.localLlmUrl === "string" && cfg.localLlmUrl.length > 0
-        ? cfg.localLlmUrl
-        : "http://localhost:1234/v1",
-    localLlmModel:
-      typeof cfg.localLlmModel === "string" && cfg.localLlmModel.length > 0
-        ? cfg.localLlmModel
-        : "local-model",
-    localLlmApiKey:
-      typeof cfg.localLlmApiKey === "string" && cfg.localLlmApiKey.length > 0
-        ? resolveEnvVars(cfg.localLlmApiKey)
-        : localLlmApiKeyEnv === undefined
-          ? undefined
-          : readEnvVar(localLlmApiKeyEnv),
-    localLlmApiKeyEnv,
-    localLlmHeaders:
-      cfg.localLlmHeaders && typeof cfg.localLlmHeaders === "object" && !Array.isArray(cfg.localLlmHeaders)
-        ? Object.fromEntries(
-            Object.entries(cfg.localLlmHeaders as Record<string, unknown>)
-              .filter(([, value]) => typeof value === "string")
-              .map(([key, value]) => [key, String(value)]),
-          )
-        : undefined,
-    localLlmAuthHeader: cfg.localLlmAuthHeader !== false,
-    localLlmFallback: cfg.localLlmFallback !== false, // default: true
-    localLlmHomeDir:
-      typeof cfg.localLlmHomeDir === "string" && cfg.localLlmHomeDir.length > 0
-        ? cfg.localLlmHomeDir
-        : undefined,
-    localLmsCliPath:
-      typeof cfg.localLmsCliPath === "string" && cfg.localLmsCliPath.length > 0
-        ? cfg.localLmsCliPath
-        : undefined,
-    localLmsBinDir:
-      typeof cfg.localLmsBinDir === "string" && cfg.localLmsBinDir.length > 0
-        ? cfg.localLmsBinDir
-        : undefined,
-    localLlmTimeoutMs:
-      parseBoundedIntegerMs(cfg.localLlmTimeoutMs, 180_000, 1, 86_400_000),
-    localLlmMaxContext:
-      parseOptionalIntegerAtLeast(cfg.localLlmMaxContext, 1024, "localLlmMaxContext"),
+    ...parseLocalLlmConfig(cfg, localLlmApiKeyEnv, resolveEnvVars),
     // Observability (disabled by default to avoid log spam)
     slowLogEnabled: cfg.slowLogEnabled === true,
     slowLogThresholdMs:

package/src/extraction-prompt.ts

@@ -45,9 +45,19 @@ export const EXTRACTION_RESPONSE_SHAPE = `{
   "episodeTitle": "<six-to-eight word conversation segment title>",
   "relationships": [{"source": "<normalized-name>", "target": "<normalized-name>", "label": "<source-grounded relationship>"}]
 }`;
+export function extractionResponseShape(spanMode: "off" | "shadow" | "on" = "off"): string {
+  if (spanMode === "off") return EXTRACTION_RESPONSE_SHAPE;
+  return EXTRACTION_RESPONSE_SHAPE.replace(
+    `"eventTime": "<optional source temporal expression>"`,
+    `"eventTime": "<optional source temporal expression>",
+    "span": {"sourceMessageIndex": <message number>, "charStart": <number>, "charEnd": <number>, "frame": "<at most 15 words>"}`,
+  );
+}
 export const EXTRACTION_RESPONSE_PLACEHOLDERS: Record<string, true> = {};
-for (const placeholder of EXTRACTION_RESPONSE_SHAPE.match(/<[^<>\r\n]+>/g) ?? []) {
-  EXTRACTION_RESPONSE_PLACEHOLDERS[placeholder] = true;
+for (const shape of [EXTRACTION_RESPONSE_SHAPE, extractionResponseShape("on")]) {
+  for (const placeholder of shape.match(/<[^<>\r\n]+>/g) ?? []) {
+    EXTRACTION_RESPONSE_PLACEHOLDERS[placeholder] = true;
+  }
 }
 export const CUE_ANCHOR_PROMPT_INSTRUCTION = `- For each fact, emit at most 3 "cueAnchors".
 - Each cue anchor value must be a source-grounded search hook of at most 120 characters, in the form "<main entity or topic> <key aspect>".
@@ -76,6 +86,30 @@ export function eventTimePromptInstruction(config: PluginConfig): string {
 When a fact states when it became or stopped being true, copy that explicit temporal expression verbatim into "eventTime". Omit "eventTime" when no such expression appears; never infer dates.`;
 }
 
+/**
+ * Span-mode extraction instructions (issue #2333 Phase B). Empty when the
+ * mode is off (byte-identical prompt). The offset convention mirrors the
+ * Phase A bench: messages numbered in order from 0, offsets index the
+ * message's own text (after the `[role] ` tag), [charStart, charEnd)
+ * end-exclusive.
+ */
+export function spanModePromptSection(spanMode: "off" | "shadow" | "on"): string {
+  if (spanMode === "off") return "";
+  const requestBoth = spanMode === "shadow";
+  return `
+Span mode (numbered-message offsets):
+- Conversation messages are numbered in order starting at 0 (the "[role] text" blocks).
+- For each fact, return "span": {"sourceMessageIndex": <message number>, "charStart": <number>, "charEnd": <number>, "frame": "<at most 15 words>"} pointing at the verbatim supporting words in that message's text (the text after the "] "). Offsets are [charStart, charEnd), end-exclusive, counting from the first character of the message text.
+- "frame" is at most 15 generated words that make the span self-contained: resolve pronouns to names, name the subject (e.g., frame "Maya's relocation:" + span "moved to Seattle last spring…").${
+    requestBoth
+      ? `
+- This run is in SHADOW mode: ALSO include the full standalone "content" restatement exactly as you would normally write it, in addition to "span".`
+      : `
+- Set "content" to the frame text (the short ≤15-word frame) — do not restate the full fact; the span plus frame carry it.`
+  }
+- Invalid spans fall back to the frame, so keep the frame informative.`;
+}
+
 /**
  * Build extraction instructions shared between local and cloud LLM.
  *
@@ -170,7 +204,7 @@ When you see something that matches a known entity, use THAT name exactly. Only
 `
     : ""
 }
-${eventTimePromptInstruction(config)}
+${eventTimePromptInstruction(config)}${spanModePromptSection(config.extraction?.spanMode ?? "off")}
 Also extract relationships between entities mentioned in the conversation.
 - Format: {source: "entity-name", target: "entity-name", label: "relationship description"}
 - Max 5 relationships per extraction

package/src/extraction-source-grounding-rules.ts

@@ -29,6 +29,7 @@ import {
 export interface ExtractionSourceGroundingOptions {
   sourceGrounding: boolean;
   anchorTemporalExpressions: boolean;
+  spanMode?: "off" | "shadow" | "on";
 }
 
 export function applyExtractionSourceGrounding(
@@ -60,6 +61,7 @@ export function applyExtractionSourceGrounding(
     anchorSource(assertionSourceText),
     anchoredRoleSources,
     options.anchorTemporalExpressions ? anchorSource : undefined,
+    options.spanMode ?? "off",
   );
 }
 
@@ -653,18 +655,51 @@ function filterGroundedFact(
   assertionSource: string | undefined,
   roleAssertionSources: ExtractionGroundingRoleSources | undefined,
   eventTimeNormalizer: ((eventTime: string) => string) | undefined,
+  spanMode: "off" | "shadow" | "on" = "off",
 ): ExtractionResult["facts"][number] | undefined {
-  const factContext = selectGroundingContext(
+  // Span-materialized content (issue #2333) is "frame: verbatim slice". The
+  // frame is generated deixis, so full-content gating would drop a verbatim
+  // fact. Only on-mode facts whose content is that materialized shape may
+  // fall back to the slice. Shadow stays zero-diff with off.
+  const quote = typeof fact.quote === "string" ? fact.quote : "";
+  const materializedPrefix = quote.length > 0 && fact.content.endsWith(quote) && fact.content !== quote
+    ? fact.content.slice(0, fact.content.length - quote.length)
+    : "";
+  const embeddedQuote =
+    spanMode === "on"
+    && quote.trim().length > 0
+    && (materializedPrefix.endsWith(": ") || /[:—-]\s$/.test(materializedPrefix))
+      ? quote
+      : undefined;
+  const contentContext = selectGroundingContext(
     fact.content,
     resolveGroundingContext(fact.content, source, assertionSource, roleAssertionSources),
   );
-  if (!isGroundedCandidate(
-    fact.content,
-    factContext.source,
-    factContext.assertionSource,
-    false,
-    factContext.allowRoleNormalization,
-  )) return undefined;
+  let gateContent = fact.content;
+  let factContext = contentContext;
+  if (
+    !isGroundedCandidate(
+      fact.content,
+      contentContext.source,
+      contentContext.assertionSource,
+      false,
+      contentContext.allowRoleNormalization,
+    )
+  ) {
+    if (embeddedQuote === undefined) return undefined;
+    gateContent = embeddedQuote;
+    factContext = selectGroundingContext(
+      gateContent,
+      resolveGroundingContext(gateContent, source, assertionSource, roleAssertionSources),
+    );
+    if (!isGroundedCandidate(
+      gateContent,
+      factContext.source,
+      factContext.assertionSource,
+      false,
+      factContext.allowRoleNormalization,
+    )) return undefined;
+  }
   const factEventTime = fact.eventTime;
   const groundedFactEventTime = factEventTime === undefined
     ? undefined
@@ -672,7 +707,7 @@ function filterGroundedFact(
   const factSupportingSentences = sourceSentences(factContext.assertionSource ?? factContext.source).filter(
     (sentence) =>
     isGroundedCandidate(
-      fact.content,
+      gateContent,
       sentence,
       undefined,
       false,
@@ -1068,6 +1103,7 @@ export function filterExtractionResultBySource(
   assertionSource?: string,
   roleAssertionSources?: ExtractionGroundingRoleSources,
   eventTimeNormalizer?: (eventTime: string) => string,
+  spanMode: "off" | "shadow" | "on" = "off",
 ): ExtractionResult {
   if (source.trim().length === 0) {
     return {
@@ -1088,6 +1124,7 @@ export function filterExtractionResultBySource(
       assertionSource,
       roleAssertionSources,
       eventTimeNormalizer,
+      spanMode,
     );
     return grounded === undefined ? [] : [grounded];
   });

package/src/extraction-source-grounding.ts

@@ -99,6 +99,7 @@ export function filterExtractionResultBySource(
   assertionSource?: string,
   roleAssertionSources?: ExtractionGroundingRoleSources,
   eventTimeNormalizer?: (eventTime: string) => string,
+  spanMode: "off" | "shadow" | "on" = "off",
 ): ExtractionResult {
-  return filterGroundingRules(result, source, assertionSource, roleAssertionSources, eventTimeNormalizer);
+  return filterGroundingRules(result, source, assertionSource, roleAssertionSources, eventTimeNormalizer, spanMode);
 }

package/src/extraction-span-config.ts

@@ -0,0 +1,81 @@
+/**
+ * Span-mode extraction config + fact span-ref types (issue #2333 Phase B,
+ * bench-gated). Sibling module so config.ts and types.ts stay under their
+ * structural ratchets (#1995); types.ts re-exports both.
+ *
+ * Default "off"; an unrecognized value is REJECTED, never defaulted
+ * (checklist #39) — silently ignoring it would run an unbenched extraction
+ * contract.
+ */
+
+import { z } from "zod";
+import { parseExtractionLivenessConfig } from "./extraction-liveness.js";
+
+export interface ExtractionSpanConfig {
+  /**
+   * - "off" (default): extraction generates content as before.
+   * - "shadow": request spans AND content; materialize + compare + log
+   *   agreement telemetry; persist the generated content (zero behavior
+   *   change).
+   * - "on": persist materialized frame+span content, falling back per fact
+   *   to generated content when the span fails validation.
+   */
+  spanMode: "off" | "shadow" | "on";
+}
+
+export interface ExtractedFactSpanRef {
+  /** Ordinal of the source turn in the extraction segment (0-based). */
+  sourceMessageIndex: number;
+  /** [charStart, charEnd) into the turn's exact in-memory text (UTF-16 code units). */
+  charStart: number;
+  charEnd: number;
+  /** ≤ 15 generated words making the span self-contained (resolves deixis). */
+  frame: string;
+  /** Optional prompt-source identity; when present it must match the captured stamp. */
+  sourceHash?: string;
+  sourceLength?: number;
+}
+
+export const EXTRACTION_SPAN_MODES = ["off", "shadow", "on"] as const;
+const extractionSpanBlockSchema = z.object({ spanMode: z.unknown().optional() }).strict();
+
+/** Span + liveness fields for parseConfig. One spread keeps config.ts at the ratchet. */
+export function parseExtractionFields(cfg: Record<string, unknown>) {
+  return {
+    extraction: parseExtractionSpanConfig(cfg.extraction),
+    extractionLiveness: parseExtractionLivenessConfig(cfg),
+  };
+}
+
+export function parseExtractionSpanConfig(raw: unknown): ExtractionSpanConfig {
+  if (raw === undefined) {
+    return { spanMode: "off" };
+  }
+  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
+    throw new Error(
+      `extraction must be an object with optional spanMode (got ${JSON.stringify(raw)}). Omit the key for the default "off".`,
+    );
+  }
+  const block = raw as Record<string, unknown>;
+  const parsed = extractionSpanBlockSchema.safeParse(block);
+  if (!parsed.success) {
+    const unknownKeys = parsed.error.issues.flatMap((issue) =>
+      issue.code === "unrecognized_keys" ? issue.keys : []
+    );
+    throw new Error(
+      `extraction has unknown propert${unknownKeys.length === 1 ? "y" : "ies"}: ${unknownKeys.join(", ")}. Allowed: "spanMode".`,
+    );
+  }
+  const spanMode = block.spanMode;
+  if (spanMode === undefined) {
+    return { spanMode: "off" };
+  }
+  for (const mode of EXTRACTION_SPAN_MODES) {
+    if (spanMode === mode) {
+      return { spanMode: mode };
+    }
+  }
+  throw new Error(
+    `extraction.spanMode must be one of "off", "shadow", "on" (got ${JSON.stringify(spanMode)}). Omit the key for the default "off".`,
+  );
+}