@remnic/core 9.65.8 → 9.66.1

package/src/dedup/merge-on-write.test.ts

@@ -0,0 +1,91 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import {
+  type MergeJudgeVerdict,
+  type MergeOnWritePair,
+  applyMergeOnWrite,
+  judgeMergeDecision,
+  shouldConsiderMerge,
+} from "./merge-on-write.js";
+
+function pair(overrides: Partial<MergeOnWritePair> = {}): MergeOnWritePair {
+  return {
+    category: "fact",
+    score: 0.85,
+    incomingContent: "User prefers dark mode on laptops.",
+    existingContent: "User likes dark mode.",
+    existingId: "mem-1",
+    ...overrides,
+  };
+}
+
+const mergeJudge = async () => "merge" as const;
+
+test("merge-on-write: skip band is not considered", () => {
+  assert.equal(shouldConsiderMerge({ score: 0.92 }), false);
+  assert.equal(shouldConsiderMerge({ score: 0.99 }), false);
+});
+
+test("merge-on-write: merge band is considered", () => {
+  assert.equal(shouldConsiderMerge({ score: 0.8 }), true);
+  assert.equal(shouldConsiderMerge({ score: 0.85 }), true);
+  assert.equal(shouldConsiderMerge({ score: 0.919 }), true);
+});
+
+test("merge-on-write: score below mergeMin is not considered", () => {
+  assert.equal(shouldConsiderMerge({ score: 0.79 }), false);
+});
+
+test("merge-on-write: refused categories always create", async () => {
+  for (const category of ["procedure", "reasoning_trace", "moment", "correction"]) {
+    const decision = await applyMergeOnWrite({
+      enabled: true,
+      pair: pair({ category }),
+      judge: mergeJudge,
+    });
+    assert.equal(decision, "create", category);
+  }
+});
+
+test("merge-on-write: judge throw creates", async () => {
+  const viaJudge = await judgeMergeDecision(pair(), async () => {
+    throw new Error("timeout");
+  });
+  assert.equal(viaJudge, "create");
+
+  const viaApply = await applyMergeOnWrite({
+    enabled: true,
+    pair: pair(),
+    judge: async () => {
+      throw new Error("timeout");
+    },
+  });
+  assert.equal(viaApply, "create");
+});
+
+test("merge-on-write: uncertain judge creates", async () => {
+  const decision = await judgeMergeDecision(pair(), async (): Promise<MergeJudgeVerdict> => "uncertain");
+  assert.equal(decision, "create");
+});
+
+test("merge-on-write: disabled or mergeMin 0 creates", async () => {
+  let called = 0;
+  const judge = async () => {
+    called += 1;
+    return "merge" as const;
+  };
+
+  assert.equal(await applyMergeOnWrite({ pair: pair(), judge }), "create");
+  assert.equal(await applyMergeOnWrite({ enabled: false, pair: pair(), judge }), "create");
+  assert.equal(await applyMergeOnWrite({ enabled: true, mergeMin: 0, pair: pair(), judge }), "create");
+  assert.equal(called, 0);
+});
+
+test("merge-on-write: enabled in-band merge applies", async () => {
+  const decision = await applyMergeOnWrite({
+    enabled: true,
+    pair: pair(),
+    judge: mergeJudge,
+  });
+  assert.equal(decision, "merge");
+});

package/src/dedup/merge-on-write.ts

@@ -0,0 +1,75 @@
+/**
+ * Isolated merge-on-write judge (issue #2330 first slice).
+ *
+ * Default off. Persist wiring comes later. Failures create.
+ */
+
+export const DEFAULT_MERGE_MIN = 0.8;
+export const DEFAULT_SKIP_THRESHOLD = 0.92;
+
+export const REFUSED_MERGE_CATEGORIES = [
+  "procedure",
+  "reasoning_trace",
+  "moment",
+  "correction",
+] as const;
+
+export type MergeOnWriteDecision = "merge" | "create";
+export type MergeJudgeVerdict = MergeOnWriteDecision | "uncertain";
+
+export interface MergeOnWritePair {
+  category: string;
+  score: number;
+  incomingContent?: string;
+  existingContent?: string;
+  existingId?: string;
+}
+
+export type MergeJudge = (
+  pair: MergeOnWritePair,
+) => Promise<MergeJudgeVerdict> | MergeJudgeVerdict;
+
+export function shouldConsiderMerge(opts: {
+  score: number;
+  skipThreshold?: number;
+  mergeMin?: number;
+}): boolean {
+  const mergeMin = opts.mergeMin ?? DEFAULT_MERGE_MIN;
+  const skipThreshold = opts.skipThreshold ?? DEFAULT_SKIP_THRESHOLD;
+  if (!(mergeMin > 0)) return false;
+  if (!Number.isFinite(opts.score) || !Number.isFinite(skipThreshold)) return false;
+  return mergeMin <= opts.score && opts.score < skipThreshold;
+}
+
+export async function judgeMergeDecision(
+  pair: MergeOnWritePair,
+  judge: MergeJudge,
+): Promise<MergeOnWriteDecision> {
+  try {
+    const verdict = await judge(pair);
+    return verdict === "merge" ? "merge" : "create";
+  } catch {
+    return "create";
+  }
+}
+
+export async function applyMergeOnWrite(opts: {
+  pair: MergeOnWritePair;
+  judge: MergeJudge;
+  enabled?: boolean;
+  mergeMin?: number;
+  skipThreshold?: number;
+}): Promise<MergeOnWriteDecision> {
+  if (opts.enabled !== true) return "create";
+  if ((REFUSED_MERGE_CATEGORIES as readonly string[]).includes(opts.pair.category)) return "create";
+  if (
+    !shouldConsiderMerge({
+      score: opts.pair.score,
+      mergeMin: opts.mergeMin,
+      skipThreshold: opts.skipThreshold,
+    })
+  ) {
+    return "create";
+  }
+  return judgeMergeDecision(opts.pair, opts.judge);
+}

package/src/dedup/novelty-gate.test.ts

@@ -0,0 +1,143 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { parseConfig } from "../config.js";
+import {
+  applyNoveltyGate,
+  DEFAULT_NOVELTY_ADD_THRESHOLD,
+  DEFAULT_NOVELTY_NOOP_THRESHOLD,
+  embeddingsFromCosineHits,
+  scoreNovelty,
+  type NoveltyNeighbor,
+} from "./novelty-gate.js";
+
+function neighborAtCosine(id: string, cosine: number): NoveltyNeighbor {
+  const s = Math.min(1, Math.max(-1, cosine));
+  return { id, embedding: [s, Math.sqrt(Math.max(0, 1 - s * s))] };
+}
+
+const CANDIDATE = [1, 0];
+
+test("novelty gate: parseConfig defaults keep the gate off with a wide uncertain band", () => {
+  const config = parseConfig({});
+  assert.equal(config.noveltyGateEnabled, false);
+  assert.equal(config.noveltyAddThreshold, DEFAULT_NOVELTY_ADD_THRESHOLD);
+  assert.equal(config.noveltyNoopThreshold, DEFAULT_NOVELTY_NOOP_THRESHOLD);
+  assert.ok(config.noveltyAddThreshold - config.noveltyNoopThreshold > 0.3);
+});
+
+test("novelty gate: conservative preset stays off", () => {
+  const config = parseConfig({ memoryOsPreset: "conservative" });
+  assert.equal(config.noveltyGateEnabled, false);
+});
+
+test("novelty gate: gate off does not call lookup", async () => {
+  let called = 0;
+  const decision = await applyNoveltyGate({
+    enabled: false,
+    addThreshold: DEFAULT_NOVELTY_ADD_THRESHOLD,
+    noopThreshold: DEFAULT_NOVELTY_NOOP_THRESHOLD,
+    lookup: async () => {
+      called += 1;
+      throw new Error("lookup must not run when the gate is off");
+    },
+  });
+  assert.equal(called, 0);
+  assert.equal(decision.decision, "uncertain");
+});
+
+test("novelty gate: paraphrase routes noop", () => {
+  const decision = scoreNovelty(CANDIDATE, [neighborAtCosine("paraphrase", 0.98)]);
+  assert.equal(decision.decision, "noop");
+  assert.equal(decision.neighborId, "paraphrase");
+  assert.ok(decision.score <= DEFAULT_NOVELTY_NOOP_THRESHOLD);
+});
+
+test("novelty gate: unrelated routes add", () => {
+  const decision = scoreNovelty(CANDIDATE, [neighborAtCosine("other", 0.05)]);
+  assert.equal(decision.decision, "add");
+  assert.equal(decision.neighborId, "other");
+  assert.ok(decision.score >= DEFAULT_NOVELTY_ADD_THRESHOLD);
+});
+
+test("novelty gate: exactly at addThreshold is add", () => {
+  const neighbor = neighborAtCosine("edge-add", 1 - DEFAULT_NOVELTY_ADD_THRESHOLD);
+  const measured = scoreNovelty(CANDIDATE, [neighbor], { addThreshold: 1, noopThreshold: -1 });
+  const decision = scoreNovelty(CANDIDATE, [neighbor], {
+    addThreshold: measured.score,
+    noopThreshold: DEFAULT_NOVELTY_NOOP_THRESHOLD,
+  });
+  assert.equal(decision.decision, "add");
+});
+
+test("novelty gate: exactly at noopThreshold is noop", () => {
+  const neighbor = neighborAtCosine("edge-noop", 1 - DEFAULT_NOVELTY_NOOP_THRESHOLD);
+  const measured = scoreNovelty(CANDIDATE, [neighbor], { addThreshold: 1, noopThreshold: -1 });
+  const decision = scoreNovelty(CANDIDATE, [neighbor], {
+    addThreshold: DEFAULT_NOVELTY_ADD_THRESHOLD,
+    noopThreshold: measured.score,
+  });
+  assert.equal(decision.decision, "noop");
+});
+
+test("novelty gate: mid-band routes uncertain", () => {
+  const mid = (DEFAULT_NOVELTY_ADD_THRESHOLD + DEFAULT_NOVELTY_NOOP_THRESHOLD) / 2;
+  const decision = scoreNovelty(CANDIDATE, [neighborAtCosine("mid", 1 - mid)]);
+  assert.equal(decision.decision, "uncertain");
+});
+
+test("novelty gate: empty neighborhood is add", () => {
+  const decision = scoreNovelty(CANDIDATE, []);
+  assert.equal(decision.decision, "add");
+  assert.equal(decision.score, 1);
+  assert.equal(decision.neighborId, undefined);
+});
+
+test("novelty gate: thrown lookup is uncertain", async () => {
+  const decision = await applyNoveltyGate({
+    enabled: true,
+    addThreshold: DEFAULT_NOVELTY_ADD_THRESHOLD,
+    noopThreshold: DEFAULT_NOVELTY_NOOP_THRESHOLD,
+    lookup: async () => {
+      throw new Error("backend down");
+    },
+  });
+  assert.equal(decision.decision, "uncertain");
+});
+
+test("novelty gate: cross-connector neighbor cannot noop (PR #2564 review)", () => {
+  // Identical content already stored by connector A (cosine ~1) must not
+  // suppress connector B's candidate: neighborhood scoped to same-connector
+  // neighbors only (mirrors the PR #1852 semantic skip gate).
+  const identicalFromA = { id: "a-dup", score: 1, sourceConnector: "connector-a" };
+  const { neighborhood } = embeddingsFromCosineHits([identicalFromA], {
+    candidateConnector: "connector-b",
+  });
+  assert.deepEqual(neighborhood, []);
+  const decision = scoreNovelty(CANDIDATE, neighborhood);
+  assert.equal(decision.decision, "add");
+});
+
+test("novelty gate: unattributed neighbor is scoped out when candidate has a connector", () => {
+  const unattributed = { id: "anon-dup", score: 1 };
+  const { neighborhood } = embeddingsFromCosineHits([unattributed], {
+    candidateConnector: "connector-b",
+  });
+  assert.deepEqual(neighborhood, []);
+});
+
+test("novelty gate: same-connector paraphrase still noops", () => {
+  const sameConnector = { id: "b-dup", score: 0.98, sourceConnector: "connector-b" };
+  const { neighborhood } = embeddingsFromCosineHits([sameConnector], {
+    candidateConnector: "connector-b",
+  });
+  assert.equal(neighborhood.length, 1);
+  assert.equal(scoreNovelty(CANDIDATE, neighborhood).decision, "noop");
+});
+
+test("novelty gate: candidate without connector keeps unscoped neighborhood", () => {
+  const foreign = { id: "a-dup", score: 0.98, sourceConnector: "connector-a" };
+  const { neighborhood } = embeddingsFromCosineHits([foreign]);
+  assert.equal(neighborhood.length, 1);
+  assert.equal(scoreNovelty(CANDIDATE, neighborhood).decision, "noop");
+});

package/src/dedup/novelty-gate.ts

@@ -0,0 +1,163 @@
+import { coerceBooleanLike, coerceNumber } from "../connectors/coerce.js";
+
+/**
+ * Embedding-density novelty gate (issue #1953, SAGE).
+ *
+ * score = 1 - mean(cosine of k nearest). Higher score = more novel.
+ *   score >= noveltyAddThreshold  → add (skip semantic/LLM dedup)
+ *   score <= noveltyNoopThreshold → noop (drop; do not touch contentHashIndex)
+ *   else                          → uncertain (fall through to semantic dedup)
+ *
+ * Defaults keep a wide UNCERTAIN band. noveltyGateEnabled defaults false.
+ * Backend failure is not handled here — callers treat a thrown lookup as uncertain.
+ */
+export const DEFAULT_NOVELTY_ADD_THRESHOLD = 0.55;
+export const DEFAULT_NOVELTY_NOOP_THRESHOLD = 0.15;
+export const DEFAULT_NOVELTY_K = 5;
+
+export interface NoveltyNeighbor {
+  id: string;
+  embedding: number[];
+}
+
+export interface NoveltyDecision {
+  score: number;
+  decision: "add" | "noop" | "uncertain";
+  neighborId?: string;
+}
+
+export interface NoveltyThresholds {
+  addThreshold?: number;
+  noopThreshold?: number;
+  k?: number;
+}
+
+export interface WritePathDedupConfig {
+  semanticDedupEnabled: boolean;
+  semanticDedupThreshold: number;
+  semanticDedupCandidates: number;
+  noveltyGateEnabled: boolean;
+  noveltyAddThreshold: number;
+  noveltyNoopThreshold: number;
+}
+
+function clamp01(value: number | undefined, fallback: number): number {
+  return value !== undefined && Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : fallback;
+}
+
+function cosine(a: number[], b: number[]): number {
+  if (a.length === 0 || a.length !== b.length) return 0;
+  let dot = 0;
+  let magA = 0;
+  let magB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  if (denom === 0) return 0;
+  const raw = dot / denom;
+  if (!Number.isFinite(raw)) return 0;
+  if (raw < 0) return 0;
+  if (raw > 1) return 1;
+  return raw;
+}
+
+export function parseWritePathDedupConfig(cfg: Record<string, unknown>): WritePathDedupConfig {
+  const rawCandidates = coerceNumber(cfg.semanticDedupCandidates, "semanticDedupCandidates");
+  let semanticDedupCandidates = 5;
+  if (rawCandidates !== undefined && rawCandidates >= 0) {
+    const n = Math.floor(rawCandidates);
+    semanticDedupCandidates = rawCandidates > 0 && n === 0 ? 1 : n;
+  }
+  return {
+    semanticDedupEnabled: coerceBooleanLike(cfg.semanticDedupEnabled, "semanticDedupEnabled") !== false,
+    semanticDedupThreshold: clamp01(
+      coerceNumber(cfg.semanticDedupThreshold, "semanticDedupThreshold"),
+      0.92,
+    ),
+    semanticDedupCandidates,
+    noveltyGateEnabled: coerceBooleanLike(cfg.noveltyGateEnabled, "noveltyGateEnabled") === true,
+    noveltyAddThreshold: clamp01(
+      coerceNumber(cfg.noveltyAddThreshold, "noveltyAddThreshold"),
+      DEFAULT_NOVELTY_ADD_THRESHOLD,
+    ),
+    noveltyNoopThreshold: clamp01(
+      coerceNumber(cfg.noveltyNoopThreshold, "noveltyNoopThreshold"),
+      DEFAULT_NOVELTY_NOOP_THRESHOLD,
+    ),
+  };
+}
+
+export function embeddingsFromCosineHits(
+  hits: ReadonlyArray<{ id: string; score: number; sourceConnector?: string }>,
+  opts: { candidateConnector?: string } = {},
+): { embedding: number[]; neighborhood: NoveltyNeighbor[] } {
+  // Connector-aware scoping (PR #2564 review finding on 54d865cb9): a
+  // neighbor from a different (or unattributed) connector must not drive a
+  // noop — same contract as the semantic skip gate (PR #1852). Candidates
+  // without a connector keep the unscoped neighborhood.
+  const candidateConnector = opts.candidateConnector?.trim() || undefined;
+  const scoped = candidateConnector
+    ? hits.filter((hit) => (hit.sourceConnector?.trim() || undefined) === candidateConnector)
+    : hits;
+  return {
+    embedding: [1, 0],
+    neighborhood: scoped.map((hit) => {
+      const s = Number.isFinite(hit.score) ? Math.min(1, Math.max(-1, hit.score)) : 0;
+      return { id: hit.id, embedding: [s, Math.sqrt(Math.max(0, 1 - s * s))] };
+    }),
+  };
+}
+
+export function scoreNovelty(
+  embedding: number[],
+  neighborhood: NoveltyNeighbor[],
+  thresholds: NoveltyThresholds = {},
+): NoveltyDecision {
+  const addThreshold = clamp01(thresholds.addThreshold ?? DEFAULT_NOVELTY_ADD_THRESHOLD, DEFAULT_NOVELTY_ADD_THRESHOLD);
+  const noopThreshold = clamp01(thresholds.noopThreshold ?? DEFAULT_NOVELTY_NOOP_THRESHOLD, DEFAULT_NOVELTY_NOOP_THRESHOLD);
+  const k = thresholds.k !== undefined && Number.isFinite(thresholds.k) && thresholds.k > 0
+    ? Math.floor(thresholds.k)
+    : DEFAULT_NOVELTY_K;
+
+  if (neighborhood.length === 0) {
+    return { score: 1, decision: "add" };
+  }
+
+  const ranked = neighborhood
+    .map((neighbor) => ({
+      id: neighbor.id,
+      cosine: cosine(embedding, neighbor.embedding),
+    }))
+    .sort((a, b) => b.cosine - a.cosine);
+  const nearest = ranked.slice(0, Math.max(1, k));
+  const density = nearest.reduce((sum, row) => sum + row.cosine, 0) / nearest.length;
+  const score = 1 - density;
+  const neighborId = ranked[0]?.id;
+
+  if (score >= addThreshold) return { score, decision: "add", neighborId };
+  if (score <= noopThreshold) return { score, decision: "noop", neighborId };
+  return { score, decision: "uncertain", neighborId };
+}
+
+export async function applyNoveltyGate(opts: {
+  enabled: boolean;
+  addThreshold: number;
+  noopThreshold: number;
+  lookup: () => Promise<{ embedding: number[]; neighborhood: NoveltyNeighbor[] }>;
+}): Promise<NoveltyDecision> {
+  if (!opts.enabled) {
+    return { score: 0, decision: "uncertain" };
+  }
+  try {
+    const { embedding, neighborhood } = await opts.lookup();
+    return scoreNovelty(embedding, neighborhood, {
+      addThreshold: opts.addThreshold,
+      noopThreshold: opts.noopThreshold,
+    });
+  } catch {
+    return { score: 0, decision: "uncertain" };
+  }
+}

package/src/extraction-prompt-safety.test.ts

@@ -1,5 +1,6 @@
 import assert from "node:assert/strict";
 import test from "node:test";
+import type { FallbackLlmOptions } from "./fallback-llm.js";
 
 import { parseConfig } from "./config.js";
 import { ExtractionEngine } from "./extraction.js";
@@ -111,7 +112,11 @@ type LocalLlmFixture = {
 };
 
 type GatewayFixture = {
-  parseWithSchemaDetailed(messages: ChatMessage[]): Promise<unknown>;
+  parseWithSchemaDetailed(
+    messages: ChatMessage[],
+    schema: unknown,
+    options?: FallbackLlmOptions,
+  ): Promise<unknown>;
 };
 
 type DirectClientFixture = {
@@ -283,11 +288,13 @@ test("gateway extraction prompt uses placeholders and accepts an empty question
   const engine = new ExtractionEngine(parseConfig({ modelSource: "gateway" }));
   let calls = 0;
   let prompt = "";
+  let nativeSchema: Record<string, unknown> | undefined;
 
   const fallbackLlm: GatewayFixture = {
-    async parseWithSchemaDetailed(messages: ChatMessage[]) {
+    async parseWithSchemaDetailed(messages: ChatMessage[], _schema, options) {
       calls += 1;
       prompt = messages[0]?.content ?? "";
+      nativeSchema = options?.jsonSchema?.schema;
       return { modelUsed: "fixture-gateway", result: SOURCE_GROUNDED_EXTRACTION };
     },
   };
@@ -298,6 +305,11 @@ test("gateway extraction prompt uses placeholders and accepts an empty question
   assert.equal(calls, 1);
   assertSourceGroundedResult(result);
   assertSafeExtractionPrompt(prompt);
+  assert.match(prompt, /Return only valid JSON matching this shape/);
+  assert.equal(nativeSchema?.type, "object");
+  const properties = nativeSchema?.properties as Record<string, Record<string, unknown>>;
+  assert.equal(properties.facts?.type, "array");
+  assert.equal((properties.facts?.items as Record<string, unknown>)?.type, "object");
 });
 
 test("direct extraction prompt uses placeholders and accepts an empty question list", async () => {

package/src/extraction.ts

@@ -7,6 +7,7 @@ import { shouldRunProactivePass } from "./proactive-contention.js";
 import { FallbackLlmClient, fallbackLlmRuntimeContextFromConfig, gatewayTaskChainOptions } from "./fallback-llm.js";
 import { revalidateDependentsViaLlm, type RevalidationFastChatCompletion } from "./orchestration/dependency-revalidation.js";
 import {
+  ExtractionProviderOutputSchema,
   ExtractionResultSchema,
   ConsolidationResultSchema,
   IdentityConsolidationResultSchema,
@@ -60,6 +61,7 @@ import {
   buildProfileConsolidationSystemPrompt,
   eventTimePromptInstruction,
 } from "./extraction-prompt.js";
+import { zodToJsonSchema } from "zod-to-json-schema";
 import {
   containsExtractionPlaceholder,
   extractionAttributes,
@@ -77,6 +79,14 @@ type ExtractedRelationshipResult = NonNullable<ExtractionResult["relationships"]
 
 const PROACTIVE_MIN_CONFIDENCE = 0.8;
 
+const EXTRACTION_PROVIDER_JSON_SCHEMA = zodToJsonSchema(
+  ExtractionProviderOutputSchema,
+  {
+    $refStrategy: "none",
+    effectStrategy: "input",
+  },
+) as Record<string, unknown>;
+
 function extractionEntityType(value: unknown): ExtractedEntityResult["type"] | undefined {
   const type = extractionText(value);
   if (
@@ -1289,7 +1299,12 @@ export class ExtractionEngine {
     this.profiler.startSpan("gateway-fallback", extractionTraceId);
     try {
       const messages = [
-        { role: "system" as const, content: buildExtractionInstructions(this.config, existingEntities, ambientCapture) },
+        {
+          role: "system" as const,
+          content:
+            buildExtractionInstructions(this.config, existingEntities, ambientCapture) +
+            `\n\nReturn only valid JSON matching this shape. Placeholder text describes field shape only and is never source evidence:\n${EXTRACTION_RESPONSE_SHAPE}`,
+        },
         { role: "user" as const, content: conversation },
       ];
 
@@ -1302,6 +1317,11 @@ export class ExtractionEngine {
           temperature: 0.3,
           maxTokens: this.config.extractionMaxOutputTokens,
           timeoutMs: this.config.localLlmTimeoutMs,
+          jsonSchema: {
+            name: "extraction_result",
+            schema: EXTRACTION_PROVIDER_JSON_SCHEMA,
+            strict: false,
+          },
           signal,
         }),
       );