@remnic/core 10.0.84 → 10.0.86

package/src/extraction-quote-grounding.ts

@@ -67,7 +67,18 @@ function normalizeWhitespaceWithMap(source: string): { text: string; originalInd
   return { text: chars.join(""), originalIndex };
 }
 
-function findQuoteAnchors(quote: string, source: string): QuoteAnchor[] {
+function turnStartOffsets(source: string, turns: readonly string[]): readonly number[] | undefined {
+  const offsets: number[] = [];
+  let at = 0;
+  for (const turn of turns) {
+    if (!source.startsWith(turn, at)) return undefined;
+    offsets.push(at);
+    at += turn.length + TURN_SEPARATOR.length;
+  }
+  return at - TURN_SEPARATOR.length === source.length ? offsets : undefined;
+}
+
+function findQuoteAnchors(quote: string, source: string, bounds?: readonly number[]): QuoteAnchor[] {
   const needle = quote.replace(/\s+/gu, " ").trim();
   if (needle.length < QUOTE_ANCHOR_MIN_CHARS) return [];
   const normalized = normalizeWhitespaceWithMap(source);
@@ -79,17 +90,36 @@ function findQuoteAnchors(quote: string, source: string): QuoteAnchor[] {
     from = at + 1;
     const start = normalized.originalIndex[at]!;
     const end = normalized.originalIndex[at + needle.length - 1]! + 1;
-    if (source.slice(start, end).includes(TURN_SEPARATOR)) continue;
-
-    let before = source.slice(Math.max(0, start - QUOTE_ANCHOR_CONTEXT_CHARS), start);
-    const turnStart = before.lastIndexOf(TURN_SEPARATOR);
-    if (turnStart >= 0) before = before.slice(turnStart + TURN_SEPARATOR.length);
-    else if (start > QUOTE_ANCHOR_CONTEXT_CHARS) before = before.replace(/^\S+/u, "");
 
-    let after = source.slice(end, end + QUOTE_ANCHOR_CONTEXT_CHARS);
-    const turnEnd = after.indexOf(TURN_SEPARATOR);
-    if (turnEnd >= 0) after = after.slice(0, turnEnd);
-    else if (source.length - end > QUOTE_ANCHOR_CONTEXT_CHARS) after = after.replace(/\S+$/u, "");
+    let before: string;
+    let after: string;
+    if (bounds === undefined) {
+      if (source.slice(start, end).includes(TURN_SEPARATOR)) continue;
+      before = source.slice(Math.max(0, start - QUOTE_ANCHOR_CONTEXT_CHARS), start);
+      const sepBefore = before.lastIndexOf(TURN_SEPARATOR);
+      if (sepBefore >= 0) before = before.slice(sepBefore + TURN_SEPARATOR.length);
+      else if (start > QUOTE_ANCHOR_CONTEXT_CHARS) before = before.replace(/^\S+/u, "");
+      after = source.slice(end, end + QUOTE_ANCHOR_CONTEXT_CHARS);
+      const sepAfter = after.indexOf(TURN_SEPARATOR);
+      if (sepAfter >= 0) after = after.slice(0, sepAfter);
+      else if (source.length - end > QUOTE_ANCHOR_CONTEXT_CHARS) after = after.replace(/\S+$/u, "");
+    } else {
+      let turnStart = 0;
+      let turnEnd = source.length;
+      for (const bound of bounds) {
+        if (bound <= start) turnStart = bound;
+        else {
+          turnEnd = bound - TURN_SEPARATOR.length;
+          break;
+        }
+      }
+      if (turnEnd < end) continue;
+      before = source.slice(Math.max(turnStart, start - QUOTE_ANCHOR_CONTEXT_CHARS), start);
+      if (start - QUOTE_ANCHOR_CONTEXT_CHARS > turnStart) before = before.replace(/^\S+/u, "");
+      const contextEnd = Math.min(turnEnd, end + QUOTE_ANCHOR_CONTEXT_CHARS);
+      after = source.slice(end, contextEnd);
+      if (end + QUOTE_ANCHOR_CONTEXT_CHARS < turnEnd) after = after.replace(/\S+$/u, "");
+    }
 
     anchors.push({ quote: needle, window: `${before}${needle.replace(/;/gu, ",")}${after}` });
   }
@@ -109,16 +139,22 @@ function findQuoteAnchors(quote: string, source: string): QuoteAnchor[] {
  * `source` may have been rewritten before grounding (temporal anchoring,
  * pronoun resolution) while the model quoted the original words. When the raw
  * quote is not found, `transformQuote` applies the same rewrite to the quote.
+ *
+ * `turns` are the texts `source` was joined from; when they tile `source`
+ * exactly, quotes and windows cut at real turn boundaries instead of any
+ * blank line. Without them, every `TURN_SEPARATOR` run is a boundary.
  */
 export function quoteAnchors(
   quote: string | undefined,
   source: string,
   transformQuote?: (quote: string) => string,
+  turns?: readonly string[],
 ): QuoteAnchor[] {
   if (typeof quote !== "string") return [];
-  const exact = findQuoteAnchors(quote, source);
+  const bounds = turns === undefined ? undefined : turnStartOffsets(source, turns);
+  const exact = findQuoteAnchors(quote, source, bounds);
   if (exact.length > 0 || transformQuote === undefined) return exact;
-  return findQuoteAnchors(transformQuote(quote), source);
+  return findQuoteAnchors(transformQuote(quote), source, bounds);
 }
 
 export function tokenPool(tokens: ReadonlySet<string>): TokenPool {
@@ -130,24 +166,62 @@ function hasNegation(text: string): boolean {
   return NEGATOR_PATTERN.test(text.replace(/[\u2018\u2019]/gu, "'")) || hasDenialReporting(text);
 }
 
+const CLAUSE_PATTERN = /[^;.!?\n]+[;.!?]*/gu;
+/** Split "X, and Y" claim coordination without touching list commas. */
+const CLAIM_COORD_SPLIT = /,\s{1,8}(?:and|but|or|nor)\s{1,8}/iu;
+
+function splitClaimClauses(content: string): string[] {
+  return (content.match(CLAUSE_PATTERN) ?? [content]).flatMap((clause) => {
+    const parts = clause
+      .split(CLAIM_COORD_SPLIT)
+      .map((part) => part.trim())
+      .filter((part) => part.length > 0);
+    return parts.length > 0 ? parts : [clause];
+  });
+}
+
+function clauseOverlap(clauseTokens: readonly string[], target: string) {
+  const has = tokenPool(tokenize(target));
+  const shared = clauseTokens.filter(has).length;
+  return { shared, ratio: clauseTokens.length === 0 ? 0 : shared / clauseTokens.length };
+}
+
+/** What a quote clause's first negator governs: the words after it. */
+function negatedScope(clause: string): string {
+  const match = NEGATOR_PATTERN.exec(clause.replace(/[\u2018\u2019]/gu, "'"));
+  return match ? clause.slice(match.index + match[0].length) : clause;
+}
+
 /**
  * True when the claim asserts what a quote clause denies or does not assert:
  * "did not approve the budget" or "denied approving it" restated as "approved
  * the budget", "suspects Bob stole funds" or "Is Alice the CEO?" restated as a
- * plain statement. A claim that keeps a negator, exclusion word, or reporting
- * marker ("do not X" reworded as "X is excluded") is not a misstatement.
+ * plain statement. The claim must restate the quote clause, and none of its
+ * clauses that share a word with what the negator governs may keep the
+ * negator, exclusion word, or reporting marker ("do not X" reworded as "X is
+ * excluded", or "Alice reviewed it, but she did not approve it"). A negator in
+ * a clause about something else does not count, so a claim that keeps one
+ * quote negation can still misstate another quote clause, and a claim clause
+ * that shares only the subject does not satisfy the check.
  */
 function misstatesQuote(content: string, quote: string): boolean {
-  const keepsNegation = hasNegation(content) || EXCLUSION_PATTERN.test(content);
-  const keepsReporting = hasNonAssertiveReporting(content) || content.trim().endsWith("?");
-  const contentHas = tokenPool(tokenize(content));
-  return (quote.match(/[^;.!?\n]+[;.!?]*/gu) ?? []).some((clause) => {
-    const negated = hasNegation(clause) && !keepsNegation;
-    const unasserted = (clause.trim().endsWith("?") || hasNonAssertiveReporting(clause)) && !keepsReporting;
-    if (!negated && !unasserted) return false;
-    const clauseTokens = [...tokenize(clause)];
-    const shared = clauseTokens.filter(contentHas).length;
-    return shared >= GROUNDING_MIN_SHARED_TOKENS && shared / clauseTokens.length >= 0.5;
+  const claimClauses = splitClaimClauses(content);
+  return (quote.match(CLAUSE_PATTERN) ?? []).some((quoteClause) => {
+    const needsNegation = hasNegation(quoteClause);
+    const needsReporting = quoteClause.trim().endsWith("?") || hasNonAssertiveReporting(quoteClause);
+    if (!needsNegation && !needsReporting) return false;
+    const clauseTokens = [...tokenize(quoteClause)];
+    const whole = clauseOverlap(clauseTokens, content);
+    if (whole.shared < GROUNDING_MIN_SHARED_TOKENS || whole.ratio < 0.5) return false;
+    const clausesSharing = (tokens: readonly string[]) =>
+      claimClauses.filter((claimClause) => clauseOverlap(tokens, claimClause).shared > 0);
+    if (needsNegation) {
+      const scopeTokens = [...tokenize(negatedScope(quoteClause))];
+      const related = clausesSharing(scopeTokens.length > 0 ? scopeTokens : clauseTokens);
+      if (!related.some((text) => hasNegation(text) || EXCLUSION_PATTERN.test(text))) return true;
+    }
+    return needsReporting
+      && !clausesSharing(clauseTokens).some((text) => hasNonAssertiveReporting(text) || text.trim().endsWith("?"));
   });
 }
 

package/src/extraction-source-grounding-rules.ts

@@ -43,6 +43,8 @@ export interface ExtractionSourceGroundingOptions {
   spanMode?: "off" | "shadow" | "on";
   /** Rewrite applied to the source before this stage, so a verbatim quote can still be located in it. */
   quoteTransform?: (quote: string) => string;
+  /** Turn texts the sources were joined from, so quote anchors cut at real turn boundaries. */
+  turns?: readonly string[];
 }
 
 export function applyExtractionSourceGrounding(
@@ -68,6 +70,7 @@ export function applyExtractionSourceGrounding(
         ? undefined
         : anchorSource(roleAssertionSources.identity),
     };
+  const anchoredTurns = options.turns === undefined ? undefined : options.turns.map((turn) => anchorSource(turn));
   return filterExtractionResultBySource(
     result,
     anchorSource(sourceText),
@@ -78,6 +81,7 @@ export function applyExtractionSourceGrounding(
     options.anchorTemporalExpressions
       ? (quote) => anchorSource(options.quoteTransform?.(quote) ?? quote)
       : undefined,
+    anchoredTurns,
   );
 }
 
@@ -1117,6 +1121,7 @@ export function filterExtractionResultBySource(
   eventTimeNormalizer?: (eventTime: string) => string,
   spanMode: "off" | "shadow" | "on" = "off",
   quoteTransform?: (quote: string) => string,
+  turns?: readonly string[],
 ): ExtractionResult {
   if (source.trim().length === 0) {
     return {
@@ -1136,7 +1141,7 @@ export function filterExtractionResultBySource(
     result.facts.some((fact) => typeof fact.quote === "string") ? tokenize(quoteSource) : new Set(),
   );
   const facts = result.facts.flatMap((fact) => {
-    const anchors = quoteAnchors(fact.quote, quoteSource, quoteTransform);
+    const anchors = quoteAnchors(fact.quote, quoteSource, quoteTransform, turns);
     const grounded = filterGroundedFact(
       fact,
       source,

package/src/extraction-source-grounding.ts

@@ -101,6 +101,7 @@ export function filterExtractionResultBySource(
   eventTimeNormalizer?: (eventTime: string) => string,
   spanMode: "off" | "shadow" | "on" = "off",
   quoteTransform?: (quote: string) => string,
+  turns?: readonly string[],
 ): ExtractionResult {
   return filterGroundingRules(
     result,
@@ -110,5 +111,6 @@ export function filterExtractionResultBySource(
     eventTimeNormalizer,
     spanMode,
     quoteTransform,
+    turns,
   );
 }

package/src/extraction.ts

@@ -1122,10 +1122,8 @@ export class ExtractionEngine {
     const groundingSource = boundedTurns
       .map((turn) => turn.content)
       .join("\n\n");
-    const assertionSource = boundedTurns
-      .filter((turn) => turn.extractionContextOnly !== true)
-      .map((turn) => turn.content)
-      .join("\n\n");
+    const assertionTurns = boundedTurns.filter((turn) => turn.extractionContextOnly !== true);
+    const assertionSource = assertionTurns.map((turn) => turn.content).join("\n\n");
     const roleAssertionSources: ExtractionGroundingRoleSources = {
       profile: boundedTurns
         .filter((turn) => turn.role === "user" && turn.extractionContextOnly !== true)
@@ -1171,6 +1169,7 @@ export class ExtractionEngine {
       sourceConnector,
       ambientCapture,
       scopeClassificationEnabled: lifecycleCaps.extractionScopeClassification,
+      turns: assertionTurns.map((turn) => turn.content),
     };
 
     const traceId = crypto.randomUUID();

package/src/fact-dedup-window.test.ts

@@ -1,14 +1,27 @@
-import test from "node:test";
+import test, { mock } from "node:test";
 import assert from "node:assert/strict";
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { StorageManager } from "./storage.js";
 import { withHeldFileLock } from "./utils/serialize-mutations.js";
 import { confirmActiveFactDuplicate, persistUnderFactDedupWindow } from "./orchestration/fact-dedup-window.js";
 
-function probe(storage: StorageManager) {
-  return { hasContentHashDedup: (s: StorageManager, key: string) => s.hasFactContentHash(key) };
+const SKY_OPTS = {
+  category: "fact",
+  contentHashDedupKey: "the sky is blue",
+  connector: undefined,
+  citationEnabled: false,
+  citationTemplate: "",
+} as const;
+const CONTENDED_LOCK_OPTIONS = { maxWaitMs: 40, pollMs: 10, retryMaxAttempts: 2, retryBaseMs: 10 };
+
+/** Hold a fresh fact-hash index lock the way a peer mid-publish does; returns the release. */
+async function holdIndexLock(root: string): Promise<() => Promise<void>> {
+  const lockPath = path.join(root, "state", "fact-hashes.txt.lock");
+  await mkdir(path.dirname(lockPath), { recursive: true });
+  await writeFile(lockPath, `99999 peer-owner ${new Date().toISOString()}\n`);
+  return () => rm(lockPath, { force: true });
 }
 
 async function makeStorage(): Promise<{ storage: StorageManager; cleanup: () => Promise<void> }> {
@@ -19,14 +32,7 @@ async function makeStorage(): Promise<{ storage: StorageManager; cleanup: () =>
 test("confirmActiveFactDuplicate sees a fact another session committed into the same namespace", async () => {
   const { storage, cleanup } = await makeStorage();
   try {
-    const recheck = () =>
-      confirmActiveFactDuplicate(probe(storage), storage, {
-        category: "fact",
-        contentHashDedupKey: "the sky is blue",
-        connector: undefined,
-        citationEnabled: false,
-        citationTemplate: "",
-      });
+    const recheck = () => confirmActiveFactDuplicate(storage, SKY_OPTS);
     assert.equal(await recheck(), false, "clean namespace must not report a duplicate");
 
     // A concurrent session (separate manager, same dir) commits the fact.
@@ -48,14 +54,7 @@ test("persistUnderFactDedupWindow suppresses on recheck hit and writes on a miss
       await storage.writeMemory("fact", "the sky is blue");
       return "written";
     };
-    const recheck = () =>
-      confirmActiveFactDuplicate(probe(storage), storage, {
-        category: "fact",
-        contentHashDedupKey: "the sky is blue",
-        connector: undefined,
-        citationEnabled: false,
-        citationTemplate: "",
-      });
+    const recheck = () => confirmActiveFactDuplicate(storage, SKY_OPTS);
 
     // First session: recheck misses, write proceeds.
     const first = await persistUnderFactDedupWindow(storage["baseDir"], recheck, write);
@@ -114,3 +113,87 @@ test("persistUnderFactDedupWindow fails closed when a peer holds fact-dedup.lock
     await cleanup();
   }
 });
+
+test("lookupContentHash never pairs a stale MISS with authority while a peer holds the index lock", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    storage.factHashIndexLockOptions = CONTENDED_LOCK_OPTIONS;
+    assert.equal(await storage.isFactContentHashAuthoritative(), true, "starts authoritative");
+    const peer = new StorageManager(storage["baseDir"]);
+    await peer.writeMemory("fact", "The sky is BLUE!");
+    const release = await holdIndexLock(storage["baseDir"]);
+    try {
+      assert.deepEqual(
+        await storage.lookupContentHash(SKY_OPTS.contentHashDedupKey),
+        { present: false, authoritative: false },
+        "the rebuild cannot run, so the stale MISS must not claim authority",
+      );
+      assert.equal(await confirmActiveFactDuplicate(storage, SKY_OPTS), true, "the corpus confirms the peer's fact");
+    } finally {
+      await release();
+    }
+    assert.deepEqual(
+      await storage.lookupContentHash(SKY_OPTS.contentHashDedupKey),
+      { present: true, authoritative: true },
+      "once the lock is free the rebuild publishes and the hit is authoritative",
+    );
+  } finally {
+    await cleanup();
+  }
+});
+
+test("confirmActiveFactDuplicate stays a duplicate when the peer releases the index lock right after the lookup", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    storage.factHashIndexLockOptions = CONTENDED_LOCK_OPTIONS;
+    assert.equal(await storage.isFactContentHashAuthoritative(), true, "starts authoritative");
+    const peer = new StorageManager(storage["baseDir"]);
+    await peer.writeMemory("fact", "The sky is BLUE!");
+    const release = await holdIndexLock(storage["baseDir"]);
+    const lookup = storage.lookupContentHash.bind(storage);
+    mock.method(storage, "lookupContentHash", async (content: string) => {
+      const answer = await lookup(content);
+      await release();
+      return answer;
+    });
+    assert.equal(
+      await confirmActiveFactDuplicate(storage, SKY_OPTS),
+      true,
+      "a lock released after the lookup must not turn the stale MISS into a clean not-duplicate",
+    );
+  } finally {
+    await cleanup();
+  }
+});
+
+test("two managers on one directory commit the same fact through the window exactly once", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const peer = new StorageManager(storage["baseDir"]);
+    const commit = (manager: StorageManager) =>
+      persistUnderFactDedupWindow(
+        manager.dir,
+        () => confirmActiveFactDuplicate(manager, SKY_OPTS),
+        () => manager.writeMemory("fact", "The sky is blue"),
+      );
+    const outcomes = await Promise.all([commit(storage), commit(peer)]);
+    assert.equal(outcomes.filter((outcome) => !outcome.suppressed).length, 1, "one writer wins, the other is suppressed");
+    const stored = await new StorageManager(storage["baseDir"]).readAllMemories();
+    assert.equal(stored.filter((memory) => memory.content.includes("sky is blue")).length, 1);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("confirmActiveFactDuplicate keeps the authoritative MISS fast path", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    assert.equal(await storage.isFactContentHashAuthoritative(), true, "warm the index so no rebuild reads the corpus");
+    const corpusReads = mock.method(storage, "readAllMemories");
+    assert.equal(await confirmActiveFactDuplicate(storage, SKY_OPTS), false);
+    assert.equal(corpusReads.mock.callCount(), 0, "authoritative MISS must not scan the corpus");
+  } finally {
+    await cleanup();
+  }
+});
+