@remnic/core 10.0.84 → 10.0.85

package/src/buffer-persistence.ts

@@ -1,4 +1,5 @@
-import type { BufferEntryState, BufferState, BufferSurpriseEvent, BufferTurn } from "./types.js";
+import type { DrainedTurnSets } from "./buffer-turn-helpers.js";
+import type { BufferEntryState, BufferState, BufferSurpriseEvent } from "./types.js";
 import type { BufferSessionSaveOutcome } from "./storage/buffer-drain-claims.js";
 
 /**
@@ -26,8 +27,8 @@ export interface BufferStorage {
 export interface BufferPersistOutcome {
   /** Sessions whose in-memory entry moved into a drain claim: the caller must evict them (a claimed session no longer lives in this process). */
   handedOver: string[];
-  /** Per session, the turns a drain already extracted: the caller subtracts them from its in-memory copy. */
-  drainedTurns: Map<string, BufferTurn[]>;
+  /** Per session, the turns a drain extracted, kept in their original lists. */
+  drainedTurns: Map<string, DrainedTurnSets>;
 }
 
 /**
@@ -73,17 +74,20 @@ export class BufferSessionPersistence {
     storage: BufferStorage,
     state: BufferState,
     peekEntry: (key: string) => BufferEntryState | undefined,
+    reconcile?: { key: string; drained: DrainedTurnSets },
   ): Promise<BufferPersistOutcome> {
+    const drainedTurns = new Map<string, DrainedTurnSets>(
+      reconcile ? [[reconcile.key, reconcile.drained]] : [],
+    );
     const saveSession = storage.saveBufferSession?.bind(storage);
     const deleteSession = storage.deleteBufferSession?.bind(storage);
     if (!saveSession || !deleteSession) {
       await storage.saveBuffer(state);
       this.dirty.clear();
       this.deleted.clear();
-      return { handedOver: [], drainedTurns: new Map() };
+      return { handedOver: [], drainedTurns };
     }
     const handedOver: string[] = [];
-    const drainedTurns = new Map<string, BufferTurn[]>();
     for (const key of [...this.deleted]) {
       await deleteSession(key);
       this.deleted.delete(key);
@@ -104,7 +108,12 @@ export class BufferSessionPersistence {
       if (outcome?.status === "handed_over") handedOver.push(key);
       // A drain tombstone: the turns it extracted must leave this process's
       // copy too, or the owner would extract them a second time.
-      if (outcome?.drainedTurns && outcome.drainedTurns.length > 0) drainedTurns.set(key, outcome.drainedTurns);
+      if (
+        outcome?.drainedTurns &&
+        (outcome.drainedTurns.turns.length > 0 || outcome.drainedTurns.retainedTurns.length > 0)
+      ) {
+        drainedTurns.set(key, outcome.drainedTurns);
+      }
     }
     return { handedOver, drainedTurns };
   }

package/src/buffer-turn-helpers.ts

@@ -1,5 +1,11 @@
 import type { BufferEntryState, BufferState, BufferTurn } from "./types.js";
 
+/** Extracted identities retain whether they came from active or retained turns. */
+export interface DrainedTurnSets {
+  turns: BufferTurn[];
+  retainedTurns: BufferTurn[];
+}
+
 export interface BufferTurnOwner {
   /**
    * Server-resolved principal that owns this session at the authenticated

package/src/buffer.ts

@@ -300,16 +300,16 @@ export class SmartBuffer {
     this.loaded = true;
   }
 
-  private async saveUnlocked(): Promise<void> {
-    const { handedOver, drainedTurns } = await this.persistence.persist(this.storage, this.state, (key) => this.peekEntry(key) ?? undefined);
+  private async saveUnlocked(reconcile?: { key: string; drained: import("./buffer-turn-helpers.js").DrainedTurnSets }): Promise<void> {
+    const { handedOver, drainedTurns } = await this.persistence.persist(this.storage, this.state, (key) => this.peekEntry(key) ?? undefined, reconcile);
     // A drain tombstone: the turns the drain extracted leave this copy too,
     // so a stale owner never extracts (or persists) them a second time.
     for (const [key, drained] of drainedTurns) {
       const entry = this.peekEntry(key);
       if (!entry) continue;
-      entry.turns = subtractBufferTurns(entry.turns, drained);
+      entry.turns = subtractBufferTurns(entry.turns, drained.turns);
       if (entry.retainedTurns && entry.retainedTurns.length > 0) {
-        entry.retainedTurns = subtractBufferTurns(entry.retainedTurns, drained);
+        entry.retainedTurns = subtractBufferTurns(entry.retainedTurns, drained.retainedTurns);
         if (entry.retainedTurns.length === 0) delete entry.retainedTurns;
       }
       if (key === "default") this.state.turns = entry.turns;
@@ -340,8 +340,8 @@ export class SmartBuffer {
     return bufferEntryCounts(this.state);
   }
 
-  async save(): Promise<void> {
-    await this.enqueueMutation(async () => this.saveUnlocked());
+  async save(reconcile?: { key: string; drained: import("./buffer-turn-helpers.js").DrainedTurnSets }): Promise<void> {
+    await this.enqueueMutation(async () => this.saveUnlocked(reconcile));
   }
 
   /**

package/src/cli.ts

@@ -4895,7 +4895,7 @@ export function registerCli(
             // No sidecar — attempt in-memory decryption if the secure-store is
             // unlocked.  decryptCapsuleFileInMemory returns the plaintext gzip
             // bytes; we then gunzip and parse exactly like the plaintext path.
-            const { gunzipSync } = await import("node:zlib");
+            const { gunzipCapsuleArchive } = await import("./transfer/capsule-gunzip.js");
             const { parseExportBundle } = await import("./transfer/types.js");
             let decryptedBuf: Buffer;
             try {
@@ -4917,7 +4917,7 @@ export function registerCli(
               process.exitCode = 1;
               return;
             }
-            const json = gunzipSync(decryptedBuf).toString("utf-8");
+            const json = gunzipCapsuleArchive(decryptedBuf, "capsule inspect", archivePath);
             const parsed2 = parseExportBundle(JSON.parse(json));
             if (parsed2.capsuleVersion !== 2) {
               process.stderr.write(
@@ -4929,10 +4929,10 @@ export function registerCli(
             manifest = (parsed2.bundle as { manifest: Record<string, unknown> }).manifest;
           } else {
             const { readFile } = await import("node:fs/promises");
-            const { gunzipSync } = await import("node:zlib");
+            const { gunzipCapsuleArchive } = await import("./transfer/capsule-gunzip.js");
             const { parseExportBundle } = await import("./transfer/types.js");
             const buf = await readFile(archivePath);
-            const json = gunzipSync(buf).toString("utf-8");
+            const json = gunzipCapsuleArchive(buf, "capsule inspect", archivePath);
             const parsed2 = parseExportBundle(JSON.parse(json));
             if (parsed2.capsuleVersion !== 2) {
               process.stderr.write(

package/src/extraction-quote-grounding.test.ts

@@ -410,6 +410,55 @@ test("a claim that inverts a negated quote is rejected, a restated exclusion is
   assert.deepEqual(keptContents(grounded), ["Alice would avoid approving the budget."]);
 });
 
+test("a claim that keeps one quote negation and flips another is rejected", () => {
+  const source = "Notes: Alice did not approve the budget. Bob did not review the ledger.";
+  const quote = "Alice did not approve the budget. Bob did not review the ledger.";
+  const grounded = filterExtractionResultBySource(
+    resultOf([
+      { content: "Alice did not approve the budget, and Bob reviewed the ledger.", quote },
+      {
+        content: "Alice did not approve the budget, and Bob did not review the ledger.",
+        quote,
+      },
+    ]),
+    source,
+    source,
+  );
+  assert.deepEqual(keptContents(grounded), [
+    "Alice did not approve the budget, and Bob did not review the ledger.",
+  ]);
+});
+
+test("a claim that keeps the denial in a pronoun clause is kept", () => {
+  const source = "Notes: Alice did not approve the budget.";
+  const quote = "Alice did not approve the budget.";
+  const claims = [
+    "Alice discussed the budget; she did not approve it.",
+    "Alice reviewed the budget, but she did not approve it.",
+  ];
+  const grounded = filterExtractionResultBySource(
+    resultOf(claims.map((content) => ({ content, quote }))),
+    source,
+    source,
+  );
+  assert.deepEqual(keptContents(grounded), claims);
+});
+
+test("a negator in a claim clause that shares only the subject does not keep the denial", () => {
+  const source = "Notes: Alice did not approve the budget. Alice did not review the contract.";
+  const grounded = filterExtractionResultBySource(
+    resultOf([
+      {
+        content: "Alice did not review the contract. She approved the budget.",
+        quote: "Alice did not approve the budget.",
+      },
+    ]),
+    source,
+    source,
+  );
+  assert.equal(grounded.facts.length, 0);
+});
+
 test("a claim that inverts a denial verb in the quote is rejected", () => {
   const source = "Notes: Alice denied approving the budget. Bob reviewed it.";
   const grounded = filterExtractionResultBySource(

package/src/extraction-quote-grounding.ts

@@ -130,24 +130,62 @@ function hasNegation(text: string): boolean {
   return NEGATOR_PATTERN.test(text.replace(/[\u2018\u2019]/gu, "'")) || hasDenialReporting(text);
 }
 
+const CLAUSE_PATTERN = /[^;.!?\n]+[;.!?]*/gu;
+/** Split "X, and Y" claim coordination without touching list commas. */
+const CLAIM_COORD_SPLIT = /,\s{1,8}(?:and|but|or|nor)\s{1,8}/iu;
+
+function splitClaimClauses(content: string): string[] {
+  return (content.match(CLAUSE_PATTERN) ?? [content]).flatMap((clause) => {
+    const parts = clause
+      .split(CLAIM_COORD_SPLIT)
+      .map((part) => part.trim())
+      .filter((part) => part.length > 0);
+    return parts.length > 0 ? parts : [clause];
+  });
+}
+
+function clauseOverlap(clauseTokens: readonly string[], target: string) {
+  const has = tokenPool(tokenize(target));
+  const shared = clauseTokens.filter(has).length;
+  return { shared, ratio: clauseTokens.length === 0 ? 0 : shared / clauseTokens.length };
+}
+
+/** What a quote clause's first negator governs: the words after it. */
+function negatedScope(clause: string): string {
+  const match = NEGATOR_PATTERN.exec(clause.replace(/[\u2018\u2019]/gu, "'"));
+  return match ? clause.slice(match.index + match[0].length) : clause;
+}
+
 /**
  * True when the claim asserts what a quote clause denies or does not assert:
  * "did not approve the budget" or "denied approving it" restated as "approved
  * the budget", "suspects Bob stole funds" or "Is Alice the CEO?" restated as a
- * plain statement. A claim that keeps a negator, exclusion word, or reporting
- * marker ("do not X" reworded as "X is excluded") is not a misstatement.
+ * plain statement. The claim must restate the quote clause, and none of its
+ * clauses that share a word with what the negator governs may keep the
+ * negator, exclusion word, or reporting marker ("do not X" reworded as "X is
+ * excluded", or "Alice reviewed it, but she did not approve it"). A negator in
+ * a clause about something else does not count, so a claim that keeps one
+ * quote negation can still misstate another quote clause, and a claim clause
+ * that shares only the subject does not satisfy the check.
  */
 function misstatesQuote(content: string, quote: string): boolean {
-  const keepsNegation = hasNegation(content) || EXCLUSION_PATTERN.test(content);
-  const keepsReporting = hasNonAssertiveReporting(content) || content.trim().endsWith("?");
-  const contentHas = tokenPool(tokenize(content));
-  return (quote.match(/[^;.!?\n]+[;.!?]*/gu) ?? []).some((clause) => {
-    const negated = hasNegation(clause) && !keepsNegation;
-    const unasserted = (clause.trim().endsWith("?") || hasNonAssertiveReporting(clause)) && !keepsReporting;
-    if (!negated && !unasserted) return false;
-    const clauseTokens = [...tokenize(clause)];
-    const shared = clauseTokens.filter(contentHas).length;
-    return shared >= GROUNDING_MIN_SHARED_TOKENS && shared / clauseTokens.length >= 0.5;
+  const claimClauses = splitClaimClauses(content);
+  return (quote.match(CLAUSE_PATTERN) ?? []).some((quoteClause) => {
+    const needsNegation = hasNegation(quoteClause);
+    const needsReporting = quoteClause.trim().endsWith("?") || hasNonAssertiveReporting(quoteClause);
+    if (!needsNegation && !needsReporting) return false;
+    const clauseTokens = [...tokenize(quoteClause)];
+    const whole = clauseOverlap(clauseTokens, content);
+    if (whole.shared < GROUNDING_MIN_SHARED_TOKENS || whole.ratio < 0.5) return false;
+    const clausesSharing = (tokens: readonly string[]) =>
+      claimClauses.filter((claimClause) => clauseOverlap(tokens, claimClause).shared > 0);
+    if (needsNegation) {
+      const scopeTokens = [...tokenize(negatedScope(quoteClause))];
+      const related = clausesSharing(scopeTokens.length > 0 ? scopeTokens : clauseTokens);
+      if (!related.some((text) => hasNegation(text) || EXCLUSION_PATTERN.test(text))) return true;
+    }
+    return needsReporting
+      && !clausesSharing(clauseTokens).some((text) => hasNonAssertiveReporting(text) || text.trim().endsWith("?"));
   });
 }
 

package/src/fact-dedup-window.test.ts

@@ -1,14 +1,27 @@
-import test from "node:test";
+import test, { mock } from "node:test";
 import assert from "node:assert/strict";
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { StorageManager } from "./storage.js";
 import { withHeldFileLock } from "./utils/serialize-mutations.js";
 import { confirmActiveFactDuplicate, persistUnderFactDedupWindow } from "./orchestration/fact-dedup-window.js";
 
-function probe(storage: StorageManager) {
-  return { hasContentHashDedup: (s: StorageManager, key: string) => s.hasFactContentHash(key) };
+const SKY_OPTS = {
+  category: "fact",
+  contentHashDedupKey: "the sky is blue",
+  connector: undefined,
+  citationEnabled: false,
+  citationTemplate: "",
+} as const;
+const CONTENDED_LOCK_OPTIONS = { maxWaitMs: 40, pollMs: 10, retryMaxAttempts: 2, retryBaseMs: 10 };
+
+/** Hold a fresh fact-hash index lock the way a peer mid-publish does; returns the release. */
+async function holdIndexLock(root: string): Promise<() => Promise<void>> {
+  const lockPath = path.join(root, "state", "fact-hashes.txt.lock");
+  await mkdir(path.dirname(lockPath), { recursive: true });
+  await writeFile(lockPath, `99999 peer-owner ${new Date().toISOString()}\n`);
+  return () => rm(lockPath, { force: true });
 }
 
 async function makeStorage(): Promise<{ storage: StorageManager; cleanup: () => Promise<void> }> {
@@ -19,14 +32,7 @@ async function makeStorage(): Promise<{ storage: StorageManager; cleanup: () =>
 test("confirmActiveFactDuplicate sees a fact another session committed into the same namespace", async () => {
   const { storage, cleanup } = await makeStorage();
   try {
-    const recheck = () =>
-      confirmActiveFactDuplicate(probe(storage), storage, {
-        category: "fact",
-        contentHashDedupKey: "the sky is blue",
-        connector: undefined,
-        citationEnabled: false,
-        citationTemplate: "",
-      });
+    const recheck = () => confirmActiveFactDuplicate(storage, SKY_OPTS);
     assert.equal(await recheck(), false, "clean namespace must not report a duplicate");
 
     // A concurrent session (separate manager, same dir) commits the fact.
@@ -48,14 +54,7 @@ test("persistUnderFactDedupWindow suppresses on recheck hit and writes on a miss
       await storage.writeMemory("fact", "the sky is blue");
       return "written";
     };
-    const recheck = () =>
-      confirmActiveFactDuplicate(probe(storage), storage, {
-        category: "fact",
-        contentHashDedupKey: "the sky is blue",
-        connector: undefined,
-        citationEnabled: false,
-        citationTemplate: "",
-      });
+    const recheck = () => confirmActiveFactDuplicate(storage, SKY_OPTS);
 
     // First session: recheck misses, write proceeds.
     const first = await persistUnderFactDedupWindow(storage["baseDir"], recheck, write);
@@ -114,3 +113,87 @@ test("persistUnderFactDedupWindow fails closed when a peer holds fact-dedup.lock
     await cleanup();
   }
 });
+
+test("lookupContentHash never pairs a stale MISS with authority while a peer holds the index lock", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    storage.factHashIndexLockOptions = CONTENDED_LOCK_OPTIONS;
+    assert.equal(await storage.isFactContentHashAuthoritative(), true, "starts authoritative");
+    const peer = new StorageManager(storage["baseDir"]);
+    await peer.writeMemory("fact", "The sky is BLUE!");
+    const release = await holdIndexLock(storage["baseDir"]);
+    try {
+      assert.deepEqual(
+        await storage.lookupContentHash(SKY_OPTS.contentHashDedupKey),
+        { present: false, authoritative: false },
+        "the rebuild cannot run, so the stale MISS must not claim authority",
+      );
+      assert.equal(await confirmActiveFactDuplicate(storage, SKY_OPTS), true, "the corpus confirms the peer's fact");
+    } finally {
+      await release();
+    }
+    assert.deepEqual(
+      await storage.lookupContentHash(SKY_OPTS.contentHashDedupKey),
+      { present: true, authoritative: true },
+      "once the lock is free the rebuild publishes and the hit is authoritative",
+    );
+  } finally {
+    await cleanup();
+  }
+});
+
+test("confirmActiveFactDuplicate stays a duplicate when the peer releases the index lock right after the lookup", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    storage.factHashIndexLockOptions = CONTENDED_LOCK_OPTIONS;
+    assert.equal(await storage.isFactContentHashAuthoritative(), true, "starts authoritative");
+    const peer = new StorageManager(storage["baseDir"]);
+    await peer.writeMemory("fact", "The sky is BLUE!");
+    const release = await holdIndexLock(storage["baseDir"]);
+    const lookup = storage.lookupContentHash.bind(storage);
+    mock.method(storage, "lookupContentHash", async (content: string) => {
+      const answer = await lookup(content);
+      await release();
+      return answer;
+    });
+    assert.equal(
+      await confirmActiveFactDuplicate(storage, SKY_OPTS),
+      true,
+      "a lock released after the lookup must not turn the stale MISS into a clean not-duplicate",
+    );
+  } finally {
+    await cleanup();
+  }
+});
+
+test("two managers on one directory commit the same fact through the window exactly once", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const peer = new StorageManager(storage["baseDir"]);
+    const commit = (manager: StorageManager) =>
+      persistUnderFactDedupWindow(
+        manager.dir,
+        () => confirmActiveFactDuplicate(manager, SKY_OPTS),
+        () => manager.writeMemory("fact", "The sky is blue"),
+      );
+    const outcomes = await Promise.all([commit(storage), commit(peer)]);
+    assert.equal(outcomes.filter((outcome) => !outcome.suppressed).length, 1, "one writer wins, the other is suppressed");
+    const stored = await new StorageManager(storage["baseDir"]).readAllMemories();
+    assert.equal(stored.filter((memory) => memory.content.includes("sky is blue")).length, 1);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("confirmActiveFactDuplicate keeps the authoritative MISS fast path", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    assert.equal(await storage.isFactContentHashAuthoritative(), true, "warm the index so no rebuild reads the corpus");
+    const corpusReads = mock.method(storage, "readAllMemories");
+    assert.equal(await confirmActiveFactDuplicate(storage, SKY_OPTS), false);
+    assert.equal(corpusReads.mock.callCount(), 0, "authoritative MISS must not scan the corpus");
+  } finally {
+    await cleanup();
+  }
+});
+