@remnic/core 10.0.66 → 10.0.67

package/src/fact-dedup-window.test.ts

@@ -0,0 +1,116 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { StorageManager } from "./storage.js";
+import { withHeldFileLock } from "./utils/serialize-mutations.js";
+import { confirmActiveFactDuplicate, persistUnderFactDedupWindow } from "./orchestration/fact-dedup-window.js";
+
+function probe(storage: StorageManager) {
+  return { hasContentHashDedup: (s: StorageManager, key: string) => s.hasFactContentHash(key) };
+}
+
+async function makeStorage(): Promise<{ storage: StorageManager; cleanup: () => Promise<void> }> {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-fact-dedup-window-"));
+  return { storage: new StorageManager(root), cleanup: () => rm(root, { recursive: true, force: true }) };
+}
+
+test("confirmActiveFactDuplicate sees a fact another session committed into the same namespace", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const recheck = () =>
+      confirmActiveFactDuplicate(probe(storage), storage, {
+        category: "fact",
+        contentHashDedupKey: "the sky is blue",
+        connector: undefined,
+        citationEnabled: false,
+        citationTemplate: "",
+      });
+    assert.equal(await recheck(), false, "clean namespace must not report a duplicate");
+
+    // A concurrent session (separate manager, same dir) commits the fact.
+    const peer = new StorageManager(storage["baseDir"]);
+    await peer.writeMemory("fact", "The sky is BLUE!");
+
+    assert.equal(await recheck(), true, "the committed fact must confirm as an active duplicate");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("persistUnderFactDedupWindow suppresses on recheck hit and writes on a miss", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    let writes = 0;
+    const write = async () => {
+      writes += 1;
+      await storage.writeMemory("fact", "the sky is blue");
+      return "written";
+    };
+    const recheck = () =>
+      confirmActiveFactDuplicate(probe(storage), storage, {
+        category: "fact",
+        contentHashDedupKey: "the sky is blue",
+        connector: undefined,
+        citationEnabled: false,
+        citationTemplate: "",
+      });
+
+    // First session: recheck misses, write proceeds.
+    const first = await persistUnderFactDedupWindow(storage["baseDir"], recheck, write);
+    assert.deepEqual(first, { suppressed: false, result: "written" });
+    assert.equal(writes, 1);
+
+    // Second session racing the same content: recheck hits, write suppressed.
+    const second = await persistUnderFactDedupWindow(storage["baseDir"], recheck, write);
+    assert.deepEqual(second, { suppressed: true });
+    assert.equal(writes, 1, "the raced duplicate must not be written");
+    assert.equal(await storage.hasFactContentHash("the sky is blue"), true);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("persistUnderFactDedupWindow fails closed when a peer holds fact-dedup.lock", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const holderEntered = Promise.withResolvers<void>();
+    const holderRelease = Promise.withResolvers<void>();
+    const holder = withHeldFileLock(
+      path.join(storage["baseDir"], "fact-dedup.lock"),
+      { staleMs: 60_000 },
+      async (acquired) => {
+        assert.ok(acquired, "test holder must acquire the lock");
+        holderEntered.resolve();
+        await holderRelease.promise;
+      },
+    );
+    await holderEntered.promise;
+
+    let wrote = false;
+    await assert.rejects(
+      persistUnderFactDedupWindow(
+        storage["baseDir"],
+        async () => false,
+        async () => {
+          wrote = true;
+          return "written";
+        },
+      ),
+      /fact-dedup\.lock not acquired/,
+    );
+    assert.equal(wrote, false, "a locked-out writer must not write unlocked");
+
+    holderRelease.resolve();
+    await holder;
+    const outcome = await persistUnderFactDedupWindow(
+      storage["baseDir"],
+      async () => false,
+      async () => "written",
+    );
+    assert.deepEqual(outcome, { suppressed: false, result: "written" });
+  } finally {
+    await cleanup();
+  }
+});

package/src/orchestration/consolidation-profile-race.test.ts

@@ -0,0 +1,178 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { StorageManager } from "../storage.js";
+import { ConsolidationRunCoordinator } from "./consolidation-run.js";
+import type { ExtractionEngine } from "../extraction.js";
+import type { PluginConfig } from "../types.js";
+
+function asPluginConfig(fake: Partial<PluginConfig>): PluginConfig {
+  // SAFETY: the coordinator reads only the three members raceConfig sets.
+  return fake as PluginConfig;
+}
+
+function raceConfig(): PluginConfig {
+  return asPluginConfig({
+    memoryDir: "/synthetic/remnic-profile-race-tests",
+    queryAwareIndexingEnabled: false,
+    dependencyPropagation: { enabled: false, maxDependents: 0, linkTypes: [], timeoutMs: 0, dryRun: true },
+  });
+}
+
+const PROFILE_SECTION = { consolidateTriggerLines: 2, consolidateTargetLines: 5 };
+
+function makeRaceCoordinator(
+  storage: StorageManager,
+  extraction: ExtractionEngine,
+): ConsolidationRunCoordinator {
+  return new ConsolidationRunCoordinator({
+    config: raceConfig(),
+    getStorage: () => storage,
+    // SAFETY: records nothing in this test.
+    getStorageRouter: () => ({ recordWrite: async () => {} }) as never,
+    getExtraction: () => extraction,
+    storageDirNamespace: () => "namespace-a",
+    // SAFETY: propagation is disabled in this config; the port is never used.
+    getDependencyPropagationDelivery: () =>
+      ({
+        prepare: async () => null,
+        afterMutation: async () => {},
+        cancel: () => {},
+        deferPrepared: async () => {},
+      }) as never,
+    // SAFETY: index removal unused in this test.
+    embeddingFallback: { removeFromIndex: async () => {} } as never,
+    // SAFETY: node rebuild unused in this test.
+    tmtBuilder: { maybeRebuildNodes: async () => {} } as never,
+    consolidationObservers: new Set(),
+    getAccessTrackingBuffer: () => new Map(),
+    // SAFETY: unused passes in this config.
+    lifecyclePolicyCoordinator: {
+      runFactArchival: async () => 0,
+      runSummarization: async () => {},
+      runTopicExtraction: async () => {},
+    } as never,
+    // SAFETY: unused pass in this config.
+    compressionGuidelineCoordinator: {
+      runCompressionGuidelineLearningPass: async () => {},
+    } as never,
+    // SAFETY: unused pass in this config.
+    semanticConsolidationCoordinator: {} as never,
+    // SAFETY: unused pass in this config.
+    entitySynthesisCoordinator: {} as never,
+    // SAFETY: profile entry only.
+    recallSectionCoordinator: {
+      getRecallSectionEntry: (sectionId: string) =>
+        sectionId === "profile" ? PROFILE_SECTION : undefined,
+    } as never,
+    // SAFETY: unused pass in this config.
+    tierMigrationCoordinator: {
+      runCycle: async () => ({ migrated: 0 }),
+    } as never,
+    flushAccessTracking: async () => {},
+    indexPersistedMemory: async () => {},
+    autoConsolidateIdentity: async () => {},
+    fastChatCompletion: async () => null,
+  });
+}
+
+/** Extraction fake whose consolidateProfile gates AFTER receiving the old
+ * profile, exactly like a slow LLM call, so the test can interleave an append
+ * inside the real await window (no fake timing). */
+// SAFETY: the stub implements exactly the two members the profile branch
+// calls; the tests assert their effects instead of the stub's other surface.
+function asExtractionEngine(
+  fake: Pick<ExtractionEngine, "consolidate" | "consolidateProfile">,
+): ExtractionEngine {
+  // SAFETY: callers stub only the members this Pick names.
+  return fake as ExtractionEngine;
+}
+
+function makeGatedConsolidationExtraction(
+  profilePassedToModel: PromiseWithResolvers<string>,
+  releaseConsolidation: PromiseWithResolvers<void>,
+  consolidatedProfile: string,
+): ExtractionEngine {
+  return asExtractionEngine({
+    async consolidate() {
+      return { items: [], profileUpdates: [], entityUpdates: [] };
+    },
+    async consolidateProfile(fullProfileContent: string) {
+      profilePassedToModel.resolve(fullProfileContent);
+      await releaseConsolidation.promise;
+      return { consolidatedProfile, removedCount: 2, summary: "test consolidation" };
+    },
+  });
+}
+
+async function makeStorage(): Promise<{ storage: StorageManager; cleanup: () => Promise<void> }> {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-profile-consolidation-race-"));
+  const storage = new StorageManager(root);
+  // The consolidation pass early-returns below five memories; seed enough to
+  // reach the profile branch.
+  for (let i = 0; i < 5; i++) await storage.writeMemory("fact", `seed claim ${i}`);
+  return { storage, cleanup: () => rm(root, { recursive: true, force: true }) };
+}
+
+const BASE_PROFILE_LINES = ["# Behavioral Profile", "", "- trait one", "- trait two", "- trait three", ""];
+
+test("profile consolidation keeps a concurrent append instead of clobbering it", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    await storage.writeProfile(BASE_PROFILE_LINES.join("\n"));
+
+    const profilePassedToModel = Promise.withResolvers<string>();
+    const releaseConsolidation = Promise.withResolvers<void>();
+    const extraction = makeGatedConsolidationExtraction(
+      profilePassedToModel,
+      releaseConsolidation,
+      "# Behavioral Profile\n\n- consolidated summary\n",
+    );
+    const coordinator = makeRaceCoordinator(storage, extraction);
+
+    const run = coordinator.run();
+    const baseline = await profilePassedToModel.promise;
+    assert.equal(baseline, await storage.readProfile(), "model must receive the pre-consolidation profile");
+
+    // Concurrent extractor appends during the real LLM await window.
+    await storage.appendToProfile(["unique concurrent bullet"]);
+
+    releaseConsolidation.resolve();
+    await run;
+
+    const profile = await storage.readProfile();
+    assert.match(profile, /unique concurrent bullet/, "append made during consolidation must survive");
+    assert.doesNotMatch(profile, /consolidated summary/, "stale consolidation must not be committed");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("profile consolidation commits when the profile is unchanged during the LLM call", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    await storage.writeProfile(BASE_PROFILE_LINES.join("\n"));
+
+    const profilePassedToModel = Promise.withResolvers<string>();
+    const releaseConsolidation = Promise.withResolvers<void>();
+    const extraction = makeGatedConsolidationExtraction(
+      profilePassedToModel,
+      releaseConsolidation,
+      "# Behavioral Profile\n\n- consolidated summary\n",
+    );
+    const coordinator = makeRaceCoordinator(storage, extraction);
+
+    const run = coordinator.run();
+    await profilePassedToModel.promise;
+    releaseConsolidation.resolve();
+    await run;
+
+    const profile = await storage.readProfile();
+    assert.match(profile, /consolidated summary/, "unchanged profile must take the consolidation");
+    assert.doesNotMatch(profile, /trait one/, "consolidation replaces the old bullets");
+  } finally {
+    await cleanup();
+  }
+});

package/src/orchestration/consolidation-run.ts

@@ -930,14 +930,26 @@ export class ConsolidationRunCoordinator {
           profileConsolidationTargetLines,
         );
         if (profileResult) {
-          await storage.writeProfile(profileResult.consolidatedProfile);
-          // Profile consolidation rewrites profile.md — a durable namespace
-          // mutation; record it for the catalog touch even when it is the only
-          // change in the pass (codex). Otherwise lastWriteAt goes stale.
-          memoryItemMutated = true;
-          log.info(
-            `profile.md consolidated: removed ${profileResult.removedCount} items — ${profileResult.summary}`,
+          const outcome = await storage.commitConsolidatedProfile(
+            currentProfile,
+            profileResult.consolidatedProfile,
           );
+          if (outcome === "conflict") {
+            // A concurrent append landed during the LLM call. Committing the
+            // stale rewrite would drop it, so skip; the line cap still trips
+            // next pass, so consolidation retries against the fresh profile.
+            log.info(
+              "profile.md changed during consolidation — skipping stale rewrite (retries next pass)",
+            );
+          } else {
+            // Profile consolidation rewrites profile.md — a durable namespace
+            // mutation; record it for the catalog touch even when it is the only
+            // change in the pass (codex). Otherwise lastWriteAt goes stale.
+            memoryItemMutated = true;
+            log.info(
+              `profile.md consolidated: removed ${profileResult.removedCount} items — ${profileResult.summary}`,
+            );
+          }
         }
       }
     }

package/src/orchestration/extraction-drain.ts

@@ -13,6 +13,7 @@ import { describeError } from "../buffer-turn-helpers.js";
 import { chunkExtractionWindow, classifyDrainResult, mergeDrainResults } from "../extraction-window-drain.js";
 import { log } from "../logger.js";
 import type { BufferTurn } from "../types.js";
+import type { ExtractionPreemptHandle } from "./extraction-queue-coordinator.js";
 import type { ExtractionRunCoordinator, ExtractionRunResult } from "./extraction-run.js";
 
 type RunOptions = Parameters<ExtractionRunCoordinator["runExtraction"]>[1];
@@ -42,7 +43,6 @@ export interface ExtractionDrainDeps {
   buffer: {
     dropExtractionChunk(bufferKey: string, turns: readonly BufferTurn[], options?: { allowNonPrefix?: boolean }): Promise<number>;
   };
-  extractionQueueCoordinator: { beginPreemptable(abort: () => void): void };
 }
 
 /** Identity of a chunk: its failure count must not follow a buffer key to different content. */
@@ -111,8 +111,15 @@ export class ExtractionDrain {
    * persist step ends the host's preemption registration, so the drain registers
    * it again before every later chunk; a fail-closed flush can then abort the
    * next chunk's model request instead of waiting behind the whole window.
+   * `preemptHandle`: the queue task's own preemption slot (per task, so two
+   * concurrent automatic extractions cannot clobber each other's registration).
    */
-  async run(turns: BufferTurn[], runOptions: DrainRunOptions, preemptable = false): Promise<ExtractionRunResult> {
+  async run(
+    turns: BufferTurn[],
+    runOptions: DrainRunOptions,
+    preemptable = false,
+    preemptHandle?: ExtractionPreemptHandle,
+  ): Promise<ExtractionRunResult> {
     const budgetChars = this.deps.extractionWindowBudgetChars?.() ?? Number.POSITIVE_INFINITY;
     const chunkable =
       runOptions.clearBufferAfterExtraction &&
@@ -155,7 +162,7 @@ export class ExtractionDrain {
       if (preemptable && position > 0) {
         const local = new AbortController();
         preempt = local;
-        this.deps.extractionQueueCoordinator.beginPreemptable(() => local.abort(preemptedExtractionError()));
+        preemptHandle?.arm(() => local.abort(preemptedExtractionError()));
         chunkOptions = {
           ...chunkOptions,
           abortSignal: runOptions.abortSignal ? AbortSignal.any([runOptions.abortSignal, local.signal]) : local.signal,