@remnic/core 10.0.66 → 10.0.68

package/src/storage/profile-mutation.ts

@@ -0,0 +1,119 @@
+import path from "node:path";
+import { serializeMutations, withHeldFileLock } from "../utils/serialize-mutations.js";
+
+/**
+ * Ports over the StorageManager surface the profile mutation domain needs.
+ * Kept as ports so this module stays out of the storage.ts god-file (issue
+ * #1533/#1995) while sharing ONE serialization domain per baseDir: the
+ * `profile:<baseDir>` key is the in-process FIFO and `profile.md.lock` is the
+ * cross-process gate, matching updateMetaSerialized's meta.json.lock shape
+ * (bounded, fail-closed BEFORE any read/write, no LLM inside).
+ */
+export interface ProfileMutationPorts {
+  readonly baseDir: string;
+  readProfile(): Promise<string>;
+  writeProfile(content: string): Promise<void>;
+}
+
+export type ProfileCommitOutcome = "committed" | "conflict";
+
+function normalizeForDedup(s: string): string {
+  return s
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/**
+ * Check if a new bullet is a fuzzy duplicate of any existing bullet.
+ * Returns true if the new bullet should be skipped.
+ */
+function isFuzzyDuplicate(newNorm: string, existingNorms: string[]): boolean {
+  for (const existing of existingNorms) {
+    // Exact normalized match
+    if (newNorm === existing) return true;
+
+    // Containment check: shorter must be >60% length of longer
+    const shorter = newNorm.length <= existing.length ? newNorm : existing;
+    const longer = newNorm.length > existing.length ? newNorm : existing;
+    if (shorter.length > 20 && shorter.length / longer.length > 0.6 && longer.includes(shorter)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+async function appendToProfileUnlocked(ports: ProfileMutationPorts, updates: string[]): Promise<void> {
+  // Filter out empty entries the LLM may return (persist screens strings already).
+  updates = updates.filter((u) => u.trim().length > 0);
+  if (updates.length === 0) return;
+  const existing = await ports.readProfile();
+
+  const existingNorms = (existing ? existing.split("\n") : [])
+    .filter((l) => l.startsWith("- "))
+    .map((l) => normalizeForDedup(l.slice(2).trim()));
+  const newBullets = updates.filter((u) => {
+    const norm = normalizeForDedup(u);
+    return !isFuzzyDuplicate(norm, existingNorms);
+  });
+  if (newBullets.length === 0) return;
+
+  await ports.writeProfile(
+    existing
+      ? existing.trimEnd() + "\n" + newBullets.map((b) => `- ${b}`).join("\n") + "\n"
+      : ["# Behavioral Profile", "", ...newBullets.map((b) => `- ${b}`), ""].join("\n"),
+  );
+}
+
+/**
+ * Cross-process profile mutation domain: fail-closed `profile.md.lock` for the
+ * brief mutation window only. Lock contention throws (`profile update skipped:
+ * profile.md.lock not acquired`) — never a silent unlocked RMW; callers treat
+ * it like a meta lock skip (retain and retry).
+ */
+async function withProfileMutationLock<T>(ports: ProfileMutationPorts, task: () => Promise<T>): Promise<T> {
+  return withHeldFileLock(
+    path.join(ports.baseDir, "profile.md.lock"),
+    // Bounded: a profile RMW is a short read-merge-write, never a scan.
+    { staleMs: 5_000, maxWaitMs: 2_000, pollMs: 50 },
+    async (acquired) => {
+      if (!acquired) {
+        throw new Error(`profile update skipped: profile.md.lock not acquired (${ports.baseDir})`);
+      }
+      return task();
+    },
+  );
+}
+
+export async function appendToProfileSerialized(ports: ProfileMutationPorts, updates: string[]): Promise<void> {
+  // Per-dir in-process chain, then the cross-process lock around the whole
+  // RMW: serialized, the second append re-reads after the first's write.
+  // Errors propagate: the extraction retry path retains the accepted turns.
+  await serializeMutations(`profile:${ports.baseDir}`, () =>
+    withProfileMutationLock(ports, () => appendToProfileUnlocked(ports, updates)),
+  );
+}
+
+/**
+ * Commit an LLM profile consolidation only if the profile is unchanged since
+ * `baseline` was read; when a concurrent append landed during the LLM call,
+ * return "conflict" and leave the file untouched (the append survives and the
+ * line cap re-triggers consolidation on fresh data next pass). The LLM call
+ * itself stays OUTSIDE this serialization domain; only this short
+ * read-compare-write is serialized + locked.
+ */
+export async function commitConsolidatedProfileSerialized(
+  ports: ProfileMutationPorts,
+  baseline: string,
+  consolidated: string,
+): Promise<ProfileCommitOutcome> {
+  return serializeMutations(`profile:${ports.baseDir}`, () =>
+    withProfileMutationLock(ports, async () => {
+      const current = await ports.readProfile();
+      if (current !== baseline) return "conflict";
+      await ports.writeProfile(consolidated);
+      return "committed";
+    }),
+  );
+}

package/src/storage-append-profile.test.ts

@@ -0,0 +1,114 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { StorageManager } from "./storage.js";
+import { withHeldFileLock } from "./utils/serialize-mutations.js";
+
+async function makeStorage(): Promise<{ storage: StorageManager; cleanup: () => Promise<void> }> {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-profile-append-"));
+  return { storage: new StorageManager(root), cleanup: () => rm(root, { recursive: true, force: true }) };
+}
+
+test("concurrent appendToProfile bullets both survive the shared profile read-modify-write", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    // Hold the FIRST write open while the second append runs a full RMW. Under
+    // a shared unlocked RMW both reads see the same (missing) profile and the
+    // second write silently drops the first task's bullet.
+    const realWrite = storage.writeProfile.bind(storage);
+    const firstWriteEntered = Promise.withResolvers<void>();
+    const firstWriteGate = Promise.withResolvers<void>();
+    let writes = 0;
+    // SAFETY: the test drives only readProfile/writeProfile on the real manager;
+    // instance overrides keep the RMW interception local to this test.
+    storage.writeProfile = async (content: string) => {
+      writes += 1;
+      if (writes === 1) {
+        firstWriteEntered.resolve();
+        await firstWriteGate.promise;
+      }
+      await realWrite(content);
+    };
+
+    const first = storage.appendToProfile(["bullet-a"]);
+    await firstWriteEntered.promise;
+    const second = storage.appendToProfile(["bullet-b"]);
+    firstWriteGate.resolve();
+    await Promise.all([first, second]);
+
+    const profile = await storage.readProfile();
+    assert.match(profile ?? "", /bullet-a/, "first task's bullet must survive");
+    assert.match(profile ?? "", /bullet-b/, "second task's bullet must survive");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("appendToProfile fails closed when a cross-process peer holds profile.md.lock", async () => {
+  const { storage, cleanup } = await makeStorage();
+  const peer = new StorageManager(storage["baseDir"]);
+  try {
+    const holderEntered = Promise.withResolvers<void>();
+    const holderRelease = Promise.withResolvers<void>();
+    // Foreign holder (simulated external drain process) owns the lock file.
+    const holder = withHeldFileLock(
+      path.join(storage["baseDir"], "profile.md.lock"),
+      { staleMs: 60_000 },
+      async (acquired) => {
+        assert.ok(acquired, "test holder must acquire the lock");
+        holderEntered.resolve();
+        await holderRelease.promise;
+      },
+    );
+    await holderEntered.promise;
+
+    // The peer must NOT fall back to an unlocked RMW: fail closed, file untouched.
+    await assert.rejects(
+      peer.appendToProfile(["lost bullet"]),
+      /profile\.md\.lock not acquired/,
+    );
+    const before = await storage.readProfile();
+    assert.doesNotMatch(before, /lost bullet/, "locked-out append must not write");
+
+    holderRelease.resolve();
+    await holder;
+
+    // Once the peer releases, the append goes through normally.
+    await peer.appendToProfile(["late bullet"]);
+    const after = await storage.readProfile();
+    assert.match(after, /late bullet/, "append after contention window must succeed");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("commitConsolidatedProfile commits unchanged baseline and conflicts on a stale one", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    await storage.appendToProfile(["trait one", "trait two"]);
+    const baseline = await storage.readProfile();
+
+    // Unchanged profile → commit takes the consolidated rewrite.
+    const committed = await storage.commitConsolidatedProfile(
+      baseline,
+      "# Behavioral Profile\n\n- consolidated summary\n",
+    );
+    assert.equal(committed, "committed");
+    assert.match(await storage.readProfile(), /consolidated summary/);
+
+    // Stale baseline (profile moved on since the LLM saw it) → conflict, file untouched.
+    await storage.appendToProfile(["fresh concurrent trait"]);
+    const current = await storage.readProfile();
+    const conflict = await storage.commitConsolidatedProfile(
+      baseline,
+      "# Behavioral Profile\n\n- older stale rewrite\n",
+    );
+    assert.equal(conflict, "conflict");
+    assert.equal(await storage.readProfile(), current, "conflicting commit must not touch the file");
+    assert.match(await storage.readProfile(), /fresh concurrent trait/);
+  } finally {
+    await cleanup();
+  }
+});

package/src/storage.ts

@@ -28,13 +28,14 @@ import { createTombstoneMigrationSourceContents } from "./storage/tombstone-migr
 import { MemoryReadStore, readWindowedMemories, type WindowedMemoryReadOptions, type WindowedMemoryReadResult } from "./storage/memory-read-store.js";
 import { runCommittedInvalidation } from "./storage/committed-invalidation.js";
 import { renderProfileWithLastUpdated, stripOkfProfileFrontmatter, withOkfProfileFrontmatter } from "./storage/profile-header.js";
+import { appendToProfileSerialized, commitConsolidatedProfileSerialized, type ProfileMutationPorts } from "./storage/profile-mutation.js";
 import { parseQuestionFile as parseQuestionFileText } from "./storage/questions-file.js";
 import { assertNotOkfReservedBasename, OKF_QUESTION_TYPE, okfTypeForEntityKind, okfTypeForMemory } from "./okf/type-mapping.js";
 import { appendContextTransformRecords, readContextTransformRecords } from "./storage/context-transform-ledger.js";
 import type { ContextTransformTelemetryRecord } from "./active-context-transform.js";
 import { readMaybeEncryptedLines, readMemoryActionEventRowsFromLines } from "./storage/secure-line-reader.js";
 import { parseOfflineSyncDigestCache, type OfflineSyncDigestCacheEntry } from "./offline-sync-digest-cache.js";
-import { updateMetaSerialized, withHeldFileLock } from "./utils/serialize-mutations.js";
+import { updateMetaSerialized } from "./utils/serialize-mutations.js";
 import {
   appendLifecycleEventsSerialized,
   type DrainPendingLifecycleForSyncResult,
@@ -4296,57 +4297,25 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     log.debug("updated profile.md");
   }
 
-  private static normalizeForDedup(s: string): string {
-    if (typeof s !== "string") return "";
-    return s
-      .toLowerCase()
-      .replace(/[^a-z0-9\s]/g, " ")
-      .replace(/\s+/g, " ")
-      .trim();
+  async appendToProfile(updates: string[]): Promise<void> {
+    await appendToProfileSerialized(this.profileMutationPorts(), updates);
   }
 
-  /**
-   * Check if a new bullet is a fuzzy duplicate of any existing bullet.
-   * Returns true if the new bullet should be skipped.
-   */
-  private static isFuzzyDuplicate(newNorm: string, existingNorms: string[]): boolean {
-    for (const existing of existingNorms) {
-      // Exact normalized match
-      if (newNorm === existing) return true;
-
-      // Containment check: shorter must be >60% length of longer
-      const shorter = newNorm.length <= existing.length ? newNorm : existing;
-      const longer = newNorm.length > existing.length ? newNorm : existing;
-      if (shorter.length > 20 && shorter.length / longer.length > 0.6 && longer.includes(shorter)) {
-        return true;
-      }
-    }
-    return false;
+  /** Commit an LLM profile consolidation only if the profile is unchanged
+   * since `baseline` was read; see profile-mutation.ts for the race domain. */
+  async commitConsolidatedProfile(
+    baseline: string,
+    consolidated: string,
+  ): Promise<"committed" | "conflict"> {
+    return commitConsolidatedProfileSerialized(this.profileMutationPorts(), baseline, consolidated);
   }
 
-  async appendToProfile(updates: string[]): Promise<void> {
-    // Filter out non-string entries that the LLM may return
-    updates = updates.filter((u) => typeof u === "string" && u.trim().length > 0);
-    if (updates.length === 0) return;
-    const existing = await this.readProfile();
-
-    const lines = existing ? existing.split("\n") : [];
-    const existingBulletRaw = lines.filter((l) => l.startsWith("- ")).map((l) => l.slice(2).trim());
-    const existingNorms = existingBulletRaw.map(StorageManager.normalizeForDedup);
-
-    const newBullets = updates.filter((u) => {
-      const norm = StorageManager.normalizeForDedup(u);
-      return !StorageManager.isFuzzyDuplicate(norm, existingNorms);
-    });
-    if (newBullets.length === 0) return;
-
-    if (!existing) {
-      const content = ["# Behavioral Profile", "", ...newBullets.map((b) => `- ${b}`), ""].join("\n");
-      await this.writeProfile(content);
-    } else {
-      const withBullets = existing.trimEnd() + "\n" + newBullets.map((b) => `- ${b}`).join("\n") + "\n";
-      await this.writeProfile(withBullets);
-    }
+  private profileMutationPorts(): ProfileMutationPorts {
+    return {
+      baseDir: this.baseDir,
+      readProfile: () => this.readProfile(),
+      writeProfile: (content) => this.writeProfile(content),
+    };
   }
 
   /** Check if profile.md exceeds the max line cap and needs LLM consolidation */

package/src/types.ts

@@ -722,6 +722,8 @@ export interface PluginConfig
   /** Distinct sessions the extraction buffer admits before backpressure (#3303). */
   bufferMaxSessions: number;
   bufferMaxMinutes: number;
+  /** Queue depth cap; default 1 = serial queue, same-buffer tasks never overlap. Applies at daemon start. */
+  extractionQueueConcurrency: number;
   /**
    * Debounce window (ms) for persisting the smart buffer to `state/buffer.json`
    * (issue #1909). Steady-state buffering coalesces the whole-state serialize