@remnic/core 9.3.594 → 9.3.596

package/src/dedup/semantic.ts

@@ -138,7 +138,13 @@ export async function decideSemanticDedup(
   // Defensive: callers ought to return sorted, but don't trust it.
   let top: SemanticDedupHit | undefined;
   for (const hit of hits) {
-    if (!hit || typeof hit.score !== "number" || !Number.isFinite(hit.score)) {
+    if (
+      !hit ||
+      typeof hit.id !== "string" ||
+      hit.id.trim().length === 0 ||
+      typeof hit.score !== "number" ||
+      !Number.isFinite(hit.score)
+    ) {
       continue;
     }
     if (!top || hit.score > top.score) {

package/src/entity-retrieval.ts

@@ -12,6 +12,8 @@ const RECENT_TRANSCRIPT_LOOKBACK_HOURS = 24;
 const INSTRUCTION_LIKE_RE = /\b(always|never|must|should|remember to|do not|don't|process|workflow|template|checklist|instruction)\b/i;
 const METADATA_WRAPPER_RE = /^(source|context|metadata|notes?):/i;
 const ENTITY_PRONOUN_RE = /\b(he|him|his|she|her|they|them|their|it|its)\b/i;
+const BELIEF_LEDGER_SECTION_KEY = "belief_ledger";
+const BELIEF_LEDGER_FACT_RE = /^claim=([^;]+);\s*status=([^;]+);\s*updatedAt=([^;]+);\s*(.+)$/;
 
 type EntityQueryMode = "direct" | "timeline" | "follow_up";
 
@@ -110,6 +112,59 @@ function dedupeHintSnippetsByText(snippets: EntityHintSnippet[]): EntityHintSnip
   return result;
 }
 
+function isBeliefLedgerSection(section: Pick<EntityStructuredSection, "key">): boolean {
+  return normalizeEntityText(section.key).replace(/\s+/g, "_") === BELIEF_LEDGER_SECTION_KEY;
+}
+
+function beliefLedgerFactKeys(sections: EntityStructuredSection[]): Set<string> {
+  const keys = new Set<string>();
+  for (const section of sections) {
+    if (!isBeliefLedgerSection(section)) continue;
+    for (const fact of section.facts) {
+      keys.add(normalizeEntityText(fact));
+    }
+  }
+  return keys;
+}
+
+function recallFactsForStructuredSection(section: EntityStructuredSection): string[] {
+  if (!isBeliefLedgerSection(section)) return section.facts;
+  return currentActiveBeliefLedgerFactTexts(section.facts);
+}
+
+function currentActiveBeliefLedgerFactTexts(facts: string[]): string[] {
+  const byClaim = new Map<string, { status: string; updatedAtMs: number; texts: string[] }>();
+  for (const fact of facts) {
+    const match = BELIEF_LEDGER_FACT_RE.exec(fact.trim());
+    if (!match) continue;
+    const [, claimId, status, updatedAt, text] = match;
+    const updatedAtMs = Date.parse(updatedAt);
+    if (!claimId?.trim() || !status?.trim() || !Number.isFinite(updatedAtMs) || !text?.trim()) continue;
+    const normalizedStatus = status.trim().toLowerCase();
+    const normalizedClaimId = claimId.trim();
+    const current = byClaim.get(normalizedClaimId);
+    const inactiveTieWins =
+      current &&
+      updatedAtMs === current.updatedAtMs &&
+      current.status === "active" &&
+      normalizedStatus !== "active";
+    if (!current || updatedAtMs > current.updatedAtMs || inactiveTieWins) {
+      byClaim.set(normalizedClaimId, { status: normalizedStatus, updatedAtMs, texts: [text.trim()] });
+      continue;
+    }
+    if (updatedAtMs === current.updatedAtMs && current.status === normalizedStatus) {
+      current.texts.push(text.trim());
+    }
+  }
+  const result: string[] = [];
+  for (const current of byClaim.values()) {
+    if (current.status === "active") {
+      result.push(...current.texts);
+    }
+  }
+  return result;
+}
+
 function relationLine(entry: EntityMentionIndexEntry, relationship: { target: string; label: string }): string {
   const normalizedLabel = relationship.label.replace(/\s+/g, " ").trim();
   if (normalizedLabel.length === 0) return `${entry.name} is connected to ${relationship.target}`;
@@ -408,7 +463,13 @@ async function buildEntityMentionIndex(
   const entities = new Map<string, EntityMentionIndexEntry>();
   for (const entity of entityFiles) {
     const canonicalId = normalizeEntityName(entity.name, entity.type);
-    const sanitizedFacts = entity.facts.map((fact) => sanitizeEntityFact(fact)).filter(Boolean).map((fact) => compactLine(fact, 180));
+    const rawStructuredSections = entity.structuredSections ?? [];
+    const rawBeliefLedgerFactKeys = beliefLedgerFactKeys(rawStructuredSections);
+    const sanitizedFacts = entity.facts
+      .map((fact) => sanitizeEntityFact(fact))
+      .filter(Boolean)
+      .filter((fact) => !rawBeliefLedgerFactKeys.has(normalizeEntityText(fact)))
+      .map((fact) => compactLine(fact, 180));
     const sanitizedTimelineFacts = entity.timeline
       .map((entry) => sanitizeEntityFact(entry.text))
       .filter(Boolean)
@@ -421,10 +482,10 @@ async function buildEntityMentionIndex(
       summary: entity.synthesis?.trim() || entity.summary?.trim() || undefined,
       facts: sanitizedFacts,
       timelineFacts: uniqueStrings(sanitizedTimelineFacts),
-      structuredSections: (entity.structuredSections ?? []).map((section) => ({
+      structuredSections: rawStructuredSections.map((section) => ({
         key: section.key,
         title: section.title,
-        facts: section.facts
+        facts: recallFactsForStructuredSection(section)
           .map((fact) => sanitizeEntityFact(fact))
           .filter(Boolean)
           .map((fact) => compactLine(fact, 180)),

package/src/storage.ts

@@ -3226,6 +3226,35 @@ export class StorageManager {
     return ContentHashIndex.computeHash(sanitizeMemoryContent(hashSource).text);
   }
 
+  private async addActiveFactContentHash(memory: MemoryFile): Promise<void> {
+    if (memory.frontmatter.category !== "fact") return;
+    if (inferMemoryStatus(memory.frontmatter, memory.path) !== "active") return;
+    const hash = this.factContentHashForRemoval(memory);
+    if (!hash) return;
+
+    await this.ensureFactHashIndexAuthoritative();
+    const factHashIndex = await this.getFactHashIndex();
+    factHashIndex.addByHash(hash);
+    await factHashIndex.save();
+  }
+
+  private async syncFactHashIndexAfterRewrite(before: MemoryFile, after: MemoryFile): Promise<void> {
+    if (before.frontmatter.category !== "fact" && after.frontmatter.category !== "fact") return;
+
+    const beforeHash = this.factContentHashForRemoval(before);
+    const afterHash = this.factContentHashForRemoval(after);
+    const beforeStatus = inferMemoryStatus(before.frontmatter, before.path);
+    const afterStatus = inferMemoryStatus(after.frontmatter, after.path);
+    if (beforeHash === afterHash && beforeStatus === afterStatus) return;
+
+    if (beforeStatus === "active" && beforeHash && (beforeHash !== afterHash || afterStatus !== "active")) {
+      await this.removeFactContentHashesForMemories([before]);
+    }
+    if (afterStatus === "active" && afterHash && (beforeHash !== afterHash || beforeStatus !== "active")) {
+      await this.addActiveFactContentHash(after);
+    }
+  }
+
   async removeFactContentHashesForMemories(memories: MemoryFile[]): Promise<void> {
     await this.ensureFactHashIndexAuthoritative();
     const factHashIndex = await this.getFactHashIndex();
@@ -4492,6 +4521,17 @@ export class StorageManager {
     if (memory.path.includes(`${path.sep}cold${path.sep}`)) {
       this.invalidateColdMemoriesCache();
     }
+    try {
+      await this.syncFactHashIndexAfterRewrite(
+        memory,
+        {
+          ...memory,
+          frontmatter: updated,
+        },
+      );
+    } catch (err) {
+      log.warn(`storage.writeMemoryFrontmatter completed but failed to update fact hash index: ${err}`);
+    }
     await this.appendGeneratedMemoryLifecycleEventFailOpen(
       "storage.writeMemoryFrontmatter",
       {

package/src/temporal-index.test.ts

@@ -0,0 +1,191 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { mkdir, mkdtemp, readFile, rm, symlink, utimes, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { setTimeout as delay } from "node:timers/promises";
+
+import { queryByDateRangeAsync, queryByTagsAsync } from "./temporal-index.js";
+
+async function runIndexWorker(moduleUrl: string, memoryDir: string, workerId: number, count: number): Promise<void> {
+  const workerSource = `
+const { indexMemory } = await import(process.argv[1]);
+const memoryDir = process.argv[2];
+const workerId = Number(process.argv[3]);
+const count = Number(process.argv[4]);
+for (let i = 0; i < count; i += 1) {
+  indexMemory(
+    memoryDir,
+    \`/tmp/remnic-temporal-worker-\${workerId}-memory-\${i}.md\`,
+    "2026-03-09T12:00:00.000Z",
+    ["concurrency/shared", \`concurrency/worker-\${workerId}\`],
+  );
+}
+`;
+
+  await new Promise<void>((resolve, reject) => {
+    const child = spawn(
+      process.execPath,
+      ["--import", "tsx", "-e", workerSource, moduleUrl, memoryDir, String(workerId), String(count)],
+      {
+        stdio: ["ignore", "pipe", "pipe"],
+      }
+    );
+    let stderr = "";
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve();
+        return;
+      }
+      reject(new Error(`index worker ${workerId} exited ${code}: ${stderr}`));
+    });
+  });
+}
+
+test("temporal index concurrent writers retain every date and tag path", async () => {
+  const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-index-concurrent-"));
+  const moduleUrl = new URL("./temporal-index.ts", import.meta.url).href;
+  const workerCount = 4;
+  const entriesPerWorker = 12;
+  const expectedPaths = new Set<string>();
+
+  for (let workerId = 0; workerId < workerCount; workerId += 1) {
+    for (let i = 0; i < entriesPerWorker; i += 1) {
+      expectedPaths.add(`/tmp/remnic-temporal-worker-${workerId}-memory-${i}.md`);
+    }
+  }
+
+  await Promise.all(
+    Array.from({ length: workerCount }, (_, workerId) =>
+      runIndexWorker(moduleUrl, memoryDir, workerId, entriesPerWorker)
+    )
+  );
+
+  const dateMatches = await queryByDateRangeAsync(memoryDir, "2026-03-09", "2026-03-10");
+  const tagMatches = await queryByTagsAsync(memoryDir, ["concurrency/shared"]);
+
+  assert.deepEqual(dateMatches, expectedPaths);
+  assert.deepEqual(tagMatches, expectedPaths);
+
+  const timeIndex = JSON.parse(await readFile(join(memoryDir, "state", "index_time.json"), "utf8"));
+  const tagIndex = JSON.parse(await readFile(join(memoryDir, "state", "index_tags.json"), "utf8"));
+  assert.equal(timeIndex.dates["2026-03-09"].length, expectedPaths.size);
+  assert.equal(tagIndex.tags["concurrency/shared"].paths.length, expectedPaths.size);
+});
+
+test("temporal index writers wait for old locks owned by live processes", async () => {
+  const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-index-live-lock-"));
+  const stateDir = join(memoryDir, "state");
+  const lockDir = join(stateDir, "index_time.json.lock.d");
+  const moduleUrl = new URL("./temporal-index.ts", import.meta.url).href;
+  await mkdir(lockDir, { recursive: true });
+  await writeFile(join(lockDir, "owner.json"), JSON.stringify({ pid: process.pid }), "utf8");
+  const oldLockTime = new Date(Date.now() - 120_000);
+  await utimes(lockDir, oldLockTime, oldLockTime);
+
+  const workerSource = `
+const { indexMemory } = await import(process.argv[1]);
+indexMemory(
+  process.argv[2],
+  "/tmp/remnic-temporal-live-lock-memory.md",
+  "2026-03-10T12:00:00.000Z",
+  ["concurrency/live-lock"],
+);
+`;
+  let closed = false;
+  const workerDone = new Promise<void>((resolve, reject) => {
+    const child = spawn(process.execPath, ["--import", "tsx", "-e", workerSource, moduleUrl, memoryDir], {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stderr = "";
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      closed = true;
+      if (code === 0) {
+        resolve();
+        return;
+      }
+      reject(new Error(`live-lock worker exited ${code}: ${stderr}`));
+    });
+  });
+
+  await delay(150);
+  assert.equal(closed, false);
+  await rm(lockDir, { recursive: true, force: true });
+  await workerDone;
+
+  const dateMatches = await queryByDateRangeAsync(memoryDir, "2026-03-10", "2026-03-11");
+  const tagMatches = await queryByTagsAsync(memoryDir, ["concurrency/live-lock"]);
+  assert.deepEqual(dateMatches, new Set(["/tmp/remnic-temporal-live-lock-memory.md"]));
+  assert.deepEqual(tagMatches, new Set(["/tmp/remnic-temporal-live-lock-memory.md"]));
+});
+
+test("temporal index writers clear stale locks whose owner pid was recycled", async () => {
+  const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-index-recycled-pid-"));
+  const stateDir = join(memoryDir, "state");
+  const lockDir = join(stateDir, "index_time.json.lock.d");
+  const moduleUrl = new URL("./temporal-index.ts", import.meta.url).href;
+  await mkdir(lockDir, { recursive: true });
+  await writeFile(
+    join(lockDir, "owner.json"),
+    JSON.stringify({
+      pid: process.pid,
+      processStartedAtMs: Date.now() - 7 * 86_400_000,
+      createdAt: new Date(Date.now() - 120_000).toISOString(),
+    }),
+    "utf8"
+  );
+  const oldLockTime = new Date(Date.now() - 120_000);
+  await utimes(lockDir, oldLockTime, oldLockTime);
+
+  await runIndexWorker(moduleUrl, memoryDir, 99, 1);
+
+  const dateMatches = await queryByDateRangeAsync(memoryDir, "2026-03-09", "2026-03-10");
+  const tagMatches = await queryByTagsAsync(memoryDir, ["concurrency/shared"]);
+  assert.deepEqual(dateMatches, new Set(["/tmp/remnic-temporal-worker-99-memory-0.md"]));
+  assert.deepEqual(tagMatches, new Set(["/tmp/remnic-temporal-worker-99-memory-0.md"]));
+});
+
+test("temporal index writers remove regular file lock blockers", async () => {
+  const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-index-file-lock-"));
+  const stateDir = join(memoryDir, "state");
+  const lockPath = join(stateDir, "index_time.json.lock.d");
+  const moduleUrl = new URL("./temporal-index.ts", import.meta.url).href;
+  await mkdir(stateDir, { recursive: true });
+  await writeFile(lockPath, "not a lock directory", "utf8");
+
+  await runIndexWorker(moduleUrl, memoryDir, 100, 1);
+
+  const dateMatches = await queryByDateRangeAsync(memoryDir, "2026-03-09", "2026-03-10");
+  const tagMatches = await queryByTagsAsync(memoryDir, ["concurrency/shared"]);
+  assert.deepEqual(dateMatches, new Set(["/tmp/remnic-temporal-worker-100-memory-0.md"]));
+  assert.deepEqual(tagMatches, new Set(["/tmp/remnic-temporal-worker-100-memory-0.md"]));
+});
+
+test("temporal index writers fail open on symlink lock blockers", async () => {
+  const memoryDir = await mkdtemp(join(tmpdir(), "remnic-temporal-index-symlink-lock-"));
+  const stateDir = join(memoryDir, "state");
+  const lockPath = join(stateDir, "index_time.json.lock.d");
+  const symlinkTarget = join(memoryDir, "outside-lock-target");
+  const moduleUrl = new URL("./temporal-index.ts", import.meta.url).href;
+  await mkdir(stateDir, { recursive: true });
+  await writeFile(symlinkTarget, "do not follow", "utf8");
+  await symlink(symlinkTarget, lockPath);
+
+  await runIndexWorker(moduleUrl, memoryDir, 101, 1);
+
+  const dateMatches = await queryByDateRangeAsync(memoryDir, "2026-03-09", "2026-03-10");
+  const tagMatches = await queryByTagsAsync(memoryDir, ["concurrency/shared"]);
+  assert.equal(dateMatches, null);
+  assert.deepEqual(tagMatches, new Set(["/tmp/remnic-temporal-worker-101-memory-0.md"]));
+});