@remnic/core 9.45.3 → 9.45.5

package/src/storage/entity-canonical-id-migration.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import {
   access,
   lstat,
@@ -21,6 +21,7 @@ import { withHeldFileLock } from "../utils/serialize-mutations.js";
 import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
 import { assertPathInsideRoot } from "../utils/path-containment.js";
 import { withEntityCanonicalMutationLock } from "./entity-canonical-id-lock.js";
+import { assertSafeEntityId } from "./entity-id-safety.js";
 
 import {
   ENTITY_CANONICAL_ID_MIGRATION_FILE,
@@ -95,33 +96,63 @@ async function fingerprintPath(filePath: string): Promise<string> {
     : "missing";
 }
 
+async function fingerprintEntityEntries(entitiesDir: string): Promise<string> {
+  let names: string[];
+  try {
+    names = (await readdir(entitiesDir))
+      .filter((name) => path.extname(name) === ".md")
+      .sort();
+  } catch (error) {
+    if (isErrnoCode(error, "ENOENT")) return "missing";
+    throw error;
+  }
+  const hash = createHash("sha256");
+  for (const name of names) {
+    hash.update(`${name.length}:${name}:`);
+    try {
+      const entry = await lstat(path.join(entitiesDir, name), { bigint: true });
+      hash.update(`${entry.dev}:${entry.ino}:${entry.size}:${entry.mtimeNs}:${entry.ctimeNs}`);
+    } catch (error) {
+      if (!isErrnoCode(error, "ENOENT")) throw error;
+      hash.update("missing");
+    }
+    hash.update("\n");
+  }
+  return hash.digest("hex");
+}
+
 /**
- * Completion fingerprint for the migration runner's skip gate. Incorporates
- * only state that can change what a re-run would DO: the entities directory
- * (creates/deletes/renames), the alias table (normalization inputs), the
- * caller-supplied entity-mutation version (store-mediated entity content
- * edits, which touch neither of the first two), and the reconcile-pending
- * marker raw-byte writers touch. It must NOT incorporate the memory-corpus
- * write version: plain memory creates cannot introduce legacy entity
- * references (writes normalize through the historical mapping table), and
- * keying the gate to them re-ran the migration after every fact write — the
- * issue #2213 hot loop.
+ * Completion fingerprint for migration inputs only. Entity entry identities
+ * catch same-path edits that do not change the directory inode; the cooperative
+ * sentinel covers managed writes. Memory-corpus writes stay excluded so plain
+ * fact writes do not reopen migration.
  */
 export async function getFingerprint(
   baseDir: string,
   entitiesDir: string,
   getEntityMutationVersion: () => string,
 ): Promise<string> {
-  const [entityFingerprint, aliasFingerprint, reconcileFingerprint, consumingFingerprint] = await Promise.all([
+  const [
+    entityDirectoryFingerprint,
+    entityEntriesFingerprint,
+    aliasFingerprint,
+    reconcileFingerprint,
+    consumingFingerprint,
+  ] = await Promise.all([
     fingerprintPath(entitiesDir),
+    fingerprintEntityEntries(entitiesDir),
     fingerprintPath(path.join(baseDir, "config", "aliases.json")),
     fingerprintPath(path.join(baseDir, "state", ENTITY_CANONICAL_ID_RECONCILE_MARKER)),
-    // A `.consuming` generation stranded by a crashed PEER process must also
-    // re-invoke a live runner whose cached fingerprint predates it.
     fingerprintPath(path.join(baseDir, "state", ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER)),
   ]);
-  return [entityFingerprint, aliasFingerprint, reconcileFingerprint, consumingFingerprint, getEntityMutationVersion()]
-    .join(":");
+  return [
+    entityDirectoryFingerprint,
+    entityEntriesFingerprint,
+    aliasFingerprint,
+    reconcileFingerprint,
+    consumingFingerprint,
+    getEntityMutationVersion(),
+  ].join(":");
 }
 
 export async function validateRoots(
@@ -238,6 +269,12 @@ async function readState(
   }
   const result = EntityCanonicalIdMigrationStateSchema.safeParse(parsed);
   if (!result.success) throw new Error("Invalid entity canonical-id migration state.");
+  for (const mappings of [result.data.mappings, result.data.blocked ?? {}]) {
+    for (const [legacyId, canonicalId] of Object.entries(mappings)) {
+      assertSafeEntityId(legacyId);
+      assertSafeEntityId(canonicalId);
+    }
+  }
   return result.data;
 }
 
@@ -292,12 +329,6 @@ async function sameFileIdentity(leftPath: string, rightPath: string): Promise<bo
   }
 }
 
-function assertSafeEntityId(entityId: string): void {
-  if (/[\/\\\0]/.test(entityId)) {
-    throw new Error(`Refusing entity canonical-id migration through unsafe entity id: ${entityId}.`);
-  }
-}
-
 async function assertNotSymlink(filePath: string, label: string): Promise<void> {
   let stat;
   try {

package/src/storage/entity-canonical-id-references.ts

@@ -32,6 +32,7 @@ import { log } from "../logger.js";
 import { isErrnoCode } from "../utils/errno.js";
 import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
 import { withEntityCanonicalMutationLock } from "./entity-canonical-id-lock.js";
+import { assertSafeEntityId } from "./entity-id-safety.js";
 import { normalizeSupersessionKey } from "../temporal-supersession.js";
 
 export const ENTITY_CANONICAL_ID_MIGRATION_FILE = "entity-canonical-id-migration-v1.json";
@@ -51,6 +52,8 @@ function parseJournalMappings(raw: string): Readonly<Record<string, string>> {
   const cleaned: Record<string, string> = Object.create(null) as Record<string, string>;
   for (const [legacyId, canonicalId] of Object.entries(mappings)) {
     if (legacyId.length > 0 && typeof canonicalId === "string" && canonicalId.length > 0) {
+      assertSafeEntityId(legacyId);
+      assertSafeEntityId(canonicalId);
       cleaned[legacyId] = canonicalId;
     }
   }

package/src/storage/entity-id-safety.ts

@@ -0,0 +1,7 @@
+const UNSAFE_ENTITY_ID_PATTERN = /[\/\\\p{Cc}\p{Zl}\p{Zp}]/u;
+
+export function assertSafeEntityId(entityId: string): void {
+  if (UNSAFE_ENTITY_ID_PATTERN.test(entityId)) {
+    throw new Error("Refusing unsafe entity id containing a path separator or control character.");
+  }
+}

package/src/storage/entity-store.ts

@@ -58,6 +58,7 @@ export interface EntityStoreDeps {
   currentHistoricalIds(): Readonly<Record<string, string>>;
   findMatchingEntity(proposedName: string, type: string): Promise<string | null>;
   getEntityCacheSecureStoreKey(): string;
+  getEntityMutationVersion(): number;
   getMemoryStatusVersion(): number;
   invalidateKnowledgeIndexCache(): void;
   knowledgeIndexCache: { result: string; builtAt: number } | null;
@@ -483,8 +484,11 @@ export class EntityStore {
    */
   async readAllEntityFiles(): Promise<EntityFile[]> {
     const currentVersion = this.deps.getMemoryStatusVersion();
-    const schemaCacheKey = buildEntitySchemaCacheKey(this.deps.entitySchemas);
-    const cacheKey = `${this.deps.getEntityCacheSecureStoreKey()}\u0000${schemaCacheKey}`;
+    const cacheKey = [
+      this.deps.getEntityCacheSecureStoreKey(),
+      buildEntitySchemaCacheKey(this.deps.entitySchemas),
+      this.deps.getEntityMutationVersion(),
+    ].join("\u0000");
     const cached = getCachedEntities(this.deps.baseDir, currentVersion, cacheKey);
     if (cached) return cached;
 

package/src/storage/memory-frontmatter-metadata.ts

@@ -1,17 +1,12 @@
 export type RawFrontmatterOwner = object;
 
-const rawFrontmatterByMemory = new WeakMap<RawFrontmatterOwner, string>();
-
-export function extractRawFrontmatter(raw: string): string | null {
-  return raw.match(/^---\n([\s\S]*?)\n---\n?/)?.[1] ?? null;
-}
+const rawDocumentByMemory = new WeakMap<RawFrontmatterOwner, string>();
 
 export function rememberRawFrontmatter<T extends RawFrontmatterOwner>(owner: T, raw: string): T {
-  const frontmatter = extractRawFrontmatter(raw);
-  if (frontmatter !== null) rawFrontmatterByMemory.set(owner, frontmatter);
+  if (/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/.test(raw)) rawDocumentByMemory.set(owner, raw);
   return owner;
 }
 
-export function readRawFrontmatter(owner: RawFrontmatterOwner): string | undefined {
-  return rawFrontmatterByMemory.get(owner);
+export function readRawMemoryDocument(owner: RawFrontmatterOwner): string | undefined {
+  return rawDocumentByMemory.get(owner);
 }

package/src/storage/memory-migration-serialization.ts

@@ -1,5 +1,6 @@
 import type { MemoryFile, MemoryFrontmatter } from "../types.js";
-import { readRawFrontmatter } from "./memory-frontmatter-metadata.js";
+import { readRawMemoryDocument } from "./memory-frontmatter-metadata.js";
+import { canonicalizeEntityRefFrontmatter } from "./entity-canonical-id-references.js";
 
 export type MemoryEntityRefSerializer = (memory: MemoryFile, entityRef: string) => string;
 
@@ -8,20 +9,14 @@ export function serializeMemoryWithEntityRef(
   entityRef: string,
   fallback: MemoryEntityRefSerializer,
 ): string {
-  const rawFrontmatter = readRawFrontmatter(memory);
-  if (rawFrontmatter === undefined) return fallback(memory, entityRef);
-  const lines = rawFrontmatter.split("\n");
-  let entityRefLine = -1;
-  for (let index = 0; index < lines.length; index += 1) {
-    if (/^\s*entityRef\s*:/.test(lines[index] ?? "")) entityRefLine = index;
-  }
-  if (entityRefLine >= 0) {
-    const indent = lines[entityRefLine]!.match(/^\s*/)?.[0] ?? "";
-    lines[entityRefLine] = `${indent}entityRef: ${entityRef}`;
-  } else {
-    lines.push(`entityRef: ${entityRef}`);
-  }
-  return `---\n${lines.join("\n")}\n---\n\n${memory.content}\n`;
+  const rawDocument = readRawMemoryDocument(memory);
+  if (rawDocument === undefined) return fallback(memory, entityRef);
+  const currentEntityRef = memory.frontmatter.entityRef;
+  if (currentEntityRef === undefined) return fallback(memory, entityRef);
+  if (currentEntityRef === entityRef) return rawDocument;
+  const rewritten = canonicalizeEntityRefFrontmatter(rawDocument, { [currentEntityRef]: entityRef });
+  if (rewritten !== rawDocument) return rewritten;
+  return fallback(memory, entityRef);
 }
 
 export function createMemoryEntityRefSerializer(

package/src/storage/tombstone-blocked-capture-mutation.ts

@@ -30,9 +30,15 @@ export function buildExplicitCaptureDedupKey(
   ].join(" ");
 }
 
+export function buildCapturePathLockIdentity(pathname: string): string {
+  return `path ${pathname.length} ${pathname}`;
+}
+
 export function captureIdentityHash(memory: MemoryFile): string {
   return createHash("sha256")
-    .update(buildExplicitCaptureDedupKey(memory.content, memory.frontmatter.category, memory.frontmatter.sourceConnector))
+    .update(
+      buildExplicitCaptureDedupKey(memory.content, memory.frontmatter.category, memory.frontmatter.sourceConnector)
+    )
     .digest("hex");
 }
 
@@ -48,9 +54,7 @@ export class TombstoneBlockedCaptureWriteLock {
 
   private captureWriteLockPathForHash(identityHash: string): string {
     const filename =
-      identityHash.length === 0
-        ? "explicit-capture-write.lock"
-        : `explicit-capture-write-${identityHash}.lock`;
+      identityHash.length === 0 ? "explicit-capture-write.lock" : `explicit-capture-write-${identityHash}.lock`;
     return path.join(this.options.stateDir, "tombstone-blocked-capture", filename);
   }
 
@@ -87,7 +91,7 @@ export class TombstoneBlockedCaptureWriteLock {
     task: () => Promise<T>,
     identityHashes: readonly string[]
   ): Promise<T> {
-    const lockPaths = [...new Set(identityHashes.map((hash) => this.captureWriteLockPathForHash(hash)))].sort();
+    const lockPaths = [...new Set(identityHashes.map((hash) => this.captureWriteLockPathForHash(hash)))];
     const acquire = async (index: number): Promise<T> => {
       if (index === lockPaths.length) return await task();
       return await this.withSingleCaptureWriteLock(lockPaths[index], () => acquire(index + 1));
@@ -95,20 +99,13 @@ export class TombstoneBlockedCaptureWriteLock {
     return await acquire(0);
   }
 
-
-  async withCaptureWriteLock<T>(
-    task: () => Promise<T>,
-    identity?: string | readonly string[]
-  ): Promise<T> {
+  async withCaptureWriteLock<T>(task: () => Promise<T>, identity?: string | readonly string[]): Promise<T> {
     const identities =
       Array.isArray(identity) && identity.length > 0 ? identity : [typeof identity === "string" ? identity : ""];
     return await this.withCaptureWriteLocks(task, identities, []);
   }
 
-  async withCaptureWriteLockHashes<T>(
-    task: () => Promise<T>,
-    identityHashes: readonly string[]
-  ): Promise<T> {
+  async withCaptureWriteLockHashes<T>(task: () => Promise<T>, identityHashes: readonly string[]): Promise<T> {
     return await this.withCaptureWriteLocks(task, [], identityHashes);
   }
 
@@ -134,17 +131,44 @@ export class TombstoneBlockedCaptureWriteLock {
       key.length === 0 ? "" : createHash("sha256").update(key).digest("hex");
     const identityKeys = [...new Set(identities)];
     const hashes = [...new Set(identityHashes)];
-    const missingIdentityHashes = identityKeys
-      .filter((key) => !current || (!current.has(key) && !current.has(hashToken(key))))
-      .map(identityHash);
-    const missingHashes = hashes.filter((hash) => !current || !current.has(`\u0000${hash}`));
-    const physicalMissing = [...new Set([...missingIdentityHashes, ...missingHashes])];
-    if (current && physicalMissing.length > 0) {
-      const heldLockPaths = [...current]
+    const requested = [
+      ...identityKeys
+        .filter((key) => !current || (!current.has(key) && !current.has(hashToken(key))))
+        .map((key) => ({ hash: identityHash(key), pathLock: key.startsWith("path ") })),
+      ...hashes.filter((hash) => !current || !current.has(`\u0000${hash}`)).map((hash) => ({ hash, pathLock: false })),
+    ]
+      .filter((entry, index, entries) => entries.findIndex((candidate) => candidate.hash === entry.hash) === index)
+      .sort(
+        (left, right) =>
+          Number(left.pathLock) - Number(right.pathLock) ||
+          this.captureWriteLockPathForHash(left.hash).localeCompare(this.captureWriteLockPathForHash(right.hash))
+      );
+    const physicalMissing = requested.map((entry) => entry.hash);
+    if (current && requested.length > 0) {
+      const held = [...current]
         .filter((token) => token.startsWith("\u0000"))
-        .map((token) => this.captureWriteLockPathForHash(token.slice(1)));
-      const requestedLockPaths = physicalMissing.map((hash) => this.captureWriteLockPathForHash(hash));
-      if (heldLockPaths.some((held) => requestedLockPaths.some((requested) => requested < held))) {
+        .map((token) => {
+          const hash = token.slice(1);
+          const pathLock = [...current].some(
+            (key) => !key.startsWith("\u0000") && key.startsWith("path ") && hashToken(key) === token
+          );
+          return { hash, pathLock };
+        });
+      const requestedFirst = requested[0];
+      const heldLast = held
+        .sort(
+          (left, right) =>
+            Number(left.pathLock) - Number(right.pathLock) ||
+            this.captureWriteLockPathForHash(left.hash).localeCompare(this.captureWriteLockPathForHash(right.hash))
+        )
+        .at(-1);
+      if (
+        heldLast &&
+        requestedFirst &&
+        (Number(requestedFirst.pathLock) < Number(heldLast.pathLock) ||
+          (requestedFirst.pathLock === heldLast.pathLock &&
+            this.captureWriteLockPathForHash(requestedFirst.hash) < this.captureWriteLockPathForHash(heldLast.hash)))
+      ) {
         throw new Error(CAPTURE_WRITE_LOCK_BUSY_MESSAGE);
       }
     }
@@ -160,7 +184,6 @@ export class TombstoneBlockedCaptureWriteLock {
   }
 }
 
-
 export type TombstoneBlockedMutationHost = {
   readCurrent: () => Promise<MemoryFile | null>;
   isBlocked: (memory: MemoryFile | null) => boolean;
@@ -187,17 +210,16 @@ export type TombstoneBlockedMutation = {
 
 export async function runTombstoneBlockedMutation(
   host: TombstoneBlockedMutationHost,
-  mutation: TombstoneBlockedMutation,
+  mutation: TombstoneBlockedMutation
 ): Promise<void> {
   let lockIdentity: string | readonly string[] = mutation.identity;
-  let mustLock = mutation.blocked || mutation.coordinate === true;
   for (;;) {
     let retryIdentity: string | undefined;
     const mutate = async (): Promise<void> => {
       const current = await host.readCurrent();
       const currentBlocked = host.isBlocked(current);
       const currentQueuedReview = host.isQueuedReview(current);
-      if (current !== null && (currentBlocked || currentQueuedReview)) {
+      if (current !== null && (mutation.coordinate === true || currentBlocked || currentQueuedReview)) {
         const currentIdentity = host.memoryIdentity(current);
         const heldIdentities = Array.isArray(lockIdentity) ? lockIdentity : [lockIdentity];
         if (!heldIdentities.includes(currentIdentity)) {
@@ -230,17 +252,10 @@ export async function runTombstoneBlockedMutation(
       if (rebuildMarker) await mutation.beforeIndexUpdate?.();
       await mutation.updateIndex(rebuildMarker, currentBlocked && current !== null ? current : undefined);
     };
-    if (mustLock) {
-      await host.withCaptureWriteLock(mutate, lockIdentity);
-    } else {
-      await mutate();
-    }
+    await host.withCaptureWriteLock(mutate, lockIdentity);
     if (retryIdentity === undefined) return;
-    const identities = Array.isArray(lockIdentity)
-      ? [...lockIdentity, retryIdentity]
-      : [lockIdentity, retryIdentity];
+    const identities = Array.isArray(lockIdentity) ? [...lockIdentity, retryIdentity] : [lockIdentity, retryIdentity];
     lockIdentity = [...new Set(identities)];
-    mustLock = true;
   }
 }
 
@@ -255,30 +270,3 @@ export function isQueuedReviewFrontmatter(frontmatter: MemoryFrontmatter): boole
 export function isQueuedReviewMemory(memory: MemoryFile | null): memory is MemoryFile {
   return memory !== null && isQueuedReviewFrontmatter(memory.frontmatter);
 }
-
-export type TombstoneBlockedChunkMutation = {
-  blocked: boolean;
-  id: string;
-  identity: string;
-  findDuplicate: () => Promise<MemoryFile | null>;
-  persistMemory: () => Promise<void>;
-  afterWrite: () => Promise<void>;
-  withCaptureWriteLock: (task: () => Promise<string>, identity: string) => Promise<string>;
-};
-
-export async function runTombstoneBlockedChunkMutation(
-  mutation: TombstoneBlockedChunkMutation,
-): Promise<string> {
-  const persist = async (): Promise<string> => {
-    if (mutation.blocked) {
-      const duplicate = await mutation.findDuplicate();
-      if (duplicate) return duplicate.frontmatter.id;
-    }
-    await mutation.persistMemory();
-    await mutation.afterWrite();
-    return mutation.id;
-  };
-  return mutation.blocked
-    ? await mutation.withCaptureWriteLock(persist, mutation.identity)
-    : await persist();
-}