@remnic/core 9.35.1 → 9.35.3

package/src/storage/entity-ref-repair.ts

@@ -0,0 +1,171 @@
+/**
+ * Post-persist entity-reference repair (issue #2213), extracted from
+ * storage.ts (#1520 god-file ratchet).
+ *
+ * Every store-mediated writer that persists an `entityRef` resolves it at
+ * the write; when the canonical-id journal moves ACROSS the write (a peer
+ * migration publishing between resolve and persist), the repair re-resolves
+ * from the caller's ORIGINAL ref and rewrites through the blocked-capture
+ * surface. On repair failure the caller-declared rollback runs BEFORE the
+ * error propagates (AGENTS.md §14): a caller that reports failure must
+ * leave disk in the state peers/caches were told about.
+ */
+import { unlink } from "node:fs/promises";
+import { log } from "../logger.js";
+import type { MemoryFile, MemoryFrontmatter } from "../types.js";
+import * as entityRefs from "./entity-canonical-id-references.js";
+import {
+  applyTombstoneResurrectionGate,
+  type TombstoneMatch,
+  type TombstoneStore,
+} from "../lifecycle/tombstones.js";
+import { supersessionKeysForFact } from "../temporal-supersession.js";
+import { ContentHashIndex } from "./content-hash-index.js";
+import { rewriteProjectedMemoryEntityReference } from "../memory-projection-mutations.js";
+
+export interface EntityRefRepairDeps {
+  readonly stateDir: string;
+  readonly baseDir: string;
+  currentHistoricalIds(): Readonly<Record<string, string>>;
+  serializeFrontmatter(fm: MemoryFrontmatter): string;
+  writeTombstoneBlockedMemory(
+    pathname: string,
+    fileContent: string,
+    frontmatter: MemoryFrontmatter,
+    content: string,
+  ): Promise<void>;
+  invalidateAllMemoriesCache(): void;
+  getTombstoneStore(): Promise<TombstoneStore>;
+  tombstonesEnabled(): boolean;
+  tombstonesNamespace(): string;
+}
+
+export interface EntityRefRepairOptions {
+  /** Re-run the add-only resurrection gate under the FINAL ref before the
+   * rewrite (chunk writes, where `body` IS the hash source). */
+  regateFact?: boolean;
+  /** Rollback: rewrite this pre-mutation record on repair failure. */
+  onFailRestore?: MemoryFile;
+  /** Rollback: unlink a file this mutation created on repair failure. */
+  onFailRemove?: string;
+}
+
+export class EntityRefRepair {
+  constructor(private readonly deps: EntityRefRepairDeps) {}
+
+  /**
+   * Shared lookup+apply core for the write-time resurrection gate (#1579 /
+   * #2213): fail-open, add-only; writeMemory's gate + chunk-repair re-gate
+   * consult this one implementation (rule 43).
+   */
+  async gate(
+    fm: MemoryFrontmatter,
+    hashSource: string,
+    structuredAttributes?: Record<string, string>,
+  ): Promise<TombstoneMatch | null> {
+    if (!this.deps.tombstonesEnabled()) return null;
+    try {
+      const store = await this.deps.getTombstoneStore();
+      // Pass EVERY derived key (thread Ociag/Oci-W): emitters register one
+      // tombstone per key, so the block can be on any of them.
+      const supersessionKeys =
+        fm.entityRef && structuredAttributes
+          ? supersessionKeysForFact({ entityRef: fm.entityRef, structuredAttributes })
+          : [];
+      const match = applyTombstoneResurrectionGate(store, fm, {
+        normalizedText: ContentHashIndex.normalizeContent(hashSource),
+        supersessionKeys,
+        namespace: this.deps.tombstonesNamespace(),
+      });
+      if (match) {
+        log.info(
+          `tombstone: blocked resurrection of fact ${fm.id} (tier=${match.matchedTier}, tombstone=${match.tombstoneId}, reason=${match.reason})`
+        );
+      }
+      return match;
+    } catch (err) {
+      // Fail-open (rule 34): a lookup error must not block the write.
+      log.warn(`tombstone lookup failed for fact ${fm.id} (fail-open): ${err}`);
+      return null;
+    }
+  }
+
+  /** Post-persist repair delegate — see the references module for semantics. */
+  async repair(
+    filePath: string,
+    fm: MemoryFrontmatter,
+    rawRef: string,
+    refIds: Readonly<Record<string, string>>,
+    body: string,
+    opts: EntityRefRepairOptions = {},
+  ): Promise<void> {
+    const refBeforeRepair = fm.entityRef;
+    try {
+      await entityRefs.repairEntityRefAfterJournalMove({
+        stateDir: this.deps.stateDir,
+        currentIds: () => this.deps.currentHistoricalIds(),
+        idsAtResolve: refIds,
+        rawRef,
+        frontmatter: fm,
+        rewrite: async () => {
+          if (opts.regateFact && fm.category === "fact") {
+            await this.gate(fm, body, undefined);
+          }
+          // Blocked-capture surface: a repair rewrite of a tombstone-blocked
+          // record must keep TombstoneBlockedCaptureIndex consistent.
+          await this.deps.writeTombstoneBlockedMemory(
+            filePath,
+            `${this.deps.serializeFrontmatter(fm)}\n\n${body}\n`,
+            fm,
+            body,
+          );
+          this.deps.invalidateAllMemoriesCache();
+        },
+      });
+    } catch (err) {
+      if (opts.onFailRestore) {
+        await this.restore(opts.onFailRestore);
+      } else if (opts.onFailRemove) {
+        await unlink(opts.onFailRemove).catch(() => undefined);
+        this.deps.invalidateAllMemoriesCache();
+      }
+      throw err;
+    }
+    await this.syncProjection(fm.id, refBeforeRepair, fm.entityRef);
+  }
+
+  /**
+   * Restore a record's pre-mutation bytes through the blocked-capture
+   * surface when a post-persist repair fails (§14).
+   */
+  async restore(before: MemoryFile): Promise<void> {
+    try {
+      await this.deps.writeTombstoneBlockedMemory(
+        before.path,
+        `${this.deps.serializeFrontmatter(before.frontmatter)}\n\n${before.content}\n`,
+        before.frontmatter,
+        before.content,
+      );
+    } catch (restoreErr) {
+      log.warn(`failed to restore ${before.frontmatter.id} after repair failure: ${restoreErr}`);
+    }
+    this.deps.invalidateAllMemoriesCache();
+  }
+
+  /**
+   * Projection rows written under the pre-repair claimant cannot be restored
+   * once the mapping is parked (Codex P1). Fail-open: projection is rebuildable.
+   */
+  async syncProjection(
+    memoryId: string,
+    refBefore: string | undefined,
+    refAfter: string | undefined,
+  ): Promise<void> {
+    if (typeof refBefore !== "string" || typeof refAfter !== "string" || refBefore === refAfter) return;
+    try {
+      await rewriteProjectedMemoryEntityReference(this.deps.baseDir, memoryId, refBefore, refAfter);
+    } catch (err) {
+      log.warn(`projection entityRef sync failed for ${memoryId}: ${err}`);
+    }
+  }
+}

package/src/storage/entity-store.ts

@@ -13,13 +13,25 @@
 import { readdir, unlink } from "node:fs/promises";
 import path from "node:path";
 import { normalizeEntityStructuredSection, sortStructuredSectionsBySchema } from "../entity-schema.js";
-import { withEntityCanonicalMutationLock } from "./entity-canonical-id-migration.js";
+import { withEntityCanonicalMutationLock } from "./entity-canonical-id-lock.js";
 import { log } from "../logger.js";
 import { getCachedEntities, setCachedEntities } from "../memory-cache.js";
 import type { VersionTrigger } from "../page-versioning.js";
 import { SecureStoreLockedError } from "../secure-store/secure-fs.js";
-import type { EntityFile, EntityStructuredSection, PluginConfig, ScoredEntity } from "../types.js";
+import type {
+  EntityActivityEntry,
+  EntityFile,
+  EntityRelationship,
+  EntityStructuredSection,
+  PluginConfig,
+  ScoredEntity,
+} from "../types.js";
 import { isErrnoCode } from "../utils/errno.js";
+import {
+  reconcileIfJournalMovedSync,
+  requestEntityCanonicalIdReconcileSync,
+  resolveHistoricalEntityCanonicalId,
+} from "./entity-canonical-id-references.js";
 import {
   buildEntitySchemaCacheKey,
   compareEntityTimestamps,
@@ -38,9 +50,12 @@ export interface EntityStoreDeps {
   readonly storageManagerClass: typeof StorageManager;
   readonly baseDir: string;
   bumpMemoryStatusVersion(): void;
+  /** Entity-mutation sentinel for the migration fingerprint (issue #2213). */
+  bumpEntityMutationVersion(): void;
   ensureDirectories(): Promise<void>;
   readonly entitiesDir: string;
   readonly entitySchemas: PluginConfig["entitySchemas"] | undefined;
+  currentHistoricalIds(): Readonly<Record<string, string>>;
   findMatchingEntity(proposedName: string, type: string): Promise<string | null>;
   getEntityCacheSecureStoreKey(): string;
   getMemoryStatusVersion(): number;
@@ -66,6 +81,118 @@ export class EntityStore {
     return normalizeEntityNameShared(raw, type, this.deps.userAliases);
   }
 
+  /**
+   * Read an entity file by id, resolving through the migration journal with a
+   * fallback to the caller's original id (issue #2213): mid-migration, the
+   * journal maps legacy → canonical BEFORE the file moves — writing the
+   * still-existing legacy file is safe (the rename retry carries it), while
+   * resolving only the canonical path silently drops the mutation.
+   */
+  private async readEntityForMutation(
+    name: string,
+    method: string,
+    historicalIds: Readonly<Record<string, string>>,
+  ): Promise<{ filePath: string; entity: EntityFile } | null> {
+    const canonical = resolveHistoricalEntityCanonicalId(name, historicalIds);
+    for (const candidate of canonical === name ? [name] : [canonical, name]) {
+      const filePath = path.join(this.deps.entitiesDir, `${candidate}.md`);
+      try {
+        const entity = parseEntityFile(await this.deps.readStorageSecureFile(filePath), this.deps.entitySchemas);
+        return { filePath, entity };
+      } catch (err) {
+        if (err instanceof SecureStoreLockedError) throw err;
+        if (!isErrnoCode(err, "ENOENT")) throw err;
+      }
+    }
+    log.debug(`${method}: entity file ${canonical}.md not found`);
+    return null;
+  }
+
+  /**
+   * Locked read-modify-write for the entity-file mutators (issue #2213):
+   * holding the entity-mutation lock keeps `migrateEntityFilePair` (which
+   * takes the same lock) from moving the file between the fallback read and
+   * this write — an unlocked write could recreate a just-renamed legacy file.
+   * The journal itself is NOT serialized by this lock (`pruneBlocked` can
+   * park a mapping mid-mutation), so the journal identity is REVALIDATED
+   * just before committing — a moved journal restarts resolution against the
+   * fresh table instead of writing into a since-contested claimant — and
+   * checked once more after the write for a move across the write itself.
+   * `mutate` returns false to skip the write. Resolves true when written.
+   */
+  private async mutateEntityFile(
+    name: string,
+    method: string,
+    mutate: (entity: EntityFile, historicalIds: Readonly<Record<string, string>>) => boolean,
+  ): Promise<boolean> {
+    const stateDir = path.join(this.deps.baseDir, "state");
+    return withEntityCanonicalMutationLock(stateDir, async () => {
+      for (let attempt = 0; attempt < 3; attempt += 1) {
+        const idsAtResolve = this.deps.currentHistoricalIds();
+        const located = await this.readEntityForMutation(name, method, idsAtResolve);
+        const mutated = located !== null && mutate(located.entity, idsAtResolve);
+        // Revalidate BEFORE honoring either verdict: a journal that moved
+        // makes both the resolution AND a dedupe-based no-op stale (a parked
+        // mapping can make "already present on the claimant" wrong).
+        if (this.deps.currentHistoricalIds() !== idsAtResolve) continue;
+        if (!located || !mutated) return false;
+        located.entity.updated = new Date().toISOString();
+        await this.deps.writeStorageSecureFile(
+          located.filePath,
+          serializeEntityFile(located.entity, this.deps.entitySchemas),
+        );
+        reconcileIfJournalMovedSync(stateDir, idsAtResolve, this.deps.currentHistoricalIds());
+        this.deps.invalidateKnowledgeIndexCache();
+        this.deps.bumpEntityMutationVersion();
+        return true;
+      }
+      // The journal kept moving under us. A silent false would DROP the
+      // requested mutation (the reconcile pass rewrites existing references;
+      // it cannot replay a mutation that was never written) — surface a
+      // retryable error so the caller's persist machinery logs/retries it.
+      requestEntityCanonicalIdReconcileSync(stateDir);
+      throw new Error(
+        `${method}: entity canonical-id journal kept changing; mutation not applied — retry`,
+      );
+    });
+  }
+
+  /** Add a relationship to an entity file. Deduplicates by target+label. */
+  async addEntityRelationship(name: string, rel: EntityRelationship): Promise<void> {
+    await this.mutateEntityFile(name, "addEntityRelationship", (entity, historicalIds) => {
+      // Resolve the target with the SAME snapshot the locked read used, and
+      // dedupe against CANONICALIZED stored targets — an existing edge the
+      // migration has not rewritten yet still names the legacy id, and a
+      // plain equality check would miss it and leave duplicates after the
+      // rewrite (issue #2213).
+      const target = resolveHistoricalEntityCanonicalId(rel.target, historicalIds);
+      const duplicate = entity.relationships.some(
+        (r) => resolveHistoricalEntityCanonicalId(r.target, historicalIds) === target && r.label === rel.label,
+      );
+      if (duplicate) return false;
+      entity.relationships.push({ ...rel, target });
+      return true;
+    });
+  }
+
+  /** Add an activity entry to an entity file; prunes oldest beyond maxEntries. */
+  async addEntityActivity(name: string, entry: EntityActivityEntry, maxEntries: number): Promise<void> {
+    await this.mutateEntityFile(name, "addEntityActivity", (entity) => {
+      entity.activity.unshift(entry);
+      if (entity.activity.length > maxEntries) entity.activity = entity.activity.slice(0, maxEntries);
+      return true;
+    });
+  }
+
+  async addEntityAlias(name: string, alias: string): Promise<void> {
+    const wrote = await this.mutateEntityFile(name, "addEntityAlias", (entity) => {
+      if (entity.aliases.includes(alias)) return false;
+      entity.aliases.push(alias);
+      return true;
+    });
+    if (wrote) this.deps.bumpMemoryStatusVersion();
+  }
+
   async writeEntity(
     name: string,
     type: string,
@@ -212,6 +339,7 @@ export class EntityStore {
     await this.deps.writeStorageSecureFile(filePath, serializeEntityFile(entity, this.deps.entitySchemas));
     this.deps.invalidateKnowledgeIndexCache();
     this.deps.bumpMemoryStatusVersion(); // invalidate entity cache
+    this.deps.bumpEntityMutationVersion();
     log.debug(`wrote entity ${normalized}`);
     return normalized;
   }
@@ -243,6 +371,37 @@ export class EntityStore {
     }
   }
 
+  /** Re-serialize every entity file into the compiled-truth timeline shape. */
+  async migrateEntityFilesToCompiledTruthTimeline(): Promise<{ total: number; migrated: number }> {
+    const entityNames = await this.listEntityNames();
+    const stateDir = path.join(this.deps.baseDir, "state");
+    let migrated = 0;
+    for (const entityName of entityNames) {
+      // Read+rewrite under the entity MUTATION lock (issue #2213): an
+      // unlocked whole-record rewrite could restore relationship targets a
+      // peer migration rewrote between the read and the write, and this
+      // path bumps only the status version, which never re-triggers the
+      // completed migration's reference scan.
+      const wrote = await withEntityCanonicalMutationLock(stateDir, async () => {
+        const raw = await this.readEntity(entityName);
+        if (!raw) return false;
+        const serialized = serializeEntityFile(parseEntityFile(raw, this.deps.entitySchemas), this.deps.entitySchemas);
+        if (raw.trimEnd() === serialized.trimEnd()) return false;
+        const filePath = this.deps.resolveEntityFilePath(entityName);
+        if (filePath === null) return false;
+        await this.deps.writeStorageSecureFile(filePath, serialized);
+        this.deps.bumpEntityMutationVersion();
+        return true;
+      });
+      if (wrote) migrated += 1;
+    }
+    if (migrated > 0) {
+      this.deps.invalidateKnowledgeIndexCache();
+      this.deps.bumpMemoryStatusVersion();
+    }
+    return { total: entityNames.length, migrated };
+  }
+
   /**
    * Set or rewrite the synthesis layer of an entity file.
    */
@@ -315,6 +474,7 @@ export class EntityStore {
     ]);
     this.deps.invalidateKnowledgeIndexCache();
     this.deps.bumpMemoryStatusVersion(); // invalidate entity cache
+    this.deps.bumpEntityMutationVersion();
   }
 
   /**
@@ -693,6 +853,7 @@ export class EntityStore {
 
         const canonicalPath = path.join(this.deps.entitiesDir, `${canonical}.md`);
         await this.deps.writeStorageSecureFile(canonicalPath, serializeEntityFile(mergedEntity, this.deps.entitySchemas));
+        this.deps.bumpEntityMutationVersion();
 
         // Remove non-canonical files
         for (const file of files) {

package/src/storage/tombstone-blocked-capture-sync.ts

@@ -7,6 +7,10 @@ import type { MemoryFile, MemoryFrontmatter } from "../types.js";
 import { markProjectedMemoryPathInvalid } from "../memory-projection-store.js";
 import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
 import { isErrnoCode } from "../utils/errno.js";
+import {
+  pathMayCarryEntityRefs,
+  requestEntityCanonicalIdReconcile,
+} from "./entity-canonical-id-references.js";
 import {
   readMaybeEncryptedFileFromChunks,
   writeMaybeEncryptedFileFromChunks,
@@ -822,11 +826,25 @@ export abstract class TombstoneBlockedCaptureIndexHost {
   async writeOfflineSyncFile(filePath: string, content: Buffer): Promise<void> {
     const target = this.assertManagedStoragePath(filePath, "storage.writeOfflineSyncFile");
     await this.writeTombstoneBlockedOfflineSyncFile(target, content);
+    await this.requestSyncReconcileIfMemoryPath(target);
   }
 
   async writeOfflineSyncFileChunks(filePath: string, chunks: AsyncIterable<Buffer>): Promise<void> {
     const target = this.assertManagedStoragePath(filePath, "storage.writeOfflineSyncFileChunks");
     await this.writeTombstoneBlockedOfflineSyncFileChunks(target, chunks);
+    await this.requestSyncReconcileIfMemoryPath(target);
+  }
+
+  /**
+   * Replicated bytes are opaque (possibly encrypted) and can carry legacy
+   * entity references a completed migration already renamed (issue #2213) —
+   * but only memory-tier markdown can, so transcript/state/other sync traffic
+   * must not trigger the full-corpus reconcile pass.
+   */
+  private async requestSyncReconcileIfMemoryPath(target: string): Promise<void> {
+    const stateDir = this.tombstoneBlockedCaptureIndexOptions().stateDir;
+    if (!pathMayCarryEntityRefs(path.dirname(stateDir), target)) return;
+    await requestEntityCanonicalIdReconcile(stateDir);
   }
 
   async deleteOfflineSyncFile(filePath: string): Promise<void> {