@remnic/core 9.26.0 → 9.28.0

package/src/storage.ts

@@ -1,6 +1,5 @@
 import { readProjectionRebuiltAt } from "./maintenance/projection-support.js";
 import {
-  access,
   lstat,
   readdir,
   readFile,
@@ -9,7 +8,6 @@ import {
   writeFile,
   mkdir,
   unlink,
-  rename,
   appendFile,
   open,
 } from "node:fs/promises";
@@ -20,6 +18,7 @@ import path from "node:path";
 import { log } from "./logger.js";
 import { assertMemoryFrontmatterId, warnProjectionFallback } from "./storage-guards.js";
 import { MemoryReadStore } from "./storage/memory-read-store.js";
+import { renderProfileWithLastUpdated } from "./storage/profile-header.js";
 import { readMaybeEncryptedLines, readMemoryActionEventRowsFromLines } from "./storage/secure-line-reader.js";
 import {
   appendLifecycleEventsSerialized,
@@ -34,10 +33,17 @@ import {
 import { selfDeps } from "./orchestration/self-deps.js";
 import { EntityStore } from "./storage/entity-store.js";
 import { IdentityContinuityStore } from "./storage/identity-continuity-store.js";
+import * as entityMigration from "./storage/entity-canonical-id-migration.js";
+import {
+  runLegacyEntityCanonicalIdMigration,
+  type EntityCanonicalIdMigrationHost,
+} from "./storage/entity-canonical-id-migration-adapter.js";
+import { EntityCanonicalIdMigrationRunner } from "./storage/entity-canonical-id-migration-runner.js";
+import { rememberRawFrontmatter } from "./storage/memory-frontmatter-metadata.js";
+import { createMemoryEntityRefSerializer } from "./storage/memory-migration-serialization.js";
+export { normalizeEntityName } from "./entity-id-normalization.js";
 import { isErrnoCode } from "./utils/errno.js";
-import { withHeldFileLock } from "./utils/serialize-mutations.js";
-import { RECALL_FALLBACK_DIRS, getCategoryDir, categoryDirName } from "./utils/category-dir.js";
-import { assertPathInsideRoot } from "./utils/path-containment.js";
+import { getCategoryDir, categoryDirName } from "./utils/category-dir.js";
 import { decodeYamlScalar } from "./utils/yaml-scalar.js";
 import {
   qmdCollectionPathParts,
@@ -1094,52 +1100,6 @@ function inferCurrentStateStatus(
   return inferMemoryStatus(frontmatter, pathRel, fallbackStatus);
 }
 
-/** Built-in aliases for common structural normalizations (no personal data) */
-const BUILTIN_ALIASES: Record<string, string> = {
-  openclaw: "openclaw",
-  "open-claw": "openclaw",
-};
-
-/**
- * Normalize an entity name to a canonical form.
- * Strips non-alphanumeric chars, collapses hyphens, removes type prefix duplication.
- * e.g. "My Project" → "my-project"
- *
- * Checks caller-provided user aliases first, then built-in aliases. Alias
- * tables are instance state on StorageManager (issue #1534) — use
- * `storageManager.normalizeEntityName(raw, type)` when normalizing within a
- * store, or pass `storageManager.entityAliases` explicitly. Without an
- * aliases argument only the built-in structural aliases apply.
- */
-export function normalizeEntityName(raw: string, type: string, aliases?: Readonly<Record<string, string>>): string {
-  // Strip type prefix if present (e.g. name="person-jane-doe", type="person")
-  const rawStr = typeof raw === "string" ? raw : "";
-  const typeStr = typeof type === "string" && type.trim().length > 0 ? type : "entity";
-
-  let name = rawStr.toLowerCase().trim();
-  const typePrefix = `${typeStr.toLowerCase()}-`;
-  if (name.startsWith(typePrefix)) {
-    name = name.slice(typePrefix.length);
-  }
-
-  // Replace non-alphanumeric with hyphens, collapse multiples, trim edges
-  let normalized = name
-    .replace(/[^a-z0-9]+/g, "-")
-    .replace(/-+/g, "-")
-    .replace(/^-|-$/g, "");
-
-  // Check caller-provided user aliases first, then built-in. Own-property and
-  // string guards keep inherited object keys (e.g. an entity literally named
-  // "constructor") and malformed alias values from corrupting canonical ids.
-  const userAlias = aliases !== undefined && Object.hasOwn(aliases, normalized) ? aliases[normalized] : undefined;
-  if (typeof userAlias === "string" && userAlias.length > 0) {
-    normalized = userAlias;
-  } else if (Object.hasOwn(BUILTIN_ALIASES, normalized)) {
-    normalized = BUILTIN_ALIASES[normalized];
-  }
-
-  return `${typeStr.toLowerCase()}-${normalized}`;
-}
 
 /**
  * Simple Levenshtein distance between two strings.
@@ -2134,24 +2094,39 @@ export class StorageManager {
    * Set or clear the at-rest encryption key.
    *
    * Pass a 32-byte Buffer to enable encryption; pass null to clear
-   * (lock) the store.  The caller is responsible for key lifecycle —
+   * (lock) the store. The caller is responsible for key lifecycle —
    * this method does not zero the buffer on replacement; the keyring
-   * module (`keyring.ts`) owns zeroization.
+   * module (`keyring.ts`) owns zeroization. The setter remains synchronous
+   * for existing consumers; call `setSecureStoreKeyAndWait` when unlock must
+   * await migration.
    */
   setSecureStoreKey(key: Buffer | null, encryptOnWrite = true): void {
+    if (!this.applySecureStoreKey(key, encryptOnWrite) || key === null) return;
+    void this.entityCanonicalIdMigration.triggerAfterUnlock().catch((error: unknown) => {
+      log.warn(
+        `secure-store unlock migration failed: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    });
+  }
+
+  public async setSecureStoreKeyAndWait(key: Buffer | null, encryptOnWrite = true): Promise<void> {
+    if (!this.applySecureStoreKey(key, encryptOnWrite) || key === null) return;
+    await this.entityCanonicalIdMigration.triggerAfterUnlock();
+  }
+
+  private applySecureStoreKey(key: Buffer | null, encryptOnWrite: boolean): boolean {
+    const sameKey =
+      this._secureStoreKey === key ||
+      (this._secureStoreKey !== null && key !== null && this._secureStoreKey.equals(key));
+    if (sameKey && this._secureStoreEncryptOnWrite === encryptOnWrite) return false;
+
     this._secureStoreKey = key;
     this._secureStoreEncryptOnWrite = encryptOnWrite;
-    // Route through the invalidation chokepoint (issue #1535): a key change
-    // (or store lock) must evict every cache layer that may hold content
-    // decrypted under the previous key, not just the entity layer.
     invalidateAllForDir(this.baseDir);
     this.invalidateKnowledgeIndexCache();
-    // A key change flips resolveWriteKey(), so a file's plain/encrypted class on
-    // the next full rewrite may differ from what was cached (issue #1909).
     this.secureFileEncryptionSniffCache.clear();
-    // Same rationale for the behavior-signals dedup key cache (issue #1909):
-    // drop it on a key change so a re-encrypted/re-decrypted ledger is reloaded.
     this.behaviorSignalsKeyCache = null;
+    return true;
   }
 
   private getEntityCacheSecureStoreKey(): string {
@@ -2339,6 +2314,7 @@ export class StorageManager {
     // loadAliases() explicitly — every creation path must still get the
     // store's own aliases, never an empty or foreign table.
     this.loadAliasesSync();
+    this.loadHistoricalEntityCanonicalIdsSync();
   }
 
   /** Set the process-wide hot-memories cache default (issue #1902). The
@@ -3596,9 +3572,17 @@ export class StorageManager {
    * store loaded last rewrite every other store's canonical entity ids.
    */
   private userAliases: Record<string, string> = {};
-
+  private historicalEntityCanonicalIds: Readonly<Record<string, string>> = {};
+  private readonly entityCanonicalIdMigration = new EntityCanonicalIdMigrationRunner(
+    () => !(this._secureStoreRequired && !this.isSecureStoreUnlocked()),
+    () => this.runLegacyEntityCanonicalIdMigration(),
+    () => entityMigration.getFingerprint(this.baseDir, this.entitiesDir, () => this.getCorpusScanVersion()),
+  );
   normalizeEntityName(raw: string, type: string): string {
-    return this.entityStore.normalizeEntityName(raw, type);
+    return entityMigration.resolveHistoricalEntityCanonicalId(
+      this.entityStore.normalizeEntityName(raw, type),
+      this.historicalEntityCanonicalIds,
+    );
   }
 
   /**
@@ -3645,8 +3629,24 @@ export class StorageManager {
       log.debug("no config/aliases.json found — using built-in aliases only");
     }
   }
+  private loadHistoricalEntityCanonicalIdsSync(): void {
+    this.historicalEntityCanonicalIds = entityMigration.loadHistoricalEntityCanonicalIds(this.stateDir);
+  }
+
+  private async runLegacyEntityCanonicalIdMigration(): Promise<string> {
+    const completionFingerprint = await runLegacyEntityCanonicalIdMigration(
+      this as unknown as EntityCanonicalIdMigrationHost,
+      (content) => parseEntityFile(content, this.entitySchemas),
+      (entity) => serializeEntityFile(entity, this.entitySchemas),
+      createMemoryEntityRefSerializer(serializeFrontmatter),
+    );
+    this.loadHistoricalEntityCanonicalIdsSync();
+    return completionFingerprint;
+  }
 
   async ensureDirectories(): Promise<void> {
+    await mkdir(this.baseDir, { recursive: true });
+    await entityMigration.validateRoots(this.baseDir, this.entitiesDir, this.stateDir, this.archiveDir);
     const today = new Date().toISOString().slice(0, 10);
     await mkdir(path.join(this.factsDir, today), { recursive: true });
     await mkdir(path.join(this.proceduresDir, today), { recursive: true });
@@ -3661,6 +3661,7 @@ export class StorageManager {
     await mkdir(this.identityAuditsWeeklyDir, { recursive: true });
     await mkdir(this.identityAuditsMonthlyDir, { recursive: true });
     await mkdir(path.join(this.baseDir, "config"), { recursive: true });
+    await this.entityCanonicalIdMigration.ensure();
     // Activate the corpus sentinel at setup (issue #1902, Cursor Medium): an
     // existing corpus predating this feature has a size-0/absent sentinel, which
     // getCachedMemories treats as a miss — so a read-heavy workload would never
@@ -3674,6 +3675,7 @@ export class StorageManager {
     if (this.hotMemoriesCacheEnabled && this.getMemoryCorpusVersion() === 0) {
       this.bumpMemoryCorpusVersion();
     }
+    await this.entityCanonicalIdMigration.markDirectoriesInitialized();
   }
 
   /**
@@ -4251,15 +4253,13 @@ export class StorageManager {
   }
 
   async writeProfile(content: string): Promise<void> {
+    const stampedContent = renderProfileWithLastUpdated(content, new Date().toISOString());
     await this.ensureDirectories();
     await this.snapshotBeforeWrite(this.profilePath, "consolidation");
-    await this.writeStorageSecureFile(this.profilePath, content);
+    await this.writeStorageSecureFile(this.profilePath, stampedContent);
     log.debug("updated profile.md");
   }
 
-  /**
-   * Normalize a string for fuzzy profile dedup: lowercase, strip punctuation, collapse whitespace.
-   */
   private static normalizeForDedup(s: string): string {
     if (typeof s !== "string") return "";
     return s
@@ -4308,15 +4308,13 @@ export class StorageManager {
       const content = [
         "# Behavioral Profile",
         "",
-        `*Last updated: ${new Date().toISOString()}*`,
-        "",
         ...newBullets.map((b) => `- ${b}`),
         "",
       ].join("\n");
       await this.writeProfile(content);
     } else {
-      const updatedTimestamp = existing.replace(/\*Last updated:.*\*/, `*Last updated: ${new Date().toISOString()}*`);
-      const withBullets = updatedTimestamp.trimEnd() + "\n" + newBullets.map((b) => `- ${b}`).join("\n") + "\n";
+      const withBullets =
+        existing.trimEnd() + "\n" + newBullets.map((b) => `- ${b}`).join("\n") + "\n";
       await this.writeProfile(withBullets);
     }
   }
@@ -4685,7 +4683,7 @@ export class StorageManager {
             const raw = await readMaybeEncryptedFile(fullPath, this._secureStoreKey, this.baseDir);
             const parsed = parseFrontmatter(raw);
             if (!parsed) return null;
-            return {
+            return rememberRawFrontmatter({
               path: fullPath,
               frontmatter: normalizeFrontmatterForPath(
                 parsed.frontmatter,
@@ -4693,7 +4691,7 @@ export class StorageManager {
                 parsed.content
               ),
               content: parsed.content,
-            } satisfies MemoryFile;
+            } satisfies MemoryFile, raw);
           } catch (err) {
             // Re-throw store-locked errors so a locked encrypted store fails
             // loudly rather than appearing as an empty memory corpus (Cursor
@@ -4844,7 +4842,7 @@ export class StorageManager {
               const raw = await readMaybeEncryptedFile(fullPath, this._secureStoreKey, this.baseDir);
               const parsed = parseFrontmatter(raw);
               if (parsed) {
-                memories.push({
+                memories.push(rememberRawFrontmatter({
                   path: fullPath,
                   frontmatter: normalizeFrontmatterForPath(
                     parsed.frontmatter,
@@ -4852,7 +4850,7 @@ export class StorageManager {
                     parsed.content
                   ),
                   content: parsed.content,
-                });
+                }, raw));
               }
             } catch (err) {
               // Re-throw store-locked errors — a locked encrypted store
@@ -6136,9 +6134,6 @@ export class StorageManager {
     this.invalidateKnowledgeIndexCache();
   }
 
-  /**
-   * Add an alias to an entity file. Deduplicates.
-   */
   async addEntityAlias(name: string, alias: string): Promise<void> {
     const filePath = path.join(this.entitiesDir, `${name}.md`);
     let entity: EntityFile;
@@ -6157,6 +6152,7 @@ export class StorageManager {
     entity.updated = new Date().toISOString();
     await this.writeStorageSecureFile(filePath, serializeEntityFile(entity, this.entitySchemas));
     this.invalidateKnowledgeIndexCache();
+    this.bumpMemoryStatusVersion();
   }
 
   async updateEntitySynthesis(

package/src/testing/subjects/profile-header-write.test.ts

@@ -0,0 +1,71 @@
+/**
+ * Profile-header write-boundary subject for the lifecycle scenario matrix.
+ * Profile content is not session state, so only the session-end mutation row
+ * applies; the remaining canonical rows are explicit skips rather than false
+ * extraction-lifecycle coverage.
+ */
+
+import assert from "node:assert/strict";
+
+import { StorageManager } from "../../storage.js";
+import { type LifecycleSubject, type MatrixRow, runLifecycleMatrix } from "../lifecycle-matrix.js";
+import { cleanupDir, mkTempMemoryDir } from "../orchestrator-lite.js";
+
+interface ProfileHeaderState {
+  dir: string;
+  storage: StorageManager;
+  writeStartedAt?: number;
+}
+
+const STALE_PROFILE = [
+  "# Behavioral Profile",
+  "",
+  "*Last updated: 2024-01-02T03:04:05.000Z*",
+  "",
+  "- Profile content survives the write boundary.",
+  "",
+].join("\n");
+
+const subject: LifecycleSubject<ProfileHeaderState> = {
+  appliesTo(row: MatrixRow): boolean | string {
+    return row.id === "session-end"
+      ? true
+      : "profile writes have no provider, restart, or replay variants; direct profile tests cover those boundaries";
+  },
+
+  async setup(row: MatrixRow): Promise<ProfileHeaderState> {
+    const dir = await mkTempMemoryDir(`profile-header-${row.id}`);
+    try {
+      return { dir, storage: new StorageManager(dir) };
+    } catch (error) {
+      await cleanupDir(dir);
+      throw error;
+    }
+  },
+
+  async exercise(state: ProfileHeaderState): Promise<void> {
+    state.writeStartedAt = Date.now();
+    await state.storage.writeProfile(STALE_PROFILE);
+  },
+
+  async invariants(state: ProfileHeaderState): Promise<void> {
+    const profile = await state.storage.readProfile();
+    const headers = profile.match(/^\*Last updated: .*\*$/gm) ?? [];
+    assert.equal(headers.length, 1, "profile writes leave one canonical Last updated header");
+    assert.notEqual(headers[0], "*Last updated: 2024-01-02T03:04:05.000Z*");
+    assert.match(profile, /^# Behavioral Profile$/m);
+
+    const timestamp = headers[0]?.match(/^\*Last updated: ([^*]+)\*$/)?.[1];
+    assert.ok(timestamp, "the canonical header contains an ISO timestamp");
+    const updatedAt = Date.parse(timestamp);
+    assert.ok(Number.isFinite(updatedAt), "the profile header timestamp is parseable");
+    assert.ok(updatedAt >= (state.writeStartedAt ?? 0), "the header advances at the write boundary");
+    assert.ok(updatedAt <= Date.now(), "the header is not in the future");
+  },
+
+  async teardown(state: ProfileHeaderState): Promise<void> {
+    await cleanupDir(state.dir);
+  },
+};
+
+runLifecycleMatrix("profile-header-write", subject);

package/src/types.ts

@@ -3,6 +3,7 @@ import type { MeetingsConfig } from "./meetings/types.js";
 import type { ActivityConfig } from "./activity/types.js";
 import type { WearablesConfig } from "./wearables/types.js";
 import type { ExtractionLivenessConfig } from "./extraction-liveness.js";
+import type { ReplicaPeersConfig } from "./replica-peers-config.js";
 
 export type ReasoningEffort = "none" | "low" | "medium" | "high";
 export type TriggerMode = "smart" | "every_n" | "time_based";
@@ -1208,9 +1209,8 @@ export interface PluginConfig extends BoundedJsonlStateConfig {
   dreamsPhases: DreamsPhasesConfig;
   procedural: ProceduralConfig;
   extractionLiveness: ExtractionLivenessConfig;
-  /** Claim-level provenance spans (#1575); see `ProvenanceConfig` for defaults.
-   * PR 1 parses + persists only; extraction wiring lands in PR 2.
-   */
+  replicaPeers: ReplicaPeersConfig;
+  /** Claim-level provenance spans (#1575); see `ProvenanceConfig` for defaults. */
   provenance: ProvenanceConfig;
   /**
    * Wearable transcript ingestion (Limitless / Bee / Omi connectors).