@remnic/core 1.1.11 → 1.1.13

package/src/peers/migrate-from-identity-anchor.test.ts

@@ -0,0 +1,291 @@
+/**
+ * Tests for the identity-anchor → peer-registry migration.
+ *
+ * Issue #679 PR 5/5. All fixtures are synthetic — no real user data.
+ */
+
+import assert from "node:assert/strict";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { readPeer } from "./storage.js";
+import { migrateFromIdentityAnchor } from "./migrate-from-identity-anchor.js";
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ──────────────────────────────────────────────────────────────────────────────
+
+async function makeTempDir(): Promise<string> {
+  return fs.mkdtemp(path.join(os.tmpdir(), "peer-migrate-test-"));
+}
+
+/** Write a file, creating intermediate directories as needed. */
+async function writeFixture(filePath: string, content: string): Promise<void> {
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.writeFile(filePath, content, "utf8");
+}
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Fixture strings (synthetic, no personal data)
+// ──────────────────────────────────────────────────────────────────────────────
+
+const SAMPLE_ANCHOR = `# Identity Continuity Anchor
+
+## Identity Traits
+
+Prefers clear, structured communication.
+
+## Communication Preferences
+
+Bullet points over prose for technical topics.
+
+## Operating Principles
+
+Write tests before implementation.
+`;
+
+const SAMPLE_IDENTITY_MD = `## Learning patterns
+
+Tends to iterate on implementation until tests pass cleanly.
+`;
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Tests
+// ──────────────────────────────────────────────────────────────────────────────
+
+test("migrate: creates peers/self/identity.md from identity-anchor.md fixture", async () => {
+  const dir = await makeTempDir();
+  await writeFixture(
+    path.join(dir, "identity", "identity-anchor.md"),
+    SAMPLE_ANCHOR,
+  );
+
+  const result = await migrateFromIdentityAnchor({ memoryDir: dir });
+
+  assert.equal(result.written, true, "expected written=true");
+  assert.equal(result.skipped, false);
+  assert.equal(result.dryRun, false);
+  assert.ok(result.identityAnchorSource !== null, "identityAnchorSource should be set");
+  assert.equal(result.peer.id, "self");
+  assert.equal(result.peer.kind, "self");
+
+  // Verify the file exists on disk and round-trips through readPeer.
+  const loaded = await readPeer(dir, "self");
+  assert.ok(loaded !== null, "peers/self/identity.md should exist after migration");
+  assert.equal(loaded.id, "self");
+  assert.equal(loaded.kind, "self");
+
+  // Notes should contain the anchor content.
+  assert.ok(
+    loaded.notes?.includes("Identity Traits"),
+    "notes should include migrated anchor content",
+  );
+  assert.ok(
+    loaded.notes?.includes("Migrated from identity-anchor.md"),
+    "notes should have the section label",
+  );
+});
+
+test("migrate: includes IDENTITY.md content when both sources present", async () => {
+  const dir = await makeTempDir();
+  await writeFixture(
+    path.join(dir, "identity", "identity-anchor.md"),
+    SAMPLE_ANCHOR,
+  );
+  await writeFixture(path.join(dir, "IDENTITY.md"), SAMPLE_IDENTITY_MD);
+
+  const result = await migrateFromIdentityAnchor({ memoryDir: dir });
+
+  assert.equal(result.written, true);
+  assert.ok(result.identityAnchorSource !== null);
+  assert.ok(result.identityMdSource !== null, "identityMdSource should be set");
+
+  const loaded = await readPeer(dir, "self");
+  assert.ok(loaded !== null);
+  assert.ok(loaded.notes?.includes("Migrated from IDENTITY.md"));
+  assert.ok(loaded.notes?.includes("Learning patterns"));
+});
+
+test("migrate: works with no legacy files (fresh install)", async () => {
+  const dir = await makeTempDir();
+
+  const result = await migrateFromIdentityAnchor({ memoryDir: dir });
+
+  assert.equal(result.written, true);
+  assert.equal(result.skipped, false);
+  assert.equal(result.dryRun, false);
+  assert.equal(result.identityAnchorSource, null);
+  assert.equal(result.identityMdSource, null);
+
+  const loaded = await readPeer(dir, "self");
+  assert.ok(loaded !== null, "self peer should exist even with no source data");
+  assert.equal(loaded.id, "self");
+  assert.equal(loaded.kind, "self");
+  // No notes when both sources are absent.
+  assert.equal(loaded.notes, undefined);
+});
+
+test("dry-run: prints but does not write", async () => {
+  const dir = await makeTempDir();
+  await writeFixture(
+    path.join(dir, "identity", "identity-anchor.md"),
+    SAMPLE_ANCHOR,
+  );
+
+  const result = await migrateFromIdentityAnchor({
+    memoryDir: dir,
+    dryRun: true,
+  });
+
+  assert.equal(result.dryRun, true);
+  assert.equal(result.written, false);
+  assert.equal(result.skipped, false);
+
+  // Proposed peer is returned.
+  assert.equal(result.peer.id, "self");
+  assert.equal(result.peer.kind, "self");
+  assert.ok(result.peer.notes?.includes("Identity Traits"));
+
+  // Nothing should have been written to disk.
+  const loaded = await readPeer(dir, "self");
+  assert.equal(loaded, null, "no file should exist on disk after dry-run");
+});
+
+test("dry-run: returns proposed peer even with no legacy files", async () => {
+  const dir = await makeTempDir();
+
+  const result = await migrateFromIdentityAnchor({
+    memoryDir: dir,
+    dryRun: true,
+  });
+
+  assert.equal(result.dryRun, true);
+  assert.equal(result.written, false);
+  assert.equal(result.skipped, false);
+  assert.equal(result.peer.id, "self");
+  assert.equal(result.peer.kind, "self");
+
+  const loaded = await readPeer(dir, "self");
+  assert.equal(loaded, null, "no file should exist on disk after dry-run");
+});
+
+test("idempotent: second run returns skipped=true without overwriting", async () => {
+  const dir = await makeTempDir();
+  await writeFixture(
+    path.join(dir, "identity", "identity-anchor.md"),
+    SAMPLE_ANCHOR,
+  );
+
+  // First run — writes.
+  const first = await migrateFromIdentityAnchor({ memoryDir: dir });
+  assert.equal(first.written, true);
+  assert.equal(first.skipped, false);
+
+  // Read the createdAt that was assigned.
+  const afterFirst = await readPeer(dir, "self");
+  assert.ok(afterFirst !== null);
+  const createdAt = afterFirst.createdAt;
+
+  // Second run — should skip.
+  const second = await migrateFromIdentityAnchor({ memoryDir: dir });
+  assert.equal(second.written, false);
+  assert.equal(second.skipped, true);
+  assert.equal(second.dryRun, false);
+
+  // The on-disk record should be unchanged (same createdAt).
+  const afterSecond = await readPeer(dir, "self");
+  assert.ok(afterSecond !== null);
+  assert.equal(afterSecond.createdAt, createdAt, "createdAt must not change on second run");
+});
+
+test("dryRun flag is preserved on skip path (P2 codex fix)", async () => {
+  // Codex P2: the early-return for an existing self peer previously hard-coded
+  // dryRun: false, contradicting the MigrateFromIdentityAnchorResult contract.
+  const dir = await makeTempDir();
+  // First: real write to seed the self peer.
+  const first = await migrateFromIdentityAnchor({ memoryDir: dir });
+  assert.equal(first.written, true);
+
+  // Second: dry-run on top of an existing peer — dryRun must reflect the
+  // caller's request, not be hard-coded to false.
+  const second = await migrateFromIdentityAnchor({ memoryDir: dir, dryRun: true });
+  assert.equal(second.skipped, true);
+  assert.equal(second.dryRun, true, "dryRun must be true when caller passed dryRun: true");
+  assert.equal(second.written, false);
+});
+
+test("idempotent: running dry-run twice does not write on either run", async () => {
+  const dir = await makeTempDir();
+
+  await migrateFromIdentityAnchor({ memoryDir: dir, dryRun: true });
+  await migrateFromIdentityAnchor({ memoryDir: dir, dryRun: true });
+
+  const loaded = await readPeer(dir, "self");
+  assert.equal(loaded, null, "no file should exist after two dry-runs");
+});
+
+test("respects custom displayName option", async () => {
+  const dir = await makeTempDir();
+
+  const result = await migrateFromIdentityAnchor({
+    memoryDir: dir,
+    displayName: "Primary Operator",
+  });
+
+  assert.equal(result.peer.displayName, "Primary Operator");
+  const loaded = await readPeer(dir, "self");
+  assert.ok(loaded !== null);
+  assert.equal(loaded.displayName, "Primary Operator");
+});
+
+test("symlinked identity-anchor.md is silently skipped", async () => {
+  const dir = await makeTempDir();
+  // Create the identity directory, then a symlink to a file outside memoryDir.
+  const identityDir = path.join(dir, "identity");
+  await fs.mkdir(identityDir, { recursive: true });
+  const externalFile = path.join(dir, "_external.md");
+  await fs.writeFile(externalFile, SAMPLE_ANCHOR, "utf8");
+  await fs.symlink(externalFile, path.join(identityDir, "identity-anchor.md"));
+
+  const result = await migrateFromIdentityAnchor({ memoryDir: dir });
+
+  // Symlinked source should be silently skipped.
+  assert.equal(result.identityAnchorSource, null);
+  // Migration should still succeed (no legacy data, but self peer created).
+  assert.equal(result.written, true);
+  assert.equal(result.peer.notes, undefined, "no notes from symlinked source");
+});
+
+test("symlinked IDENTITY.md is silently skipped", async () => {
+  const dir = await makeTempDir();
+  const externalFile = path.join(dir, "_external-identity.md");
+  await fs.writeFile(externalFile, SAMPLE_IDENTITY_MD, "utf8");
+  await fs.symlink(externalFile, path.join(dir, "IDENTITY.md"));
+
+  const result = await migrateFromIdentityAnchor({ memoryDir: dir });
+
+  assert.equal(result.identityMdSource, null);
+  assert.equal(result.written, true);
+});
+
+test("symlinked identity parent directory is silently skipped (P1 codex fix)", async () => {
+  // Codex P1: safeReadRegularFile only checked lstat on the final path
+  // component; a symlinked parent like `memoryDir/identity -> /tmp/outside`
+  // could let the migrator read outside memoryDir. safeReadLegacyFile now
+  // rejects symlinked parent directories before opening the file.
+  const dir = await makeTempDir();
+  const externalDir = await makeTempDir();
+  // Write the anchor in the external directory.
+  await fs.writeFile(path.join(externalDir, "identity-anchor.md"), SAMPLE_ANCHOR, "utf8");
+  // Point `memoryDir/identity` at the external directory via a symlink.
+  await fs.symlink(externalDir, path.join(dir, "identity"));
+
+  const result = await migrateFromIdentityAnchor({ memoryDir: dir });
+
+  // The symlinked parent must be silently skipped.
+  assert.equal(result.identityAnchorSource, null, "symlinked parent dir must be rejected");
+  assert.equal(result.written, true, "migration should still succeed with no notes");
+  assert.equal(result.peer.notes, undefined, "no notes from symlinked parent source");
+});

package/src/peers/migrate-from-identity-anchor.ts

@@ -0,0 +1,350 @@
+/**
+ * Identity-anchor → peer-registry migration — issue #679 PR 5/5.
+ *
+ * Reads the legacy `identity/identity-anchor.md` file (written by
+ * `StorageManager.writeIdentityAnchor`) and the `IDENTITY.md`
+ * reflections file, then seeds the peer registry with a `self` peer
+ * whose `peers/self/identity.md` kernel captures the relevant
+ * source material.
+ *
+ * Design principles:
+ *
+ * 1. **Idempotent** — if `peers/self/identity.md` already exists this
+ *    function returns `{ skipped: true }` without overwriting it. The
+ *    second-run guarantee prevents accidental double-migration and
+ *    makes `remnic peer migrate` safe to run in CI / post-install hooks.
+ *
+ * 2. **Non-destructive** — the legacy files are never deleted. The
+ *    operator is responsible for archive / clean-up after verifying the
+ *    migration result. Legacy `engram.identity_anchor_*` MCP tools
+ *    continue to work against the on-disk anchor file.
+ *
+ * 3. **Dry-run** — when `options.dryRun` is `true` the function
+ *    computes the proposed `Peer` record and returns it alongside a
+ *    `{ dryRun: true }` marker, without writing anything to disk.
+ *
+ * 4. **Transparent** — the return value always includes the full
+ *    proposed `Peer` so callers can print what was (or would be) written.
+ *
+ * ## Source data
+ *
+ * The migrator reads two optional legacy sources:
+ *
+ * - `{memoryDir}/identity/identity-anchor.md` — structured sections
+ *   (`## Identity Traits`, `## Communication Preferences`, etc.) written
+ *   by `identityAnchorUpdate`. When present, the full content is
+ *   embedded in `peer.notes` under a clearly labelled subsection.
+ *
+ * - `{memoryDir}/IDENTITY.md` — free-form reflection entries appended
+ *   by the extraction engine. When present it is summarised as
+ *   a second subsection of `peer.notes`.
+ *
+ * If neither file exists the `self` peer is still created with sensible
+ * defaults so that the peer registry is bootstrapped for fresh installs.
+ *
+ * ## Path safety
+ *
+ * All file reads use `path.join` relative to the caller-supplied
+ * `memoryDir`. The migration never follows symlinks for the source files
+ * (uses `lstat` to detect + skip them). Writes go through the standard
+ * `writePeer` helper which enforces all path-traversal and symlink guards.
+ */
+
+import { promises as fs, constants as fsConstants } from "node:fs";
+import path from "node:path";
+
+import { readPeer, writePeer, PEERS_DIR_NAME } from "./storage.js";
+import type { Peer } from "./types.js";
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Public types
+// ──────────────────────────────────────────────────────────────────────────────
+
+/** Options for `migrateFromIdentityAnchor`. */
+export interface MigrateFromIdentityAnchorOptions {
+  /**
+   * Memory directory (same value as `config.memoryDir`). All source and
+   * destination paths are resolved relative to this root.
+   */
+  memoryDir: string;
+
+  /**
+   * When `true`, compute and return the proposed peer record without
+   * writing anything to disk. The `written` and `skipped` flags in the
+   * result will both be `false`; the `peer` field carries the proposed
+   * record.
+   */
+  dryRun?: boolean;
+
+  /**
+   * Optional override for the `self` peer's display name. Defaults to
+   * `"Self"`.
+   */
+  displayName?: string;
+
+  /**
+   * Optional ISO-8601 timestamp to use as `createdAt`. When omitted the
+   * current time is used.
+   */
+  createdAt?: string;
+}
+
+/** Result returned by `migrateFromIdentityAnchor`. */
+export interface MigrateFromIdentityAnchorResult {
+  /**
+   * The peer record that was (or would be) written for `peers/self/`.
+   * Always present regardless of `skipped` or `dryRun`.
+   */
+  peer: Peer;
+
+  /**
+   * `true` when the migration wrote `peers/self/identity.md` to disk.
+   * `false` when `dryRun` is set or the file already existed (`skipped`).
+   */
+  written: boolean;
+
+  /**
+   * `true` when an existing `peers/self/identity.md` was detected and
+   * the migration was a no-op. When `skipped` is `true`, `written` is
+   * always `false`.
+   */
+  skipped: boolean;
+
+  /**
+   * `true` when `options.dryRun` was set. No files were written; the
+   * `peer` field holds the proposed record.
+   */
+  dryRun: boolean;
+
+  /**
+   * The identity-anchor source file path that was read (if found).
+   * `null` when the file did not exist or was a symlink (skipped).
+   */
+  identityAnchorSource: string | null;
+
+  /**
+   * The `IDENTITY.md` source file path that was read (if found).
+   * `null` when the file did not exist or was a symlink (skipped).
+   */
+  identityMdSource: string | null;
+}
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Internal helpers
+// ──────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Read a legacy source file with the same security posture as the peer storage
+ * module:
+ *
+ * 1. **Parent-directory symlink rejection** — lstat the parent directory and
+ *    return null if it is a symlink. This closes the case where e.g.
+ *    `memoryDir/identity/` is itself a symlink pointing outside `memoryDir`.
+ *
+ * 2. **Kernel-level O_NOFOLLOW** — open the file with `O_NOFOLLOW` so the
+ *    kernel atomically rejects a symlink at the final path component. This
+ *    eliminates the TOCTOU race between a separate lstat check and the
+ *    subsequent read that `safeReadLegacyFile` previously had.
+ *
+ * Returns `{ content, filePath }` on success, or `{ content: null, filePath: null }`
+ * when the file is missing, is a symlink, the parent is a symlink, or the
+ * path is not a regular file. Re-throws unexpected I/O errors (EACCES, EIO,
+ * etc.) so the caller can surface real filesystem problems.
+ */
+async function safeReadLegacyFile(
+  filePath: string,
+): Promise<{ content: string; filePath: string } | { content: null; filePath: null }> {
+  // 1. Reject a symlinked parent directory.
+  const parent = path.dirname(filePath);
+  try {
+    const parentStat = await fs.lstat(parent);
+    if (parentStat.isSymbolicLink()) {
+      return { content: null, filePath: null };
+    }
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return { content: null, filePath: null };
+    }
+    throw err;
+  }
+
+  // 2. Open with O_NOFOLLOW so the kernel refuses a symlink at the target
+  //    path, atomically closing the lstat-then-read TOCTOU window.
+  let fh: import("node:fs/promises").FileHandle;
+  try {
+    fh = await fs.open(filePath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    // ENOENT — file does not exist; ELOOP — O_NOFOLLOW detected a symlink.
+    if (code === "ENOENT" || code === "ELOOP" || code === "ENOTDIR") {
+      return { content: null, filePath: null };
+    }
+    throw err;
+  }
+
+  try {
+    // Verify it is a regular file (not a directory, FIFO, device, etc.).
+    const stat = await fh.stat();
+    if (!stat.isFile()) {
+      return { content: null, filePath: null };
+    }
+    const content = await fh.readFile("utf8");
+    return { content, filePath };
+  } finally {
+    await fh.close();
+  }
+}
+
+/**
+ * Build the `notes` markdown body for the `self` peer from the two
+ * optional legacy sources.
+ *
+ * The resulting notes are kept intentionally terse — the peer kernel is
+ * meant to hold stable identity facts, not a full dump of every reflection
+ * entry. We label the sections clearly so operators know exactly what came
+ * from where.
+ */
+function buildSelfNotes(
+  anchorContent: string | null,
+  identityMdContent: string | null,
+): string | undefined {
+  const parts: string[] = [];
+
+  if (anchorContent !== null && anchorContent.trim().length > 0) {
+    parts.push(
+      "## Migrated from identity-anchor.md\n\n" + anchorContent.trim(),
+    );
+  }
+
+  if (identityMdContent !== null && identityMdContent.trim().length > 0) {
+    // IDENTITY.md can be quite long. Embed it as a labelled section so the
+    // operator can prune it manually after migration rather than losing data.
+    parts.push(
+      "## Migrated from IDENTITY.md\n\n" + identityMdContent.trim(),
+    );
+  }
+
+  if (parts.length === 0) return undefined;
+  return parts.join("\n\n");
+}
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Public API
+// ──────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Migrate legacy identity-anchor data into `peers/self/identity.md`.
+ *
+ * @see {@link MigrateFromIdentityAnchorOptions} for full option documentation.
+ * @see {@link MigrateFromIdentityAnchorResult} for return value semantics.
+ */
+export async function migrateFromIdentityAnchor(
+  options: MigrateFromIdentityAnchorOptions,
+): Promise<MigrateFromIdentityAnchorResult> {
+  const { memoryDir, dryRun = false } = options;
+  const now = new Date().toISOString();
+  const createdAt = options.createdAt ?? now;
+  const displayName = options.displayName ?? "Self";
+
+  // 1. Guard: if peers/self/identity.md already exists, skip.
+  //    We read through the standard `readPeer` helper (which enforces all
+  //    path-traversal guards) rather than stat-ing the file directly.
+  const existing = await readPeer(memoryDir, "self");
+  if (existing !== null) {
+    return {
+      peer: existing,
+      written: false,
+      skipped: true,
+      // P2 (codex): preserve the caller-requested dryRun flag even on the
+      // skip path. Previously hard-coded `false`, which contradicted the
+      // MigrateFromIdentityAnchorResult contract for dry-run callers.
+      dryRun,
+      identityAnchorSource: null,
+      identityMdSource: null,
+    };
+  }
+
+  // 2. Read legacy sources (both optional; symlinks are silently skipped).
+  const anchorPath = path.join(memoryDir, "identity", "identity-anchor.md");
+  const identityMdPath = path.join(memoryDir, "IDENTITY.md");
+
+  const [anchorResult, identityMdResult] = await Promise.all([
+    safeReadLegacyFile(anchorPath),
+    safeReadLegacyFile(identityMdPath),
+  ]);
+
+  // 3. Compose the self peer record.
+  const notes = buildSelfNotes(
+    anchorResult.content,
+    identityMdResult.content,
+  );
+
+  const peer: Peer = {
+    id: "self",
+    kind: "self",
+    displayName,
+    createdAt,
+    updatedAt: now,
+    ...(notes !== undefined ? { notes } : {}),
+  };
+
+  // 4. Write (unless dry-run).
+  //
+  // P2 (codex): the previous readPeer-then-writePeer pattern had a
+  // check-then-act TOCTOU race: two concurrent `remnic peer migrate`
+  // processes could both observe "missing" and the later write would
+  // overwrite the file, violating the non-destructive / idempotent
+  // guarantee.
+  //
+  // Mitigation: before calling writePeer we attempt to exclusively create
+  // (O_CREAT | O_EXCL | O_NOFOLLOW) the identity file path. If the open
+  // succeeds we are the sole creator; close the handle immediately and let
+  // writePeer fill in the real content (it mkdirs + rewrites atomically).
+  // If O_EXCL fails with EEXIST, another process won the race — re-read the
+  // file and return `skipped`. The window between our exclusive open and the
+  // writePeer overwrite is deliberate: we've already locked out concurrent
+  // migrate calls via EEXIST, and `peer set self` targeting the same file
+  // path will succeed (it is a legitimate overwrite, not a migrate race).
+  if (!dryRun) {
+    const selfPeerDir = path.join(memoryDir, PEERS_DIR_NAME, "self");
+    await fs.mkdir(selfPeerDir, { recursive: true });
+    const identityFilePath = path.join(selfPeerDir, "identity.md");
+    let exclusiveFh: import("node:fs/promises").FileHandle | null = null;
+    try {
+      exclusiveFh = await fs.open(
+        identityFilePath,
+        fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW,
+      );
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === "EEXIST") {
+        // Another process created the file between our readPeer check and
+        // now. Re-read and return skipped.
+        const raceWinner = await readPeer(memoryDir, "self");
+        return {
+          peer: raceWinner ?? peer,
+          written: false,
+          skipped: true,
+          dryRun,
+          identityAnchorSource: null,
+          identityMdSource: null,
+        };
+      }
+      throw err;
+    } finally {
+      if (exclusiveFh !== null) await exclusiveFh.close();
+    }
+    // We hold the exclusive-create claim. Now writePeer fills in the
+    // real content (it calls mkdirPeerDirAtomic + writeFileNoFollow).
+    await writePeer(memoryDir, peer);
+  }
+
+  return {
+    peer,
+    written: !dryRun,
+    skipped: false,
+    dryRun,
+    identityAnchorSource: anchorResult.filePath,
+    identityMdSource: identityMdResult.filePath,
+  };
+}