hypomnema 1.7.0 → 1.7.1

package/scripts/rename.mjs

@@ -39,18 +39,20 @@ import {
   existsSync,
   readFileSync,
   writeFileSync,
-  rmSync,
   mkdirSync,
   readdirSync,
   renameSync,
   statSync,
   lstatSync,
   realpathSync,
+  chmodSync,
+  rmSync,
 } from 'fs';
 import { join, basename, dirname, normalize, isAbsolute, sep } from 'path';
 import { resolveHypoRoot, expandHome } from './lib/hypo-root.mjs';
 import { loadHypoIgnore } from './lib/hypo-ignore.mjs';
 import { collectPagesRename, slugForms } from './lib/wikilink.mjs';
+import { RENAME_MARKER_REL, renameMarkerPath, readRenameMarker } from './lib/rename-marker.mjs';
 
 // ── arg parsing ───────────────────────────────────────────────────────────────
 
@@ -227,6 +229,141 @@ function fail(args, msg) {
   process.exit(1);
 }
 
+/** Atomic write via tmp+rename (same pattern as crystallize.mjs/proposal.mjs's
+ * atomicWrite): a crash or full disk mid-write leaves the ORIGINAL bytes at
+ * `path` untouched, never a truncated/partial file. Needed for every rewrite
+ * this script lands on a page that already has content on disk — including a
+ * page's own self-referential rewrite — because a plain writeFileSync
+ * truncates before writing and a crash in that gap destroys the one copy. */
+function atomicWrite(path, content) {
+  const tmp = `${path}.${process.pid}.${Math.random().toString(36).slice(2, 10)}.tmp`;
+  // Swapping in a fresh inode also swaps in fresh permissions, so a 0444 or 0600
+  // page would come back 0644 and this rewrite would quietly widen it. Carry the
+  // existing mode over. Ownership, ACLs and hard-link identity are NOT carried;
+  // a vault page with any of those is outside what this script claims to handle.
+  let mode = null;
+  try {
+    mode = statSync(path).mode;
+  } catch {
+    mode = null; // new file, nothing to preserve
+  }
+  // `wx` so a collision is an error rather than a silent clobber of somebody
+  // else's temp, and drop the temp if anything after it fails (a full disk,
+  // a rename that cannot land) instead of leaving `.tmp` litter in the vault.
+  try {
+    writeFileSync(tmp, content, { flag: 'wx' });
+    if (mode !== null) chmodSync(tmp, mode);
+    renameSync(tmp, path);
+  } catch (err) {
+    try {
+      rmSync(tmp, { force: true });
+    } catch {
+      // best effort; the original at `path` is untouched either way
+    }
+    throw err;
+  }
+}
+
+// ── crash-recovery marker ────────────────────────────────────────────────────
+// Written as the FIRST disk write of an --apply run (right after the last guard
+// passes, before any inbound-link rewrite lands) and cleared as the LAST action
+// (right after the terminal renameSync). If the process dies in between, this
+// is the only trace left that the rename is incomplete: --from still resolves
+// at that point (the move itself hasn't happened yet — see the module
+// docstring's move-last invariant), so a re-run of the identical command
+// converges on its own. Nothing else in the vault says "you need to re-run it".
+//
+// Fail-closed by construction: writeRenameMarker calls fail() (process.exit(1))
+// on any write error, so --apply never proceeds without a marker on disk. A
+// detector with no reliable marker is worse than no detector at all.
+function writeRenameMarker(args, marker) {
+  const path = join(args.hypoDir, RENAME_MARKER_REL);
+  try {
+    mkdirSync(dirname(path), { recursive: true });
+    atomicWrite(path, JSON.stringify(marker, null, 2));
+  } catch (err) {
+    fail(
+      args,
+      `could not write the rename-in-progress marker (${err.message}) — refusing --apply without it: it is the only signal a crash mid-rewrite would leave behind.`,
+    );
+  }
+}
+
+// codex BLOCKER: writeRenameMarker above used to atomicWrite unconditionally,
+// so a SECOND rename whose own --apply started after a FIRST one crashed mid-run
+// would silently clobber the first rename's marker — permanently erasing the
+// only trace that the first rename's move/rewrite is incomplete. Called right
+// before writeRenameMarker in both modes, so the check runs before this run's
+// marker (or anything else) touches disk.
+//
+// Three outcomes:
+//   - no marker on disk                → nothing to conflict with, proceed.
+//   - marker names THIS exact command  → this is the legitimate re-run path the
+//     whole mechanism exists to let converge (identical mode/from/to). Proceed;
+//     writeRenameMarker below simply overwrites it with a fresh started_at —
+//     mode/from/to don't change, only the timestamp does.
+//   - marker names a DIFFERENT command, or can't be parsed at all → refuse. A
+//     marker we can't parse might belong to this command or a different one;
+//     "can't tell" must resolve to refuse, not to a silent overwrite.
+function guardExistingMarker(args, thisMarker) {
+  const path = renameMarkerPath(args.hypoDir);
+  if (!existsSync(path)) return;
+  const marker = readRenameMarker(args.hypoDir);
+  if (!marker || !marker.from || !marker.to) {
+    fail(
+      args,
+      `${path} exists but could not be parsed (or is missing from/to), so it cannot be told apart from this rename's own marker. Inspect it by hand, finish or discard the rename it describes, then delete the marker before retrying.`,
+    );
+  }
+  const same =
+    marker.mode === thisMarker.mode && marker.from === thisMarker.from && marker.to === thisMarker.to;
+  if (!same) {
+    fail(
+      args,
+      `a rename is already in progress (${marker.mode === 'directory' ? 'directory' : 'page'} '${marker.from}' → '${marker.to}', marker at ${path}) — finish that rename first (re-run the identical command), or confirm it is safe and delete the marker, before starting a different one.`,
+    );
+  }
+}
+
+// A failure here is not fatal — the move already succeeded — but a marker left
+// behind would make doctor misreport a finished rename as incomplete, so it is
+// surfaced (stderr) rather than swallowed.
+function clearRenameMarker(args) {
+  const path = join(args.hypoDir, RENAME_MARKER_REL);
+  try {
+    rmSync(path, { force: true });
+  } catch (err) {
+    console.error(
+      `⚠ rename completed, but could not remove the in-progress marker (${err.message}) — remove ${path} by hand.`,
+    );
+  }
+}
+
+// Refuse an --apply that would have to cross a filesystem/mount boundary,
+// BEFORE any rewrite is written to disk. renameSync cannot move across
+// devices (throws EXDEV); falling back to write-then-remove in that case
+// reintroduces the exact non-convergent "both paths exist, --to-already-exists
+// forever" crash window a single renameSync exists to close. Call this first,
+// so a device mismatch is refused with the vault untouched — not discovered
+// mid-move with inbound links already rewritten.
+function assertSameDevice(args, fromAbsPath, toAbsPath) {
+  let ancestor = dirname(toAbsPath);
+  while (!existsSync(ancestor)) ancestor = dirname(ancestor);
+  let fromDev, toDev;
+  try {
+    fromDev = statSync(fromAbsPath).dev;
+    toDev = statSync(ancestor).dev;
+  } catch {
+    return; // can't tell in advance — let the real move surface the real error
+  }
+  if (fromDev !== toDev) {
+    fail(
+      args,
+      `--from and --to are on different filesystems/mounts — an atomic move is not possible across that boundary. Move it by hand instead of --apply.`,
+    );
+  }
+}
+
 // ── directory mode ─────────────────────────────────────────────────────────────
 // A directory rename relocates a whole subtree (`projects/old/**` → `projects/new/**`)
 // and rewrites inbound links across the vault. Key facts that shape the algorithm:
@@ -545,7 +682,7 @@ function runDirectory(args, fromDirRel, ignorePatterns) {
 
   // Rewrite inbound references across the vault.
   const externalWrites = new Map(); // abs path → content (non-moved source files)
-  const movedBodies = new Map(); // new rel → content (moved page bodies, written post-move)
+  const movedBodies = new Map(); // OLD rel → content (moved-page bodies, written pre-rename)
   const fileResults = [];
   const ambiguities = [];
   let totalRewrites = 0;
@@ -577,22 +714,55 @@ function runDirectory(args, fromDirRel, ignorePatterns) {
       const landRel = inSubtree ? movedByRel.get(p.rel).toPage.rel : p.rel;
       fileResults.push({ file: landRel, rewrites });
       totalRewrites += rewrites.length;
-      if (inSubtree) movedBodies.set(movedByRel.get(p.rel).toPage.rel, content);
+      // Keyed by the OLD rel: this gets written at the OLD path, before the
+      // subtree rename, so the content travels with the directory move instead
+      // of racing it (see the apply block below).
+      if (inSubtree) movedBodies.set(p.rel, content);
       else externalWrites.set(p.path, content);
     }
     if (ambiguous.length > 0) ambiguities.push({ file: p.rel, ambiguous });
   }
 
-  // Apply: external rewrites in place → renameSync the whole subtree (carries
-  // non-.md assets) → write rewritten moved bodies at their new paths.
+  // Apply: refuse a cross-device move up front (assertSameDevice) → external
+  // rewrites in place → moved-page bodies rewritten at their OLD paths → THEN
+  // renameSync the whole subtree in one syscall (carries non-.md assets and the
+  // just-rewritten bodies along with it).
+  //
+  // Writing moved bodies before the rename (not after, as this used to) matters
+  // for convergence: a crash after renameSync but before those writes used to
+  // leave `projects/new/*` links still pointing at `projects/old/*` forever —
+  // --from no longer resolves (the directory already moved), so a re-run failed
+  // with "did not resolve to a unique existing page" instead of finishing the
+  // job. Writing pre-rename means the content moves atomically with the
+  // directory: there is no window where the subtree has moved but its own
+  // internal links have not been rewritten yet.
   let moved = false;
   if (args.apply) {
-    for (const [path, content] of externalWrites) writeFileSync(path, content);
+    assertSameDevice(args, fromAbs, toAbs);
+    const thisMarker = {
+      mode: 'directory',
+      from: fromDirRel,
+      to: toDirRel,
+      started_at: new Date().toISOString(),
+    };
+    guardExistingMarker(args, thisMarker);
+    writeRenameMarker(args, thisMarker);
+    for (const [path, content] of externalWrites) atomicWrite(path, content);
+    for (const [oldRel, content] of movedBodies) {
+      atomicWrite(join(args.hypoDir, oldRel), content);
+    }
     mkdirSync(dirname(toAbs), { recursive: true });
-    renameSync(fromAbs, toAbs);
-    for (const [newRel, content] of movedBodies) {
-      writeFileSync(join(args.hypoDir, newRel), content);
+    try {
+      renameSync(fromAbs, toAbs);
+    } catch (err) {
+      // assertSameDevice above should already have refused this. A device
+      // change mid-run is the only way to reach EXDEV here — fail loudly
+      // instead of an unsafe write-then-remove fallback; every write above is
+      // already atomic, so a re-run picks up cleanly.
+      if (err.code !== 'EXDEV') throw err;
+      fail(args, `--from and --to ended up on different filesystems mid-run — refusing an unsafe fallback move.`);
     }
+    clearRenameMarker(args);
     moved = true;
   }
 
@@ -730,6 +900,22 @@ function run(args) {
     }
   }
 
+  // Refuse a cross-device --apply before anything below writes a single byte
+  // (see assertSameDevice) — the external rewrite loop right after this can
+  // otherwise land partial vault changes ahead of a move that turns out to be
+  // impossible.
+  if (args.apply) {
+    assertSameDevice(args, fromPage.path, toPath);
+    const thisMarker = {
+      mode: 'page',
+      from: fromPage.rel,
+      to: toRel,
+      started_at: new Date().toISOString(),
+    };
+    guardExistingMarker(args, thisMarker);
+    writeRenameMarker(args, thisMarker);
+  }
+
   // Rewrite inbound references across every NON-preserved page (skip the moved
   // page itself — self-references are rewritten on its own content separately).
   const fileResults = [];
@@ -748,12 +934,12 @@ function run(args) {
       fileResults.push({ file: p.rel, rewrites });
       totalRewrites += rewrites.length;
       if (args.apply && content !== raw) {
-        // The from-page is about to move; write its rewritten body to the NEW
-        // path below, not the old one.
+        // The from-page is about to move; stash its own rewritten body — it is
+        // written to the OLD path below, right before the rename, not here.
         if (p.rel === fromPage.rel) {
           fromPage._rewritten = content;
         } else {
-          writeFileSync(p.path, content);
+          atomicWrite(p.path, content);
         }
       } else if (p.rel === fromPage.rel) {
         fromPage._rewritten = content;
@@ -764,15 +950,34 @@ function run(args) {
     }
   }
 
-  // Move the file (--apply only): write the (possibly self-rewritten) body at the
-  // new path, then drop the old one. Done as write-then-remove rather than a raw
-  // rename so the carried-over self-reference rewrites are preserved.
+  // Move the file (--apply only): if the from-page carries a self-rewritten body
+  // (its own [[foo]]-style links needed retargeting), write it to the OLD path
+  // first (atomically — a crash mid-write must never corrupt the one copy),
+  // then renameSync the file into place. A rename is one syscall with no
+  // observable intermediate state, so a crash either lands before it (old path
+  // still has the rewritten body, re-run converges) or after it (new path exists,
+  // nothing to redo) — never both paths present at once, which is the state a
+  // prior write-then-remove could leave and that the --to-already-exists guard
+  // above then refuses to recover from.
   let moved = false;
   if (args.apply) {
+    if (fromPage._rewritten !== undefined) atomicWrite(fromPage.path, fromPage._rewritten);
     mkdirSync(dirname(toPath), { recursive: true });
-    const body = fromPage._rewritten ?? readFileSync(fromPage.path, 'utf-8');
-    writeFileSync(toPath, body);
-    if (toPath !== fromPage.path) rmSync(fromPage.path, { force: true });
+    if (toPath !== fromPage.path) {
+      try {
+        renameSync(fromPage.path, toPath);
+      } catch (err) {
+        // assertSameDevice above should already have refused a cross-device
+        // move before any write happened. Reaching EXDEV here means the mount
+        // changed mid-run — fail loudly rather than silently falling back to
+        // write-then-remove, which is exactly the non-atomic state this fix
+        // removes. fromPage.path still holds the valid (possibly rewritten)
+        // body, so a re-run picks up cleanly once the device issue is gone.
+        if (err.code !== 'EXDEV') throw err;
+        fail(args, `--from and --to ended up on different filesystems mid-run — refusing an unsafe fallback move.`);
+      }
+    }
+    clearRenameMarker(args);
     moved = true;
   }
 

package/scripts/stats.mjs

@@ -98,7 +98,12 @@ function getLastActivity(hypoDir) {
 const args = parseArgs(process.argv);
 // Only validate the auto-resolved path (env/marker/default). An explicit
 // --hypo-dir=<path> (tests, other tooling) is trusted as-is, valid or not.
-if (args.hypoDirSource) checkVaultOrExit(args.hypoDir, args.hypoDirSource);
+// See lint.mjs's matching comment: source 'default'/stale-'marker' stays
+// exit-0 (CI-safe), but the all-zero counts below must not be printed as if a
+// real (empty) vault had been scanned.
+const vaultMissing = args.hypoDirSource
+  ? checkVaultOrExit(args.hypoDir, args.hypoDirSource)
+  : false;
 
 const ignorePatterns = loadHypoIgnore(args.hypoDir);
 const pageFiles = collectMdFiles(join(args.hypoDir, 'pages'), [], args.hypoDir, ignorePatterns);
@@ -165,7 +170,14 @@ const stats = {
 };
 
 if (args.json) {
-  console.log(JSON.stringify(stats, null, 2));
+  // vaultFound lets a machine consumer of `--json` tell "scanned, genuinely
+  // empty" apart from "nothing was scanned" without needing stderr — the
+  // same reasoning as lint.mjs's matching field.
+  console.log(JSON.stringify({ ...stats, vaultFound: !vaultMissing }, null, 2));
+} else if (vaultMissing) {
+  // checkVaultOrExit already printed the "No Hypomnema vault found" notice on
+  // stderr. An all-zero stats block here would read as "scanned an empty
+  // vault", which never happened.
 } else {
   console.log(`Pages:    ${pageFiles.length} total`);
   const typeEntries = Object.entries(typeCounts).sort((a, b) => b[1] - a[1]);

package/templates/gitignore

@@ -8,3 +8,7 @@
 # writes (crystallize apply, deriveRootLogEntries). Normally deleted immediately;
 # a crashed holder can leave a stale one, which must never be committed.
 *.lock
+# The lock is published by linking a staged sibling into place. `*.lock` does not
+# match the staging name, and a crash between the link and its cleanup leaves one
+# behind — where `git add -A` would happily pick it up.
+*.lock.*.tmp

package/templates/hypo-config.md

@@ -1,7 +1,7 @@
 ---
 title: Hypomnema Config
 type: config
-version: "1.7.0"
+version: "1.7.1"
 created: YYYY-MM-DD
 ---