mover-os 4.7.8 → 4.7.9

package/src/dashboard/lib/state-core/backfill.js

@@ -0,0 +1,298 @@
+"use strict";
+
+// backfill.js — CORE-LOOP PHASE 2 (SCHEMA-SPEC v0.2 section 4, migration).
+// Turns a vault COPY's existing records into v0.2 events. Copy-first is the
+// owner-approved default #5: nothing in this module is wired to any workflow,
+// hook, or server route, and applyState (dryrun.js) refuses a non-pristine
+// target. The live vault is never touched until the owner has inspected a
+// dry-run manifest on a copy.
+//
+// WHAT v1 MIGRATES (mechanically faithful sources only):
+//   - ~/.mover/pattern-manifest.json override_history[] -> override.logged
+//   - Dailies "## Tasks" checkboxes -> plan.observed + plan.ticked
+//
+// WHAT v1 DOES NOT MIGRATE (spec amendment, recorded 2026-07-11): the v0.1
+// spec table sourced receipt.filed from "Metrics_Log receipt lines", but the
+// real Metrics_Log.md is a Fitbit vitality analysis with no receipts section
+// — its dated lines are sleep scores. Fabricating receipts from them (or
+// from session-log prose via keyword heuristics) would inflate the record
+// this system exists to keep honest. Undercount over inflate: prose receipts
+// are declared non-recoverable; receipt.filed/ship.published begin at
+// Close-time capture going forward. Every skip is returned in `skipped`
+// with a reason — absence is reported, never papered over.
+//
+// Every migrated event carries source {path, sourceHash, parserVersion,
+// locator} per v0.2 amendment 2 — a locator for audit, never an identity.
+
+const fs = require("fs");
+const path = require("path");
+const { sha256Hex } = require("./event-log");
+const { overrideLogged, planTicked, planObserved } = require("./events");
+
+// v2: literal NUL bytes dropped from identity + variable-length backtick
+// fences parsed per CommonMark (terra round 3 findings 2 and 3).
+const PARSER_VERSION = 2;
+
+// ── time ────────────────────────────────────────────────────────────────────
+
+// Day-precision source dates land at UTC noon: projections.dateKeyFromTs is
+// UTC, so noon round-trips to the same YYYY-MM-DD for any writer zone within
+// +/-11h, without pretending we know the time of day.
+function tsFromDateKey(dateKey) {
+  const t = Date.parse(`${dateKey}T12:00:00Z`);
+  return Number.isFinite(t) ? t : null;
+}
+
+// Datetime strings ("2026-04-20T11:11") parse as local time by JS spec —
+// deterministic per machine, which is all a backfill needs.
+function tsFromLoose(value) {
+  if (typeof value !== "string" || !value.trim()) return null;
+  if (/^\d{4}-\d{2}-\d{2}$/.test(value.trim())) return tsFromDateKey(value.trim());
+  const t = Date.parse(value);
+  return Number.isFinite(t) ? t : null;
+}
+
+// ── shared ──────────────────────────────────────────────────────────────────
+
+function sourceFor(relPath, fileText, locator) {
+  return {
+    path: relPath,
+    sourceHash: sha256Hex(fileText),
+    parserVersion: PARSER_VERSION,
+    locator,
+  };
+}
+
+// Code spans open with a run of N backticks and close at the next run of
+// EXACTLY N (CommonMark). The old single-backtick regex corrupted
+// ``a`b``-style spans (terra round 3 finding 3). An unclosed run stays
+// literal text.
+function extractCodeSpans(text, codeSpans) {
+  let out = "";
+  let i = 0;
+  while (i < text.length) {
+    if (text[i] !== "`") { out += text[i]; i += 1; continue; }
+    let n = 0;
+    while (text[i + n] === "`") n += 1;
+    let j = i + n;
+    let close = -1;
+    while (j < text.length) {
+      if (text[j] !== "`") { j += 1; continue; }
+      let m = 0;
+      while (text[j + m] === "`") m += 1;
+      if (m === n) { close = j; break; }
+      j += m;
+    }
+    if (close === -1) { out += text.slice(i, i + n); i += n; continue; }
+    codeSpans.push(text.slice(i + n, close));
+    out += "\u0000" + (codeSpans.length - 1) + "\u0000";
+    i = close + n;
+  }
+  return out;
+}
+
+function normalizeTask(text) {
+  // Markdown canonicalization (terra attack 7 + re-verdict #7): `Send
+  // **proposal**` and `Send proposal` are the same task; formatting must not
+  // fork identity. But a CODE SPAN is literal text - stripping its backticks
+  // AND its inner asterisks merged `Run \`a*b*\`` with `Run a*b` (distinct
+  // tasks). Code spans are held out verbatim (minus the backticks), emphasis
+  // stripping applies only OUTSIDE them; links tolerate one level of nested
+  // brackets in the label.
+  // Literal NUL bytes are dropped BEFORE the sentinel pass (terra round 3
+  // finding 2: task text containing a literal NUL-digit-NUL sequence
+  // resolved as a sentinel and collapsed to a colliding identity). NUL is
+  // not representable markdown content, so dropping it is canonicalization,
+  // not data loss.
+  const codeSpans = [];
+  let t = extractCodeSpans(text.replace(/\u0000/g, ""), codeSpans);
+  t = t
+    .replace(/\[((?:[^\[\]]|\[[^\]]*\])*)\]\([^)]*\)/g, "$1") // [text [nested]](url) -> text [nested]
+    .replace(/(\*\*|__|~~)/g, "")
+    .replace(/(^|\s)[*_](\S)/g, "$1$2") // leading emphasis marker on a word
+    .replace(/(\S)[*_](\s|$)/g, "$1$2"); // trailing emphasis marker on a word
+  t = t.replace(/\u0000(\d+)\u0000/g, (_, i) => codeSpans[Number(i)] || "");
+  return t.trim().replace(/\s+/g, " ");
+}
+
+// Stable across backfill runs AND across [x]/[ ] flips of the same task on
+// the same note date — the taskId is (date, text), never checkbox state.
+function taskIdFor(noteDate, text) {
+  return sha256Hex(`${noteDate}\n${normalizeTask(text)}`).slice(0, 16);
+}
+
+// ── override_history -> override.logged ─────────────────────────────────────
+
+// Field fallbacks mirror lib/override-summary.js normalizeOverrides — the
+// manifest has grown several field spellings over its life. A missing text
+// field becomes the composed absence "not recorded" (the constructor requires
+// non-empty strings; an honest placeholder beats dropping a real override).
+// A missing/unparseable date is different: a fabricated timestamp would
+// corrupt every date-keyed projection, so the entry is SKIPPED with a reason.
+function backfillOverrides(manifestText, { manifestPath = "pattern-manifest.json" } = {}) {
+  const events = [];
+  const skipped = [];
+
+  let manifest;
+  try {
+    manifest = JSON.parse(manifestText);
+  } catch (e) {
+    return { events, skipped: [{ locator: manifestPath, reason: `manifest is not valid JSON: ${e.message}` }] };
+  }
+
+  const raw = (manifest && (manifest.override_history || manifest.overrideHistory)) || [];
+  if (!Array.isArray(raw)) {
+    return { events, skipped: [{ locator: `${manifestPath}#override_history`, reason: "override_history is not an array" }] };
+  }
+
+  raw.forEach((o, i) => {
+    const locator = `override_history[${i}]`;
+    if (o === null || typeof o !== "object") {
+      skipped.push({ locator, reason: "entry is not an object" });
+      return;
+    }
+    const ts = tsFromLoose(o.date || o.timestamp);
+    if (ts === null) {
+      skipped.push({ locator, reason: `no parseable date (got ${JSON.stringify(o.date || o.timestamp || null)})` });
+      return;
+    }
+    const str = (v) => (typeof v === "string" && v.trim() ? v.trim() : null);
+    events.push(
+      overrideLogged(
+        {
+          what: str(o.what) || str(o.action) || str(o.task) || "not recorded",
+          over: str(o.over) || str(o.plan_item) || str(o.conflicted_with) || "not recorded",
+          reason: str(o.reason) || str(o.user_override) || str(o.justification) || "not recorded",
+          level: typeof o.level === "number" ? o.level : str(o.level) || "UNKNOWN",
+        },
+        { ts, source: sourceFor(manifestPath, manifestText, locator) }
+      )
+    );
+  });
+
+  return { events, skipped };
+}
+
+// ── Daily Note "## Tasks" -> plan.observed + plan.ticked ────────────────────
+
+const NOTE_NAME_RE = /^Daily - (\d{4}-\d{2}-\d{2})\.md$/;
+const CHECKBOX_RE = /^\s*[-*]\s*\[([ xX~])\]\s+(.+?)\s*$/;
+
+// One plan.observed (the honest denominator, v0.2 amendment 5) + one
+// plan.ticked per [x] line. [ ] contributes to the denominator only.
+// [~] means "written, not verified" in this house — it is NEVER migrated as
+// done; only /morning or the user promotes it, and backfill does not get to
+// promote what a human never verified.
+function backfillDailyTasks(noteText, { noteDate, relPath }) {
+  const events = [];
+  const skipped = [];
+  const ts = tsFromDateKey(noteDate);
+  if (ts === null) {
+    return { events, skipped: [{ locator: relPath, reason: `invalid note date ${JSON.stringify(noteDate)}` }] };
+  }
+
+  const lines = noteText.split("\n");
+  const headIdx = lines.findIndex((l) => /^#{1,4}\s*Tasks\b/i.test(l));
+  if (headIdx === -1) {
+    return { events, skipped: [{ locator: `${relPath}#tasks`, reason: "no Tasks section" }] };
+  }
+
+  const taskIds = [];
+  const texts = [];
+  const ticks = [];
+  for (let i = headIdx + 1; i < lines.length; i++) {
+    if (/^#{1,4}\s/.test(lines[i])) break; // next heading ends the section
+    const m = lines[i].match(CHECKBOX_RE);
+    if (!m) continue;
+    const text = normalizeTask(m[2]);
+    const taskId = taskIdFor(noteDate, text);
+    if (taskIds.includes(taskId)) continue; // duplicate line, one denominator slot
+    taskIds.push(taskId);
+    texts.push(text);
+    if (m[1] === "x" || m[1] === "X") ticks.push({ taskId, task: text, line: i + 1 });
+  }
+
+  if (taskIds.length === 0) {
+    return { events, skipped: [{ locator: `${relPath}#tasks`, reason: "Tasks section has no checkboxes" }] };
+  }
+
+  events.push(
+    planObserved({ noteDate, taskIds, texts }, { ts, source: sourceFor(relPath, noteText, "tasks") })
+  );
+  for (const t of ticks) {
+    events.push(
+      planTicked(
+        { taskId: t.taskId, task: t.task, note_date: noteDate, done: true },
+        { ts, source: sourceFor(relPath, noteText, `tasks:line:${t.line}`) }
+      )
+    );
+  }
+  return { events, skipped };
+}
+
+// ── whole-vault sweep ────────────────────────────────────────────────────────
+
+function collectDailyNotes(vaultRoot) {
+  const dir = path.join(vaultRoot, "02_Areas", "Engine", "Dailies");
+  const notes = [];
+  if (!fs.existsSync(dir)) return notes;
+  for (const month of fs.readdirSync(dir).sort()) {
+    const monthDir = path.join(dir, month);
+    let entries;
+    try {
+      entries = fs.readdirSync(monthDir);
+    } catch {
+      continue; // a stray file at the month level, not a folder
+    }
+    for (const name of entries.sort()) {
+      const m = name.match(NOTE_NAME_RE);
+      if (!m) continue;
+      notes.push({
+        noteDate: m[1],
+        absPath: path.join(monthDir, name),
+        relPath: path.join("02_Areas", "Engine", "Dailies", month, name),
+      });
+    }
+  }
+  return notes;
+}
+
+// The full copy-first backfill: manifest overrides + every Daily's tasks,
+// returned in a deterministic order (ts ascending, locator as tiebreak) so
+// two runs over the same copy produce the same append sequence and the same
+// dedupeKey set (ids are fresh UUIDs by design — identity is opaque).
+function backfillVault(vaultRoot, { manifestPath } = {}) {
+  const events = [];
+  const skipped = [];
+
+  if (manifestPath && fs.existsSync(manifestPath)) {
+    const text = fs.readFileSync(manifestPath, "utf8");
+    const r = backfillOverrides(text, { manifestPath: path.basename(manifestPath) });
+    events.push(...r.events);
+    skipped.push(...r.skipped);
+  } else if (manifestPath) {
+    skipped.push({ locator: manifestPath, reason: "manifest file not found" });
+  }
+
+  for (const note of collectDailyNotes(vaultRoot)) {
+    const text = fs.readFileSync(note.absPath, "utf8");
+    const r = backfillDailyTasks(text, { noteDate: note.noteDate, relPath: note.relPath });
+    events.push(...r.events);
+    skipped.push(...r.skipped);
+  }
+
+  events.sort((a, b) => a.ts - b.ts || String(a.source && a.source.locator).localeCompare(String(b.source && b.source.locator)) || a.type.localeCompare(b.type));
+  return { events, skipped };
+}
+
+module.exports = {
+  PARSER_VERSION,
+  tsFromDateKey,
+  tsFromLoose,
+  taskIdFor,
+  normalizeTask,
+  backfillOverrides,
+  backfillDailyTasks,
+  collectDailyNotes,
+  backfillVault,
+};

package/src/dashboard/lib/state-core/dryrun.js

@@ -0,0 +1,188 @@
+"use strict";
+
+// dryrun.js — CORE-LOOP PHASE 2 (SCHEMA-SPEC v0.2 section 4, steps 2-3).
+// The migration contract the owner inspects BEFORE anything touches a live
+// vault: predictState computes the exact bytes a migration would write
+// (events.jsonl + the four projection files) WITHOUT writing; applyState
+// writes them to a pristine copy, re-reads every file from disk, and throws
+// unless predicted and actual sha256 match byte-for-byte. The manifest is
+// the lab's dry-run.js pattern promoted to the product path.
+//
+// Determinism: the caller captures ONE `now` and passes it to both predict
+// and apply (rebuiltAt is operational metadata outside the hashed projection
+// body, but it is inside the FILE bytes, so byte-identity needs the same
+// clock reading). Event ids are fresh UUIDs per backfill run, so the
+// manifest binds one run's events to one run's files — rerunning backfill
+// produces the same dedupeKeys and facts, not the same ids.
+
+const fs = require("fs");
+const path = require("path");
+const { serializeEnvelope, sha256Hex, stableStringify, isValidEnvelope, acquireLock } = require("./event-log");
+const { resolveEffectiveEvents, computeMetrics, computeStreak, computeBet, computeDrift, rebuildProjections, PROJECTION_VERSION } = require("./projections");
+
+const STATE_DIR = path.join("02_Areas", "Engine", "State");
+const LOG_NAME = "events.jsonl";
+const PROJECTION_NAMES = {
+  metrics: "projection-metrics.json",
+  streak: "projection-streak.json",
+  bet: "projection-bet.json",
+  drift: "projection-drift.json",
+};
+
+// One projection file's content. The hashed body carries the deterministic
+// watermark (v0.2 amendment 3: throughEventId/throughEventTs inside, never
+// rebuiltAt); `hash` must reproduce across rebuilds of the same log.
+function projectionFileBody(kind, data, watermark, eventCount) {
+  return {
+    v: PROJECTION_VERSION,
+    kind,
+    throughEventId: watermark.id,
+    throughEventTs: watermark.ts,
+    eventCount,
+    data,
+  };
+}
+
+function projectionFileText(kind, data, watermark, eventCount, rebuiltAt) {
+  const body = projectionFileBody(kind, data, watermark, eventCount);
+  const hash = sha256Hex(stableStringify(body));
+  // rebuiltAt and hash sit OUTSIDE the hashed body; stableStringify keeps
+  // the bytes reproducible for identical inputs.
+  return stableStringify({ rebuiltAt, hash, body }) + "\n";
+}
+
+// Predicts every byte a migration write would produce. Pure of the target
+// filesystem: takes the events, returns { files: [{file, bytes, sha256}] }.
+function predictState(events, { now } = {}) {
+  if (!Array.isArray(events) || !events.every((e) => isValidEnvelope(e))) {
+    throw new Error("dryrun.predictState: events must all be valid v0.2 envelopes");
+  }
+  if (typeof now !== "number") {
+    throw new Error("dryrun.predictState: opts.now (epoch ms) is required: predict and apply must share one clock reading for byte identity");
+  }
+
+  const logText = events.map((e) => serializeEnvelope(e) + "\n").join("");
+
+  const effective = resolveEffectiveEvents(events);
+  const last = events.length ? events[events.length - 1] : null;
+  const watermark = { id: last ? last.id : null, ts: last ? last.ts : null };
+  const eventCount = effective.length;
+
+  const data = {
+    metrics: computeMetrics(effective),
+    streak: computeStreak(effective),
+    bet: computeBet(effective),
+    drift: computeDrift(effective, {}),
+  };
+
+  const files = [{ file: path.join(STATE_DIR, LOG_NAME), text: logText }];
+  for (const [kind, name] of Object.entries(PROJECTION_NAMES)) {
+    files.push({ file: path.join(STATE_DIR, name), text: projectionFileText(kind, data[kind], watermark, eventCount, now) });
+  }
+
+  return {
+    files: files.map((f) => ({ file: f.file, bytes: Buffer.byteLength(f.text, "utf8"), sha256: sha256Hex(f.text), text: f.text })),
+  };
+}
+
+// Writes the predicted state onto a vault COPY and verifies byte identity by
+// re-reading RAW BUFFERS from disk (a utf8-decode round trip can mask an
+// invalid byte, terra attack 3). Refuses a non-pristine target; the whole
+// check-write-verify sequence runs under the log's own advisory lock so two
+// concurrent migrations cannot both pass the pristine check (terra attack 4),
+// and an `.applying` marker makes a crash mid-write recoverable: a rerun
+// that finds the marker knows the previous apply never completed and clears
+// the partial State files instead of refusing forever.
+//
+// SCOPE (terra re-verdict #3, acknowledged not patched): verification is
+// point-in-time. The lock binds every COOPERATIVE state-core writer; a
+// hostile direct filesystem writer can always mutate a file between the last
+// check and the caller's next read - no userspace check-then-return closes
+// that. The durable integrity token is IN each file: every projection
+// carries its own body hash, and the log's lines are self-validating
+// envelopes, so a READER can re-verify at read time. verified:true means
+// "the bytes I wrote were the bytes on disk when I looked", nothing more.
+function applyState(vaultRoot, events, { now } = {}) {
+  const predicted = predictState(events, { now });
+  const logAbs = path.join(vaultRoot, STATE_DIR, LOG_NAME);
+  const applyingMarker = path.join(vaultRoot, STATE_DIR, ".applying");
+
+  fs.mkdirSync(path.dirname(logAbs), { recursive: true });
+  const lock = acquireLock(logAbs);
+  try {
+    if (fs.existsSync(applyingMarker)) {
+      // A previous apply crashed mid-write: everything under State/ from
+      // that run is unpublished garbage. Clear exactly the files we own.
+      for (const f of predicted.files) {
+        try { fs.rmSync(path.join(vaultRoot, f.file), { force: true }); } catch (_) {}
+      }
+      fs.rmSync(applyingMarker, { force: true });
+    }
+    if (fs.existsSync(logAbs)) {
+      throw new Error(`dryrun.applyState: ${logAbs} already exists. applyState only writes to a pristine copy, never over an existing log`);
+    }
+
+    fs.writeFileSync(applyingMarker, JSON.stringify({ pid: process.pid, startedAt: now }), "utf8");
+    for (const f of predicted.files) {
+      fs.writeFileSync(path.join(vaultRoot, f.file), f.text, "utf8");
+    }
+
+    const mismatches = [];
+    const manifest = predicted.files.map((f) => {
+      const actualBuf = fs.readFileSync(path.join(vaultRoot, f.file)); // raw Buffer, no decode
+      const actual = cryptoSha(actualBuf);
+      if (actual !== f.sha256 || actualBuf.length !== f.bytes) {
+        mismatches.push({ file: f.file, predicted: f.sha256, actual, predictedBytes: f.bytes, actualBytes: actualBuf.length });
+      }
+      return { file: f.file, bytes: f.bytes, sha256: f.sha256, actual };
+    });
+
+    if (mismatches.length) {
+      throw new Error(`dryrun.applyState: predicted vs actual bytes diverge for ${mismatches.map((m) => m.file).join(", ")}. Do not proceed to a live vault`);
+    }
+
+    // Cross-check the file assembly against the product reducer itself: the
+    // rebuilt projection HASH (the full reduced body, not just the watermark
+    // id — terra attack 3b showed a mid-window event mutation slipping past
+    // an id-only check) must be reproducible from the bytes on disk.
+    const rebuilt = rebuildProjections(logAbs, { now: () => now });
+    const last = events.length ? events[events.length - 1] : null;
+    if ((rebuilt.body.throughEventId || null) !== (last ? last.id : null)) {
+      throw new Error("dryrun.applyState: rebuildProjections watermark disagrees with the written log: dryrun assembly has drifted from the reducer");
+    }
+    const effective = resolveEffectiveEvents(events);
+    const expectedBody = {
+      v: PROJECTION_VERSION,
+      throughEventId: last ? last.id : null,
+      throughEventTs: last ? last.ts : null,
+      eventCount: effective.length,
+      metrics: computeMetrics(effective),
+      streak: computeStreak(effective),
+      bet: computeBet(effective),
+      drift: computeDrift(effective, {}),
+    };
+    if (rebuilt.hash !== sha256Hex(stableStringify(expectedBody))) {
+      throw new Error("dryrun.applyState: the reducer's rebuilt hash does not match the predicted reduction. Do not proceed to a live vault");
+    }
+
+    fs.rmSync(applyingMarker, { force: true });
+    return { manifest: manifest.map(({ file, bytes, sha256 }) => ({ file, bytes, sha256 })), verified: true, rebuiltHash: rebuilt.hash };
+  } finally {
+    lock.release();
+  }
+}
+
+// Raw-buffer sha256 (predictState hashes the utf8 TEXT, which encodes to the
+// same bytes it writes; verification must hash what is actually on disk).
+function cryptoSha(buf) {
+  return require("crypto").createHash("sha256").update(buf).digest("hex");
+}
+
+module.exports = {
+  STATE_DIR,
+  LOG_NAME,
+  PROJECTION_NAMES,
+  projectionFileText,
+  predictState,
+  applyState,
+};