brainclaw 1.8.0 → 1.9.0

package/dist/core/events/checkpoint.js

@@ -0,0 +1,258 @@
+/**
+ * Journal-derived checkpoints (pln#543 Phase 3 / pln#566 Inc0, slice 1).
+ *
+ * A checkpoint is a snapshot of the materialized live-entity set at a covered
+ * `head_seq`, plus a manifest that BINDS it to the journal lineage. Per the
+ * round-3 review (NF1): checkpoints are derived from the JOURNAL ONLY — never
+ * from projection files, never consulting `projection_watermark` — so the
+ * journal is the single root of trust and there is no checkpoint↔watermark
+ * cycle. The cold-read fast path (a later slice) loads a checkpoint + replays
+ * only the sealed tail (head_seq+1..tail); on ANY trust-chain failure it falls
+ * back to projection files.
+ *
+ * This slice is strictly ADDITIVE: it builds/verifies/replays checkpoints but
+ * is not yet wired into the read path, so it carries zero read-path risk.
+ *
+ * @module
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { journalDir, readJournalRecords, journalHeadSeq } from './journal.js';
+import { applyRecordsToLive, projectLiveToState } from './materialize.js';
+import { loadConfig } from '../config.js';
+import { writeFileAtomic } from '../io.js';
+import { nowISO } from '../ids.js';
+const CHECKPOINT_SCHEMA_VERSION = 1;
+export const BASELINE_CAPABILITIES = {
+    checkpointRead: false,
+    readReconcile: false,
+    tombstoneDelete: false,
+    perEntityPatch: false,
+};
+function checkpointsDir(cwd) {
+    return path.join(journalDir(cwd), 'checkpoints');
+}
+function sha256(s) {
+    return crypto.createHash('sha256').update(s).digest('hex');
+}
+/**
+ * pln#566 F4 guard: materialize (the reducer behind both checkpoint build and
+ * checkpoint+tail replay) only consumes inline `rec.payload`; it does NOT yet
+ * dereference `payload_ref`. So if ANY journal record externalized its payload,
+ * a checkpoint built/served from the journal would silently DROP that entity.
+ * Until payload_ref dereference lands, refuse to build or serve a checkpoint
+ * when externalized payloads exist — the read path falls back to projection
+ * files, which always carry the full content. Conservative (whole-journal, not
+ * just memory-tier) and cheap.
+ */
+function journalHasExternalizedPayload(records) {
+    return records.some(r => r.payload_ref != null);
+}
+/** Identity of a record for journal-lineage binding: stable across reads, changes if the head record changes. */
+function recordIdentity(rec) {
+    return sha256(`${rec.seq}|${rec.writer}|${rec.ts}|${rec.action}|${rec.item_type}|${rec.item_id ?? ''}`);
+}
+/** The record carrying the maximum seq (the covered head). Undefined for an empty journal. */
+function headRecord(records) {
+    let head;
+    for (const rec of records) {
+        if (!head || rec.seq > head.seq)
+            head = rec;
+    }
+    return head;
+}
+function manifestPath(cwd, headSeq) {
+    return path.join(checkpointsDir(cwd), `${String(headSeq).padStart(12, '0')}.manifest.json`);
+}
+function snapshotPath(cwd, headSeq) {
+    return path.join(checkpointsDir(cwd), `${String(headSeq).padStart(12, '0')}.snapshot.json`);
+}
+/**
+ * Build a journal-derived checkpoint at the current journal head. Materializes
+ * the live entity set from the journal (NOT from projections), writes the
+ * snapshot blob, then publishes the manifest (the manifest is the commit
+ * point — an orphan snapshot without a manifest is ignored).
+ *
+ * NOTE (F6): this slice materializes from the full journal on demand. The
+ * incremental "latest verified checkpoint + sealed tail, built outside the hot
+ * lock" optimization is a later slice; on-demand build here is not on the hot
+ * write path.
+ */
+export function createCheckpoint(cwd, capabilities = BASELINE_CAPABILITIES) {
+    // Cap to the COMMITTED head (meta.next_seq-1, published after fsync). The
+    // raw segment read is lock-free, so a concurrent append may have written —
+    // but not yet fsync'd/published — records beyond the committed head; excluding
+    // them keeps the manifest bound to durable journal state (codex review MED).
+    const committedHead = journalHeadSeq(cwd);
+    const records = readJournalRecords(cwd).filter(r => r.seq <= committedHead);
+    const head = headRecord(records);
+    if (!head)
+        return { created: false, reason: 'empty journal — nothing to checkpoint' };
+    if (journalHasExternalizedPayload(records)) {
+        return { created: false, reason: 'journal has externalized payload_ref records; materialize cannot dereference them yet (pln#566 F4)' };
+    }
+    const live = applyRecordsToLive(records, new Map());
+    const entities = [...live.values()];
+    const snapshot = JSON.stringify(entities);
+    const manifest = {
+        schema_version: CHECKPOINT_SCHEMA_VERSION,
+        store_id: loadConfig(cwd).project_id ?? 'unknown',
+        head_seq: head.seq,
+        head_identity: recordIdentity(head),
+        snapshot_sha256: sha256(snapshot),
+        entity_count: entities.length,
+        capability_vector: capabilities,
+        created_at: nowISO(),
+    };
+    const dir = checkpointsDir(cwd);
+    fs.mkdirSync(dir, { recursive: true });
+    // Snapshot first, manifest last (commit point).
+    writeFileAtomic(snapshotPath(cwd, head.seq), snapshot);
+    writeFileAtomic(manifestPath(cwd, head.seq), JSON.stringify(manifest));
+    return { created: true, manifest };
+}
+/** Default: create a fresh checkpoint once the journal has grown this many
+ *  records past the last checkpoint head. Bounds the sealed tail so
+ *  checkpointRead's gap-replay stays cheap, without checkpointing every persist. */
+export const DEFAULT_CHECKPOINT_INTERVAL = 500;
+/**
+ * Create a checkpoint ONLY if the journal has grown >= interval records past
+ * the last checkpoint head (cheap head-seq check; no full scan unless building).
+ * Intended for off-hot-path callers (session-start maintenance). Journal-derived
+ * (F6). No-op when the journal is off/empty or hasn't grown enough.
+ */
+export function maybeCreateCheckpoint(cwd, interval = DEFAULT_CHECKPOINT_INTERVAL) {
+    const head = journalHeadSeq(cwd);
+    if (head === 0)
+        return { created: false, gap: 0, reason: 'journal empty/off' };
+    const last = loadLatestCheckpointManifest(cwd)?.head_seq ?? 0;
+    const gap = head - last;
+    if (gap < interval)
+        return { created: false, gap, reason: `gap ${gap} < interval ${interval}` };
+    const res = createCheckpoint(cwd);
+    return { created: res.created, gap, reason: res.reason };
+}
+/**
+ * Highest-head_seq manifest on disk, or undefined if none. Selects by NUMERIC
+ * head_seq parsed from the filename — lexicographic order on the 12-zero-pad
+ * breaks once seq exceeds 12 digits (codex review LOW).
+ */
+export function loadLatestCheckpointManifest(cwd) {
+    const dir = checkpointsDir(cwd);
+    if (!fs.existsSync(dir))
+        return undefined;
+    const files = fs.readdirSync(dir).filter(f => f.endsWith('.manifest.json'))
+        .sort((a, b) => (parseInt(b, 10) || 0) - (parseInt(a, 10) || 0)); // numeric desc on the seq prefix
+    for (const f of files) {
+        try {
+            return JSON.parse(fs.readFileSync(path.join(dir, f), 'utf-8'));
+        }
+        catch { /* skip corrupt manifest, try the next */ }
+    }
+    return undefined;
+}
+function isMaterializedEntityArray(v) {
+    return Array.isArray(v) && v.every(e => !!e && typeof e === 'object'
+        && typeof e.item_type === 'string'
+        && typeof e.item_id === 'string'
+        && !!e.payload && typeof e.payload === 'object');
+}
+/**
+ * Verify a checkpoint against an ALREADY-READ record set — never throws,
+ * validates snapshot SHAPE (not just sha256), and binds to the SAME journal
+ * view the caller will serve from (no verify/serve TOCTOU). WITHOUT reading
+ * projection files (F3). Returns the parsed entities when valid.
+ */
+function verifyCheckpointAgainstRecords(manifest, snapshotRaw, records, cwd) {
+    try {
+        if (manifest.schema_version !== CHECKPOINT_SCHEMA_VERSION) {
+            return { valid: false, reason: `unsupported checkpoint schema_version ${manifest.schema_version}` };
+        }
+        if (sha256(snapshotRaw) !== manifest.snapshot_sha256) {
+            return { valid: false, reason: 'snapshot sha256 mismatch (corrupt/tampered blob)' };
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(snapshotRaw);
+        }
+        catch {
+            return { valid: false, reason: 'snapshot is not valid JSON' };
+        }
+        if (!isMaterializedEntityArray(parsed))
+            return { valid: false, reason: 'snapshot is not a MaterializedEntity[]' };
+        const expectedStore = loadConfig(cwd).project_id ?? 'unknown';
+        if (manifest.store_id !== expectedStore) {
+            return { valid: false, reason: `store_id mismatch (manifest ${manifest.store_id} vs ${expectedStore}) — copied/wrong-branch checkpoint` };
+        }
+        const head = records.find(r => r.seq === manifest.head_seq);
+        if (!head)
+            return { valid: false, reason: `head_seq ${manifest.head_seq} not found in journal` };
+        if (recordIdentity(head) !== manifest.head_identity) {
+            return { valid: false, reason: 'head record identity mismatch — journal lineage diverged from checkpoint' };
+        }
+        return { valid: true, entities: parsed };
+    }
+    catch (err) {
+        return { valid: false, reason: `verification error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+}
+/**
+ * Public no-throw verification: snapshot integrity + shape, store binding, and
+ * journal-lineage binding. Reads the journal once. Returns valid:false on any
+ * failure (never throws) so callers fall back.
+ */
+export function verifyCheckpoint(manifest, snapshotRaw, cwd) {
+    let records;
+    try {
+        records = readJournalRecords(cwd);
+    }
+    catch (err) {
+        return { valid: false, reason: `journal read failed: ${err instanceof Error ? err.message : String(err)}` };
+    }
+    const r = verifyCheckpointAgainstRecords(manifest, snapshotRaw, records, cwd);
+    return { valid: r.valid, reason: r.reason };
+}
+/**
+ * Materialize state from the latest VERIFIED checkpoint + the sealed tail
+ * (records with seq > head_seq), using the same reducer + projector as
+ * full-journal materialization. Reads the journal exactly ONCE and uses that
+ * single view for verification, the F4 payload_ref guard, and the tail replay
+ * (no TOCTOU). Returns null on any failure (caller falls back to projections).
+ */
+export function materializeStateFromCheckpoint(cwd) {
+    const manifest = loadLatestCheckpointManifest(cwd);
+    if (!manifest)
+        return null;
+    let snapshotRaw;
+    try {
+        snapshotRaw = fs.readFileSync(snapshotPath(cwd, manifest.head_seq), 'utf-8');
+    }
+    catch {
+        return null; // orphan manifest without a readable snapshot
+    }
+    let records;
+    // Same committed-head cap as the build path: only durable records (seq <=
+    // meta.next_seq-1) drive verification, the F4 guard, and the tail replay.
+    try {
+        const committedHead = journalHeadSeq(cwd);
+        records = readJournalRecords(cwd).filter(r => r.seq <= committedHead);
+    }
+    catch {
+        return null;
+    }
+    const verdict = verifyCheckpointAgainstRecords(manifest, snapshotRaw, records, cwd);
+    if (!verdict.valid || !verdict.entities)
+        return null;
+    // F4 guard: an externalized payload (which materialize can't deref) means the
+    // checkpoint/tail would drop entities — refuse to serve, fall back.
+    if (journalHasExternalizedPayload(records))
+        return null;
+    const live = new Map();
+    for (const e of verdict.entities)
+        live.set(`${e.item_type}:${e.item_id}`, e);
+    // Replay only the sealed tail (head_seq+1..end) from the SAME record view.
+    applyRecordsToLive(records.filter(r => r.seq > manifest.head_seq), live);
+    return projectLiveToState(live);
+}
+//# sourceMappingURL=checkpoint.js.map

package/dist/core/events/genesis.js

@@ -0,0 +1,220 @@
+/**
+ * Journal genesis migration + rollback (pln#543 step 4, spec §4 phase 1).
+ *
+ * Genesis seeds the v2 journal from the current v1 projection store: one
+ * `journal_note` kind `genesis` followed by one `backfill` record per live
+ * memory entity (entity_rev 1), all under a single lock hold. It is the
+ * baseline the journal grows from and that materialize/verify check against.
+ *
+ * Discipline (matches the house upgrade rule, feedback_no_init_force):
+ *   - MANDATORY backup before writing — projections copied to a timestamped
+ *     park dir; nothing is ever deleted.
+ *   - Refuses to clobber an existing genesis unless `force` (which parks the
+ *     prior journal first — park-don't-delete).
+ *   - Rollback parks the journal directory; projections are untouched (in
+ *     dual mode they were always the source of truth), so it is a safe,
+ *     reversible "stop using the journal" operation.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { memoryDir } from '../io.js';
+import { loadState } from '../state.js';
+import { preparePersistedDocument } from '../migration.js';
+import { nowISO } from '../ids.js';
+import { logger } from '../logger.js';
+import { forceAppendJournalRecords, journalDir, readJournalRecords, resolveJournalMode, } from './journal.js';
+import { REGISTRY_FAMILIES } from './registry-post-image.js';
+import { listClaims } from '../claims.js';
+import { listAssignments } from '../assignments.js';
+import { listAgentRuns } from '../agentruns.js';
+import { listActionRequired } from '../actions.js';
+import { listCandidates } from '../candidates.js';
+import { listSequences } from '../sequence.js';
+import { listSharedJournaledRuntimeNotes } from '../runtime.js';
+const MEMORY_FAMILIES = [
+    { collection: 'active_constraints', itemType: 'constraint' },
+    { collection: 'recent_decisions', itemType: 'decision' },
+    { collection: 'known_traps', itemType: 'trap' },
+    { collection: 'open_handoffs', itemType: 'handoff' },
+    { collection: 'plan_items', itemType: 'plan' },
+];
+/** True once a `journal_note` kind `genesis` exists in the journal. */
+export function hasGenesis(cwd) {
+    return readJournalRecords(cwd).some(r => r.action === 'journal_note' && r.payload?.kind === 'genesis');
+}
+/**
+ * Copy the live projection store to a timestamped park dir. Returns the path.
+ * `ts` is injected (the codebase forbids Date.now()/new Date() in some layers;
+ * callers pass nowISO()-derived stamps) — defaults to a lexically-sortable ISO.
+ */
+export function backupStore(cwd, stamp) {
+    const base = memoryDir(cwd);
+    const memorySrc = path.join(base, 'memory');
+    const safeStamp = stamp.replace(/[:.]/g, '-');
+    const backupDir = path.join(base, 'migration-backups', `genesis-${safeStamp}`);
+    fs.mkdirSync(backupDir, { recursive: true });
+    if (fs.existsSync(memorySrc)) {
+        fs.cpSync(memorySrc, path.join(backupDir, 'memory'), { recursive: true });
+    }
+    return backupDir;
+}
+export function runGenesisMigration(options = {}) {
+    const cwd = options.cwd ?? process.cwd();
+    const state = loadState(cwd);
+    const perFamily = {};
+    const backfill = [];
+    for (const { collection, itemType } of MEMORY_FAMILIES) {
+        const items = state[collection];
+        perFamily[itemType] = items.length;
+        for (const item of items) {
+            backfill.push({
+                action: 'backfill',
+                item_type: itemType,
+                item_id: item.id,
+                agent: 'system',
+                payload: preparePersistedDocument(itemType, item),
+            });
+        }
+    }
+    const total = backfill.length;
+    if (options.dryRun) {
+        return { status: 'dry_run', backfilled: total, per_family: perFamily };
+    }
+    if (hasGenesis(cwd)) {
+        if (!options.force) {
+            return { status: 'already_present', backfilled: 0, per_family: perFamily };
+        }
+        parkJournal(cwd, nowISO());
+    }
+    // Genesis is the phase-1 (dual) seed (spec §4). Running it with the flag
+    // off lays the seed but mutations after will not dual-write — the journal
+    // then silently diverges from projections until BRAINCLAW_JOURNAL_MODE is
+    // flipped to dual. Warn so the operator flips the flag (or accepts the
+    // seed-then-flip sequence deliberately).
+    if (resolveJournalMode(cwd) === 'off') {
+        logger.warn('runGenesisMigration: BRAINCLAW_JOURNAL_MODE=off — genesis will seed the journal, but subsequent mutations will not dual-write, so the journal will diverge from projections until you set BRAINCLAW_JOURNAL_MODE=dual.');
+    }
+    const backupPath = backupStore(cwd, nowISO());
+    // genesis note first, then the backfill batch — all under one lock hold via
+    // a single forced append call (appendLocked stamps them with consecutive seqs).
+    const genesisNote = {
+        action: 'journal_note',
+        item_type: 'journal',
+        agent: 'system',
+        payload: {
+            kind: 'genesis',
+            migrated_from: 'v1',
+            backfill_count: total,
+            per_family: perFamily,
+            backup_path: backupPath,
+            at: nowISO(),
+        },
+    };
+    const written = forceAppendJournalRecords([genesisNote, ...backfill], cwd);
+    // Locate the genesis note by action+kind, not array position: appendLocked
+    // can prepend a `seq_repair` or `journal_note kind torn_tail_adjudicated`
+    // when meta is stale or the prior segment tail is torn, which would shift
+    // written[0] off the genesis note and report the wrong genesis_seq.
+    const genesisSeq = written.find(r => r.action === 'journal_note' && r.payload?.kind === 'genesis')?.seq;
+    logger.debug(`journal genesis: ${total} entities backfilled at seq ${genesisSeq}, backup ${backupPath}`);
+    return { status: 'migrated', genesis_seq: genesisSeq, backfilled: total, backup_path: backupPath, per_family: perFamily };
+}
+// ── Registry genesis supplement (pln#568 slice 3 — cutover signal O2) ──────
+/**
+ * The registry / coordination families backfilled by the registry genesis
+ * supplement, each mapped to its projection reader. Mirrors verify.ts's
+ * VERIFIED_REGISTRY_FAMILIES so a supplemented store passes `doctor
+ * --verify-journal` with zero registry drift. Runtime notes are shared-only
+ * (private/machine never enter the shared journal, pln#568).
+ */
+const REGISTRY_GENESIS_FAMILIES = [
+    { family: 'claim', list: listClaims },
+    // listActionRequired performs the server's sweep-on-read expiration, which can
+    // update dependent assignments/runs. Run it before snapshotting those families
+    // so the supplement cannot append stale assignment/run post-images after the
+    // sweep's fresh journal records. Dry-run disables the sweep to honor the
+    // no-write contract.
+    { family: 'action', list: (cwd, options) => listActionRequired(cwd, {}, { expireStale: !options.dryRun }) },
+    { family: 'assignment', list: (cwd) => listAssignments(cwd) },
+    { family: 'agent_run', list: (cwd) => listAgentRuns(cwd) },
+    { family: 'candidate', list: (cwd) => listCandidates('pending', cwd) },
+    { family: 'runtime_note', list: listSharedJournaledRuntimeNotes },
+    { family: 'sequence', list: listSequences },
+];
+/** True once a `journal_note` kind `registry_genesis` exists — the cutover
+ *  signal (O2) the observer reads to trust the journal as AUTHORITATIVE for the
+ *  registry families (drop the board_summary MCP seed). */
+export function hasRegistryGenesis(cwd) {
+    return readJournalRecords(cwd).some(r => r.action === 'journal_note' && r.payload?.kind === 'registry_genesis');
+}
+/**
+ * Backfill the registry / coordination families into the journal and emit the
+ * `registry_genesis` cutover marker (pln#568 slice 3). INCREMENTAL by design:
+ * it appends to the existing journal (preserving the memory genesis + all
+ * accumulated post-image history) rather than parking/re-seeding — a re-genesis
+ * would reset seq to 1 and break live observers' seq cursors. Idempotent: a
+ * second run no-ops once the marker is present.
+ *
+ * The marker is the safe authority signal: an observer must not switch a
+ * registry family from the MCP seed to the journal until EVERY pre-existing
+ * entity has a post-image, else it undercounts (the trp#559 badge regression).
+ * This backfill establishes that guarantee, then the marker announces it.
+ */
+export function runRegistryGenesisSupplement(options = {}) {
+    const cwd = options.cwd ?? process.cwd();
+    const perFamily = {};
+    const backfill = [];
+    for (const { family, list } of REGISTRY_GENESIS_FAMILIES) {
+        const spec = REGISTRY_FAMILIES[family];
+        const items = list(cwd, options);
+        perFamily[spec.journalItemType] = items.length;
+        for (const item of items) {
+            backfill.push({
+                action: 'backfill',
+                item_type: spec.journalItemType,
+                item_id: item.id,
+                agent: 'system',
+                payload: preparePersistedDocument(spec.docType, item),
+            });
+        }
+    }
+    const total = backfill.length;
+    if (options.dryRun) {
+        return { status: 'dry_run', backfilled: total, per_family: perFamily };
+    }
+    if (hasRegistryGenesis(cwd)) {
+        return { status: 'already_present', backfilled: 0, per_family: perFamily };
+    }
+    const marker = {
+        action: 'journal_note',
+        item_type: 'journal',
+        agent: 'system',
+        payload: { kind: 'registry_genesis', backfill_count: total, per_family: perFamily, at: nowISO() },
+    };
+    forceAppendJournalRecords([...backfill, marker], cwd);
+    logger.debug(`registry genesis: ${total} registry entities backfilled, cutover marker emitted`);
+    return { status: 'migrated', backfilled: total, per_family: perFamily };
+}
+/**
+ * Park the journal directory (events/) to a timestamped archive — the
+ * reversible "stop using the journal" operation. Projections are untouched.
+ * Returns the park path, or undefined if there was no journal.
+ */
+export function parkJournal(cwd, stamp) {
+    const dir = journalDir(cwd);
+    if (!fs.existsSync(dir))
+        return undefined;
+    const safeStamp = stamp.replace(/[:.]/g, '-');
+    const parked = path.join(memoryDir(cwd), 'migration-backups', `journal-parked-${safeStamp}`);
+    fs.mkdirSync(path.dirname(parked), { recursive: true });
+    fs.renameSync(dir, parked);
+    return parked;
+}
+export function rollbackJournal(options = {}) {
+    const cwd = options.cwd ?? process.cwd();
+    const parked = parkJournal(cwd, nowISO());
+    return parked
+        ? { status: 'rolled_back', parked_path: parked }
+        : { status: 'nothing_to_roll_back' };
+}
+//# sourceMappingURL=genesis.js.map