brainclaw 1.7.5 → 1.9.0

package/dist/core/event-log.js

@@ -3,15 +3,18 @@ import path from 'node:path';
 import { memoryDir } from './io.js';
 import { nowISO } from './ids.js';
 import { logger } from './logger.js';
+import { isObserverMode } from './observer-mode.js';
+import { appendJournalRecords, resolveJournalMode } from './events/journal.js';
+import { REGISTRY_POST_IMAGE_ITEM_TYPES } from './events/registry-post-image.js';
 const EVENT_LOG_FILE = 'events.jsonl';
 const CURSORS_DIR = '.cursors';
-// --- Writer ---
-export function appendEvent(event, cwd) {
+export function appendEvent(event, cwd, options = {}) {
     try {
         const full = {
             ts: event.ts ?? nowISO(),
             agent: event.agent ?? 'unknown',
             agent_id: event.agent_id,
+            session_id: event.session_id ?? (process.env.BRAINCLAW_SESSION_ID?.trim() || undefined),
             user: event.user ?? process.env.USER ?? process.env.USERNAME,
             action: event.action,
             item_type: event.item_type,
@@ -25,6 +28,56 @@ export function appendEvent(event, cwd) {
     catch (err) {
         logger.debug('Failed to write event log entry:', err);
     }
+    if (options.journalDualWrite !== false) {
+        dualWriteToJournal(event, cwd);
+    }
+}
+/**
+ * v2 journal dual-write (pln#543 step 2). Mirrors every v1 emission into
+ * the segmented journal when BRAINCLAW_JOURNAL_MODE=dual; a no-op when off.
+ * Mapping per spec §2.1.1: the coarse `update/upgrade/rollback : state`
+ * store event becomes a `journal_note` kind `store_marker` (the per-entity
+ * events it stands for arrive with step 3 dirty-tracking).
+ */
+function dualWriteToJournal(event, cwd) {
+    try {
+        if (resolveJournalMode(cwd) === 'off')
+            return;
+        const base = {
+            agent: event.agent,
+            agent_id: event.agent_id,
+            session_id: event.session_id,
+            user: event.user,
+            summary: event.summary,
+            ts: event.ts,
+        };
+        if (event.item_type === 'state') {
+            appendJournalRecords([{
+                    ...base,
+                    action: 'journal_note',
+                    item_type: 'journal',
+                    payload: { kind: 'store_marker', op: event.action, detail: event.summary },
+                }], cwd);
+            return;
+        }
+        // pln#568 — registry/coordination families now journal full entity-state
+        // post-images on their persist chokepoint (emitRegistryPostImage). Their
+        // legacy envelope-only dual-write here would be redundant noise (a
+        // registry-lifecycle record materialize ignores), so suppress it: in the v2
+        // journal these families appear ONLY as post-images. events.jsonl (above)
+        // still records the v1 lifecycle event for existing consumers.
+        if (REGISTRY_POST_IMAGE_ITEM_TYPES.has(event.item_type))
+            return;
+        appendJournalRecords([{
+                ...base,
+                action: event.action,
+                item_type: event.item_type,
+                item_id: event.item_id,
+            }], cwd);
+    }
+    catch (err) {
+        logger.debug('journal dual-write skipped:', err);
+    }
 }
 // --- Reader ---
 export function readAllEvents(cwd) {
@@ -43,39 +96,96 @@ export function readAllEvents(cwd) {
     }
     return events;
 }
+function normalizeReader(reader) {
+    return typeof reader === 'string' ? { agent: reader } : reader;
+}
 function cursorsDir(cwd) {
     return path.join(memoryDir(cwd), CURSORS_DIR);
 }
-function cursorPath(agent, cwd) {
-    return path.join(cursorsDir(cwd), `${agent}.json`);
+/** Cursor files are keyed by session_id when present, else by agent name. */
+function cursorKey(reader) {
+    return reader.session_id?.trim() || reader.agent;
 }
-function loadCursor(agent, cwd) {
-    const fp = cursorPath(agent, cwd);
-    if (!fs.existsSync(fp))
-        return { offset: 0, last_read: '' };
-    try {
-        return JSON.parse(fs.readFileSync(fp, 'utf-8'));
+function cursorPath(key, cwd) {
+    return path.join(cursorsDir(cwd), `${key}.json`);
+}
+function loadCursor(reader, cwd) {
+    const fp = cursorPath(cursorKey(reader), cwd);
+    if (fs.existsSync(fp)) {
+        try {
+            return JSON.parse(fs.readFileSync(fp, 'utf-8'));
+        }
+        catch {
+            return { offset: 0, last_read: '' };
+        }
     }
-    catch {
-        return { offset: 0, last_read: '' };
+    // name→instance migration: a session-keyed cursor that does not exist yet
+    // seeds from the legacy name-keyed cursor, so an upgraded instance does not
+    // replay the whole log. Cursors are caches — worst case is a re-read.
+    if (reader.session_id?.trim()) {
+        const legacy = cursorPath(reader.agent, cwd);
+        if (fs.existsSync(legacy)) {
+            try {
+                return JSON.parse(fs.readFileSync(legacy, 'utf-8'));
+            }
+            catch { /* fall through to fresh cursor */ }
+        }
     }
+    return { offset: 0, last_read: '' };
 }
-function saveCursor(agent, cursor, cwd) {
+function saveCursor(reader, cursor, cwd) {
     const dir = cursorsDir(cwd);
     if (!fs.existsSync(dir)) {
         fs.mkdirSync(dir, { recursive: true });
     }
-    fs.writeFileSync(cursorPath(agent, cwd), JSON.stringify(cursor), 'utf-8');
+    fs.writeFileSync(cursorPath(cursorKey(reader), cwd), JSON.stringify(cursor), 'utf-8');
+}
+/**
+ * True when this reader already has a cursor — session-keyed, or the legacy
+ * name-keyed cursor a session reader would migrate from. Absence means first
+ * contact: the reader has never consumed this store's event log.
+ */
+export function hasEventCursor(reader, cwd) {
+    const effectiveReader = normalizeReader(reader);
+    if (fs.existsSync(cursorPath(cursorKey(effectiveReader), cwd)))
+        return true;
+    if (effectiveReader.session_id?.trim()) {
+        return fs.existsSync(cursorPath(effectiveReader.agent, cwd));
+    }
+    return false;
+}
+/**
+ * Seed the reader's cursor at the current end of the event log WITHOUT
+ * reading it. First-contact path: a fresh agent on a mature store must not
+ * replay the whole history (its one chance to triage would be consumed by
+ * noise) — the arrival digest informs instead, and the diff becomes
+ * incremental from here. Returns the byte offset that was sealed.
+ */
+export function seedCursorToEnd(reader, cwd) {
+    const effectiveReader = normalizeReader(reader);
+    const logPath = eventLogPath(cwd);
+    let size = 0;
+    try {
+        size = fs.statSync(logPath).size;
+    }
+    catch { /* missing log → offset 0 */ }
+    saveCursor(effectiveReader, { offset: size, last_read: nowISO() }, cwd);
+    return size;
 }
 /**
- * Read events unseen by this agent since their last read.
+ * Read events unseen by this reader since their last read.
  * Updates the cursor after reading.
+ *
+ * Self-exclusion is by SESSION when both sides carry one (pln#562 step 4):
+ * an instance skips only its own events, not those of a same-named sibling.
+ * Events or readers without session info fall back to name exclusion.
  */
-export function readUnseenEvents(agent, cwd) {
+export function readUnseenEvents(reader, cwd) {
+    const effectiveReader = normalizeReader(reader);
     const logPath = eventLogPath(cwd);
     if (!fs.existsSync(logPath))
         return [];
-    const cursor = loadCursor(agent, cwd);
+    const cursor = loadCursor(effectiveReader, cwd);
     const stat = fs.statSync(logPath);
     if (stat.size <= cursor.offset)
         return [];
@@ -90,8 +200,10 @@ export function readUnseenEvents(agent, cwd) {
     for (const line of lines) {
         try {
             const evt = JSON.parse(line);
-            // Exclude events from self
-            if (evt.agent !== agent) {
+            const isSelf = effectiveReader.session_id && evt.session_id
+                ? evt.session_id === effectiveReader.session_id
+                : evt.agent === effectiveReader.agent;
+            if (!isSelf) {
                 events.push(evt);
             }
         }
@@ -99,8 +211,13 @@ export function readUnseenEvents(agent, cwd) {
             // skip
         }
     }
-    // Update cursor
-    saveCursor(agent, { offset: stat.size, last_read: nowISO() }, cwd);
+    // Update cursor. Observer mode (BRAINCLAW_OBSERVER=1) skips this — the
+    // dashboard impersonating an agent must not consume that agent's cursor
+    // (the 2026-06-10 leak where the extension drained Juan's claude-code
+    // unseen-events queue on every poll).
+    if (!isObserverMode()) {
+        saveCursor(effectiveReader, { offset: stat.size, last_read: nowISO() }, cwd);
+    }
     return events;
 }
 /**

package/dist/core/events/checkpoint.js

@@ -0,0 +1,258 @@
+/**
+ * Journal-derived checkpoints (pln#543 Phase 3 / pln#566 Inc0, slice 1).
+ *
+ * A checkpoint is a snapshot of the materialized live-entity set at a covered
+ * `head_seq`, plus a manifest that BINDS it to the journal lineage. Per the
+ * round-3 review (NF1): checkpoints are derived from the JOURNAL ONLY — never
+ * from projection files, never consulting `projection_watermark` — so the
+ * journal is the single root of trust and there is no checkpoint↔watermark
+ * cycle. The cold-read fast path (a later slice) loads a checkpoint + replays
+ * only the sealed tail (head_seq+1..tail); on ANY trust-chain failure it falls
+ * back to projection files.
+ *
+ * This slice is strictly ADDITIVE: it builds/verifies/replays checkpoints but
+ * is not yet wired into the read path, so it carries zero read-path risk.
+ *
+ * @module
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { journalDir, readJournalRecords, journalHeadSeq } from './journal.js';
+import { applyRecordsToLive, projectLiveToState } from './materialize.js';
+import { loadConfig } from '../config.js';
+import { writeFileAtomic } from '../io.js';
+import { nowISO } from '../ids.js';
+const CHECKPOINT_SCHEMA_VERSION = 1;
+export const BASELINE_CAPABILITIES = {
+    checkpointRead: false,
+    readReconcile: false,
+    tombstoneDelete: false,
+    perEntityPatch: false,
+};
+function checkpointsDir(cwd) {
+    return path.join(journalDir(cwd), 'checkpoints');
+}
+function sha256(s) {
+    return crypto.createHash('sha256').update(s).digest('hex');
+}
+/**
+ * pln#566 F4 guard: materialize (the reducer behind both checkpoint build and
+ * checkpoint+tail replay) only consumes inline `rec.payload`; it does NOT yet
+ * dereference `payload_ref`. So if ANY journal record externalized its payload,
+ * a checkpoint built/served from the journal would silently DROP that entity.
+ * Until payload_ref dereference lands, refuse to build or serve a checkpoint
+ * when externalized payloads exist — the read path falls back to projection
+ * files, which always carry the full content. Conservative (whole-journal, not
+ * just memory-tier) and cheap.
+ */
+function journalHasExternalizedPayload(records) {
+    return records.some(r => r.payload_ref != null);
+}
+/** Identity of a record for journal-lineage binding: stable across reads, changes if the head record changes. */
+function recordIdentity(rec) {
+    return sha256(`${rec.seq}|${rec.writer}|${rec.ts}|${rec.action}|${rec.item_type}|${rec.item_id ?? ''}`);
+}
+/** The record carrying the maximum seq (the covered head). Undefined for an empty journal. */
+function headRecord(records) {
+    let head;
+    for (const rec of records) {
+        if (!head || rec.seq > head.seq)
+            head = rec;
+    }
+    return head;
+}
+function manifestPath(cwd, headSeq) {
+    return path.join(checkpointsDir(cwd), `${String(headSeq).padStart(12, '0')}.manifest.json`);
+}
+function snapshotPath(cwd, headSeq) {
+    return path.join(checkpointsDir(cwd), `${String(headSeq).padStart(12, '0')}.snapshot.json`);
+}
+/**
+ * Build a journal-derived checkpoint at the current journal head. Materializes
+ * the live entity set from the journal (NOT from projections), writes the
+ * snapshot blob, then publishes the manifest (the manifest is the commit
+ * point — an orphan snapshot without a manifest is ignored).
+ *
+ * NOTE (F6): this slice materializes from the full journal on demand. The
+ * incremental "latest verified checkpoint + sealed tail, built outside the hot
+ * lock" optimization is a later slice; on-demand build here is not on the hot
+ * write path.
+ */
+export function createCheckpoint(cwd, capabilities = BASELINE_CAPABILITIES) {
+    // Cap to the COMMITTED head (meta.next_seq-1, published after fsync). The
+    // raw segment read is lock-free, so a concurrent append may have written —
+    // but not yet fsync'd/published — records beyond the committed head; excluding
+    // them keeps the manifest bound to durable journal state (codex review MED).
+    const committedHead = journalHeadSeq(cwd);
+    const records = readJournalRecords(cwd).filter(r => r.seq <= committedHead);
+    const head = headRecord(records);
+    if (!head)
+        return { created: false, reason: 'empty journal — nothing to checkpoint' };
+    if (journalHasExternalizedPayload(records)) {
+        return { created: false, reason: 'journal has externalized payload_ref records; materialize cannot dereference them yet (pln#566 F4)' };
+    }
+    const live = applyRecordsToLive(records, new Map());
+    const entities = [...live.values()];
+    const snapshot = JSON.stringify(entities);
+    const manifest = {
+        schema_version: CHECKPOINT_SCHEMA_VERSION,
+        store_id: loadConfig(cwd).project_id ?? 'unknown',
+        head_seq: head.seq,
+        head_identity: recordIdentity(head),
+        snapshot_sha256: sha256(snapshot),
+        entity_count: entities.length,
+        capability_vector: capabilities,
+        created_at: nowISO(),
+    };
+    const dir = checkpointsDir(cwd);
+    fs.mkdirSync(dir, { recursive: true });
+    // Snapshot first, manifest last (commit point).
+    writeFileAtomic(snapshotPath(cwd, head.seq), snapshot);
+    writeFileAtomic(manifestPath(cwd, head.seq), JSON.stringify(manifest));
+    return { created: true, manifest };
+}
+/** Default: create a fresh checkpoint once the journal has grown this many
+ *  records past the last checkpoint head. Bounds the sealed tail so
+ *  checkpointRead's gap-replay stays cheap, without checkpointing every persist. */
+export const DEFAULT_CHECKPOINT_INTERVAL = 500;
+/**
+ * Create a checkpoint ONLY if the journal has grown >= interval records past
+ * the last checkpoint head (cheap head-seq check; no full scan unless building).
+ * Intended for off-hot-path callers (session-start maintenance). Journal-derived
+ * (F6). No-op when the journal is off/empty or hasn't grown enough.
+ */
+export function maybeCreateCheckpoint(cwd, interval = DEFAULT_CHECKPOINT_INTERVAL) {
+    const head = journalHeadSeq(cwd);
+    if (head === 0)
+        return { created: false, gap: 0, reason: 'journal empty/off' };
+    const last = loadLatestCheckpointManifest(cwd)?.head_seq ?? 0;
+    const gap = head - last;
+    if (gap < interval)
+        return { created: false, gap, reason: `gap ${gap} < interval ${interval}` };
+    const res = createCheckpoint(cwd);
+    return { created: res.created, gap, reason: res.reason };
+}
+/**
+ * Highest-head_seq manifest on disk, or undefined if none. Selects by NUMERIC
+ * head_seq parsed from the filename — lexicographic order on the 12-zero-pad
+ * breaks once seq exceeds 12 digits (codex review LOW).
+ */
+export function loadLatestCheckpointManifest(cwd) {
+    const dir = checkpointsDir(cwd);
+    if (!fs.existsSync(dir))
+        return undefined;
+    const files = fs.readdirSync(dir).filter(f => f.endsWith('.manifest.json'))
+        .sort((a, b) => (parseInt(b, 10) || 0) - (parseInt(a, 10) || 0)); // numeric desc on the seq prefix
+    for (const f of files) {
+        try {
+            return JSON.parse(fs.readFileSync(path.join(dir, f), 'utf-8'));
+        }
+        catch { /* skip corrupt manifest, try the next */ }
+    }
+    return undefined;
+}
+function isMaterializedEntityArray(v) {
+    return Array.isArray(v) && v.every(e => !!e && typeof e === 'object'
+        && typeof e.item_type === 'string'
+        && typeof e.item_id === 'string'
+        && !!e.payload && typeof e.payload === 'object');
+}
+/**
+ * Verify a checkpoint against an ALREADY-READ record set — never throws,
+ * validates snapshot SHAPE (not just sha256), and binds to the SAME journal
+ * view the caller will serve from (no verify/serve TOCTOU). WITHOUT reading
+ * projection files (F3). Returns the parsed entities when valid.
+ */
+function verifyCheckpointAgainstRecords(manifest, snapshotRaw, records, cwd) {
+    try {
+        if (manifest.schema_version !== CHECKPOINT_SCHEMA_VERSION) {
+            return { valid: false, reason: `unsupported checkpoint schema_version ${manifest.schema_version}` };
+        }
+        if (sha256(snapshotRaw) !== manifest.snapshot_sha256) {
+            return { valid: false, reason: 'snapshot sha256 mismatch (corrupt/tampered blob)' };
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(snapshotRaw);
+        }
+        catch {
+            return { valid: false, reason: 'snapshot is not valid JSON' };
+        }
+        if (!isMaterializedEntityArray(parsed))
+            return { valid: false, reason: 'snapshot is not a MaterializedEntity[]' };
+        const expectedStore = loadConfig(cwd).project_id ?? 'unknown';
+        if (manifest.store_id !== expectedStore) {
+            return { valid: false, reason: `store_id mismatch (manifest ${manifest.store_id} vs ${expectedStore}) — copied/wrong-branch checkpoint` };
+        }
+        const head = records.find(r => r.seq === manifest.head_seq);
+        if (!head)
+            return { valid: false, reason: `head_seq ${manifest.head_seq} not found in journal` };
+        if (recordIdentity(head) !== manifest.head_identity) {
+            return { valid: false, reason: 'head record identity mismatch — journal lineage diverged from checkpoint' };
+        }
+        return { valid: true, entities: parsed };
+    }
+    catch (err) {
+        return { valid: false, reason: `verification error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+}
+/**
+ * Public no-throw verification: snapshot integrity + shape, store binding, and
+ * journal-lineage binding. Reads the journal once. Returns valid:false on any
+ * failure (never throws) so callers fall back.
+ */
+export function verifyCheckpoint(manifest, snapshotRaw, cwd) {
+    let records;
+    try {
+        records = readJournalRecords(cwd);
+    }
+    catch (err) {
+        return { valid: false, reason: `journal read failed: ${err instanceof Error ? err.message : String(err)}` };
+    }
+    const r = verifyCheckpointAgainstRecords(manifest, snapshotRaw, records, cwd);
+    return { valid: r.valid, reason: r.reason };
+}
+/**
+ * Materialize state from the latest VERIFIED checkpoint + the sealed tail
+ * (records with seq > head_seq), using the same reducer + projector as
+ * full-journal materialization. Reads the journal exactly ONCE and uses that
+ * single view for verification, the F4 payload_ref guard, and the tail replay
+ * (no TOCTOU). Returns null on any failure (caller falls back to projections).
+ */
+export function materializeStateFromCheckpoint(cwd) {
+    const manifest = loadLatestCheckpointManifest(cwd);
+    if (!manifest)
+        return null;
+    let snapshotRaw;
+    try {
+        snapshotRaw = fs.readFileSync(snapshotPath(cwd, manifest.head_seq), 'utf-8');
+    }
+    catch {
+        return null; // orphan manifest without a readable snapshot
+    }
+    let records;
+    // Same committed-head cap as the build path: only durable records (seq <=
+    // meta.next_seq-1) drive verification, the F4 guard, and the tail replay.
+    try {
+        const committedHead = journalHeadSeq(cwd);
+        records = readJournalRecords(cwd).filter(r => r.seq <= committedHead);
+    }
+    catch {
+        return null;
+    }
+    const verdict = verifyCheckpointAgainstRecords(manifest, snapshotRaw, records, cwd);
+    if (!verdict.valid || !verdict.entities)
+        return null;
+    // F4 guard: an externalized payload (which materialize can't deref) means the
+    // checkpoint/tail would drop entities — refuse to serve, fall back.
+    if (journalHasExternalizedPayload(records))
+        return null;
+    const live = new Map();
+    for (const e of verdict.entities)
+        live.set(`${e.item_type}:${e.item_id}`, e);
+    // Replay only the sealed tail (head_seq+1..end) from the SAME record view.
+    applyRecordsToLive(records.filter(r => r.seq > manifest.head_seq), live);
+    return projectLiveToState(live);
+}
+//# sourceMappingURL=checkpoint.js.map