brainclaw 1.8.0 → 1.9.1

package/dist/core/state.js

@@ -1,11 +1,13 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { ConstraintSchema, DecisionSchema, TrapSchema, HandoffSchema, PlanItemSchema } from './schema.js';
-import { ensureMemoryDir, resolveEntityDir } from './io.js';
+import { ensureMemoryDir, resolveEntityDir, writeFileAtomic } from './io.js';
 import { mutate } from './mutation-pipeline.js';
 import { commitMemoryChange } from './memory-git.js';
 import { appendEvent } from './event-log.js';
-import { loadVersionedJsonFile, saveVersionedJsonFile } from './migration.js';
+import { appendJournalRecords, resolveJournalMode, resolveCheckpointRead } from './events/journal.js';
+import { materializeStateFromCheckpoint } from './events/checkpoint.js';
+import { loadVersionedJsonFile, serializeVersionedJson, preparePersistedDocument } from './migration.js';
 import { rebuildProjectMd } from './markdown.js';
 import { refreshLiveCompanions } from '../commands/export.js';
 import { logger } from './logger.js';
@@ -102,6 +104,24 @@ export function findLoadValidationWarning(entity, id, cwd) {
 export function loadState(cwd) {
     // Load from entity-aligned directories (with legacy fallback)
     const effectiveCwd = cwd ?? process.cwd();
+    // pln#566 Inc0 s2 — checkpointRead fast path. OFF by default (dual/off mode):
+    // projection files remain the read substrate. When the capability is enabled
+    // (primary soak) AND a verified journal-derived checkpoint exists, serve from
+    // checkpoint + sealed tail instead of reading every projection file. ANY
+    // failure (no checkpoint, failed verification, replay error) falls through to
+    // the projection read below — the checkpoint is never the sole truth.
+    if (resolveCheckpointRead(effectiveCwd)) {
+        try {
+            const fast = materializeStateFromCheckpoint(effectiveCwd);
+            // Merge over emptyState so the served State carries the same envelope
+            // fields (version/write_version) a projection read produces; the
+            // checkpoint only materializes the 5 entity collections. Already sorted
+            // by projectLiveToState.
+            if (fast)
+                return { ...emptyState(), ...fast };
+        }
+        catch { /* fall through to projection read */ }
+    }
     const state = emptyState();
     state.active_constraints = loadDirectoryItems(resolveEntityDir('constraints', effectiveCwd, 'read'), ConstraintSchema, 'constraint');
     state.recent_decisions = loadDirectoryItems(resolveEntityDir('decisions', effectiveCwd, 'read'), DecisionSchema, 'decision');
@@ -116,72 +136,189 @@ export function loadState(cwd) {
     state.plan_items.sort((a, b) => a.created_at.localeCompare(b.created_at));
     return state;
 }
-function syncDirectory(dirPath, items, documentType, schema) {
-    if (!fs.existsSync(dirPath)) {
-        fs.mkdirSync(dirPath, { recursive: true });
+const persistWriteStats = { written: 0, skippedUnchanged: 0 };
+export function readPersistWriteStats() {
+    return { ...persistWriteStats };
+}
+export function resetPersistWriteStats() {
+    persistWriteStats.written = 0;
+    persistWriteStats.skippedUnchanged = 0;
+}
+/**
+ * Order-insensitive canonical form of a JSON document, for the dirty-tracking
+ * skip compare. Recursively sorts object keys; returns the raw input on parse
+ * failure so corrupt/non-JSON bytes never compare equal to a valid desired doc.
+ */
+function canonicalJson(raw) {
+    const sortKeys = (value) => {
+        if (Array.isArray(value))
+            return value.map(sortKeys);
+        if (value && typeof value === 'object') {
+            return Object.fromEntries(Object.entries(value)
+                .sort(([a], [b]) => a.localeCompare(b))
+                .map(([k, v]) => [k, sortKeys(v)]));
+        }
+        return value;
+    };
+    try {
+        return JSON.stringify(sortKeys(JSON.parse(raw)));
+    }
+    catch {
+        return raw; // unparseable → never equals a canonical desired doc
     }
-    // Write all current items
+}
+/**
+ * Pure planning pass: compute which projection files WOULD change, without
+ * writing anything. Same dirty-tracking + canonical-compare + parseable-guard
+ * logic as the old syncDirectory; only the IO is deferred to applySyncPlan.
+ */
+function planSyncDirectory(dirPath, items, documentType, schema, deleteMissing) {
+    const plan = { dirPath, writes: [], deletes: [] };
+    // Write only the items whose on-disk bytes would change (dirty-tracking,
+    // pln#543 step 3). The comparison uses the writer's own serializer so a
+    // "skip" can never diverge from what saveVersionedJsonFile would produce.
+    // Safe against trp#126: a missing/byte-different file never matches, so an
+    // in-state entity whose projection is absent/corrupt is always rewritten.
     const currentIds = new Set();
     for (const item of items) {
         currentIds.add(item.id);
         const filepath = path.join(dirPath, `${item.id}.json`);
-        saveVersionedJsonFile(documentType, filepath, item);
-    }
-    // Remove files that are no longer in the state (e.g. if deleted/pruned).
-    // CRITICAL: we must distinguish "file dropped from state intentionally" from
-    // "file silently dropped by loadDirectoryItems because its schema.parse threw".
-    // Deleting the second kind corrupts data (see trap: silent-data-loss via
-    // load-swallow + write-sync-GC). So before unlinking, we re-validate the file
-    // against the schema. Parseable + not in state = intentional remove → unlink.
-    // Unparseable = preserved, operator can inspect/repair.
-    const files = fs.readdirSync(dirPath).filter(f => f.endsWith('.json'));
+        const desired = serializeVersionedJson(documentType, item);
+        let existing;
+        try {
+            existing = fs.readFileSync(filepath, 'utf-8');
+        }
+        catch {
+            existing = undefined; // missing/unreadable → write
+        }
+        // Semantic (canonical, sorted-key) compare, not byte compare: loadState
+        // re-parses through zod which can reorder keys; a byte compare would
+        // rewrite the whole store every persist. Unparseable bytes never match
+        // → rewrite (keeps trp#126 safety).
+        if (existing !== undefined && canonicalJson(existing) === canonicalJson(desired)) {
+            persistWriteStats.skippedUnchanged += 1;
+            continue;
+        }
+        plan.writes.push({ filepath, desired, item, created: existing === undefined });
+    }
+    if (!deleteMissing)
+        return plan;
+    // Plan removals of files no longer in state. CRITICAL: distinguish an
+    // intentional drop from a file silently dropped by loadDirectoryItems on a
+    // schema.parse throw (deleting the second kind corrupts data — trp#126). Only
+    // parseable + not-in-state files are unlinked; unparseable are preserved.
+    const files = fs.existsSync(dirPath)
+        ? fs.readdirSync(dirPath).filter(f => f.endsWith('.json'))
+        : [];
     for (const file of files) {
         const id = file.replace('.json', '');
         if (currentIds.has(id))
             continue;
         const filepath = path.join(dirPath, file);
-        let parseable = false;
         try {
             schema.parse(loadVersionedJsonFile(documentType, filepath).document);
-            parseable = true;
         }
         catch {
             // Already logged by loadDirectoryItems — leave the file in place.
+            continue;
         }
-        if (parseable) {
-            fs.unlinkSync(filepath);
-        }
+        plan.deletes.push({ filepath, id });
+    }
+    return plan;
+}
+/** Apply a planned sync: the actual projection-file writes/unlinks (the IO that
+ * must happen AFTER the journal append+fsync). Returns the dirty result. */
+function applySyncPlan(plan) {
+    const result = { written: [], deleted: [] };
+    // Create the entity dir unconditionally (matches the pre-split syncDirectory,
+    // which always ensured the dir even for an empty/unchanged collection).
+    if (!fs.existsSync(plan.dirPath)) {
+        fs.mkdirSync(plan.dirPath, { recursive: true });
+    }
+    for (const { filepath, desired, item, created } of plan.writes) {
+        writeFileAtomic(filepath, desired);
+        persistWriteStats.written += 1;
+        result.written.push({ item, created });
     }
+    for (const { filepath, id } of plan.deletes) {
+        fs.unlinkSync(filepath);
+        result.deleted.push(id);
+    }
+    return result;
 }
 export function saveState(state, cwd) {
     persistState(state, cwd, { writeProjectMarkdown: false });
 }
 function persistStateUnlocked(state, cwd, options = {}) {
-    writeStateDirectories(state, cwd);
+    ensureMemoryDir(cwd);
+    const effectiveCwd = cwd ?? process.cwd();
+    // pln#566 F1 — JOURNAL BEFORE PROJECTIONS (invariant I2). Persist now runs in
+    // three ordered phases so the journal can never lag the projections: a crash
+    // mid-persist leaves the journal AHEAD (the only direction lazy-reconcile can
+    // recover), never projections ahead (which materialize/verify could not
+    // explain). Phase 1 PLAN (pure compute, no IO) → Phase 2 emit+fsync the
+    // per-entity post-images to the journal → Phase 3 APPLY projection writes.
+    const { plans, legacyDeletes, dirty } = planStateDirectories(state, effectiveCwd, options.deleteMissing ?? false);
+    emitPerEntityJournalRecords(dirty, options.eventAction, effectiveCwd);
+    faultPoint('after_journal'); // test-only: crash AFTER journal, BEFORE projections
+    applyStatePlans(plans, legacyDeletes);
+    faultPoint('after_projection'); // test-only: crash AFTER projections written
     if (options.writeProjectMarkdown ?? true) {
-        rebuildProjectMd(state, cwd);
+        rebuildProjectMd(state, effectiveCwd);
     }
+    // v1 events.jsonl: keep the coarse store event for existing consumers, but
+    // suppress its envelope-only journal dual-write — the v2 per-entity emit
+    // above is the authoritative §2.8 diff choke point.
     appendEvent({
         action: options.eventAction ?? 'update',
         item_type: 'state',
         agent: 'system',
         summary: options.eventSummary,
-    }, cwd);
-    commitMemoryChange(options.commitMessage ?? 'state update', cwd);
-    // Auto-refresh live companion files (Tier B/C agents) after state mutations.
-    // Non-fatal: failures are logged but don't break the mutation.
+    }, effectiveCwd, { journalDualWrite: false });
+    // NOTE (pln#558 step 2): the git commit and refreshLiveCompanions used to
+    // run here, INSIDE the mutation lock. A single persistState was holding
+    // the lock for >5s on Juan's machine (full-store rewrite + git add -A +
+    // git commit + live-companion refresh), which serialized every other
+    // writer. They are now invoked by persistState / mutateState AFTER the
+    // lock releases — see runPostWriteHooks below. The critical section is
+    // now writes-only; commit / companion-refresh are best-effort observers.
+}
+function runPostWriteHooks(cwd, commitMessage) {
+    // git add + git commit. Safe outside the lock because (a) git itself
+    // serializes concurrent index access via .git/index.lock, (b) the
+    // implementation already swallows failures (see memory-git.ts), and
+    // (c) the commit is an audit trail, not the data itself.
+    try {
+        commitMemoryChange(commitMessage, cwd);
+    }
+    catch { /* best-effort */ }
+    // Live companion files (Tier B/C agent surfaces). Already best-effort.
     try {
         refreshLiveCompanions(cwd);
     }
     catch { /* best-effort */ }
 }
-function cleanupLegacyDir(entityName, currentIds, cwd, documentType, schema) {
+/**
+ * Test-only crash injection (pln#566 F1). No-op unless BRAINCLAW_FAULT_POINT
+ * matches the label — then it throws, simulating a process death at that exact
+ * point in the persist pipeline so crash-ordering invariants can be tested
+ * deterministically without racing a real SIGKILL.
+ */
+function faultPoint(label) {
+    if (process.env.BRAINCLAW_FAULT_POINT === label) {
+        throw new Error(`fault-injection: crashed at "${label}" (BRAINCLAW_FAULT_POINT)`);
+    }
+}
+/**
+ * Plan (do not apply) the removal of legacy-dir orphans. Read-only: matches
+ * syncDirectory's safety condition (only parseable records absent from state
+ * are deletable; unparseable are preserved for inspection/repair). The actual
+ * unlink is deferred to applyStatePlans so it lands AFTER the journal append.
+ */
+function planCleanupLegacyDir(entityName, currentIds, cwd, documentType, schema) {
+    const out = [];
     const writeDir = resolveEntityDir(entityName, cwd, 'write');
     const readDir = resolveEntityDir(entityName, cwd, 'read');
-    // If read resolves to a different (legacy) directory, clean orphans there too.
-    // Match syncDirectory's safety condition: only delete parseable records that
-    // are absent from the current state. Schema-invalid legacy files may be drifted
-    // data that operators still need to inspect or repair.
     if (readDir !== writeDir && fs.existsSync(readDir)) {
         const files = fs.readdirSync(readDir).filter(f => f.endsWith('.json'));
         for (const file of files) {
@@ -189,51 +326,110 @@ function cleanupLegacyDir(entityName, currentIds, cwd, documentType, schema) {
             if (currentIds.has(id))
                 continue;
             const filepath = path.join(readDir, file);
-            let parseable = false;
             try {
                 schema.parse(loadVersionedJsonFile(documentType, filepath).document);
-                parseable = true;
             }
             catch {
                 logger.warn(`Preserving unparseable legacy ${entityName} file ${file}`);
                 continue;
             }
-            if (parseable) {
-                fs.unlinkSync(filepath);
-            }
+            out.push({ filepath, id }); // O3: surfaced so a delete tombstone is emitted
         }
     }
+    return out;
 }
-function writeStateDirectories(state, cwd) {
-    ensureMemoryDir(cwd);
-    const effectiveCwd = cwd ?? process.cwd();
+/**
+ * Phase 1 of persist: compute every projection change WITHOUT writing. The
+ * returned `dirty` (post-images + tombstone ids) lets the journal be emitted +
+ * fsync'd before applyStatePlans touches any file (pln#566 F1 / I2).
+ */
+function planStateDirectories(state, cwd, deleteMissing) {
     const entities = [
-        { name: 'constraints', items: state.active_constraints, docType: 'constraint', schema: ConstraintSchema },
-        { name: 'decisions', items: state.recent_decisions, docType: 'decision', schema: DecisionSchema },
-        { name: 'traps', items: state.known_traps, docType: 'trap', schema: TrapSchema },
-        { name: 'handoffs', items: state.open_handoffs, docType: 'handoff', schema: HandoffSchema },
-        { name: 'plans', items: state.plan_items, docType: 'plan', schema: PlanItemSchema },
+        { name: 'constraints', itemType: 'constraint', items: state.active_constraints, docType: 'constraint', schema: ConstraintSchema },
+        { name: 'decisions', itemType: 'decision', items: state.recent_decisions, docType: 'decision', schema: DecisionSchema },
+        { name: 'traps', itemType: 'trap', items: state.known_traps, docType: 'trap', schema: TrapSchema },
+        { name: 'handoffs', itemType: 'handoff', items: state.open_handoffs, docType: 'handoff', schema: HandoffSchema },
+        { name: 'plans', itemType: 'plan', items: state.plan_items, docType: 'plan', schema: PlanItemSchema },
     ];
-    for (const { name, items, docType, schema } of entities) {
-        const writeDir = resolveEntityDir(name, effectiveCwd, 'write');
-        syncDirectory(writeDir, items, docType, schema);
-        const currentIds = new Set(items.map(item => item.id));
-        cleanupLegacyDir(name, currentIds, effectiveCwd, docType, schema);
+    const plans = [];
+    const legacyDeletes = [];
+    const dirty = [];
+    for (const { name, itemType, items, docType, schema } of entities) {
+        const writeDir = resolveEntityDir(name, cwd, 'write');
+        const plan = planSyncDirectory(writeDir, items, docType, schema, deleteMissing);
+        plans.push(plan);
+        const deleted = plan.deletes.map(d => d.id);
+        if (deleteMissing) {
+            // O3: legacy-dir orphans must also emit a delete tombstone.
+            const legacy = planCleanupLegacyDir(name, new Set(items.map(i => i.id)), cwd, docType, schema);
+            for (const l of legacy) {
+                legacyDeletes.push(l);
+                deleted.push(l.id);
+            }
+        }
+        dirty.push({ itemType, written: plan.writes.map(w => ({ item: w.item, created: w.created })), deleted });
+    }
+    return { plans, legacyDeletes, dirty };
+}
+/** Phase 3 of persist: apply the planned projection writes/unlinks (the IO that
+ * must follow the journal append+fsync). */
+function applyStatePlans(plans, legacyDeletes) {
+    for (const plan of plans)
+        applySyncPlan(plan);
+    for (const { filepath } of legacyDeletes) {
+        try {
+            fs.unlinkSync(filepath);
+        }
+        catch { /* already gone — idempotent */ }
+    }
+}
+/**
+ * Emit one journal record per dirty entity with its full post-image
+ * (entity-state class, §2.1.1 / §2.8): the persist path is where the store's
+ * source-of-truth events are minted, because only it holds the entity docs.
+ * No-op when the journal flag is off. Failures are swallowed inside
+ * appendJournalRecords (dual mode: v1 projections remain the truth).
+ */
+function emitPerEntityJournalRecords(dirty, storeAction, cwd) {
+    if (resolveJournalMode(cwd) === 'off')
+        return;
+    const records = [];
+    for (const { itemType, written, deleted } of dirty) {
+        for (const { item, created } of written) {
+            // Post-image = the prepared document (with schema_version), so the
+            // journal record is byte-faithful to the projection: materialize can
+            // reconstruct an identical file, and verify compares like-for-like.
+            records.push({
+                action: storeAction && storeAction !== 'update' ? storeAction : (created ? 'create' : 'update'),
+                item_type: itemType,
+                item_id: item.id,
+                agent: 'system',
+                payload: preparePersistedDocument(itemType, item),
+            });
+        }
+        for (const id of deleted) {
+            records.push({ action: 'delete', item_type: itemType, item_id: id, agent: 'system' });
+        }
     }
+    if (records.length > 0)
+        appendJournalRecords(records, cwd);
 }
 export function persistState(state, cwd, options = {}) {
     const effectiveCwd = cwd ?? process.cwd();
     mutate({ cwd: effectiveCwd }, () => {
         persistStateUnlocked(state, effectiveCwd, options);
     });
+    runPostWriteHooks(effectiveCwd, options.commitMessage ?? 'state update');
 }
 export function mutateState(mutateFn, cwd, options = {}) {
     const effectiveCwd = cwd ?? process.cwd();
-    return mutate({ cwd: effectiveCwd }, () => {
+    const result = mutate({ cwd: effectiveCwd }, () => {
         const state = loadState(effectiveCwd);
-        const result = mutateFn(state);
-        persistStateUnlocked(state, effectiveCwd, options);
-        return result;
+        const value = mutateFn(state);
+        persistStateUnlocked(state, effectiveCwd, { ...options, deleteMissing: true });
+        return value;
     });
+    runPostWriteHooks(effectiveCwd, options.commitMessage ?? 'state update');
+    return result;
 }
 //# sourceMappingURL=state.js.map

package/dist/core/store-resolution.js

@@ -3,7 +3,7 @@ import os from 'node:os';
 import path from 'node:path';
 import { loadActiveProject } from './active-project.js';
 import { loadConfig } from './config.js';
-import { loadCurrentSession } from './identity.js';
+import { loadCurrentSession, loadSessionById } from './identity.js';
 import { MEMORY_DIR } from './io.js';
 import { summarizeWorkspaceProjects } from './workspace-projects.js';
 /**
@@ -98,10 +98,18 @@ export function resolveTargetStore(cwd = process.cwd(), target = 'local', option
  * 6. Workspace anchor or process.cwd()
  */
 export function resolveEffectiveCwd(options = {}) {
+    return resolveEffectiveCwdInfo(options).cwd;
+}
+/**
+ * Resolve the effective cwd and explain which selector won. Use this for MCP
+ * facades that must echo their project scope to avoid silent cross-project reads.
+ */
+export function resolveEffectiveCwdInfo(options = {}) {
     const baseCwd = path.resolve(options.baseCwd ?? process.cwd());
     // 1. Explicit --cwd flag
     if (options.explicitCwd) {
-        return path.resolve(options.explicitCwd);
+        const cwd = path.resolve(options.explicitCwd);
+        return { cwd, active_source: 'explicit', resolved_project: projectInfo(cwd) };
     }
     // 2. BRAINCLAW_CWD env var — set by MCP configs to anchor resolution to the
     //    workspace regardless of the IDE's process.cwd() at launch time. It is a
@@ -118,14 +126,16 @@ export function resolveEffectiveCwd(options = {}) {
     if (envProject) {
         const resolved = resolveProjectRef(envProject, anchorCwd, options.storeChainOptions);
         if (resolved)
-            return resolved;
+            return { cwd: resolved, active_source: 'env_project', resolved_project: projectInfo(resolved) };
     }
     // 4. Session-scoped active project (per-agent, no cross-agent interference)
-    const session = loadCurrentSession(anchorCwd);
+    const session = options.sessionId
+        ? loadSessionById(options.sessionId, anchorCwd)
+        : loadCurrentSession(anchorCwd);
     if (session?.active_project) {
         const sp = session.active_project;
         if (fs.existsSync(path.join(sp.path, MEMORY_DIR, 'config.yaml'))) {
-            return sp.path;
+            return { cwd: sp.path, active_source: 'session', resolved_project: { path: sp.path, name: sp.name } };
         }
     }
     // 5. Global active-project.json from workspace root
@@ -133,11 +143,20 @@ export function resolveEffectiveCwd(options = {}) {
     if (wsRoot) {
         const active = loadActiveProject(wsRoot);
         if (active && fs.existsSync(path.join(active.path, MEMORY_DIR, 'config.yaml'))) {
-            return active.path;
+            return { cwd: active.path, active_source: 'global', resolved_project: { path: active.path, name: active.name } };
         }
     }
     // 6. Default
-    return anchorCwd;
+    return { cwd: anchorCwd, active_source: 'cwd', resolved_project: projectInfo(anchorCwd) };
+}
+function projectInfo(cwd) {
+    try {
+        const config = loadConfig(cwd);
+        return { path: cwd, name: config.project_name };
+    }
+    catch {
+        return { path: cwd };
+    }
 }
 /**
  * Find the workspace root (farthest store in the chain, or the one with
@@ -166,12 +185,24 @@ export function resolveProjectRef(ref, cwd = process.cwd(), storeChainOptions) {
         ?? resolveWorkspaceRoot(cwd, storeChainOptions);
     if (!wsRoot)
         return undefined;
-    // Try as absolute path
+    // The trust boundary for raw path refs is the provided cwd.  Callers in
+    // MCP context set cwd to the workspace root, so child projects resolve
+    // naturally.  Walking further up (to a user-level store at home) would
+    // allow path-injection to sibling or home stores — that is the vulnerability
+    // we are closing.  Name-based lookup below is unrestricted since it matches
+    // by project_name / project_id, not by arbitrary path.
+    const trustBoundary = path.resolve(cwd);
+    // Try as absolute path — only allowed if within the cwd boundary.
     if (path.isAbsolute(ref)) {
+        if (!isAtOrBelow(ref, trustBoundary))
+            return undefined;
         return fs.existsSync(path.join(ref, MEMORY_DIR, 'config.yaml')) ? ref : undefined;
     }
-    // Try as relative path from workspace root
-    const asPath = path.resolve(wsRoot, ref);
+    // Try as relative path resolved from the cwd boundary.
+    // Guards against ../ traversal (e.g. "../sibling-project").
+    const asPath = path.resolve(trustBoundary, ref);
+    if (!isAtOrBelow(asPath, trustBoundary))
+        return undefined;
     if (fs.existsSync(path.join(asPath, MEMORY_DIR, 'config.yaml'))) {
         return asPath;
     }
@@ -315,8 +346,10 @@ function findClosestStoreBelow(target, ceiling) {
  */
 function isAtOrBelow(dir, ancestor) {
     const rel = path.relative(ancestor, dir);
-    // If relative path starts with '..', dir is above ancestor
-    return !rel.startsWith('..');
+    // '..' prefix → dir is above ancestor. An absolute result means a different
+    // Windows drive (path.relative returns the absolute `to` path then), which is
+    // also outside the boundary — without this check `D:\evil` would pass.
+    return !rel.startsWith('..') && !path.isAbsolute(rel);
 }
 function resolveAbsoluteTargetPath(cwd, target) {
     if (path.isAbsolute(target)) {