brainclaw 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -11
- package/dist/brainclaw-vscode.vsix +0 -0
- package/dist/cli.js +138 -13
- package/dist/commands/add-step.js +1 -1
- package/dist/commands/bootstrap.js +2 -26
- package/dist/commands/check-security-mcp.js +50 -33
- package/dist/commands/check-security.js +86 -43
- package/dist/commands/claim.js +22 -21
- package/dist/commands/confirm.js +26 -0
- package/dist/commands/context-diff.js +1 -1
- package/dist/commands/dispatch-watch.js +142 -0
- package/dist/commands/doctor.js +113 -2
- package/dist/commands/estimation-report.js +115 -16
- package/dist/commands/harvest.js +285 -22
- package/dist/commands/init.js +123 -21
- package/dist/commands/loops-handlers.js +4 -0
- package/dist/commands/mcp-read-handlers.js +198 -29
- package/dist/commands/mcp.js +588 -92
- package/dist/commands/memory.js +21 -17
- package/dist/commands/migrate.js +81 -17
- package/dist/commands/prune.js +78 -4
- package/dist/commands/reflect.js +26 -20
- package/dist/commands/register-agent.js +57 -1
- package/dist/commands/repair.js +20 -0
- package/dist/commands/session-end.js +15 -6
- package/dist/commands/session-start.js +18 -1
- package/dist/commands/setup-security.js +39 -18
- package/dist/commands/setup.js +26 -27
- package/dist/commands/stale.js +16 -2
- package/dist/commands/uninstall.js +126 -34
- package/dist/commands/update-step.js +6 -0
- package/dist/commands/worktree.js +60 -0
- package/dist/core/actions.js +12 -3
- package/dist/core/agent-capability.js +11 -13
- package/dist/core/agent-files.js +844 -547
- package/dist/core/agent-integrations.js +0 -3
- package/dist/core/agent-inventory.js +67 -0
- package/dist/core/agent-registry.js +163 -29
- package/dist/core/agentrun-reconciler.js +33 -2
- package/dist/core/agentruns.js +7 -1
- package/dist/core/ai-agent-detection.js +31 -44
- package/dist/core/archival.js +15 -9
- package/dist/core/assignment-reconciler.js +56 -0
- package/dist/core/assignment-sweeper.js +127 -4
- package/dist/core/assignments.js +69 -11
- package/dist/core/bootstrap.js +233 -67
- package/dist/core/brainclaw-version.js +22 -0
- package/dist/core/candidates.js +21 -1
- package/dist/core/claims.js +313 -150
- package/dist/core/config.js +6 -1
- package/dist/core/context-diff.js +148 -20
- package/dist/core/context.js +129 -8
- package/dist/core/coordination.js +22 -3
- package/dist/core/dispatch-status.js +79 -5
- package/dist/core/dispatcher.js +64 -11
- package/dist/core/entity-operations.js +45 -24
- package/dist/core/entity-registry.js +31 -5
- package/dist/core/event-log.js +138 -21
- package/dist/core/events/checkpoint.js +258 -0
- package/dist/core/events/genesis.js +220 -0
- package/dist/core/events/journal.js +507 -0
- package/dist/core/events/materialize.js +126 -0
- package/dist/core/events/registry-post-image.js +110 -0
- package/dist/core/events/verify.js +109 -0
- package/dist/core/execution-adapters.js +23 -0
- package/dist/core/facade-schema.js +38 -0
- package/dist/core/gc-semantic.js +130 -5
- package/dist/core/handoff-snapshot.js +68 -0
- package/dist/core/ids.js +19 -8
- package/dist/core/instruction-templates.js +34 -115
- package/dist/core/io.js +39 -3
- package/dist/core/json-store.js +10 -1
- package/dist/core/lock.js +153 -28
- package/dist/core/loops/bootstrap-acquire.js +25 -1
- package/dist/core/loops/facade-schema.js +2 -0
- package/dist/core/loops/hooks/survey-signals-baseline.js +36 -0
- package/dist/core/loops/index.js +1 -0
- package/dist/core/loops/presets/bootstrap.js +7 -0
- package/dist/core/loops/store.js +17 -0
- package/dist/core/loops/verbs.js +24 -1
- package/dist/core/markdown.js +8 -76
- package/dist/core/mcp-command-resolution.js +245 -0
- package/dist/core/memory-compactor.js +5 -3
- package/dist/core/memory-lifecycle.js +282 -0
- package/dist/core/merge-risk.js +150 -0
- package/dist/core/messaging.js +8 -1
- package/dist/core/migration.js +11 -1
- package/dist/core/observer-mode.js +26 -0
- package/dist/core/operations/memory-mutation.js +90 -65
- package/dist/core/operations/plan.js +27 -1
- package/dist/core/protocol-skills.js +210 -0
- package/dist/core/reflection-safety.js +6 -7
- package/dist/core/reputation.js +84 -2
- package/dist/core/runtime-signals.js +71 -9
- package/dist/core/runtime.js +84 -1
- package/dist/core/schema.js +114 -0
- package/dist/core/security-detectors.js +125 -0
- package/dist/core/security-extract.js +189 -0
- package/dist/core/security-guard.js +107 -29
- package/dist/core/security-packages.js +121 -0
- package/dist/core/security-scoring.js +76 -9
- package/dist/core/security.js +34 -2
- package/dist/core/sequence.js +11 -2
- package/dist/core/setup-flow.js +141 -13
- package/dist/core/staleness.js +72 -1
- package/dist/core/state.js +250 -54
- package/dist/core/store-resolution.js +19 -5
- package/dist/core/worktree.js +72 -8
- package/dist/facts.js +8 -8
- package/dist/facts.json +7 -7
- package/docs/PROTOCOL.md +223 -0
- package/docs/cli.md +11 -10
- package/docs/concepts/coordinator-runbook.md +129 -0
- package/docs/concepts/event-log-store-critique-A.md +333 -0
- package/docs/concepts/event-log-store-critique-B.md +353 -0
- package/docs/concepts/event-log-store-phase0-measurements.md +58 -0
- package/docs/concepts/event-log-store-proposal-A.md +365 -0
- package/docs/concepts/event-log-store-proposal-B.md +404 -0
- package/docs/concepts/event-log-store.md +928 -0
- package/docs/concepts/identity-model-proposal.md +371 -0
- package/docs/concepts/memory.md +5 -4
- package/docs/concepts/observer-protocol.md +361 -0
- package/docs/concepts/parallel-merge-protocol.md +71 -0
- package/docs/concepts/plans-and-claims.md +43 -0
- package/docs/concepts/skills.md +78 -0
- package/docs/concepts/workspace-bootstrapping.md +61 -0
- package/docs/integrations/agents.md +4 -4
- package/docs/integrations/cline.md +10 -11
- package/docs/integrations/codex.md +2 -2
- package/docs/integrations/continue.md +5 -5
- package/docs/integrations/copilot.md +14 -12
- package/docs/integrations/openclaw.md +7 -6
- package/docs/integrations/overview.md +7 -7
- package/docs/integrations/roo.md +3 -3
- package/docs/integrations/windsurf.md +6 -6
- package/docs/mcp-schema-changelog.md +29 -2
- package/docs/quickstart.md +48 -47
- package/docs/security.md +174 -15
- package/docs/storage.md +4 -2
- package/package.json +8 -6
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Event journal v2 — write path (pln#543 step 2).
|
|
3
|
+
*
|
|
4
|
+
* Implements the append side of docs/concepts/event-log-store.md:
|
|
5
|
+
* segmented append-only journal under `.brainclaw/events/`, store-global
|
|
6
|
+
* seq allocated under the store lock (§2.2), single-buffer framed appends
|
|
7
|
+
* with torn-tail adjudication (§2.6), fsync-per-mutation policy (§2.7),
|
|
8
|
+
* and the action-class table with mode-gated validation (§2.1.1, R1).
|
|
9
|
+
*
|
|
10
|
+
* Feature flag: BRAINCLAW_JOURNAL_MODE=off|dual (default off — no behavior
|
|
11
|
+
* change). Config-file wiring lands with the cutover step (plan step 5);
|
|
12
|
+
* primary/registryPrimary modes are declared but resolve to 'dual' with a
|
|
13
|
+
* one-time warning until projections (step 3) and migration (step 4) exist.
|
|
14
|
+
*
|
|
15
|
+
* In dual mode the v1 store remains the source of truth: journal failures
|
|
16
|
+
* are loud (logger.warn + counter) but never fail the mutation. The §2.6
|
|
17
|
+
* "append failures are loud" rule binds at journal-primary, not rehearsal.
|
|
18
|
+
*
|
|
19
|
+
* Checkpoints at segment roll (§2.4) arrive with step 3 (they snapshot
|
|
20
|
+
* projections, which do not exist yet); rolls here create the next segment
|
|
21
|
+
* without one.
|
|
22
|
+
*/
|
|
23
|
+
import fs from 'node:fs';
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
import crypto from 'node:crypto';
|
|
26
|
+
import { z } from 'zod';
|
|
27
|
+
import { memoryDir, writeFileAtomic } from '../io.js';
|
|
28
|
+
import { mutate } from '../mutation-pipeline.js';
|
|
29
|
+
import { loadConfig } from '../config.js';
|
|
30
|
+
import { nowISO } from '../ids.js';
|
|
31
|
+
import { logger } from '../logger.js';
|
|
32
|
+
/**
|
|
33
|
+
* Normative class per action (§2.1.1). The class is NEVER serialized —
|
|
34
|
+
* `action` is the only wire discriminant (R1). `satisfies` makes an
|
|
35
|
+
* unclassified 43rd action a compile error instead of a runtime surprise.
|
|
36
|
+
*/
|
|
37
|
+
export const ACTION_CLASS_BY_ACTION = {
|
|
38
|
+
create: 'entity-state',
|
|
39
|
+
update: 'entity-state',
|
|
40
|
+
accept: 'entity-state',
|
|
41
|
+
reject: 'entity-state',
|
|
42
|
+
claim: 'entity-state',
|
|
43
|
+
release_claim: 'entity-state',
|
|
44
|
+
rollback: 'entity-state',
|
|
45
|
+
upgrade: 'entity-state',
|
|
46
|
+
backfill: 'entity-state',
|
|
47
|
+
delete: 'tombstone',
|
|
48
|
+
checkpoint_ref: 'journal-meta',
|
|
49
|
+
journal_note: 'journal-meta',
|
|
50
|
+
seq_repair: 'journal-meta',
|
|
51
|
+
federation_apply: 'journal-meta',
|
|
52
|
+
session_start: 'observability',
|
|
53
|
+
session_end: 'observability',
|
|
54
|
+
assignment_offered: 'observability',
|
|
55
|
+
assignment_progress: 'observability',
|
|
56
|
+
run_progress: 'observability',
|
|
57
|
+
assignment_created: 'registry-lifecycle',
|
|
58
|
+
assignment_accepted: 'registry-lifecycle',
|
|
59
|
+
assignment_started: 'registry-lifecycle',
|
|
60
|
+
assignment_completed: 'registry-lifecycle',
|
|
61
|
+
assignment_cancelled: 'registry-lifecycle',
|
|
62
|
+
assignment_failed: 'registry-lifecycle',
|
|
63
|
+
assignment_blocked: 'registry-lifecycle',
|
|
64
|
+
assignment_timed_out: 'registry-lifecycle',
|
|
65
|
+
assignment_expired: 'registry-lifecycle',
|
|
66
|
+
assignment_retrying: 'registry-lifecycle',
|
|
67
|
+
assignment_rerouted: 'registry-lifecycle',
|
|
68
|
+
assignment_amended: 'registry-lifecycle',
|
|
69
|
+
run_created: 'registry-lifecycle',
|
|
70
|
+
run_launching: 'registry-lifecycle',
|
|
71
|
+
run_waiting_input: 'registry-lifecycle',
|
|
72
|
+
run_running: 'registry-lifecycle',
|
|
73
|
+
run_blocked: 'registry-lifecycle',
|
|
74
|
+
run_completed: 'registry-lifecycle',
|
|
75
|
+
run_failed: 'registry-lifecycle',
|
|
76
|
+
run_cancelled: 'registry-lifecycle',
|
|
77
|
+
run_timed_out: 'registry-lifecycle',
|
|
78
|
+
run_interrupted: 'registry-lifecycle',
|
|
79
|
+
run_amended: 'registry-lifecycle',
|
|
80
|
+
};
|
|
81
|
+
const ALL_V2_ACTIONS = Object.keys(ACTION_CLASS_BY_ACTION);
|
|
82
|
+
// --- v2 record envelope (§2.1) ---
|
|
83
|
+
export const JournalRecordSchema = z.object({
|
|
84
|
+
v: z.literal(2),
|
|
85
|
+
seq: z.number().int().positive(),
|
|
86
|
+
ts: z.string(),
|
|
87
|
+
writer: z.string().min(1),
|
|
88
|
+
agent: z.string().min(1),
|
|
89
|
+
agent_id: z.string().optional(),
|
|
90
|
+
session_id: z.string().optional(),
|
|
91
|
+
user: z.string().optional(),
|
|
92
|
+
action: z.enum(ALL_V2_ACTIONS),
|
|
93
|
+
item_type: z.string().min(1),
|
|
94
|
+
item_id: z.string().optional(),
|
|
95
|
+
entity_rev: z.number().int().positive().optional(),
|
|
96
|
+
summary: z.string().optional(),
|
|
97
|
+
payload: z.record(z.string(), z.unknown()).optional(),
|
|
98
|
+
/** §2.10 — declared for forward-compat; the externalization path is phase 1. */
|
|
99
|
+
payload_ref: z.object({ sha256: z.string(), bytes: z.number().int().positive() }).optional(),
|
|
100
|
+
});
|
|
101
|
+
let warnedUnsupportedMode = false;
|
|
102
|
+
function coerceMode(raw) {
|
|
103
|
+
const v = raw?.trim().toLowerCase();
|
|
104
|
+
if (v === undefined || v === '')
|
|
105
|
+
return undefined;
|
|
106
|
+
if (v === 'off' || v === '0' || v === 'false')
|
|
107
|
+
return 'off';
|
|
108
|
+
if (v === 'dual')
|
|
109
|
+
return 'dual';
|
|
110
|
+
if (v === 'primary' || v === 'registryprimary') {
|
|
111
|
+
if (!warnedUnsupportedMode) {
|
|
112
|
+
warnedUnsupportedMode = true;
|
|
113
|
+
logger.warn(`journal mode "${v}" not available until the primary cutover (pln#543 step 5) — running dual`);
|
|
114
|
+
}
|
|
115
|
+
return 'dual';
|
|
116
|
+
}
|
|
117
|
+
return undefined; // unrecognized → let the next source decide
|
|
118
|
+
}
|
|
119
|
+
/** Read the persisted journal mode from config.yaml (best-effort, off on any failure). */
|
|
120
|
+
function configJournalMode(cwd) {
|
|
121
|
+
try {
|
|
122
|
+
return coerceMode(loadConfig(cwd).store?.journal?.mode);
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
return undefined; // uninitialized / unreadable config → no opinion
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Resolve the journal mode. Precedence: the BRAINCLAW_JOURNAL_MODE env var
|
|
130
|
+
* (a per-process override — tests and one-off runs use it) wins when set;
|
|
131
|
+
* otherwise the persisted config.yaml `store.journal.mode`; otherwise off.
|
|
132
|
+
* Config is read live (not cached) so a flip in config.yaml is picked up by a
|
|
133
|
+
* running MCP server on its next mutation — no restart, unlike an env change
|
|
134
|
+
* (trp#522 cold-start). Mutations are human-paced, so the small config read is
|
|
135
|
+
* negligible next to the persist it gates.
|
|
136
|
+
*/
|
|
137
|
+
export function resolveJournalMode(cwd) {
|
|
138
|
+
return coerceMode(process.env.BRAINCLAW_JOURNAL_MODE) ?? configJournalMode(cwd) ?? 'off';
|
|
139
|
+
}
|
|
140
|
+
function resolveFsyncPolicy(cwd) {
|
|
141
|
+
if (process.env.BRAINCLAW_JOURNAL_FSYNC?.trim() === 'never')
|
|
142
|
+
return 'never';
|
|
143
|
+
try {
|
|
144
|
+
if (loadConfig(cwd).store?.journal?.fsync === 'never')
|
|
145
|
+
return 'never';
|
|
146
|
+
}
|
|
147
|
+
catch { /* no config → default */ }
|
|
148
|
+
return 'mutation';
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Phase-3 primary capability: serve cold reads from a journal-derived
|
|
152
|
+
* checkpoint + sealed tail instead of reading every projection file (pln#566
|
|
153
|
+
* Inc0 s2). OFF by default and ONLY in a primary-family mode — in dual/off the
|
|
154
|
+
* projection files remain the read substrate, so this is a no-op until a soak
|
|
155
|
+
* explicitly enables it. A truthy boolean here is necessary but not sufficient:
|
|
156
|
+
* the read path still verifies the checkpoint and falls back on any failure.
|
|
157
|
+
*/
|
|
158
|
+
export function resolveCheckpointRead(cwd) {
|
|
159
|
+
const env = process.env.BRAINCLAW_PRIMARY_CHECKPOINT_READ?.trim().toLowerCase();
|
|
160
|
+
if (env === '1' || env === 'true' || env === 'on')
|
|
161
|
+
return true;
|
|
162
|
+
if (env === '0' || env === 'false' || env === 'off')
|
|
163
|
+
return false;
|
|
164
|
+
try {
|
|
165
|
+
return loadConfig(cwd).store?.journal?.primary?.checkpointRead === true;
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
return false;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
// --- Writer identity (§2.1: pid + start-nonce, never bare pid) ---
|
|
172
|
+
const WRITER_ID = `w_${process.pid}-${crypto.randomBytes(3).toString('hex')}`;
|
|
173
|
+
export function journalWriterId() {
|
|
174
|
+
return WRITER_ID;
|
|
175
|
+
}
|
|
176
|
+
// --- Layout (§2.3) ---
|
|
177
|
+
const SEGMENT_PREFIX = 'seg-';
|
|
178
|
+
const SEGMENT_PAD = 8;
|
|
179
|
+
const DEFAULT_SEGMENT_ROLL_BYTES = 10 * 1024 * 1024;
|
|
180
|
+
function segmentRollBytes() {
|
|
181
|
+
const raw = Number(process.env.BRAINCLAW_JOURNAL_SEGMENT_BYTES);
|
|
182
|
+
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_SEGMENT_ROLL_BYTES;
|
|
183
|
+
}
|
|
184
|
+
export function journalDir(cwd) {
|
|
185
|
+
return path.join(memoryDir(cwd), 'events');
|
|
186
|
+
}
|
|
187
|
+
function metaPath(cwd) {
|
|
188
|
+
return path.join(journalDir(cwd), 'meta.json');
|
|
189
|
+
}
|
|
190
|
+
function segmentName(firstSeq) {
|
|
191
|
+
return `${SEGMENT_PREFIX}${String(firstSeq).padStart(SEGMENT_PAD, '0')}.jsonl`;
|
|
192
|
+
}
|
|
193
|
+
function listSegments(dir) {
|
|
194
|
+
if (!fs.existsSync(dir))
|
|
195
|
+
return [];
|
|
196
|
+
return fs.readdirSync(dir)
|
|
197
|
+
.filter(f => f.startsWith(SEGMENT_PREFIX) && f.endsWith('.jsonl'))
|
|
198
|
+
.sort();
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Read every valid v2 record across all segments in (segment, file-line)
|
|
202
|
+
* order — the canonical replay order (§2.2: never sorted by seq). Torn or
|
|
203
|
+
* schema-invalid lines are skipped per the §2.6 reader rules. This is the
|
|
204
|
+
* substrate for journal→projection materialization (materialize.ts).
|
|
205
|
+
*/
|
|
206
|
+
export function readJournalRecords(cwd) {
|
|
207
|
+
const dir = journalDir(cwd);
|
|
208
|
+
const records = [];
|
|
209
|
+
for (const seg of listSegments(dir)) {
|
|
210
|
+
const lines = fs.readFileSync(path.join(dir, seg), 'utf-8').split('\n');
|
|
211
|
+
for (const line of lines) {
|
|
212
|
+
if (!line)
|
|
213
|
+
continue;
|
|
214
|
+
const rec = parseRecordLine(line);
|
|
215
|
+
if (rec)
|
|
216
|
+
records.push(rec);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return records;
|
|
220
|
+
}
|
|
221
|
+
function freshMeta() {
|
|
222
|
+
return { next_seq: 1, active_segment: segmentName(1), entity_revs: {} };
|
|
223
|
+
}
|
|
224
|
+
function parseRecordLine(line) {
|
|
225
|
+
try {
|
|
226
|
+
const parsed = JournalRecordSchema.safeParse(JSON.parse(line));
|
|
227
|
+
return parsed.success ? parsed.data : undefined;
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
return undefined;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Rebuild meta from the segment listing + a full scan (§2.3: meta is a
|
|
235
|
+
* cache; the journal is the truth). Scan cost is bounded by retention and
|
|
236
|
+
* paid only on missing/corrupt meta.
|
|
237
|
+
*/
|
|
238
|
+
function rebuildMeta(dir) {
|
|
239
|
+
const segments = listSegments(dir);
|
|
240
|
+
if (segments.length === 0)
|
|
241
|
+
return freshMeta();
|
|
242
|
+
const meta = { next_seq: 1, active_segment: segments[segments.length - 1], entity_revs: {} };
|
|
243
|
+
for (const seg of segments) {
|
|
244
|
+
const lines = fs.readFileSync(path.join(dir, seg), 'utf-8').split('\n').filter(Boolean);
|
|
245
|
+
for (const line of lines) {
|
|
246
|
+
const rec = parseRecordLine(line);
|
|
247
|
+
if (!rec)
|
|
248
|
+
continue;
|
|
249
|
+
if (rec.seq >= meta.next_seq)
|
|
250
|
+
meta.next_seq = rec.seq + 1;
|
|
251
|
+
if (rec.item_id && rec.entity_rev !== undefined) {
|
|
252
|
+
const prev = meta.entity_revs[rec.item_id] ?? 0;
|
|
253
|
+
if (rec.entity_rev > prev)
|
|
254
|
+
meta.entity_revs[rec.item_id] = rec.entity_rev;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return meta;
|
|
259
|
+
}
|
|
260
|
+
function loadOrRebuildMeta(dir) {
|
|
261
|
+
const fp = path.join(dir, 'meta.json');
|
|
262
|
+
if (fs.existsSync(fp)) {
|
|
263
|
+
try {
|
|
264
|
+
const raw = JSON.parse(fs.readFileSync(fp, 'utf-8'));
|
|
265
|
+
if (typeof raw.next_seq === 'number' && raw.next_seq >= 1 && typeof raw.active_segment === 'string') {
|
|
266
|
+
return { next_seq: raw.next_seq, active_segment: raw.active_segment, entity_revs: raw.entity_revs ?? {} };
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
catch { /* fall through to rebuild */ }
|
|
270
|
+
}
|
|
271
|
+
return rebuildMeta(dir);
|
|
272
|
+
}
|
|
273
|
+
function saveMeta(dir, meta) {
|
|
274
|
+
writeFileAtomic(path.join(dir, 'meta.json'), JSON.stringify(meta));
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Highest DURABLE seq present in the journal, cheaply — META-CACHE ONLY (no
|
|
278
|
+
* rebuild). meta.next_seq is published AFTER the append fsync, so next_seq-1 is
|
|
279
|
+
* the last committed record. Returns 0 on absent/corrupt/invalid meta — never
|
|
280
|
+
* falls back to a full segment scan (the whole point: the "should I checkpoint
|
|
281
|
+
* yet?" gate must stay O(1), pln#566 Inc0; codex review MED). Callers that need
|
|
282
|
+
* exact recovery use loadOrRebuildMeta on the writer/status path instead.
|
|
283
|
+
*/
|
|
284
|
+
export function journalHeadSeq(cwd) {
|
|
285
|
+
try {
|
|
286
|
+
const raw = JSON.parse(fs.readFileSync(metaPath(cwd), 'utf-8'));
|
|
287
|
+
if (typeof raw.next_seq === 'number' && Number.isFinite(raw.next_seq) && raw.next_seq >= 1) {
|
|
288
|
+
return raw.next_seq - 1;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
catch { /* absent/corrupt/unreadable meta → 0 (no scan) */ }
|
|
292
|
+
return 0;
|
|
293
|
+
}
|
|
294
|
+
function inspectTail(segPath) {
|
|
295
|
+
if (!fs.existsSync(segPath))
|
|
296
|
+
return { tailSeq: 0 };
|
|
297
|
+
const content = fs.readFileSync(segPath, 'utf-8');
|
|
298
|
+
if (content.length === 0)
|
|
299
|
+
return { tailSeq: 0 };
|
|
300
|
+
const lines = content.split('\n');
|
|
301
|
+
const terminated = content.endsWith('\n');
|
|
302
|
+
let tailSeq = 0;
|
|
303
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
304
|
+
if (!lines[i])
|
|
305
|
+
continue;
|
|
306
|
+
const rec = parseRecordLine(lines[i]);
|
|
307
|
+
if (rec) {
|
|
308
|
+
tailSeq = rec.seq;
|
|
309
|
+
break;
|
|
310
|
+
}
|
|
311
|
+
// keep scanning past trailing garbage to find the last valid record
|
|
312
|
+
}
|
|
313
|
+
const lastNonEmpty = [...lines].reverse().find(l => l.length > 0);
|
|
314
|
+
if (lastNonEmpty && (!terminated || !parseRecordLine(lastNonEmpty))) {
|
|
315
|
+
const fragBytes = Buffer.byteLength(lastNonEmpty, 'utf-8');
|
|
316
|
+
const total = Buffer.byteLength(content, 'utf-8');
|
|
317
|
+
const end = terminated ? total - 1 : total;
|
|
318
|
+
return {
|
|
319
|
+
tailSeq,
|
|
320
|
+
tornFragment: {
|
|
321
|
+
byte_start: end - fragBytes,
|
|
322
|
+
byte_end: end,
|
|
323
|
+
sha256: crypto.createHash('sha256').update(lastNonEmpty).digest('hex'),
|
|
324
|
+
},
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
return { tailSeq };
|
|
328
|
+
}
|
|
329
|
+
function classViolations(record) {
|
|
330
|
+
const cls = ACTION_CLASS_BY_ACTION[record.action];
|
|
331
|
+
const issues = [];
|
|
332
|
+
switch (cls) {
|
|
333
|
+
case 'entity-state':
|
|
334
|
+
if (!record.item_id)
|
|
335
|
+
issues.push('entity-state requires item_id');
|
|
336
|
+
if (record.entity_rev === undefined)
|
|
337
|
+
issues.push('entity-state requires entity_rev');
|
|
338
|
+
if (!record.payload && !record.payload_ref)
|
|
339
|
+
issues.push('entity-state requires payload (post-image)');
|
|
340
|
+
break;
|
|
341
|
+
case 'tombstone':
|
|
342
|
+
if (!record.item_id)
|
|
343
|
+
issues.push('tombstone requires item_id');
|
|
344
|
+
if (record.payload)
|
|
345
|
+
issues.push('tombstone forbids payload');
|
|
346
|
+
break;
|
|
347
|
+
case 'journal-meta':
|
|
348
|
+
if (record.item_type !== 'journal')
|
|
349
|
+
issues.push('journal-meta requires item_type "journal"');
|
|
350
|
+
if (record.item_id)
|
|
351
|
+
issues.push('journal-meta forbids item_id');
|
|
352
|
+
if (!record.payload)
|
|
353
|
+
issues.push('journal-meta requires payload');
|
|
354
|
+
break;
|
|
355
|
+
case 'observability':
|
|
356
|
+
if (record.payload)
|
|
357
|
+
issues.push('observability forbids payload');
|
|
358
|
+
break;
|
|
359
|
+
case 'registry-lifecycle':
|
|
360
|
+
if (!record.item_id)
|
|
361
|
+
issues.push('registry-lifecycle requires item_id');
|
|
362
|
+
// payload OPTIONAL until phase 1.5 (J4) — no rule here in dual.
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
return issues;
|
|
366
|
+
}
|
|
367
|
+
// --- Append path ---
|
|
368
|
+
let violationCount = 0;
|
|
369
|
+
let failureCount = 0;
|
|
370
|
+
export function journalStatus(cwd) {
|
|
371
|
+
const dir = journalDir(cwd);
|
|
372
|
+
const meta = fs.existsSync(dir) ? loadOrRebuildMeta(dir) : freshMeta();
|
|
373
|
+
return {
|
|
374
|
+
mode: resolveJournalMode(cwd),
|
|
375
|
+
next_seq: meta.next_seq,
|
|
376
|
+
segments: listSegments(dir).length,
|
|
377
|
+
violations: violationCount,
|
|
378
|
+
failures: failureCount,
|
|
379
|
+
writer: WRITER_ID,
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* Append a batch of records to the journal. Seq allocation, segment
|
|
384
|
+
* resolution, and the write all happen under the store-wide mutation lock
|
|
385
|
+
* (§2.2: no lockless append path). The lock is reentrant in-process, so
|
|
386
|
+
* calling this from inside persistState/mutateState costs a counter bump.
|
|
387
|
+
*
|
|
388
|
+
* Dual-mode posture: any failure is logged and counted, never thrown —
|
|
389
|
+
* the v1 store is still the source of truth during rehearsal.
|
|
390
|
+
*
|
|
391
|
+
* Returns the appended records (empty when mode=off or on failure).
|
|
392
|
+
*/
|
|
393
|
+
export function appendJournalRecords(inputs, cwd) {
|
|
394
|
+
const mode = resolveJournalMode(cwd);
|
|
395
|
+
if (mode === 'off' || inputs.length === 0)
|
|
396
|
+
return [];
|
|
397
|
+
try {
|
|
398
|
+
return mutate({ cwd: cwd ?? process.cwd() }, (resolvedCwd) => appendLocked(inputs, resolvedCwd));
|
|
399
|
+
}
|
|
400
|
+
catch (err) {
|
|
401
|
+
failureCount += 1;
|
|
402
|
+
logger.warn('journal append failed (dual mode, v1 store unaffected):', err instanceof Error ? err.message : String(err));
|
|
403
|
+
return [];
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Append regardless of the journal mode flag — for explicit operations that
|
|
408
|
+
* seed or repair the journal itself (genesis migration, doctor repair). The
|
|
409
|
+
* mode flag gates the automatic dual-write from mutations, not deliberate
|
|
410
|
+
* journal-authoring tools. Throws on failure (unlike the dual-write path,
|
|
411
|
+
* the operator wants to know a migration write failed).
|
|
412
|
+
*/
|
|
413
|
+
export function forceAppendJournalRecords(inputs, cwd) {
|
|
414
|
+
if (inputs.length === 0)
|
|
415
|
+
return [];
|
|
416
|
+
return mutate({ cwd: cwd ?? process.cwd() }, (resolvedCwd) => appendLocked(inputs, resolvedCwd));
|
|
417
|
+
}
|
|
418
|
+
function appendLocked(inputs, cwd) {
|
|
419
|
+
const dir = journalDir(cwd);
|
|
420
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
421
|
+
const meta = loadOrRebuildMeta(dir);
|
|
422
|
+
let segPath = path.join(dir, meta.active_segment);
|
|
423
|
+
// Tail validation (§2.2): re-derive next_seq from the journal; meta is a cache.
|
|
424
|
+
const tail = inspectTail(segPath);
|
|
425
|
+
const metaWasBehind = tail.tailSeq >= meta.next_seq;
|
|
426
|
+
let nextSeq = Math.max(meta.next_seq, tail.tailSeq + 1);
|
|
427
|
+
const pending = [];
|
|
428
|
+
const stamp = (input) => {
|
|
429
|
+
const cls = ACTION_CLASS_BY_ACTION[input.action];
|
|
430
|
+
const bumpsRev = cls === 'entity-state' || cls === 'tombstone';
|
|
431
|
+
let entityRev;
|
|
432
|
+
if (bumpsRev && input.item_id) {
|
|
433
|
+
entityRev = (meta.entity_revs[input.item_id] ?? 0) + 1;
|
|
434
|
+
meta.entity_revs[input.item_id] = entityRev;
|
|
435
|
+
}
|
|
436
|
+
const record = {
|
|
437
|
+
v: 2,
|
|
438
|
+
seq: nextSeq++,
|
|
439
|
+
ts: input.ts ?? nowISO(),
|
|
440
|
+
writer: WRITER_ID,
|
|
441
|
+
agent: input.agent ?? 'unknown',
|
|
442
|
+
agent_id: input.agent_id,
|
|
443
|
+
session_id: input.session_id ?? (process.env.BRAINCLAW_SESSION_ID?.trim() || undefined),
|
|
444
|
+
user: input.user ?? process.env.USER ?? process.env.USERNAME,
|
|
445
|
+
action: input.action,
|
|
446
|
+
item_type: input.item_type,
|
|
447
|
+
item_id: input.item_id,
|
|
448
|
+
entity_rev: entityRev,
|
|
449
|
+
summary: input.summary,
|
|
450
|
+
payload: input.payload,
|
|
451
|
+
};
|
|
452
|
+
return record;
|
|
453
|
+
};
|
|
454
|
+
if (metaWasBehind) {
|
|
455
|
+
pending.push(stamp({
|
|
456
|
+
action: 'seq_repair',
|
|
457
|
+
item_type: 'journal',
|
|
458
|
+
agent: 'system',
|
|
459
|
+
payload: { meta_next_seq: meta.next_seq, tail_seq: tail.tailSeq, repaired_next_seq: tail.tailSeq + 1 },
|
|
460
|
+
}));
|
|
461
|
+
}
|
|
462
|
+
if (tail.tornFragment) {
|
|
463
|
+
pending.push(stamp({
|
|
464
|
+
action: 'journal_note',
|
|
465
|
+
item_type: 'journal',
|
|
466
|
+
agent: 'system',
|
|
467
|
+
payload: { kind: 'torn_tail_adjudicated', segment: meta.active_segment, ...tail.tornFragment },
|
|
468
|
+
}));
|
|
469
|
+
}
|
|
470
|
+
for (const input of inputs) {
|
|
471
|
+
pending.push(stamp(input));
|
|
472
|
+
}
|
|
473
|
+
for (const record of pending) {
|
|
474
|
+
for (const rule of classViolations(record)) {
|
|
475
|
+
violationCount += 1;
|
|
476
|
+
logger.debug(`journal dual-mode violation seq=${record.seq} ${record.action}: ${rule}`);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
// Segment roll (§2.3) — checkpoint at roll arrives with step 3.
|
|
480
|
+
try {
|
|
481
|
+
if (fs.existsSync(segPath) && fs.statSync(segPath).size >= segmentRollBytes()) {
|
|
482
|
+
meta.active_segment = segmentName(pending[0].seq);
|
|
483
|
+
segPath = path.join(dir, meta.active_segment);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
catch { /* roll check best-effort; appends continue on the current segment */ }
|
|
487
|
+
// Single-buffer framed write (§2.6): leading \n caps torn-write damage
|
|
488
|
+
// at one record; short write throws (caught by the dual-mode boundary).
|
|
489
|
+
const buffer = Buffer.from('\n' + pending.map(r => JSON.stringify(Object.fromEntries(Object.entries(r).filter(([, v]) => v !== undefined)))).join('\n') + '\n', 'utf-8');
|
|
490
|
+
const fd = fs.openSync(segPath, 'a');
|
|
491
|
+
try {
|
|
492
|
+
const written = fs.writeSync(fd, buffer, 0, buffer.length);
|
|
493
|
+
if (written !== buffer.length) {
|
|
494
|
+
throw new Error(`short write: ${written}/${buffer.length} bytes`);
|
|
495
|
+
}
|
|
496
|
+
if (resolveFsyncPolicy(cwd) === 'mutation') {
|
|
497
|
+
fs.fsyncSync(fd);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
finally {
|
|
501
|
+
fs.closeSync(fd);
|
|
502
|
+
}
|
|
503
|
+
meta.next_seq = nextSeq;
|
|
504
|
+
saveMeta(dir, meta);
|
|
505
|
+
return pending;
|
|
506
|
+
}
|
|
507
|
+
//# sourceMappingURL=journal.js.map
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Journal → projection materialization (pln#543 step 3).
|
|
3
|
+
*
|
|
4
|
+
* Rebuilds entity state by replaying the segmented journal's post-images
|
|
5
|
+
* (§2.8 lazy-projection capability). In dual mode this is NOT the hot read
|
|
6
|
+
* path — projections remain the source of truth (§2.7) and reads stay
|
|
7
|
+
* projection-backed. It serves three roles:
|
|
8
|
+
* 1. Verification: `verifyProjectionsAgainstJournal` proves the dual-write
|
|
9
|
+
* is faithful — the gate that must be green before the primary cutover
|
|
10
|
+
* (step 5) can trust the journal as the read substrate.
|
|
11
|
+
* 2. Recovery: rebuild a lost/corrupt projection from the journal.
|
|
12
|
+
* 3. The future primary read path (step 5 flips it on).
|
|
13
|
+
*
|
|
14
|
+
* Replay is strictly in (segment, file-line) order; the later post-image of
|
|
15
|
+
* an item wins wholesale (snapshot semantics, §2.2 reducer). A `delete`
|
|
16
|
+
* tombstone removes the entity; a later `create` revives it.
|
|
17
|
+
*/
|
|
18
|
+
import { ACTION_CLASS_BY_ACTION, readJournalRecords } from './journal.js';
|
|
19
|
+
import { ConstraintSchema, DecisionSchema, TrapSchema, HandoffSchema, PlanItemSchema, } from '../schema.js';
|
|
20
|
+
/**
|
|
21
|
+
* Replay the journal into the live set of entities (latest post-image per
|
|
22
|
+
* id, tombstones removed). Keyed by `${item_type}:${item_id}` so the same
|
|
23
|
+
* id under different families never collides.
|
|
24
|
+
*/
|
|
25
|
+
/**
|
|
26
|
+
* The journal reducer (§2.2): apply records onto a live entity map in
|
|
27
|
+
* (segment, file-line) order — later post-image wins wholesale, tombstone
|
|
28
|
+
* removes. Extracted so the same reducer drives full-journal materialization
|
|
29
|
+
* AND checkpoint+tail replay (pln#566 Inc0) — they can never diverge.
|
|
30
|
+
*/
|
|
31
|
+
export function applyRecordsToLive(records, live) {
|
|
32
|
+
for (const rec of records) {
|
|
33
|
+
if (!rec.item_id)
|
|
34
|
+
continue;
|
|
35
|
+
const cls = ACTION_CLASS_BY_ACTION[rec.action];
|
|
36
|
+
const key = `${rec.item_type}:${rec.item_id}`;
|
|
37
|
+
if (cls === 'tombstone') {
|
|
38
|
+
live.delete(key);
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (cls === 'entity-state' && rec.payload) {
|
|
42
|
+
live.set(key, {
|
|
43
|
+
item_type: rec.item_type,
|
|
44
|
+
item_id: rec.item_id,
|
|
45
|
+
entity_rev: rec.entity_rev,
|
|
46
|
+
payload: rec.payload,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
// observability / registry-lifecycle / journal-meta carry no memory
|
|
50
|
+
// post-image — ignored by state materialization.
|
|
51
|
+
}
|
|
52
|
+
return live;
|
|
53
|
+
}
|
|
54
|
+
export function materializeEntitiesFromJournal(cwd) {
|
|
55
|
+
return applyRecordsToLive(readJournalRecords(cwd), new Map());
|
|
56
|
+
}
|
|
57
|
+
export const MEMORY_FAMILIES = [
|
|
58
|
+
{ itemType: 'constraint', schema: ConstraintSchema, collection: 'active_constraints' },
|
|
59
|
+
{ itemType: 'decision', schema: DecisionSchema, collection: 'recent_decisions' },
|
|
60
|
+
{ itemType: 'trap', schema: TrapSchema, collection: 'known_traps' },
|
|
61
|
+
{ itemType: 'handoff', schema: HandoffSchema, collection: 'open_handoffs' },
|
|
62
|
+
{ itemType: 'plan', schema: PlanItemSchema, collection: 'plan_items' },
|
|
63
|
+
];
|
|
64
|
+
/**
|
|
65
|
+
* Rebuild the 5 memory-class collections of `State` purely from the journal.
|
|
66
|
+
* Payloads failing schema validation are skipped (mirrors loadState's
|
|
67
|
+
* tolerant read) — verify treats them as drift.
|
|
68
|
+
*/
|
|
69
|
+
/**
|
|
70
|
+
* Project a materialized live-entity map into the 5 memory-class collections
|
|
71
|
+
* of `State`. Shared by full-journal materialization and checkpoint+tail
|
|
72
|
+
* replay (pln#566 Inc0) so both produce byte-identical state. Payloads failing
|
|
73
|
+
* schema validation are skipped (mirrors loadState's tolerant read).
|
|
74
|
+
*/
|
|
75
|
+
export function projectLiveToState(live) {
|
|
76
|
+
const state = {
|
|
77
|
+
active_constraints: [], recent_decisions: [], known_traps: [],
|
|
78
|
+
open_handoffs: [], plan_items: [],
|
|
79
|
+
};
|
|
80
|
+
for (const { itemType, schema, collection } of MEMORY_FAMILIES) {
|
|
81
|
+
const items = [];
|
|
82
|
+
for (const entity of live.values()) {
|
|
83
|
+
if (entity.item_type !== itemType)
|
|
84
|
+
continue;
|
|
85
|
+
const parsed = schema.safeParse(entity.payload);
|
|
86
|
+
if (parsed.success)
|
|
87
|
+
items.push(parsed.data);
|
|
88
|
+
}
|
|
89
|
+
items.sort((a, b) => a.created_at.localeCompare(b.created_at));
|
|
90
|
+
state[collection] = items;
|
|
91
|
+
}
|
|
92
|
+
return state;
|
|
93
|
+
}
|
|
94
|
+
export function materializeMemoryStateFromJournal(cwd) {
|
|
95
|
+
return projectLiveToState(materializeEntitiesFromJournal(cwd));
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Journal item_types of the registry / coordination families (pln#568). Their
|
|
99
|
+
* post-images are entity-state records (emitRegistryPostImage) the reducer
|
|
100
|
+
* already upserts, so no reducer change is needed to project them. NOTE
|
|
101
|
+
* action_required journals under item_type `state` (the slot board-projection
|
|
102
|
+
* reserved for it), not `action`.
|
|
103
|
+
*/
|
|
104
|
+
export const REGISTRY_ITEM_TYPES = ['claim', 'assignment', 'agent_run', 'state', 'candidate', 'runtime_note', 'sequence'];
|
|
105
|
+
/**
|
|
106
|
+
* Materialize the registry / coordination families from the journal: the latest
|
|
107
|
+
* post-image per id (tombstones removed), grouped by journal item_type. Drives
|
|
108
|
+
* registry verification (verify.ts) and journal-only recovery of these
|
|
109
|
+
* families. The memory store-marker (`journal_note`, no item_id) never enters
|
|
110
|
+
* the live map, so a `state` group here is purely action_required post-images.
|
|
111
|
+
*/
|
|
112
|
+
export function materializeRegistryFromJournal(cwd) {
|
|
113
|
+
const registry = new Set(REGISTRY_ITEM_TYPES);
|
|
114
|
+
const byType = new Map();
|
|
115
|
+
for (const entity of materializeEntitiesFromJournal(cwd).values()) {
|
|
116
|
+
if (!registry.has(entity.item_type))
|
|
117
|
+
continue;
|
|
118
|
+
const group = byType.get(entity.item_type);
|
|
119
|
+
if (group)
|
|
120
|
+
group.push(entity);
|
|
121
|
+
else
|
|
122
|
+
byType.set(entity.item_type, [entity]);
|
|
123
|
+
}
|
|
124
|
+
return byType;
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=materialize.js.map
|