@awebai/oats 0.61.0 → 0.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +1 -1
  2. package/bin/oats.mjs +27 -12
  3. package/docs/capabilities.md +1 -1
  4. package/docs/desktop-cli-api.md +30 -2
  5. package/docs/execution-targets.md +89 -4
  6. package/docs/implementation.md +2 -1
  7. package/docs/named-test-runs.md +382 -0
  8. package/docs/official-catalog.md +1 -1
  9. package/docs/packages.md +5 -5
  10. package/docs/release-lane.md +13 -0
  11. package/docs/release-notes/v0.62.0.md +223 -0
  12. package/docs/release-notes/v0.63.0.md +135 -0
  13. package/docs/souls-and-instances.md +6 -1
  14. package/docs/upgrading.md +135 -11
  15. package/docs/workspaces.md +1 -1
  16. package/lib/core.mjs +73 -23
  17. package/lib/doctor-capture.mjs +35 -0
  18. package/lib/session-viewer.mjs +86 -7
  19. package/lib/tree-copy.mjs +188 -7
  20. package/package-catalog.json +1 -1
  21. package/package.json +1 -1
  22. package/packages/client/cli-locator.mjs +2 -2
  23. package/packages/record/README.md +87 -13
  24. package/packages/record/bin/capture-hook.mjs +8 -0
  25. package/packages/record/bin/capture.mjs +200 -139
  26. package/packages/record/bin/recall.mjs +8 -3
  27. package/packages/record/bin/setup.mjs +116 -225
  28. package/packages/record/bin/turn-record.mjs +2 -2
  29. package/packages/record/docs/capture-service.md +1527 -28
  30. package/packages/record/docs/capture-status.schema.json +429 -12
  31. package/packages/record/lib/capture-admission.mjs +82 -0
  32. package/packages/record/lib/capture-aw-unit.mjs +137 -0
  33. package/packages/record/lib/capture-aw.mjs +31 -12
  34. package/packages/record/lib/capture-budget.mjs +31 -0
  35. package/packages/record/lib/capture-cc.mjs +39 -6
  36. package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
  37. package/packages/record/lib/capture-cleanup-work.mjs +77 -0
  38. package/packages/record/lib/capture-command-operation.mjs +37 -0
  39. package/packages/record/lib/capture-command-output.mjs +62 -0
  40. package/packages/record/lib/capture-command-worker.mjs +6 -0
  41. package/packages/record/lib/capture-command.mjs +65 -0
  42. package/packages/record/lib/capture-completion.mjs +152 -0
  43. package/packages/record/lib/capture-context.mjs +50 -0
  44. package/packages/record/lib/capture-default-roots.mjs +30 -0
  45. package/packages/record/lib/capture-discovery.mjs +114 -0
  46. package/packages/record/lib/capture-failures.mjs +45 -0
  47. package/packages/record/lib/capture-health-doctor.mjs +13 -0
  48. package/packages/record/lib/capture-health-installation.mjs +117 -0
  49. package/packages/record/lib/capture-health.mjs +116 -0
  50. package/packages/record/lib/capture-hook-ownership.mjs +58 -0
  51. package/packages/record/lib/capture-hook-runner.mjs +17 -0
  52. package/packages/record/lib/capture-hook.mjs +26 -0
  53. package/packages/record/lib/capture-index-inventory.mjs +205 -0
  54. package/packages/record/lib/capture-index-outcomes.mjs +113 -0
  55. package/packages/record/lib/capture-index-reset.mjs +110 -0
  56. package/packages/record/lib/capture-index-work.mjs +77 -0
  57. package/packages/record/lib/capture-issue-preparation.mjs +73 -0
  58. package/packages/record/lib/capture-issues.mjs +15 -2
  59. package/packages/record/lib/capture-lock.mjs +137 -17
  60. package/packages/record/lib/capture-native-unit.mjs +322 -0
  61. package/packages/record/lib/capture-pass.mjs +116 -0
  62. package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
  63. package/packages/record/lib/capture-retained-scratch.mjs +213 -0
  64. package/packages/record/lib/capture-retry.mjs +102 -0
  65. package/packages/record/lib/capture-run.mjs +125 -0
  66. package/packages/record/lib/capture-scratch.mjs +12 -3
  67. package/packages/record/lib/capture-service-definition.mjs +104 -0
  68. package/packages/record/lib/capture-service-install.mjs +362 -0
  69. package/packages/record/lib/capture-service-ownership.mjs +137 -0
  70. package/packages/record/lib/capture-service-runner.mjs +18 -0
  71. package/packages/record/lib/capture-small-snapshot.mjs +85 -0
  72. package/packages/record/lib/capture-source-discovery.mjs +319 -0
  73. package/packages/record/lib/capture-source-queue.mjs +199 -0
  74. package/packages/record/lib/capture-source-work.mjs +85 -0
  75. package/packages/record/lib/capture-state.mjs +16 -71
  76. package/packages/record/lib/capture-status.mjs +171 -0
  77. package/packages/record/lib/capture-supervisor.mjs +136 -0
  78. package/packages/record/lib/capture-watch.mjs +32 -0
  79. package/packages/record/lib/capture-work-session.mjs +241 -0
  80. package/packages/record/lib/capture-worker.mjs +74 -0
  81. package/packages/record/lib/derived-state.mjs +58 -17
  82. package/packages/record/lib/file-lines.mjs +22 -6
  83. package/packages/record/lib/formats.mjs +4 -3
  84. package/packages/record/lib/index-capture.mjs +97 -0
  85. package/packages/record/lib/index-db.mjs +121 -66
  86. package/packages/record/lib/index-reset.mjs +19 -0
  87. package/packages/record/lib/store.mjs +46 -14
  88. package/packages/record/lib/transcript-prefix.mjs +125 -0
  89. package/skills/oats-getting-started/SKILL.md +1 -1
  90. package/docs/macos-test-runs.md +0 -246
@@ -14,14 +14,13 @@
14
14
  import { closeSync, existsSync, mkdirSync, openSync, readSync, rmSync, unlinkSync } from "node:fs";
15
15
  import { join } from "node:path";
16
16
  import { keysetRows } from "./sqlite-rows.mjs";
17
+ import { captureContext } from "./capture-context.mjs";
18
+ import { DerivedState, fileVersion, sameVersion } from "./derived-state.mjs";
19
+ import { validJournalCheckpoint } from "./store.mjs";
20
+ import { INDEX_SCHEMA_VERSION as SCHEMA_VERSION, TEXT_SCHEMA, resetIndexTables } from "./index-reset.mjs";
17
21
  import { createRequire } from "node:module";
18
22
  const require = createRequire(import.meta.url);
19
23
 
20
- // Bump whenever the derived schema changes shape. The index is cache:
21
- // a version mismatch self-heals by deleting the database and rebuilding
22
- // from the record — never by in-place migration.
23
- const SCHEMA_VERSION = 1;
24
-
25
24
  import { extractCcText, extractSessionTextFor } from "./formats.mjs";
26
25
  import { parseOutfit, parseTag } from "./tags.mjs";
27
26
  import { parseSegment, parseSpawn } from "./segments.mjs";
@@ -103,9 +102,7 @@ CREATE TABLE IF NOT EXISTS spawns (
103
102
  ts TEXT NOT NULL
104
103
  );
105
104
  CREATE INDEX IF NOT EXISTS spawns_agent ON spawns(agent_thread);
106
- CREATE VIRTUAL TABLE IF NOT EXISTS turn_text USING fts5(
107
- turn_id UNINDEXED, loc UNINDEXED, role UNINDEXED, text
108
- );
105
+ ${TEXT_SCHEMA}
109
106
  `;
110
107
 
111
108
  export class RecordIndex {
@@ -144,12 +141,22 @@ export class RecordIndex {
144
141
  }
145
142
  this.db.exec(SCHEMA);
146
143
  this.db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
144
+ this.statements = new Map();
147
145
  }
148
146
 
149
147
  close() {
148
+ this.statements.clear();
150
149
  this.db.close();
151
150
  }
152
151
 
152
+ // One prepared statement per SQL text for this connection. Only for
153
+ // statements run to completion (run/get/all), never left iterating.
154
+ statement(sql) {
155
+ let prepared = this.statements.get(sql);
156
+ if (!prepared) { prepared = this.db.prepare(sql); this.statements.set(sql, prepared); }
157
+ return prepared;
158
+ }
159
+
153
160
  recordOwner() {
154
161
  if (this.store.owner) return this.store.owner;
155
162
  const row = this.db.prepare("SELECT value FROM meta WHERE key = 'owner'").get();
@@ -159,35 +166,88 @@ export class RecordIndex {
159
166
  // Old releases keep writing index_state's count. A separate cursor table
160
167
  // leaves that schema/writer compatible; differing counts invalidate our
161
168
  // cursor rather than trusting a byte position left behind by a downgrade.
169
+ //
170
+ // One capture.db connection serves the whole pass. A census without locks
171
+ // proves a stream current only when its complete validation checkpoint
172
+ // names the journal's exact current version and the search cursor agrees
173
+ // with it in bytes, count (both tables) and epoch; such a stream needs no
174
+ // lock and no validation. Every other stream is a candidate and is
175
+ // re-read under its stream lock. The census is not an atomic snapshot of
176
+ // concurrent writers: a change after a stream is observed is indexed by a
177
+ // later pass, as with any sequential pass.
162
178
  update() {
163
- let reset = false;
164
- for (const row of keysetRows(this.db, `SELECT s.stream, s.indexed_count, b.bytes,
179
+ const context = captureContext(this.store);
180
+ const state = context ? context.state : new DerivedState(this.store.root);
181
+ try {
182
+ // Skip evidence is read from this connection: it must still be the file.
183
+ if (!context) state.assertFile();
184
+ const current = this.#reconcile(state, !context);
185
+ this.#catchUp(state, !context, current);
186
+ if (!context) state.assertFile();
187
+ } finally { if (!context) state.close(); }
188
+ }
189
+
190
+ // Indexed streams whose evidence is incomplete are validated under their
191
+ // locks first: if any proves the union stale (missing or replaced journal,
192
+ // new epoch, invalid cursor, legacy count disagreement), the whole union
193
+ // resets before any stream catches up, and no stream is current against
194
+ // the new, empty generation. → the streams proven current, each with the
195
+ // search-cursor row its proof used.
196
+ #reconcile(state, owned) {
197
+ const current = new Map();
198
+ const rows = keysetRows(this.db, `SELECT s.stream, s.indexed_count, b.bytes,
165
199
  b.indexed_count AS byte_count, b.epoch FROM index_state s
166
- LEFT JOIN index_bytes b ON s.stream = b.stream`, "stream")) {
167
- const checkpoint = this.store.journalCheckpoint(row.stream);
200
+ LEFT JOIN index_bytes b ON s.stream = b.stream`, "stream");
201
+ for (const row of rows) {
202
+ const path = this.store.journalPath(row.stream);
203
+ // Inside a capture work group every stream still goes through the
204
+ // group's membership check: only a pass owning its connection skips.
205
+ if (owned && proven(row, state.get("journal", row.stream), fileVersion(path))) { current.set(row.stream, row); continue; }
206
+ const checkpoint = this.store.withStreamLock(row.stream, () => {
207
+ if (owned) state.assertFile();
208
+ return this.store.validateJournal(row.stream, state);
209
+ });
168
210
  let boundary = true;
169
211
  if (Number.isSafeInteger(row.bytes) && row.bytes > 0 && row.bytes < checkpoint.bytes) {
170
- const fd = openSync(this.store.journalPath(row.stream), "r"), byte = Buffer.alloc(1);
212
+ const fd = openSync(path, "r"), byte = Buffer.alloc(1);
171
213
  try { boundary = readSync(fd, byte, 0, 1, row.bytes - 1) === 1 && byte[0] === 10; }
172
214
  finally { closeSync(fd); }
173
215
  }
174
216
  if (!Number.isSafeInteger(row.bytes) || row.bytes < 0 || !boundary || row.indexed_count !== row.byte_count ||
175
217
  row.epoch !== checkpoint.epoch || row.bytes > checkpoint.bytes || row.byte_count > checkpoint.count) {
176
- reset = true;
177
- break;
218
+ // Deleting a stream or changing its history can change tombstone authority
219
+ // and cross-stream dedupe. Rebuild the derived union, not only its FTS rows.
220
+ this.reset();
221
+ return new Map();
178
222
  }
179
223
  }
180
- // Deleting a stream or changing its history can change tombstone authority
181
- // and cross-stream dedupe. Rebuild the derived union, not only its FTS rows.
182
- if (reset) this.reset();
224
+ return current;
225
+ }
226
+
227
+ // Index the new bytes of every live stream not proven current, each under
228
+ // its stream lock, in bounded transactions that commit rows and cursors
229
+ // together. No transaction is open while the next stream lock is taken.
230
+ // A proof holds only for the cursor it read: another index connection may
231
+ // have reset the union or moved the cursor since, so the cursor is read
232
+ // again when the stream's turn comes and a changed one makes it a candidate.
233
+ #catchUp(state, owned, current) {
183
234
  const owner = this.store.owner;
184
235
  if (owner) this.db.prepare("INSERT INTO meta (key, value) VALUES ('owner', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(owner);
185
236
  const effectiveOwner = this.recordOwner();
186
- const getState = this.db.prepare("SELECT * FROM index_bytes WHERE stream = ?");
187
- const setCount = this.db.prepare("INSERT INTO index_state VALUES (?, ?) ON CONFLICT(stream) DO UPDATE SET indexed_count=excluded.indexed_count");
188
- const setBytes = this.db.prepare("INSERT INTO index_bytes VALUES (?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET bytes=excluded.bytes, indexed_count=excluded.indexed_count, epoch=excluded.epoch");
237
+ const getState = this.statement("SELECT * FROM index_bytes WHERE stream = ?");
238
+ const setCount = this.statement("INSERT INTO index_state VALUES (?, ?) ON CONFLICT(stream) DO UPDATE SET indexed_count=excluded.indexed_count");
239
+ const setBytes = this.statement("INSERT INTO index_bytes VALUES (?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET bytes=excluded.bytes, indexed_count=excluded.indexed_count, epoch=excluded.epoch");
240
+ const cursor = this.statement(`SELECT s.indexed_count, b.bytes, b.indexed_count AS byte_count, b.epoch FROM index_state s
241
+ LEFT JOIN index_bytes b ON s.stream = b.stream WHERE s.stream = ?`);
189
242
  for (const streamId of this.store.iterateStreams()) {
190
- this.store.withStreamLock(streamId, () => this.store.withDerived(state => {
243
+ const proof = current.get(streamId);
244
+ if (proof) {
245
+ const now = cursor.get(streamId);
246
+ if (now?.indexed_count === proof.indexed_count && now.bytes === proof.bytes &&
247
+ now.byte_count === proof.byte_count && now.epoch === proof.epoch) continue;
248
+ }
249
+ this.store.withStreamLock(streamId, () => {
250
+ if (owned) state.assertFile();
191
251
  const checkpoint = this.store.validateJournal(streamId, state);
192
252
  const row = getState.get(streamId);
193
253
  const previous = row?.stream != null ? row : null;
@@ -214,23 +274,16 @@ export class RecordIndex {
214
274
  } catch (err) { this.db.exec("ROLLBACK"); throw err; }
215
275
  }
216
276
  } finally { iterator.return(); }
217
- }));
277
+ });
218
278
  }
219
279
  }
220
280
 
221
281
  // Full rebuild = reset + update from zero. Same code path as update().
222
282
  reset() {
283
+ this.statements.clear();
223
284
  this.db.exec("BEGIN");
224
285
  try {
225
- this.db.exec("DELETE FROM turns");
226
- this.db.exec("DELETE FROM turn_text");
227
- this.db.exec("DELETE FROM tombstones");
228
- this.db.exec("DELETE FROM tags");
229
- this.db.exec("DELETE FROM outfits");
230
- this.db.exec("DELETE FROM segments");
231
- this.db.exec("DELETE FROM spawns");
232
- this.db.exec("DELETE FROM index_state");
233
- this.db.exec("DELETE FROM index_bytes");
286
+ resetIndexTables(this.db);
234
287
  this.db.exec("COMMIT");
235
288
  } catch (err) {
236
289
  this.db.exec("ROLLBACK");
@@ -245,20 +298,20 @@ export class RecordIndex {
245
298
  // streams: first seen wins.
246
299
  addTurn(turn, streamId, owner) {
247
300
  if (typeof turn.id !== "string") return;
248
- const existing = this.db.prepare("SELECT stream FROM turns WHERE id = ?").get(turn.id);
301
+ const existing = this.statement("SELECT stream FROM turns WHERE id = ?").get(turn.id);
249
302
  if (existing) {
250
303
  // Duplicate id across streams. Keep attribution deterministic and in
251
304
  // agreement with RecordStore.readAll(): alphabetically first stream,
252
305
  // regardless of indexing order.
253
306
  if (streamId < existing.stream) {
254
- this.db.prepare("UPDATE turns SET stream = ? WHERE id = ?").run(streamId, turn.id);
307
+ this.statement("UPDATE turns SET stream = ? WHERE id = ?").run(streamId, turn.id);
255
308
  }
256
309
  return;
257
310
  }
258
311
 
259
312
  const fromName = String(turn.from ?? "");
260
313
  // Hidden if a valid tombstone already arrived for this id.
261
- const hidden = this.db.prepare("SELECT 1 AS found FROM tombstones WHERE ref = ? AND (from_name = ? OR from_name = ?) LIMIT 1")
314
+ const hidden = this.statement("SELECT 1 AS found FROM tombstones WHERE ref = ? AND (from_name = ? OR from_name = ?) LIMIT 1")
262
315
  .get(turn.id, fromName, owner ?? fromName)?.found === 1 ? 1 : 0;
263
316
 
264
317
  // Sessions are one turn per native event (no snapshots, nothing to
@@ -266,11 +319,10 @@ export class RecordIndex {
266
319
  const events = 0;
267
320
  const superseded = 0;
268
321
 
269
- this.db
270
- .prepare(
271
- `INSERT INTO turns (id, ts, from_name, to_name, thread, kind, source, fidelity, stream, events, hidden, superseded)
272
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
273
- )
322
+ this.statement(
323
+ `INSERT INTO turns (id, ts, from_name, to_name, thread, kind, source, fidelity, stream, events, hidden, superseded)
324
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
325
+ )
274
326
  .run(
275
327
  turn.id,
276
328
  String(turn.ts ?? ""),
@@ -289,24 +341,22 @@ export class RecordIndex {
289
341
  if (turn.kind === "tombstone") {
290
342
  for (const link of turn.links ?? []) {
291
343
  if (link.rel !== "tombstones" || typeof link.ref !== "string") continue;
292
- this.db
293
- .prepare("INSERT OR IGNORE INTO tombstones (ref, from_name) VALUES (?, ?)")
344
+ this.statement("INSERT OR IGNORE INTO tombstones (ref, from_name) VALUES (?, ?)")
294
345
  .run(link.ref, fromName);
295
346
  // Apply to a target that arrived first.
296
- const target = this.db
297
- .prepare("SELECT id, from_name FROM turns WHERE id = ? AND hidden = 0")
347
+ const target = this.statement("SELECT id, from_name FROM turns WHERE id = ? AND hidden = 0")
298
348
  .get(link.ref);
299
349
  if (target && (target.from_name === fromName || (owner && fromName === owner))) {
300
- this.db.prepare("UPDATE turns SET hidden = 1 WHERE id = ?").run(target.id);
301
- this.db.prepare("DELETE FROM turn_text WHERE turn_id = ?").run(target.id);
302
- this.db.prepare("DELETE FROM segments WHERE note_id = ?").run(target.id);
303
- this.db.prepare("DELETE FROM spawns WHERE note_id = ?").run(target.id);
350
+ this.statement("UPDATE turns SET hidden = 1 WHERE id = ?").run(target.id);
351
+ this.statement("DELETE FROM turn_text WHERE turn_id = ?").run(target.id);
352
+ this.statement("DELETE FROM segments WHERE note_id = ?").run(target.id);
353
+ this.statement("DELETE FROM spawns WHERE note_id = ?").run(target.id);
304
354
  }
305
355
  }
306
356
  }
307
357
 
308
358
  if (!hidden && !superseded) {
309
- const insText = this.db.prepare(
359
+ const insText = this.statement(
310
360
  "INSERT INTO turn_text (turn_id, loc, role, text) VALUES (?, ?, ?, ?)",
311
361
  );
312
362
  for (const doc of this.extractText(turn)) {
@@ -337,26 +387,23 @@ export class RecordIndex {
337
387
  }
338
388
  }
339
389
  if (tag) {
340
- this.db
341
- .prepare(
342
- "INSERT OR IGNORE INTO tags (tag_id, target_ref, about, acts, case_slug, note) VALUES (?, ?, ?, ?, ?, ?)",
343
- )
390
+ this.statement(
391
+ "INSERT OR IGNORE INTO tags (tag_id, target_ref, about, acts, case_slug, note) VALUES (?, ?, ?, ?, ?, ?)",
392
+ )
344
393
  .run(turn.id, tag.target, JSON.stringify(tag.about), JSON.stringify(tag.acts), tag.caseSlug, tag.note);
345
394
  }
346
395
  const outfit = outfitParsed;
347
396
  if (outfit) {
348
- this.db
349
- .prepare("INSERT OR IGNORE INTO outfits (outfit_id, task, status, members) VALUES (?, ?, ?, ?)")
397
+ this.statement("INSERT OR IGNORE INTO outfits (outfit_id, task, status, members) VALUES (?, ?, ?, ?)")
350
398
  .run(turn.id, outfit.task, outfit.status, JSON.stringify(outfit.members));
351
399
  }
352
400
  if (segParsed && !hidden) {
353
401
  const locN = (r) => Number(/line:(\d+)/.exec(r ?? "")?.[1] ?? null);
354
402
  const tsMs = Date.parse(String(turn.ts ?? ""));
355
- this.db
356
- .prepare(
357
- `INSERT OR IGNORE INTO segments (note_id, thread, start_n, end_n, type, outcome, about, established, lesson, ts, ts_ms)
358
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
359
- )
403
+ this.statement(
404
+ `INSERT OR IGNORE INTO segments (note_id, thread, start_n, end_n, type, outcome, about, established, lesson, ts, ts_ms)
405
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
406
+ )
360
407
  .run(
361
408
  turn.id,
362
409
  segParsed.thread,
@@ -371,15 +418,13 @@ export class RecordIndex {
371
418
  Number.isFinite(tsMs) ? tsMs : 0,
372
419
  );
373
420
  // Full established+lesson searchable under role "segment".
374
- this.db
375
- .prepare("INSERT INTO turn_text (turn_id, loc, role, text) VALUES (?, ?, ?, ?)")
421
+ this.statement("INSERT INTO turn_text (turn_id, loc, role, text) VALUES (?, ?, ?, ?)")
376
422
  .run(turn.id, "", "segment", segParsed.established + (segParsed.lesson ? "\n" + segParsed.lesson : ""));
377
423
  }
378
424
  if (spawnParsed && !hidden) {
379
- this.db
380
- .prepare(
381
- "INSERT OR IGNORE INTO spawns (note_id, agent_thread, outfit_id, task, harness, grant_ref, ts) VALUES (?, ?, ?, ?, ?, ?, ?)",
382
- )
425
+ this.statement(
426
+ "INSERT OR IGNORE INTO spawns (note_id, agent_thread, outfit_id, task, harness, grant_ref, ts) VALUES (?, ?, ?, ?, ?, ?, ?)",
427
+ )
383
428
  .run(turn.id, spawnParsed.agent, spawnParsed.outfit, spawnParsed.task, spawnParsed.harness, spawnParsed.grant, String(turn.ts ?? ""));
384
429
  }
385
430
  }
@@ -552,6 +597,16 @@ export function extractTurnText(turn) {
552
597
  // per-format extractors live in formats.mjs.
553
598
  export const extractSessionText = extractCcText;
554
599
 
600
+ // A stream whose search cursor is complete against a complete validation of
601
+ // the journal's exact current version. Anything missing, malformed, partial
602
+ // or disagreeing is unproven, never a reason to skip.
603
+ function proven(row, checkpoint, version) {
604
+ return version !== null && validJournalCheckpoint(checkpoint, version.size) && checkpoint.complete && !checkpoint.resetting &&
605
+ sameVersion(checkpoint.version, version) && row.epoch === checkpoint.epoch &&
606
+ Number.isSafeInteger(row.bytes) && row.bytes === checkpoint.bytes &&
607
+ row.byte_count === row.indexed_count && row.byte_count === checkpoint.count;
608
+ }
609
+
555
610
  export function dropIndex(store) {
556
611
  rmSync(join(store.root, "index"), { recursive: true, force: true });
557
612
  }
@@ -0,0 +1,19 @@
1
+ // Shared atomic-reset mutations. Caller owns BEGIN/COMMIT/ROLLBACK so a private
2
+ // command receipt can join the reset without changing public reset semantics.
3
+ // Bump on derived schema shape changes. RecordIndex rebuilds a mismatched
4
+ // cache from journals instead of migrating it in place.
5
+ export const INDEX_SCHEMA_VERSION = 1;
6
+ export const TEXT_SCHEMA = `CREATE VIRTUAL TABLE IF NOT EXISTS turn_text USING fts5(
7
+ turn_id UNINDEXED, loc UNINDEXED, role UNINDEXED, text
8
+ );`;
9
+
10
+ export function resetIndexTables(db) {
11
+ db.exec("DELETE FROM turns");
12
+ // Ordinary tables use truncate; FTS recreation avoids visiting every text
13
+ // row. Readers still see either the previous union or the committed empty one.
14
+ db.exec("DROP TABLE turn_text");
15
+ db.exec(TEXT_SCHEMA);
16
+ for (const table of ["tombstones", "tags", "outfits", "segments", "spawns", "index_state", "index_bytes"])
17
+ db.exec(`DELETE FROM ${table}`);
18
+ db.exec(`PRAGMA user_version = ${INDEX_SCHEMA_VERSION}`);
19
+ }
@@ -33,6 +33,7 @@ import {
33
33
  writeFileSync,
34
34
  } from "node:fs";
35
35
  import { DerivedState, fileVersion, sameVersion, invalidateLegacyMaps } from "./derived-state.mjs";
36
+ import { captureContext } from "./capture-context.mjs";
36
37
  import { randomBytes } from "node:crypto";
37
38
  import { hostname } from "node:os";
38
39
  import { join } from "node:path";
@@ -315,6 +316,16 @@ function describeHolder(held) {
315
316
  return `held by pid ${held.token.pid} on ${held.token.host} since ${held.token.acquiredAt}`;
316
317
  }
317
318
 
319
+ /** A journal validation checkpoint whose fields can describe a journal of
320
+ * `size` bytes. Shape only: whether it is complete and still names the
321
+ * current file version is the caller's question. */
322
+ export function validJournalCheckpoint(checkpoint, size) {
323
+ return Boolean(checkpoint) && Number.isSafeInteger(checkpoint.bytes) && checkpoint.bytes >= 0 && checkpoint.bytes <= size &&
324
+ Number.isSafeInteger(checkpoint.count) && checkpoint.count >= 0 && checkpoint.count <= checkpoint.bytes &&
325
+ typeof checkpoint.epoch === "string" && typeof checkpoint.complete === "boolean" &&
326
+ (checkpoint.resetting === undefined || typeof checkpoint.resetting === "boolean") && !(checkpoint.resetting && checkpoint.complete);
327
+ }
328
+
318
329
  export class RecordStore {
319
330
  constructor(root, { owner, lockTimeoutMs = 10000, lockStaleMs = 30000 } = {}) {
320
331
  if (!root) throw new StoreError("record root is required");
@@ -338,7 +349,13 @@ export class RecordStore {
338
349
  // concurrent writer's already-fsynced turn. Exclusive-create lockfile
339
350
  // carrying an owner token; see the stream-lock helpers above for how
340
351
  // staleness is proven and why release is ownership-checked.
341
- withStreamLock(streamId, fn) {
352
+ withStreamLock(streamId, fn, { check = () => {} } = {}) {
353
+ check();
354
+ const context = captureContext(this);
355
+ if (context) {
356
+ if (!context.held.has(streamId)) throw new StoreError(`stream outside capture work group: ${streamId}`);
357
+ return fn();
358
+ }
342
359
  const dir = join(this.streamsDir(), streamId);
343
360
  mkdirSync(dir, { recursive: true });
344
361
  const lockPath = join(dir, ".lock");
@@ -354,6 +371,7 @@ export class RecordStore {
354
371
  // buy ~25ms on the crash-recovery path and reopen that hole; it is not
355
372
  // worth it.
356
373
  for (;;) {
374
+ check();
357
375
  if (createLock(lockPath, dir, token)) break;
358
376
  const held = readLock(lockPath); // throws on anything but ENOENT
359
377
  const reclaimError =
@@ -424,13 +442,13 @@ export class RecordStore {
424
442
 
425
443
  // Absolute byte boundaries, including blank lines, permit durable seeks.
426
444
  // The final fragment never advances a cursor: it is the owner's torn tail.
427
- *iterateJournal(streamId, { start = 0, end = Infinity } = {}) {
445
+ *iterateJournal(streamId, { start = 0, end = Infinity, check = () => {} } = {}) {
428
446
  let fd;
429
447
  try { fd = openSync(this.journalPath(streamId), "r"); }
430
448
  catch (e) { if (e.code === "ENOENT") return; throw e; }
431
449
  let position = start;
432
450
  try {
433
- for (const line of bufferLines(fileChunks(fd, { start, end, size: 65536 }))) {
451
+ for (const line of bufferLines(fileChunks(fd, { start, end, size: 65536, check }))) {
434
452
  if (line[line.length - 1] !== 10) return;
435
453
  let parsed;
436
454
  try { parsed = parseJournal(line); }
@@ -445,6 +463,8 @@ export class RecordStore {
445
463
  }
446
464
 
447
465
  withDerived(fn) {
466
+ const context = captureContext(this);
467
+ if (context) return fn(context.state);
448
468
  const state = new DerivedState(this.root);
449
469
  try { return fn(state); } finally { state.close(); }
450
470
  }
@@ -453,28 +473,39 @@ export class RecordStore {
453
473
  // our fsynced append can skip history. Unknown writes (including old releases),
454
474
  // replacement and shrink start a new epoch and invalidate dependent cursors.
455
475
  // Partial validation is resumable, but cannot authorize an append.
456
- validateJournal(streamId, state) {
476
+ validateJournal(streamId, state, { maxRecords = Infinity, maxBytes = Infinity, stopped = () => false, work } = {}) {
457
477
  const path = this.journalPath(streamId), version = fileVersion(path);
458
478
  let checkpoint = state.get("journal", streamId);
459
- if (checkpoint && (!Number.isSafeInteger(checkpoint.bytes) || checkpoint.bytes < 0 || checkpoint.bytes > (version?.size ?? 0) ||
460
- !Number.isSafeInteger(checkpoint.count) || checkpoint.count < 0 || checkpoint.count > checkpoint.bytes ||
461
- typeof checkpoint.epoch !== "string" || typeof checkpoint.complete !== "boolean")) checkpoint = null;
479
+ if (checkpoint && !validJournalCheckpoint(checkpoint, version?.size ?? 0)) checkpoint = null;
462
480
  if (checkpoint && sameVersion(checkpoint.version, version) && checkpoint.complete) return checkpoint;
463
481
  if (!checkpoint || !sameVersion(checkpoint.version, version)) {
464
- checkpoint = { version, epoch: randomBytes(16).toString("hex"), bytes: 0, count: 0, complete: false };
482
+ checkpoint = { version, epoch: randomBytes(16).toString("hex"), bytes: 0, count: 0, complete: false, resetting: true };
483
+ state.transaction(() => state.set("journal", streamId, checkpoint));
484
+ }
485
+ let recordsUsed = 0, bytesUsed = 0;
486
+ // Clearing a large obsolete lookup is recovery work too. Its durable
487
+ // phase marker prevents a deadline stop from restarting that deletion.
488
+ while (checkpoint.resetting) {
489
+ if (recordsUsed >= maxRecords || stopped()) return checkpoint;
465
490
  state.transaction(() => {
466
- state.db.prepare("DELETE FROM journal_ids WHERE stream = ?").run(streamId);
467
- state.set("journal", streamId, checkpoint);
491
+ const limit = Math.min(2000, maxRecords - recordsUsed);
492
+ const removed = Number(state.statement(`DELETE FROM journal_ids WHERE rowid IN
493
+ (SELECT rowid FROM journal_ids WHERE stream = ? LIMIT ?)`).run(streamId, limit).changes);
494
+ recordsUsed += removed;
495
+ if (work) work.deletedIds = (work.deletedIds ?? 0) + removed;
496
+ if (removed < limit) { delete checkpoint.resetting; state.set("journal", streamId, checkpoint); }
468
497
  });
469
498
  }
470
- const addId = state.db.prepare("INSERT OR IGNORE INTO journal_ids VALUES (?, ?)");
471
- const iterator = this.iterateJournal(streamId, { start: checkpoint.bytes, end: version?.size ?? 0 });
499
+ const addId = state.statement("INSERT OR IGNORE INTO journal_ids VALUES (?, ?)");
500
+ const iterator = this.iterateJournal(streamId, { start: checkpoint.bytes, end: version?.size ?? 0,
501
+ check: () => { stopped(); } });
472
502
  let done = false;
473
503
  try {
474
504
  while (!done) {
505
+ if (recordsUsed >= maxRecords || bytesUsed >= maxBytes || stopped()) return checkpoint;
475
506
  state.transaction(() => {
476
507
  let batch = 0, bytes = 0;
477
- while (batch < 2000 && bytes < 8 * 1024 * 1024) {
508
+ while (batch < 2000 && bytes < 8 * 1024 * 1024 && recordsUsed < maxRecords && bytesUsed < maxBytes && !stopped()) {
478
509
  const next = iterator.next();
479
510
  if (next.done) { done = true; break; }
480
511
  const entry = next.value;
@@ -483,7 +514,8 @@ export class RecordStore {
483
514
  if (typeof entry.turn.id === "string") addId.run(streamId, entry.turn.id);
484
515
  }
485
516
  checkpoint.bytes = entry.end;
486
- bytes += entry.bytes; batch++;
517
+ bytes += entry.bytes; bytesUsed += entry.bytes; batch++; recordsUsed++;
518
+ if (work) { work.records = (work.records ?? 0) + 1; work.bytes = (work.bytes ?? 0) + entry.bytes; }
487
519
  }
488
520
  if (!sameVersion(version, fileVersion(path))) throw new StoreError(`journal changed during validation: ${path}`);
489
521
  checkpoint.complete = done;
@@ -1,6 +1,9 @@
1
1
  // Recovery proof for positional transcript journals. The temporary lookup is
2
2
  // disk-backed: comparing a long prefix retains one native record at a time.
3
3
  import { digest } from "./session-snapshot.mjs";
4
+ import { randomUUID } from "node:crypto";
5
+ import { captureContext } from "./capture-context.mjs";
6
+ import { fileVersion, sameVersion } from "./derived-state.mjs";
4
7
 
5
8
  export function compareTranscriptPrefix(store, stream, state, lines) {
6
9
  const db = state.db;
@@ -38,3 +41,125 @@ export function compareTranscriptPrefix(store, stream, state, lines) {
38
41
  if (line < lastLine) return { conflict: { line: line + 1, offset, reason: "captured-prefix-missing" } };
39
42
  return { bytes: offset, line, lastTs };
40
43
  }
44
+
45
+ // Command-only recovery proof. The caller supplies complete native lines from
46
+ // an immutable verified staged snapshot, starting at the requested byte. A
47
+ // retained allocation or a completely revalidated digest/size binds its id. This helper neither opens the
48
+ // mutable source nor transfers scratch ownership; those belong to the worker.
49
+ // No partial phase authorizes append. Checkpoints and hashes join the caller's
50
+ // existing stream-locked processing transaction, with no per-record commits.
51
+ export function advanceTranscriptPrefix(store, stream, { key, snapshot, lines, maxRecords = 2000,
52
+ maxBytes = 8 * 1024 * 1024, stopped = () => false, work } = {}) {
53
+ const context = captureContext(store), state = context?.state;
54
+ if (!state?.depth || !context.held.has(stream)) throw new Error("resumable prefix proof requires its stream work group");
55
+ if (typeof key !== "string" || !key || typeof snapshot?.id !== "string" || !snapshot.id ||
56
+ !Number.isSafeInteger(snapshot.size) || snapshot.size < 0 || typeof lines !== "function" ||
57
+ !Number.isSafeInteger(maxRecords) || maxRecords < 1 || maxRecords > 2000 ||
58
+ !Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > 8 * 1024 * 1024) {
59
+ throw new Error("invalid resumable prefix proof");
60
+ }
61
+ const db = state.db;
62
+ if (db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='transcript_prefix'").get()?.found !== 1) {
63
+ db.exec("CREATE TABLE transcript_prefix (job TEXT, line INTEGER, hash TEXT NOT NULL, PRIMARY KEY(job,line))");
64
+ // A recreated hash table must not qualify checkpoints whose rows vanished.
65
+ state.set("prefix-meta", "epoch", randomUUID());
66
+ }
67
+ let epoch = state.get("prefix-meta", "epoch");
68
+ if (epoch === null) { epoch = randomUUID(); state.set("prefix-meta", "epoch", epoch); }
69
+ if (typeof epoch !== "string" || !epoch) throw new Error("invalid transcript prefix cache epoch");
70
+ const journalVersion = fileVersion(store.journalPath(stream));
71
+ let proof = state.get("prefix-proof", key);
72
+ if (!proof || proof.epoch !== epoch || proof.stream !== stream || proof.snapshotId !== snapshot.id ||
73
+ proof.snapshotSize !== snapshot.size || !sameVersion(proof.journalVersion, journalVersion)) {
74
+ proof = { epoch, stream, snapshotId: snapshot.id, snapshotSize: snapshot.size, journalVersion,
75
+ phase: "clear", journalBytes: 0, lastLine: 0, lastTs: "", bytes: 0, line: 0 };
76
+ }
77
+ if (!["clear", "journal", "source", "ready", "conflict"].includes(proof.phase) ||
78
+ !["journalBytes", "lastLine", "bytes", "line"].every(k => Number.isSafeInteger(proof[k]) && proof[k] >= 0) ||
79
+ proof.journalBytes > (journalVersion?.size ?? 0) || proof.bytes > snapshot.size ||
80
+ proof.line > proof.lastLine || typeof proof.lastTs !== "string") throw new Error("invalid transcript prefix checkpoint");
81
+ const publish = () => {
82
+ if (!sameVersion(journalVersion, fileVersion(store.journalPath(stream)))) throw new Error("journal changed during prefix proof");
83
+ state.set("prefix-proof", key, proof);
84
+ return { ...proof, complete: proof.phase === "ready" };
85
+ };
86
+ let records = 0, bytes = 0;
87
+ const stop = () => records >= maxRecords || bytes >= maxBytes || stopped();
88
+ if (proof.phase === "clear") {
89
+ if (stop()) return publish();
90
+ const removed = Number(db.prepare(`DELETE FROM transcript_prefix WHERE rowid IN
91
+ (SELECT rowid FROM transcript_prefix WHERE job=? LIMIT ?)`).run(key, maxRecords).changes);
92
+ records += removed;
93
+ if (work) work.deletedHashes = (work.deletedHashes ?? 0) + removed;
94
+ if (removed < maxRecords) proof.phase = "journal";
95
+ }
96
+ if (proof.phase === "journal" && !stop()) {
97
+ const put = db.prepare("INSERT OR IGNORE INTO transcript_prefix VALUES (?, ?, ?)");
98
+ const get = db.prepare("SELECT hash FROM transcript_prefix WHERE job=? AND line=?");
99
+ const iterator = store.iterateJournal(stream, { start: proof.journalBytes, end: journalVersion?.size ?? 0,
100
+ check: () => { stopped(); } });
101
+ try {
102
+ while (!stop()) {
103
+ const entry = iterator.next();
104
+ if (entry.done) { proof.phase = proof.lastLine ? "source" : "ready"; break; }
105
+ const { turn, end, bytes: used } = entry.value;
106
+ if (turn?.kind === "session") {
107
+ const line = turn.provenance?.origin?.line;
108
+ if (!Number.isSafeInteger(line) || line < 1 || typeof turn.body?.line !== "string") throw new Error(`invalid positional transcript journal: ${stream}`);
109
+ const hash = digest(Buffer.from(turn.body.line)), prior = get.get(key, line);
110
+ if (prior?.hash != null && prior.hash !== hash) throw new Error(`conflicting positional transcript journal: ${stream}, line ${line}`);
111
+ put.run(key, line, hash);
112
+ if (line >= proof.lastLine) { proof.lastLine = line; proof.lastTs = turn.ts; }
113
+ }
114
+ proof.journalBytes = end; records++; bytes += used;
115
+ if (work) { work.journalRecords = (work.journalRecords ?? 0) + 1; work.journalBytes = (work.journalBytes ?? 0) + used; }
116
+ }
117
+ } finally { iterator.return(); }
118
+ }
119
+ if (proof.phase === "source" && !stop()) {
120
+ const get = db.prepare("SELECT hash FROM transcript_prefix WHERE job=? AND line=?");
121
+ const iterator = lines(proof.bytes)[Symbol.iterator]();
122
+ try {
123
+ while (!stop()) {
124
+ const next = iterator.next();
125
+ if (next.done) {
126
+ proof.phase = "conflict";
127
+ proof.conflict = { line: proof.line + 1, offset: proof.bytes, reason: "captured-prefix-missing" };
128
+ break;
129
+ }
130
+ const record = next.value;
131
+ // A line arrives as its streamed digest, or as text a caller decoded.
132
+ const hash = typeof record.hash === "string" ? record.hash : typeof record.text === "string" ? digest(Buffer.from(record.text)) : null;
133
+ if (hash === null || !Number.isSafeInteger(record.bytes) || record.bytes < 1 ||
134
+ proof.bytes + record.bytes > snapshot.size) throw new Error("invalid staged prefix line");
135
+ const expected = get.get(key, proof.line + 1);
136
+ records++; bytes += record.bytes;
137
+ if (work) { work.sourceRecords = (work.sourceRecords ?? 0) + 1; work.sourceBytes = (work.sourceBytes ?? 0) + record.bytes; }
138
+ if (expected?.hash != null && expected.hash !== hash) {
139
+ proof.phase = "conflict";
140
+ proof.conflict = { line: proof.line + 1, offset: proof.bytes, reason: "captured-prefix-changed" };
141
+ break;
142
+ }
143
+ proof.bytes += record.bytes; proof.line++;
144
+ if (proof.line === proof.lastLine) { proof.phase = "ready"; break; }
145
+ }
146
+ } finally { iterator.return?.(); }
147
+ }
148
+ return publish();
149
+ }
150
+
151
+ // Reclaim completed/invalidated proof rows in bounded units. Revocation and
152
+ // deletion share the processing commit, so interruption cannot leave a ready
153
+ // checkpoint qualifying a partially removed hash table.
154
+ export function discardTranscriptPrefix(store, stream, key, { maxRecords = 2000 } = {}) {
155
+ const context = captureContext(store), state = context?.state;
156
+ if (!state?.depth || !context.held.has(stream) || !Number.isSafeInteger(maxRecords) || maxRecords < 1 || maxRecords > 2000) {
157
+ throw new Error("prefix cleanup requires a bounded stream work group");
158
+ }
159
+ const proof = state.get("prefix-proof", key);
160
+ if (proof && proof.stream !== stream) throw new Error("prefix cleanup stream differs");
161
+ state.delete("prefix-proof", key);
162
+ if (state.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='transcript_prefix'").get()?.found !== 1) return true;
163
+ return Number(state.db.prepare(`DELETE FROM transcript_prefix WHERE rowid IN
164
+ (SELECT rowid FROM transcript_prefix WHERE job=? LIMIT ?)`).run(key, maxRecords).changes) < maxRecords;
165
+ }
@@ -60,7 +60,7 @@ members:
60
60
  - git:github.com/acme/agents # the host is a member too
61
61
  - git:github.com/acme/platform
62
62
  packages:
63
- oats.framework: v1.13.0 # bare versions resolve through the official catalog
63
+ oats.framework: v1.13.2 # bare versions resolve through the official catalog
64
64
  oats.okf: v5.2.1
65
65
  defaults:
66
66
  capabilities: { oats.core: { from: package } }