@awebai/oats 0.61.0 → 0.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/oats.mjs +26 -11
- package/docs/capabilities.md +1 -1
- package/docs/execution-targets.md +49 -3
- package/docs/implementation.md +2 -1
- package/docs/named-test-runs.md +382 -0
- package/docs/official-catalog.md +1 -1
- package/docs/packages.md +5 -5
- package/docs/release-notes/v0.62.0.md +223 -0
- package/docs/upgrading.md +133 -11
- package/docs/workspaces.md +1 -1
- package/lib/core.mjs +2 -2
- package/lib/doctor-capture.mjs +35 -0
- package/lib/session-viewer.mjs +86 -7
- package/package-catalog.json +1 -1
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/record/README.md +87 -13
- package/packages/record/bin/capture-hook.mjs +8 -0
- package/packages/record/bin/capture.mjs +143 -124
- package/packages/record/bin/recall.mjs +8 -3
- package/packages/record/bin/setup.mjs +115 -225
- package/packages/record/bin/turn-record.mjs +2 -2
- package/packages/record/docs/capture-service.md +1443 -28
- package/packages/record/docs/capture-status.schema.json +429 -12
- package/packages/record/lib/capture-admission.mjs +82 -0
- package/packages/record/lib/capture-aw-unit.mjs +137 -0
- package/packages/record/lib/capture-aw.mjs +31 -12
- package/packages/record/lib/capture-budget.mjs +31 -0
- package/packages/record/lib/capture-cc.mjs +39 -6
- package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
- package/packages/record/lib/capture-cleanup-work.mjs +77 -0
- package/packages/record/lib/capture-command-operation.mjs +37 -0
- package/packages/record/lib/capture-command-output.mjs +61 -0
- package/packages/record/lib/capture-command-worker.mjs +6 -0
- package/packages/record/lib/capture-command.mjs +48 -0
- package/packages/record/lib/capture-completion.mjs +152 -0
- package/packages/record/lib/capture-context.mjs +50 -0
- package/packages/record/lib/capture-default-roots.mjs +30 -0
- package/packages/record/lib/capture-discovery.mjs +114 -0
- package/packages/record/lib/capture-failures.mjs +45 -0
- package/packages/record/lib/capture-health-doctor.mjs +13 -0
- package/packages/record/lib/capture-health-installation.mjs +117 -0
- package/packages/record/lib/capture-health.mjs +116 -0
- package/packages/record/lib/capture-hook-ownership.mjs +58 -0
- package/packages/record/lib/capture-hook-runner.mjs +17 -0
- package/packages/record/lib/capture-hook.mjs +26 -0
- package/packages/record/lib/capture-index-inventory.mjs +205 -0
- package/packages/record/lib/capture-index-outcomes.mjs +113 -0
- package/packages/record/lib/capture-index-reset.mjs +109 -0
- package/packages/record/lib/capture-index-work.mjs +77 -0
- package/packages/record/lib/capture-issue-preparation.mjs +73 -0
- package/packages/record/lib/capture-issues.mjs +15 -2
- package/packages/record/lib/capture-lock.mjs +102 -16
- package/packages/record/lib/capture-native-unit.mjs +322 -0
- package/packages/record/lib/capture-pass.mjs +102 -0
- package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
- package/packages/record/lib/capture-retained-scratch.mjs +213 -0
- package/packages/record/lib/capture-retry.mjs +102 -0
- package/packages/record/lib/capture-run.mjs +125 -0
- package/packages/record/lib/capture-scratch.mjs +12 -3
- package/packages/record/lib/capture-service-definition.mjs +104 -0
- package/packages/record/lib/capture-service-install.mjs +362 -0
- package/packages/record/lib/capture-service-ownership.mjs +137 -0
- package/packages/record/lib/capture-service-runner.mjs +18 -0
- package/packages/record/lib/capture-small-snapshot.mjs +85 -0
- package/packages/record/lib/capture-source-discovery.mjs +319 -0
- package/packages/record/lib/capture-source-queue.mjs +199 -0
- package/packages/record/lib/capture-source-work.mjs +85 -0
- package/packages/record/lib/capture-state.mjs +16 -71
- package/packages/record/lib/capture-status.mjs +171 -0
- package/packages/record/lib/capture-supervisor.mjs +117 -0
- package/packages/record/lib/capture-watch.mjs +32 -0
- package/packages/record/lib/capture-work-session.mjs +238 -0
- package/packages/record/lib/capture-worker.mjs +58 -0
- package/packages/record/lib/derived-state.mjs +28 -15
- package/packages/record/lib/file-lines.mjs +22 -6
- package/packages/record/lib/formats.mjs +4 -3
- package/packages/record/lib/index-capture.mjs +97 -0
- package/packages/record/lib/index-db.mjs +3 -17
- package/packages/record/lib/index-reset.mjs +19 -0
- package/packages/record/lib/store.mjs +36 -11
- package/packages/record/lib/transcript-prefix.mjs +125 -0
- package/skills/oats-getting-started/SKILL.md +1 -1
- package/docs/macos-test-runs.md +0 -246
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// Internal bounded indexing units. The scheduler must visit the union of live
|
|
2
|
+
// streams and index_state streams (including deleted journals), and restart
|
|
3
|
+
// its index sweep when reset is returned. This helper alone cannot declare a
|
|
4
|
+
// root complete. Public synchronous update/rebuild keep their existing API.
|
|
5
|
+
import { closeSync, openSync, readSync } from "node:fs";
|
|
6
|
+
import { captureContext } from "./capture-context.mjs";
|
|
7
|
+
|
|
8
|
+
function cursorValid(store, stream, row, checkpoint) {
|
|
9
|
+
if (!row) return true;
|
|
10
|
+
if (!Number.isSafeInteger(row.bytes) || row.bytes < 0 ||
|
|
11
|
+
!Number.isSafeInteger(row.indexed_count) || row.indexed_count < 0 ||
|
|
12
|
+
row.indexed_count !== row.byte_count || row.epoch !== checkpoint.epoch ||
|
|
13
|
+
row.bytes > checkpoint.bytes || row.indexed_count > checkpoint.count) return false;
|
|
14
|
+
if (row.bytes > 0 && row.bytes < checkpoint.bytes) {
|
|
15
|
+
const fd = openSync(store.journalPath(stream), "r"), byte = Buffer.alloc(1);
|
|
16
|
+
try { return readSync(fd, byte, 0, 1, row.bytes - 1) === 1 && byte[0] === 10; }
|
|
17
|
+
finally { closeSync(fd); }
|
|
18
|
+
}
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function indexCaptureGroup(index, streams, {
|
|
23
|
+
maxRecords = 2000, maxBytes = 8 * 1024 * 1024, stopped = () => false, allowReset = false,
|
|
24
|
+
} = {}) {
|
|
25
|
+
if (typeof allowReset !== "boolean") throw new Error("invalid index reset admission");
|
|
26
|
+
const store = index.store, context = captureContext(store);
|
|
27
|
+
if (!context?.state.depth) throw new Error("bounded indexing requires an active capture work group");
|
|
28
|
+
const names = [...new Set(streams)].sort();
|
|
29
|
+
if (!names.length || names.length > 128 || names.some(name => !context.held.has(name)))
|
|
30
|
+
throw new Error("bounded indexing streams must belong to the active capture work group");
|
|
31
|
+
if (!Number.isSafeInteger(maxRecords) || maxRecords < names.length || !Number.isSafeInteger(maxBytes) || maxBytes < names.length)
|
|
32
|
+
throw new Error("bounded indexing must allow at least one work unit per stream");
|
|
33
|
+
const recordsEach = Math.floor(maxRecords / names.length), bytesEach = Math.floor(maxBytes / names.length);
|
|
34
|
+
const get = index.db.prepare(`SELECT s.indexed_count,b.bytes,b.indexed_count AS byte_count,b.epoch
|
|
35
|
+
FROM index_state s LEFT JOIN index_bytes b ON s.stream=b.stream WHERE s.stream=?`);
|
|
36
|
+
const prepared = [], work = { records: 0, bytes: 0, deletedIds: 0 };
|
|
37
|
+
let reset = false;
|
|
38
|
+
// Every group's locks were taken before capture.db's transaction. Validate
|
|
39
|
+
// journals before opening turns.db's transaction; never acquire another
|
|
40
|
+
// stream from inside either database transaction.
|
|
41
|
+
for (const stream of names) {
|
|
42
|
+
if (stopped()) { prepared.push({ stream, phase: "validation" }); continue; }
|
|
43
|
+
const checkpoint = store.validateJournal(stream, context.state, {
|
|
44
|
+
maxRecords: recordsEach, maxBytes: bytesEach, stopped, work,
|
|
45
|
+
});
|
|
46
|
+
const row = get.get(stream);
|
|
47
|
+
if (!cursorValid(store, stream, row?.indexed_count != null ? row : null, checkpoint)) reset = true;
|
|
48
|
+
prepared.push({ stream, checkpoint });
|
|
49
|
+
}
|
|
50
|
+
// Cross-stream tombstones and first-stream attribution make a per-stream
|
|
51
|
+
// row deletion unsound. Keep the reviewed atomic union reset. Publication
|
|
52
|
+
// of its cleared cursor tables is part of that same SQLite transaction.
|
|
53
|
+
if (reset && !allowReset) {
|
|
54
|
+
// The scheduler must end this work unit and reserve a fresh full worker
|
|
55
|
+
// allowance for atomic reset. Detection is not permission to start it in
|
|
56
|
+
// a nearly exhausted capture/index unit or the finalization reserve.
|
|
57
|
+
return { reset: false, resetRequired: true, turnsIndexed: 0, bytesIndexed: 0, validation: work,
|
|
58
|
+
streams: names.map(stream => ({ stream, phase: "reset" })) };
|
|
59
|
+
}
|
|
60
|
+
if (reset) index.reset();
|
|
61
|
+
const setCount = index.db.prepare("INSERT INTO index_state VALUES (?, ?) ON CONFLICT(stream) DO UPDATE SET indexed_count=excluded.indexed_count");
|
|
62
|
+
const setBytes = index.db.prepare("INSERT INTO index_bytes VALUES (?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET bytes=excluded.bytes,indexed_count=excluded.indexed_count,epoch=excluded.epoch");
|
|
63
|
+
let transaction = false, turnsIndexed = 0, bytesIndexed = 0;
|
|
64
|
+
const begin = () => { if (!transaction) { index.db.exec("BEGIN IMMEDIATE"); transaction = true; } };
|
|
65
|
+
try {
|
|
66
|
+
const recordedOwner = index.db.prepare("SELECT value FROM meta WHERE key='owner'").get()?.value ?? null;
|
|
67
|
+
const owner = store.owner ?? recordedOwner;
|
|
68
|
+
if (owner && recordedOwner !== owner) {
|
|
69
|
+
begin(); index.db.prepare("INSERT INTO meta VALUES ('owner',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value WHERE meta.value!=excluded.value").run(owner);
|
|
70
|
+
}
|
|
71
|
+
for (const item of prepared) {
|
|
72
|
+
const { stream, checkpoint } = item;
|
|
73
|
+
if (!checkpoint?.complete) { item.phase = "validation"; continue; }
|
|
74
|
+
const row = reset ? undefined : get.get(stream);
|
|
75
|
+
let position = row?.bytes ?? 0, count = row?.indexed_count ?? 0;
|
|
76
|
+
if (position === checkpoint.bytes) { item.phase = "complete"; continue; }
|
|
77
|
+
if (stopped()) { item.phase = "index"; continue; }
|
|
78
|
+
// Fix the validated end for this unit. Later growth is subsequent work.
|
|
79
|
+
const iterator = store.iterateJournal(stream, { start: position, end: checkpoint.bytes });
|
|
80
|
+
let records = 0, bytes = 0;
|
|
81
|
+
try {
|
|
82
|
+
while (records < recordsEach && bytes < bytesEach && !stopped()) {
|
|
83
|
+
const next = iterator.next(); if (next.done) break;
|
|
84
|
+
begin();
|
|
85
|
+
const entry = next.value;
|
|
86
|
+
if (entry.turn !== null) { index.addTurn(entry.turn, stream, owner); count++; turnsIndexed++; }
|
|
87
|
+
position = entry.end; records++; bytes += entry.bytes; bytesIndexed += entry.bytes;
|
|
88
|
+
}
|
|
89
|
+
} finally { iterator.return(); }
|
|
90
|
+
if (records) { setCount.run(stream, count); setBytes.run(stream, position, count, checkpoint.epoch); }
|
|
91
|
+
item.phase = position === checkpoint.bytes ? "complete" : "index";
|
|
92
|
+
}
|
|
93
|
+
if (transaction) index.db.exec("COMMIT");
|
|
94
|
+
} catch (err) { if (transaction) index.db.exec("ROLLBACK"); throw err; }
|
|
95
|
+
return { reset, turnsIndexed, bytesIndexed, validation: work,
|
|
96
|
+
streams: prepared.map(({ stream, phase }) => ({ stream, phase })) };
|
|
97
|
+
}
|
|
@@ -14,14 +14,10 @@
|
|
|
14
14
|
import { closeSync, existsSync, mkdirSync, openSync, readSync, rmSync, unlinkSync } from "node:fs";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { keysetRows } from "./sqlite-rows.mjs";
|
|
17
|
+
import { INDEX_SCHEMA_VERSION as SCHEMA_VERSION, TEXT_SCHEMA, resetIndexTables } from "./index-reset.mjs";
|
|
17
18
|
import { createRequire } from "node:module";
|
|
18
19
|
const require = createRequire(import.meta.url);
|
|
19
20
|
|
|
20
|
-
// Bump whenever the derived schema changes shape. The index is cache:
|
|
21
|
-
// a version mismatch self-heals by deleting the database and rebuilding
|
|
22
|
-
// from the record — never by in-place migration.
|
|
23
|
-
const SCHEMA_VERSION = 1;
|
|
24
|
-
|
|
25
21
|
import { extractCcText, extractSessionTextFor } from "./formats.mjs";
|
|
26
22
|
import { parseOutfit, parseTag } from "./tags.mjs";
|
|
27
23
|
import { parseSegment, parseSpawn } from "./segments.mjs";
|
|
@@ -103,9 +99,7 @@ CREATE TABLE IF NOT EXISTS spawns (
|
|
|
103
99
|
ts TEXT NOT NULL
|
|
104
100
|
);
|
|
105
101
|
CREATE INDEX IF NOT EXISTS spawns_agent ON spawns(agent_thread);
|
|
106
|
-
|
|
107
|
-
turn_id UNINDEXED, loc UNINDEXED, role UNINDEXED, text
|
|
108
|
-
);
|
|
102
|
+
${TEXT_SCHEMA}
|
|
109
103
|
`;
|
|
110
104
|
|
|
111
105
|
export class RecordIndex {
|
|
@@ -222,15 +216,7 @@ export class RecordIndex {
|
|
|
222
216
|
reset() {
|
|
223
217
|
this.db.exec("BEGIN");
|
|
224
218
|
try {
|
|
225
|
-
this.db
|
|
226
|
-
this.db.exec("DELETE FROM turn_text");
|
|
227
|
-
this.db.exec("DELETE FROM tombstones");
|
|
228
|
-
this.db.exec("DELETE FROM tags");
|
|
229
|
-
this.db.exec("DELETE FROM outfits");
|
|
230
|
-
this.db.exec("DELETE FROM segments");
|
|
231
|
-
this.db.exec("DELETE FROM spawns");
|
|
232
|
-
this.db.exec("DELETE FROM index_state");
|
|
233
|
-
this.db.exec("DELETE FROM index_bytes");
|
|
219
|
+
resetIndexTables(this.db);
|
|
234
220
|
this.db.exec("COMMIT");
|
|
235
221
|
} catch (err) {
|
|
236
222
|
this.db.exec("ROLLBACK");
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Shared atomic-reset mutations. Caller owns BEGIN/COMMIT/ROLLBACK so a private
|
|
2
|
+
// command receipt can join the reset without changing public reset semantics.
|
|
3
|
+
// Bump on derived schema shape changes. RecordIndex rebuilds a mismatched
|
|
4
|
+
// cache from journals instead of migrating it in place.
|
|
5
|
+
export const INDEX_SCHEMA_VERSION = 1;
|
|
6
|
+
export const TEXT_SCHEMA = `CREATE VIRTUAL TABLE IF NOT EXISTS turn_text USING fts5(
|
|
7
|
+
turn_id UNINDEXED, loc UNINDEXED, role UNINDEXED, text
|
|
8
|
+
);`;
|
|
9
|
+
|
|
10
|
+
export function resetIndexTables(db) {
|
|
11
|
+
db.exec("DELETE FROM turns");
|
|
12
|
+
// Ordinary tables use truncate; FTS recreation avoids visiting every text
|
|
13
|
+
// row. Readers still see either the previous union or the committed empty one.
|
|
14
|
+
db.exec("DROP TABLE turn_text");
|
|
15
|
+
db.exec(TEXT_SCHEMA);
|
|
16
|
+
for (const table of ["tombstones", "tags", "outfits", "segments", "spawns", "index_state", "index_bytes"])
|
|
17
|
+
db.exec(`DELETE FROM ${table}`);
|
|
18
|
+
db.exec(`PRAGMA user_version = ${INDEX_SCHEMA_VERSION}`);
|
|
19
|
+
}
|
|
@@ -33,6 +33,7 @@ import {
|
|
|
33
33
|
writeFileSync,
|
|
34
34
|
} from "node:fs";
|
|
35
35
|
import { DerivedState, fileVersion, sameVersion, invalidateLegacyMaps } from "./derived-state.mjs";
|
|
36
|
+
import { captureContext } from "./capture-context.mjs";
|
|
36
37
|
import { randomBytes } from "node:crypto";
|
|
37
38
|
import { hostname } from "node:os";
|
|
38
39
|
import { join } from "node:path";
|
|
@@ -338,7 +339,13 @@ export class RecordStore {
|
|
|
338
339
|
// concurrent writer's already-fsynced turn. Exclusive-create lockfile
|
|
339
340
|
// carrying an owner token; see the stream-lock helpers above for how
|
|
340
341
|
// staleness is proven and why release is ownership-checked.
|
|
341
|
-
withStreamLock(streamId, fn) {
|
|
342
|
+
withStreamLock(streamId, fn, { check = () => {} } = {}) {
|
|
343
|
+
check();
|
|
344
|
+
const context = captureContext(this);
|
|
345
|
+
if (context) {
|
|
346
|
+
if (!context.held.has(streamId)) throw new StoreError(`stream outside capture work group: ${streamId}`);
|
|
347
|
+
return fn();
|
|
348
|
+
}
|
|
342
349
|
const dir = join(this.streamsDir(), streamId);
|
|
343
350
|
mkdirSync(dir, { recursive: true });
|
|
344
351
|
const lockPath = join(dir, ".lock");
|
|
@@ -354,6 +361,7 @@ export class RecordStore {
|
|
|
354
361
|
// buy ~25ms on the crash-recovery path and reopen that hole; it is not
|
|
355
362
|
// worth it.
|
|
356
363
|
for (;;) {
|
|
364
|
+
check();
|
|
357
365
|
if (createLock(lockPath, dir, token)) break;
|
|
358
366
|
const held = readLock(lockPath); // throws on anything but ENOENT
|
|
359
367
|
const reclaimError =
|
|
@@ -424,13 +432,13 @@ export class RecordStore {
|
|
|
424
432
|
|
|
425
433
|
// Absolute byte boundaries, including blank lines, permit durable seeks.
|
|
426
434
|
// The final fragment never advances a cursor: it is the owner's torn tail.
|
|
427
|
-
*iterateJournal(streamId, { start = 0, end = Infinity } = {}) {
|
|
435
|
+
*iterateJournal(streamId, { start = 0, end = Infinity, check = () => {} } = {}) {
|
|
428
436
|
let fd;
|
|
429
437
|
try { fd = openSync(this.journalPath(streamId), "r"); }
|
|
430
438
|
catch (e) { if (e.code === "ENOENT") return; throw e; }
|
|
431
439
|
let position = start;
|
|
432
440
|
try {
|
|
433
|
-
for (const line of bufferLines(fileChunks(fd, { start, end, size: 65536 }))) {
|
|
441
|
+
for (const line of bufferLines(fileChunks(fd, { start, end, size: 65536, check }))) {
|
|
434
442
|
if (line[line.length - 1] !== 10) return;
|
|
435
443
|
let parsed;
|
|
436
444
|
try { parsed = parseJournal(line); }
|
|
@@ -445,6 +453,8 @@ export class RecordStore {
|
|
|
445
453
|
}
|
|
446
454
|
|
|
447
455
|
withDerived(fn) {
|
|
456
|
+
const context = captureContext(this);
|
|
457
|
+
if (context) return fn(context.state);
|
|
448
458
|
const state = new DerivedState(this.root);
|
|
449
459
|
try { return fn(state); } finally { state.close(); }
|
|
450
460
|
}
|
|
@@ -453,28 +463,42 @@ export class RecordStore {
|
|
|
453
463
|
// our fsynced append can skip history. Unknown writes (including old releases),
|
|
454
464
|
// replacement and shrink start a new epoch and invalidate dependent cursors.
|
|
455
465
|
// Partial validation is resumable, but cannot authorize an append.
|
|
456
|
-
validateJournal(streamId, state) {
|
|
466
|
+
validateJournal(streamId, state, { maxRecords = Infinity, maxBytes = Infinity, stopped = () => false, work } = {}) {
|
|
457
467
|
const path = this.journalPath(streamId), version = fileVersion(path);
|
|
458
468
|
let checkpoint = state.get("journal", streamId);
|
|
459
469
|
if (checkpoint && (!Number.isSafeInteger(checkpoint.bytes) || checkpoint.bytes < 0 || checkpoint.bytes > (version?.size ?? 0) ||
|
|
460
470
|
!Number.isSafeInteger(checkpoint.count) || checkpoint.count < 0 || checkpoint.count > checkpoint.bytes ||
|
|
461
|
-
typeof checkpoint.epoch !== "string" || typeof checkpoint.complete !== "boolean"
|
|
471
|
+
typeof checkpoint.epoch !== "string" || typeof checkpoint.complete !== "boolean" ||
|
|
472
|
+
(checkpoint.resetting !== undefined && typeof checkpoint.resetting !== "boolean") || checkpoint.resetting && checkpoint.complete)) checkpoint = null;
|
|
462
473
|
if (checkpoint && sameVersion(checkpoint.version, version) && checkpoint.complete) return checkpoint;
|
|
463
474
|
if (!checkpoint || !sameVersion(checkpoint.version, version)) {
|
|
464
|
-
checkpoint = { version, epoch: randomBytes(16).toString("hex"), bytes: 0, count: 0, complete: false };
|
|
475
|
+
checkpoint = { version, epoch: randomBytes(16).toString("hex"), bytes: 0, count: 0, complete: false, resetting: true };
|
|
476
|
+
state.transaction(() => state.set("journal", streamId, checkpoint));
|
|
477
|
+
}
|
|
478
|
+
let recordsUsed = 0, bytesUsed = 0;
|
|
479
|
+
// Clearing a large obsolete lookup is recovery work too. Its durable
|
|
480
|
+
// phase marker prevents a deadline stop from restarting that deletion.
|
|
481
|
+
while (checkpoint.resetting) {
|
|
482
|
+
if (recordsUsed >= maxRecords || stopped()) return checkpoint;
|
|
465
483
|
state.transaction(() => {
|
|
466
|
-
|
|
467
|
-
state.
|
|
484
|
+
const limit = Math.min(2000, maxRecords - recordsUsed);
|
|
485
|
+
const removed = Number(state.db.prepare(`DELETE FROM journal_ids WHERE rowid IN
|
|
486
|
+
(SELECT rowid FROM journal_ids WHERE stream = ? LIMIT ?)`).run(streamId, limit).changes);
|
|
487
|
+
recordsUsed += removed;
|
|
488
|
+
if (work) work.deletedIds = (work.deletedIds ?? 0) + removed;
|
|
489
|
+
if (removed < limit) { delete checkpoint.resetting; state.set("journal", streamId, checkpoint); }
|
|
468
490
|
});
|
|
469
491
|
}
|
|
470
492
|
const addId = state.db.prepare("INSERT OR IGNORE INTO journal_ids VALUES (?, ?)");
|
|
471
|
-
const iterator = this.iterateJournal(streamId, { start: checkpoint.bytes, end: version?.size ?? 0
|
|
493
|
+
const iterator = this.iterateJournal(streamId, { start: checkpoint.bytes, end: version?.size ?? 0,
|
|
494
|
+
check: () => { stopped(); } });
|
|
472
495
|
let done = false;
|
|
473
496
|
try {
|
|
474
497
|
while (!done) {
|
|
498
|
+
if (recordsUsed >= maxRecords || bytesUsed >= maxBytes || stopped()) return checkpoint;
|
|
475
499
|
state.transaction(() => {
|
|
476
500
|
let batch = 0, bytes = 0;
|
|
477
|
-
while (batch < 2000 && bytes < 8 * 1024 * 1024) {
|
|
501
|
+
while (batch < 2000 && bytes < 8 * 1024 * 1024 && recordsUsed < maxRecords && bytesUsed < maxBytes && !stopped()) {
|
|
478
502
|
const next = iterator.next();
|
|
479
503
|
if (next.done) { done = true; break; }
|
|
480
504
|
const entry = next.value;
|
|
@@ -483,7 +507,8 @@ export class RecordStore {
|
|
|
483
507
|
if (typeof entry.turn.id === "string") addId.run(streamId, entry.turn.id);
|
|
484
508
|
}
|
|
485
509
|
checkpoint.bytes = entry.end;
|
|
486
|
-
bytes += entry.bytes; batch++;
|
|
510
|
+
bytes += entry.bytes; bytesUsed += entry.bytes; batch++; recordsUsed++;
|
|
511
|
+
if (work) { work.records = (work.records ?? 0) + 1; work.bytes = (work.bytes ?? 0) + entry.bytes; }
|
|
487
512
|
}
|
|
488
513
|
if (!sameVersion(version, fileVersion(path))) throw new StoreError(`journal changed during validation: ${path}`);
|
|
489
514
|
checkpoint.complete = done;
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// Recovery proof for positional transcript journals. The temporary lookup is
|
|
2
2
|
// disk-backed: comparing a long prefix retains one native record at a time.
|
|
3
3
|
import { digest } from "./session-snapshot.mjs";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { captureContext } from "./capture-context.mjs";
|
|
6
|
+
import { fileVersion, sameVersion } from "./derived-state.mjs";
|
|
4
7
|
|
|
5
8
|
export function compareTranscriptPrefix(store, stream, state, lines) {
|
|
6
9
|
const db = state.db;
|
|
@@ -38,3 +41,125 @@ export function compareTranscriptPrefix(store, stream, state, lines) {
|
|
|
38
41
|
if (line < lastLine) return { conflict: { line: line + 1, offset, reason: "captured-prefix-missing" } };
|
|
39
42
|
return { bytes: offset, line, lastTs };
|
|
40
43
|
}
|
|
44
|
+
|
|
45
|
+
// Command-only recovery proof. The caller supplies complete native lines from
|
|
46
|
+
// an immutable verified staged snapshot, starting at the requested byte. A
|
|
47
|
+
// retained allocation or a completely revalidated digest/size binds its id. This helper neither opens the
|
|
48
|
+
// mutable source nor transfers scratch ownership; those belong to the worker.
|
|
49
|
+
// No partial phase authorizes append. Checkpoints and hashes join the caller's
|
|
50
|
+
// existing stream-locked processing transaction, with no per-record commits.
|
|
51
|
+
export function advanceTranscriptPrefix(store, stream, { key, snapshot, lines, maxRecords = 2000,
|
|
52
|
+
maxBytes = 8 * 1024 * 1024, stopped = () => false, work } = {}) {
|
|
53
|
+
const context = captureContext(store), state = context?.state;
|
|
54
|
+
if (!state?.depth || !context.held.has(stream)) throw new Error("resumable prefix proof requires its stream work group");
|
|
55
|
+
if (typeof key !== "string" || !key || typeof snapshot?.id !== "string" || !snapshot.id ||
|
|
56
|
+
!Number.isSafeInteger(snapshot.size) || snapshot.size < 0 || typeof lines !== "function" ||
|
|
57
|
+
!Number.isSafeInteger(maxRecords) || maxRecords < 1 || maxRecords > 2000 ||
|
|
58
|
+
!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > 8 * 1024 * 1024) {
|
|
59
|
+
throw new Error("invalid resumable prefix proof");
|
|
60
|
+
}
|
|
61
|
+
const db = state.db;
|
|
62
|
+
if (db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='transcript_prefix'").get()?.found !== 1) {
|
|
63
|
+
db.exec("CREATE TABLE transcript_prefix (job TEXT, line INTEGER, hash TEXT NOT NULL, PRIMARY KEY(job,line))");
|
|
64
|
+
// A recreated hash table must not qualify checkpoints whose rows vanished.
|
|
65
|
+
state.set("prefix-meta", "epoch", randomUUID());
|
|
66
|
+
}
|
|
67
|
+
let epoch = state.get("prefix-meta", "epoch");
|
|
68
|
+
if (epoch === null) { epoch = randomUUID(); state.set("prefix-meta", "epoch", epoch); }
|
|
69
|
+
if (typeof epoch !== "string" || !epoch) throw new Error("invalid transcript prefix cache epoch");
|
|
70
|
+
const journalVersion = fileVersion(store.journalPath(stream));
|
|
71
|
+
let proof = state.get("prefix-proof", key);
|
|
72
|
+
if (!proof || proof.epoch !== epoch || proof.stream !== stream || proof.snapshotId !== snapshot.id ||
|
|
73
|
+
proof.snapshotSize !== snapshot.size || !sameVersion(proof.journalVersion, journalVersion)) {
|
|
74
|
+
proof = { epoch, stream, snapshotId: snapshot.id, snapshotSize: snapshot.size, journalVersion,
|
|
75
|
+
phase: "clear", journalBytes: 0, lastLine: 0, lastTs: "", bytes: 0, line: 0 };
|
|
76
|
+
}
|
|
77
|
+
if (!["clear", "journal", "source", "ready", "conflict"].includes(proof.phase) ||
|
|
78
|
+
!["journalBytes", "lastLine", "bytes", "line"].every(k => Number.isSafeInteger(proof[k]) && proof[k] >= 0) ||
|
|
79
|
+
proof.journalBytes > (journalVersion?.size ?? 0) || proof.bytes > snapshot.size ||
|
|
80
|
+
proof.line > proof.lastLine || typeof proof.lastTs !== "string") throw new Error("invalid transcript prefix checkpoint");
|
|
81
|
+
const publish = () => {
|
|
82
|
+
if (!sameVersion(journalVersion, fileVersion(store.journalPath(stream)))) throw new Error("journal changed during prefix proof");
|
|
83
|
+
state.set("prefix-proof", key, proof);
|
|
84
|
+
return { ...proof, complete: proof.phase === "ready" };
|
|
85
|
+
};
|
|
86
|
+
let records = 0, bytes = 0;
|
|
87
|
+
const stop = () => records >= maxRecords || bytes >= maxBytes || stopped();
|
|
88
|
+
if (proof.phase === "clear") {
|
|
89
|
+
if (stop()) return publish();
|
|
90
|
+
const removed = Number(db.prepare(`DELETE FROM transcript_prefix WHERE rowid IN
|
|
91
|
+
(SELECT rowid FROM transcript_prefix WHERE job=? LIMIT ?)`).run(key, maxRecords).changes);
|
|
92
|
+
records += removed;
|
|
93
|
+
if (work) work.deletedHashes = (work.deletedHashes ?? 0) + removed;
|
|
94
|
+
if (removed < maxRecords) proof.phase = "journal";
|
|
95
|
+
}
|
|
96
|
+
if (proof.phase === "journal" && !stop()) {
|
|
97
|
+
const put = db.prepare("INSERT OR IGNORE INTO transcript_prefix VALUES (?, ?, ?)");
|
|
98
|
+
const get = db.prepare("SELECT hash FROM transcript_prefix WHERE job=? AND line=?");
|
|
99
|
+
const iterator = store.iterateJournal(stream, { start: proof.journalBytes, end: journalVersion?.size ?? 0,
|
|
100
|
+
check: () => { stopped(); } });
|
|
101
|
+
try {
|
|
102
|
+
while (!stop()) {
|
|
103
|
+
const entry = iterator.next();
|
|
104
|
+
if (entry.done) { proof.phase = proof.lastLine ? "source" : "ready"; break; }
|
|
105
|
+
const { turn, end, bytes: used } = entry.value;
|
|
106
|
+
if (turn?.kind === "session") {
|
|
107
|
+
const line = turn.provenance?.origin?.line;
|
|
108
|
+
if (!Number.isSafeInteger(line) || line < 1 || typeof turn.body?.line !== "string") throw new Error(`invalid positional transcript journal: ${stream}`);
|
|
109
|
+
const hash = digest(Buffer.from(turn.body.line)), prior = get.get(key, line);
|
|
110
|
+
if (prior?.hash != null && prior.hash !== hash) throw new Error(`conflicting positional transcript journal: ${stream}, line ${line}`);
|
|
111
|
+
put.run(key, line, hash);
|
|
112
|
+
if (line >= proof.lastLine) { proof.lastLine = line; proof.lastTs = turn.ts; }
|
|
113
|
+
}
|
|
114
|
+
proof.journalBytes = end; records++; bytes += used;
|
|
115
|
+
if (work) { work.journalRecords = (work.journalRecords ?? 0) + 1; work.journalBytes = (work.journalBytes ?? 0) + used; }
|
|
116
|
+
}
|
|
117
|
+
} finally { iterator.return(); }
|
|
118
|
+
}
|
|
119
|
+
if (proof.phase === "source" && !stop()) {
|
|
120
|
+
const get = db.prepare("SELECT hash FROM transcript_prefix WHERE job=? AND line=?");
|
|
121
|
+
const iterator = lines(proof.bytes)[Symbol.iterator]();
|
|
122
|
+
try {
|
|
123
|
+
while (!stop()) {
|
|
124
|
+
const next = iterator.next();
|
|
125
|
+
if (next.done) {
|
|
126
|
+
proof.phase = "conflict";
|
|
127
|
+
proof.conflict = { line: proof.line + 1, offset: proof.bytes, reason: "captured-prefix-missing" };
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
const record = next.value;
|
|
131
|
+
// A line arrives as its streamed digest, or as text a caller decoded.
|
|
132
|
+
const hash = typeof record.hash === "string" ? record.hash : typeof record.text === "string" ? digest(Buffer.from(record.text)) : null;
|
|
133
|
+
if (hash === null || !Number.isSafeInteger(record.bytes) || record.bytes < 1 ||
|
|
134
|
+
proof.bytes + record.bytes > snapshot.size) throw new Error("invalid staged prefix line");
|
|
135
|
+
const expected = get.get(key, proof.line + 1);
|
|
136
|
+
records++; bytes += record.bytes;
|
|
137
|
+
if (work) { work.sourceRecords = (work.sourceRecords ?? 0) + 1; work.sourceBytes = (work.sourceBytes ?? 0) + record.bytes; }
|
|
138
|
+
if (expected?.hash != null && expected.hash !== hash) {
|
|
139
|
+
proof.phase = "conflict";
|
|
140
|
+
proof.conflict = { line: proof.line + 1, offset: proof.bytes, reason: "captured-prefix-changed" };
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
proof.bytes += record.bytes; proof.line++;
|
|
144
|
+
if (proof.line === proof.lastLine) { proof.phase = "ready"; break; }
|
|
145
|
+
}
|
|
146
|
+
} finally { iterator.return?.(); }
|
|
147
|
+
}
|
|
148
|
+
return publish();
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Reclaim completed/invalidated proof rows in bounded units. Revocation and
|
|
152
|
+
// deletion share the processing commit, so interruption cannot leave a ready
|
|
153
|
+
// checkpoint qualifying a partially removed hash table.
|
|
154
|
+
export function discardTranscriptPrefix(store, stream, key, { maxRecords = 2000 } = {}) {
|
|
155
|
+
const context = captureContext(store), state = context?.state;
|
|
156
|
+
if (!state?.depth || !context.held.has(stream) || !Number.isSafeInteger(maxRecords) || maxRecords < 1 || maxRecords > 2000) {
|
|
157
|
+
throw new Error("prefix cleanup requires a bounded stream work group");
|
|
158
|
+
}
|
|
159
|
+
const proof = state.get("prefix-proof", key);
|
|
160
|
+
if (proof && proof.stream !== stream) throw new Error("prefix cleanup stream differs");
|
|
161
|
+
state.delete("prefix-proof", key);
|
|
162
|
+
if (state.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type='table' AND name='transcript_prefix'").get()?.found !== 1) return true;
|
|
163
|
+
return Number(state.db.prepare(`DELETE FROM transcript_prefix WHERE rowid IN
|
|
164
|
+
(SELECT rowid FROM transcript_prefix WHERE job=? LIMIT ?)`).run(key, maxRecords).changes) < maxRecords;
|
|
165
|
+
}
|
|
@@ -60,7 +60,7 @@ members:
|
|
|
60
60
|
- git:github.com/acme/agents # the host is a member too
|
|
61
61
|
- git:github.com/acme/platform
|
|
62
62
|
packages:
|
|
63
|
-
oats.framework: v1.13.
|
|
63
|
+
oats.framework: v1.13.1 # bare versions resolve through the official catalog
|
|
64
64
|
oats.okf: v5.2.1
|
|
65
65
|
defaults:
|
|
66
66
|
capabilities: { oats.core: { from: package } }
|