@awebai/oats 0.54.0 → 0.55.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oats.mjs +5 -5
- package/docs/desktop-cli-api.md +15 -4
- package/docs/integrations.md +2 -2
- package/docs/knowledge.md +14 -2
- package/docs/official-catalog.md +4 -4
- package/docs/packages.md +10 -10
- package/docs/release-notes/v0.55.0.md +158 -0
- package/docs/release-notes/v0.55.1.md +45 -0
- package/docs/servers.md +13 -3
- package/docs/workspaces.md +1 -1
- package/lib/servers.mjs +82 -34
- package/package-catalog.json +3 -3
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/client/workspace-views.mjs +4 -3
- package/packages/record/README.md +58 -13
- package/packages/record/bin/capture.mjs +107 -53
- package/packages/record/bin/recall.mjs +3 -2
- package/packages/record/docs/capture-service.md +193 -0
- package/packages/record/docs/capture-status.schema.json +100 -0
- package/packages/record/lib/capture-aw.mjs +96 -94
- package/packages/record/lib/capture-cc.mjs +109 -187
- package/packages/record/lib/capture-db-reader.mjs +39 -0
- package/packages/record/lib/capture-issues.mjs +64 -0
- package/packages/record/lib/capture-scratch.mjs +111 -0
- package/packages/record/lib/capture-state.mjs +105 -0
- package/packages/record/lib/derived-state.mjs +60 -0
- package/packages/record/lib/formats.mjs +31 -31
- package/packages/record/lib/index-db.mjs +74 -34
- package/packages/record/lib/receipt-summary.mjs +62 -0
- package/packages/record/lib/session-snapshot.mjs +56 -1
- package/packages/record/lib/sqlite-rows.mjs +14 -0
- package/packages/record/lib/store.mjs +122 -43
- package/packages/record/lib/transcript-prefix.mjs +40 -0
- package/skills/oats-getting-started/SKILL.md +1 -1
|
@@ -20,12 +20,16 @@ import process from "node:process";
|
|
|
20
20
|
import { RecordStore } from "../lib/store.mjs";
|
|
21
21
|
import { captureAllSessions, captureSessions } from "../lib/capture-cc.mjs";
|
|
22
22
|
import { sessionsForHome } from "../lib/sessions-for-home.mjs";
|
|
23
|
-
import {
|
|
23
|
+
import { SESSION_FORMATS } from "../lib/formats.mjs";
|
|
24
24
|
import { cwdOfLine } from "../lib/sessions-for-home.mjs";
|
|
25
|
-
import {
|
|
25
|
+
import { stageSnapshot } from "../lib/session-snapshot.mjs";
|
|
26
|
+
import { cleanupCaptureScratch } from "../lib/capture-scratch.mjs";
|
|
27
|
+
import { bufferLines } from "../lib/file-lines.mjs";
|
|
26
28
|
import { captureAwLogs, defaultCommLogDir } from "../lib/capture-aw.mjs";
|
|
29
|
+
import { withReceiptSummaries } from "../lib/receipt-summary.mjs";
|
|
27
30
|
import { RecordIndex } from "../lib/index-db.mjs";
|
|
28
31
|
import { IgnoreError, ignoreFilePath, loadIgnore } from "../lib/ignore.mjs";
|
|
32
|
+
import { CAPTURE_SOURCES, captureHealth, resetSourceCursors, retrySetAside, writeCaptureStatus } from "../lib/capture-state.mjs";
|
|
29
33
|
import { acquireCaptureLock } from "../lib/capture-lock.mjs";
|
|
30
34
|
|
|
31
35
|
// Fail closed but actionably: an unreadable ignore file must stop capture,
|
|
@@ -45,8 +49,11 @@ function loadIgnoreOrExit(recordRoot) {
|
|
|
45
49
|
// hostname-derived owner, which forked the whole record into a second owner
|
|
46
50
|
// namespace: 571 duplicate journals from one typo. Parsing must refuse what
|
|
47
51
|
// it does not understand before anything can be written.
|
|
48
|
-
const VALUE_FLAGS = new Set(["root", "owner", "home", "file", "format"]);
|
|
52
|
+
const VALUE_FLAGS = new Set(["root", "owner", "home", "file", "format", "source"]);
|
|
49
53
|
const BOOL_FLAGS = new Set([
|
|
54
|
+
"reset-cursors",
|
|
55
|
+
"retry-set-aside",
|
|
56
|
+
"health",
|
|
50
57
|
"watch",
|
|
51
58
|
"status",
|
|
52
59
|
"install-hint",
|
|
@@ -67,6 +74,12 @@ const USAGE = `capture — land sessions and aw client logs in the turn record.
|
|
|
67
74
|
capture --watch pass now, then re-pass on filesystem change
|
|
68
75
|
(debounced) and every 15 minutes regardless
|
|
69
76
|
capture --status show store/stream summary, capture nothing
|
|
77
|
+
capture --health [--json] read capture set-aside diagnosis, capture nothing
|
|
78
|
+
capture --reset-cursors [--source aw|cc|pi|codex]
|
|
79
|
+
reset selected source cursors and exit; the next
|
|
80
|
+
pass reprocesses them with journal deduplication
|
|
81
|
+
capture --retry-set-aside revalidate set-aside inputs in one reconciliation;
|
|
82
|
+
does not authorize changing captured transcripts
|
|
70
83
|
capture --home <dir> capture the sessions that ran inside <dir> (an
|
|
71
84
|
OATS instance home) and print them as JSON:
|
|
72
85
|
thread, stream, turn count, first/last turn id;
|
|
@@ -146,6 +159,10 @@ function parseArgs(argv) {
|
|
|
146
159
|
let args;
|
|
147
160
|
try {
|
|
148
161
|
args = parseArgs(process.argv.slice(2));
|
|
162
|
+
const recovery = ["reset-cursors", "retry-set-aside", "health"].filter(key => args[key]);
|
|
163
|
+
if (recovery.length > 1) throw new UsageError("choose one recovery or health operation");
|
|
164
|
+
if (args.source !== undefined && (!args["reset-cursors"] || !CAPTURE_SOURCES.includes(args.source))) throw new UsageError("--source requires --reset-cursors and aw|cc|pi|codex");
|
|
165
|
+
if (recovery.length && ["home", "file", "format", "watch", "status", "install-hint", "current-roots", "sessions-only", "aw-only"].some(key => args[key])) throw new UsageError("recovery/health cannot be combined with another capture mode");
|
|
149
166
|
} catch (err) {
|
|
150
167
|
if (!(err instanceof UsageError)) throw err;
|
|
151
168
|
// --file --json answers JSON even for a usage error: it is for programs.
|
|
@@ -165,9 +182,43 @@ if (args.help) {
|
|
|
165
182
|
const root = args.root ?? process.env.TURN_RECORD_ROOT ?? join(homedir(), ".turn-record");
|
|
166
183
|
const ownerExplicit = args.owner !== undefined || process.env.TURN_RECORD_OWNER !== undefined;
|
|
167
184
|
const owner = args.owner ?? process.env.TURN_RECORD_OWNER ?? hostname().split(".")[0];
|
|
168
|
-
const quiet = Boolean(args.quiet);
|
|
185
|
+
const quiet = Boolean(args.quiet || args.json && (args["reset-cursors"] || args["retry-set-aside"]));
|
|
169
186
|
const store = new RecordStore(root, { owner });
|
|
170
187
|
|
|
188
|
+
if (args.health) {
|
|
189
|
+
try {
|
|
190
|
+
const health = captureHealth(root);
|
|
191
|
+
if (args.json) console.log(JSON.stringify(health, null, 2));
|
|
192
|
+
else {
|
|
193
|
+
console.log(`capture: ${health.status}`);
|
|
194
|
+
for (const row of health.rows) console.log(`${row.code}: ${row.message}`);
|
|
195
|
+
for (const entry of health.backlog.setAside) console.log(`${entry.source}: ${entry.path}:${entry.line ?? "?"} offset ${entry.offset}: ${entry.reason}`);
|
|
196
|
+
if (health.backlog.truncated) console.log("showing first 100 set-aside inputs");
|
|
197
|
+
}
|
|
198
|
+
} catch (err) {
|
|
199
|
+
if (args.json) console.log(JSON.stringify({ status: "failed", error: err.message }));
|
|
200
|
+
else console.error(`capture: ${err.message}`);
|
|
201
|
+
process.exitCode = 1;
|
|
202
|
+
}
|
|
203
|
+
process.exit(process.exitCode ?? 0);
|
|
204
|
+
}
|
|
205
|
+
if (args["reset-cursors"] || args["retry-set-aside"]) {
|
|
206
|
+
try {
|
|
207
|
+
const result = args["reset-cursors"]
|
|
208
|
+
? withCaptureLock(() => { resetSourceCursors(root, owner, args.source); return { reset: true, sources: args.source ? [args.source] : CAPTURE_SOURCES }; })
|
|
209
|
+
: pass({ retry: true });
|
|
210
|
+
if (result.skipped) throw new Error("capture lock held; recovery was not performed");
|
|
211
|
+
if (process.exitCode) throw new Error("recovery failed to release its capture lock");
|
|
212
|
+
if (args.json) console.log(JSON.stringify(result));
|
|
213
|
+
else if (args["reset-cursors"]) log(`capture: reset source cursors (${result.sources.join(", ")})`);
|
|
214
|
+
} catch (err) {
|
|
215
|
+
if (args.json) console.log(JSON.stringify({ status: "failed", complete: false, error: err.message }));
|
|
216
|
+
else console.error(`capture: ${err.message}`);
|
|
217
|
+
process.exitCode = 1;
|
|
218
|
+
}
|
|
219
|
+
process.exit(process.exitCode ?? 0);
|
|
220
|
+
}
|
|
221
|
+
|
|
171
222
|
// The owner decides the stream namespace, and turn ids hash it into the
|
|
172
223
|
// canonical core — so the same conversation captured under two owners yields
|
|
173
224
|
// two sets of ids for identical content, which the index cannot dedupe. When
|
|
@@ -178,21 +229,18 @@ const store = new RecordStore(root, { owner });
|
|
|
178
229
|
// root, no other owners, and nothing is printed.
|
|
179
230
|
function warnOnStrangerOwner() {
|
|
180
231
|
if (ownerExplicit) return;
|
|
181
|
-
let
|
|
232
|
+
let example;
|
|
182
233
|
try {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
return; // unreadable or absent root: not this function's business
|
|
191
|
-
}
|
|
192
|
-
if (others.size === 0 || others.has(owner)) return;
|
|
234
|
+
for (const stream of store.iterateStreams()) {
|
|
235
|
+
const existing = stream.slice(0, stream.indexOf("~"));
|
|
236
|
+
if (existing === owner) return;
|
|
237
|
+
if (existing) example ??= existing;
|
|
238
|
+
}
|
|
239
|
+
} catch { return; }
|
|
240
|
+
if (!example) return;
|
|
193
241
|
console.error(
|
|
194
242
|
`capture: writing as owner "${owner}" (from hostname), but this record holds ` +
|
|
195
|
-
`streams
|
|
243
|
+
`streams under another owner such as "${example}". ` +
|
|
196
244
|
`Pass --owner or set TURN_RECORD_OWNER if that is not what you meant — ` +
|
|
197
245
|
`capturing under a second owner duplicates the record rather than extending it.`,
|
|
198
246
|
);
|
|
@@ -240,7 +288,16 @@ function withCaptureLock(fn) {
|
|
|
240
288
|
return { appended: 0, skipped: true, lock: lock.held };
|
|
241
289
|
}
|
|
242
290
|
try {
|
|
243
|
-
|
|
291
|
+
let result, failure;
|
|
292
|
+
try { cleanupCaptureScratch(root, lock); result = fn(); return result; }
|
|
293
|
+
catch (err) { failure = err; throw err; }
|
|
294
|
+
finally {
|
|
295
|
+
try { writeCaptureStatus(root, { pending: Boolean(failure || result?.pending || result?.held || result?.incomplete || result?.failed) }); }
|
|
296
|
+
catch (err) {
|
|
297
|
+
if (failure) throw new Error(`${failure.message}; capture status persistence also failed: ${err.message}`, { cause: failure });
|
|
298
|
+
throw err;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
244
301
|
} finally {
|
|
245
302
|
const r = lock.release();
|
|
246
303
|
if (!r.released) {
|
|
@@ -256,15 +313,17 @@ function withCaptureLock(fn) {
|
|
|
256
313
|
}
|
|
257
314
|
}
|
|
258
315
|
|
|
259
|
-
function pass() {
|
|
316
|
+
function pass({ retry = false } = {}) {
|
|
260
317
|
// The privacy loader fails closed by exiting; it runs before the lock is
|
|
261
318
|
// taken so that exit never leaves the lock behind.
|
|
262
319
|
const ignore = loadIgnoreOrExit(root);
|
|
263
320
|
return withCaptureLock(() => {
|
|
264
|
-
|
|
321
|
+
if (retry) retrySetAside(root);
|
|
322
|
+
const out = { appended: 0, pending: false };
|
|
265
323
|
if (!args["aw-only"]) {
|
|
266
|
-
for (const r of captureAllSessions(store, { owner, ignore })) {
|
|
324
|
+
for (const r of captureAllSessions(store, { owner, ignore, collectIssues: false })) {
|
|
267
325
|
out.appended += r.appended;
|
|
326
|
+
out.pending ||= !r.complete;
|
|
268
327
|
const extras = [r.ignored ? `${r.ignored} ignored` : "", r.held ? `${r.held} held` : "", r.incomplete ? `${r.incomplete} incomplete` : ""]
|
|
269
328
|
.filter(Boolean)
|
|
270
329
|
.join(", ");
|
|
@@ -279,7 +338,7 @@ function pass() {
|
|
|
279
338
|
let awFailed = 0;
|
|
280
339
|
let awFiles = 0;
|
|
281
340
|
let awIgnored = 0;
|
|
282
|
-
for (const r of captureAwLogs(store, { owner, ignore })) {
|
|
341
|
+
for (const r of captureAwLogs(store, { owner, ignore, collect: false })) {
|
|
283
342
|
if (r.ignored) {
|
|
284
343
|
awIgnored++;
|
|
285
344
|
continue;
|
|
@@ -288,6 +347,7 @@ function pass() {
|
|
|
288
347
|
awEntries += r.entries;
|
|
289
348
|
awAppended += r.appended;
|
|
290
349
|
awFailed += r.failed;
|
|
350
|
+
out.pending ||= r.failed > 0;
|
|
291
351
|
out.appended += r.appended;
|
|
292
352
|
if (r.failed) console.error(`aw-log ${r.account}: ${r.failed} entries failed to project`);
|
|
293
353
|
}
|
|
@@ -302,7 +362,7 @@ function pass() {
|
|
|
302
362
|
const index = new RecordIndex(store);
|
|
303
363
|
try {
|
|
304
364
|
index.update();
|
|
305
|
-
log(
|
|
365
|
+
log("index: updated");
|
|
306
366
|
} finally {
|
|
307
367
|
index.close();
|
|
308
368
|
}
|
|
@@ -335,12 +395,6 @@ function captureUnderLock(recordStore, recordOwner, batches, ignore, outcome, is
|
|
|
335
395
|
});
|
|
336
396
|
}
|
|
337
397
|
|
|
338
|
-
/** A stream's turns that no tombstone hides. A tombstoned turn is hidden everywhere; a boundary naming one
|
|
339
|
-
* would be refused by recall, so boundaries come from the visible turns only. */
|
|
340
|
-
function visibleTurns(recordStore, stream, claims = recordStore.tombstoneClaims()) {
|
|
341
|
-
return recordStore.readStream(stream).filter((t) => !recordStore.claimHides(claims, t));
|
|
342
|
-
}
|
|
343
|
-
|
|
344
398
|
/** The outcome vocabulary: complete only with no failure, skip, hold or incomplete tail (and, for --home, no
|
|
345
399
|
* unattributed source). */
|
|
346
400
|
function statusOf(outcome, unattributed = 0) {
|
|
@@ -366,21 +420,18 @@ function isFormat(found, stated) {
|
|
|
366
420
|
/** Records (non-blank lines, an undecodable one included) and the formats whose session header a COMPLETE
|
|
367
421
|
* line carries. A pi or codex file is settled at its header; a cc file is judged whole. A line its writer
|
|
368
422
|
* never terminated is no header. */
|
|
369
|
-
function sessionHeaders(
|
|
423
|
+
function sessionHeaders(chunks, stated) {
|
|
370
424
|
let records = 0;
|
|
371
425
|
const found = new Set();
|
|
372
|
-
const
|
|
373
|
-
|
|
426
|
+
for (const line of bufferLines(chunks)) {
|
|
427
|
+
const complete = line[line.length - 1] === 10;
|
|
428
|
+
let text = null;
|
|
429
|
+
try { text = (complete ? line.subarray(0, -1) : line).toString("utf8"); }
|
|
430
|
+
catch (err) { if (err.code !== "ERR_STRING_TOO_LONG") throw err; }
|
|
374
431
|
if (text === null || text.trim() !== "") records++;
|
|
375
432
|
if (complete && text !== null) for (const source of HEADER_ORDER) if (cwdOfLine(source, text) !== undefined) found.add(source);
|
|
376
|
-
|
|
377
|
-
let pending;
|
|
378
|
-
for (const { text } of jsonlLines(bytes)) {
|
|
379
|
-
if (pending !== undefined) take(pending, true);
|
|
380
|
-
if (stated !== "cc" && found.has(stated)) return { records, found };
|
|
381
|
-
pending = text;
|
|
433
|
+
if (stated !== "cc" && found.has(stated)) break;
|
|
382
434
|
}
|
|
383
|
-
if (pending !== undefined) take(pending, terminated);
|
|
384
435
|
return { records, found };
|
|
385
436
|
}
|
|
386
437
|
|
|
@@ -430,17 +481,18 @@ function fileCapture() {
|
|
|
430
481
|
if (err.code === "ELOOP" || err.code === "EMLINK") return fail("E_NOT_REGULAR_FILE", `${path} is a symbolic link; pass the file it names`, where);
|
|
431
482
|
return fail("E_FILE_UNREADABLE", `cannot open ${path}: ${err.message}`, where);
|
|
432
483
|
}
|
|
433
|
-
let stat,
|
|
484
|
+
let stat, pinned;
|
|
434
485
|
try {
|
|
435
486
|
stat = fstatSync(fd);
|
|
436
487
|
if (!stat.isFile()) return fail("E_NOT_REGULAR_FILE", `${path} is ${fileKind(stat)}, not a regular file`, where);
|
|
437
|
-
|
|
488
|
+
pinned = stageSnapshot(fd, stat, path, undefined, root);
|
|
438
489
|
} catch (err) {
|
|
439
490
|
return fail("E_FILE_UNREADABLE", `cannot read ${path}: ${err.message}`, where);
|
|
440
491
|
} finally { closeSync(fd); }
|
|
441
492
|
|
|
442
|
-
|
|
443
|
-
|
|
493
|
+
try {
|
|
494
|
+
const { records, found } = sessionHeaders(pinned.chunks(), args.format);
|
|
495
|
+
if (!records) return fail("E_NO_TURNS", `${path} has no records${stat.size ? " (only blank lines)" : " (it is empty)"}`, where);
|
|
444
496
|
if (!isFormat(found, args.format)) {
|
|
445
497
|
const other = HEADER_ORDER.find((f) => f !== args.format && isFormat(found, f));
|
|
446
498
|
return fail("E_FORMAT", other
|
|
@@ -448,12 +500,12 @@ function fileCapture() {
|
|
|
448
500
|
: `${path} has no ${args.format} session header (${SESSION_HEADER[args.format]})`, where);
|
|
449
501
|
}
|
|
450
502
|
|
|
451
|
-
const sha256 =
|
|
503
|
+
const sha256 = pinned.hash;
|
|
452
504
|
const fileStore = new RecordStore(root, { owner: fileOwner });
|
|
453
505
|
const outcome = { appended: 0, skipped: false, held: 0, incomplete: 0, failed: 0, ignored: 0 };
|
|
454
506
|
const issues = [];
|
|
455
507
|
try {
|
|
456
|
-
Object.assign(outcome, captureUnderLock(fileStore, fileOwner, [[args.format, [{ path, pinned
|
|
508
|
+
Object.assign(outcome, captureUnderLock(fileStore, fileOwner, [[args.format, [{ path, pinned }]]], ignore, outcome, issues));
|
|
457
509
|
} catch (err) {
|
|
458
510
|
// A failed pass may have appended part of the file: never claim a count for it.
|
|
459
511
|
return fail("E_CAPTURE_FAILED", err.message || String(err), { ...where, sha256, appended: null });
|
|
@@ -461,17 +513,18 @@ function fileCapture() {
|
|
|
461
513
|
if (process.exitCode) return fail("E_CAPTURE_FAILED", "capture lock release failed; see stderr for recovery", { ...where, sha256 });
|
|
462
514
|
|
|
463
515
|
const stream = `${fileOwner}~${fmt.source}.${sessionId}`;
|
|
464
|
-
const turns =
|
|
516
|
+
const turns = withReceiptSummaries(fileStore, summary => summary(stream));
|
|
465
517
|
const status = statusOf(outcome);
|
|
466
518
|
const { lock: _lock, ...counts } = outcome;
|
|
467
519
|
const receipt = {
|
|
468
520
|
...where, instance, thread: `${fmt.source}:session:${sessionId}`, stream, sessionId,
|
|
469
|
-
turns: turns.
|
|
521
|
+
turns: turns.count, firstTurnId: turns.first?.id ?? null, lastTurnId: turns.last?.id ?? null,
|
|
470
522
|
...counts, status, complete: status === "complete", sha256, ...(issues.length ? { issues } : {}),
|
|
471
523
|
};
|
|
472
524
|
if (json) console.log(JSON.stringify(receipt, null, 2));
|
|
473
525
|
else console.log(`capture --file: ${status}, ${receipt.turns} turns (${outcome.appended} new) in ${stream}, sha256 ${sha256}`);
|
|
474
526
|
return 0;
|
|
527
|
+
} finally { pinned.close(); }
|
|
475
528
|
}
|
|
476
529
|
|
|
477
530
|
if (args.file !== undefined || args.format !== undefined || args.json) process.exit(fileCapture());
|
|
@@ -535,11 +588,11 @@ if (args.home) {
|
|
|
535
588
|
formats.get(s.source).push(s);
|
|
536
589
|
}
|
|
537
590
|
Object.assign(outcome, captureUnderLock(store, owner, formats, ignore, outcome, issues));
|
|
538
|
-
|
|
591
|
+
withReceiptSummaries(store, summary => {
|
|
539
592
|
for (const s of found) {
|
|
540
593
|
const stream = `${owner}~${s.source}.${s.sessionId}`;
|
|
541
|
-
const turns =
|
|
542
|
-
if (!turns.
|
|
594
|
+
const turns = summary(stream);
|
|
595
|
+
if (!turns.count) continue; // ignored by rule, nothing capturable yet, or all hidden
|
|
543
596
|
sessions.push({
|
|
544
597
|
thread: s.thread,
|
|
545
598
|
source: s.source,
|
|
@@ -547,12 +600,13 @@ if (args.home) {
|
|
|
547
600
|
path: s.path,
|
|
548
601
|
cwd: s.cwd,
|
|
549
602
|
stream,
|
|
550
|
-
turns: turns.
|
|
551
|
-
firstTurnId: turns
|
|
552
|
-
lastTurnId: turns
|
|
553
|
-
lastTs: turns
|
|
603
|
+
turns: turns.count,
|
|
604
|
+
firstTurnId: turns.first.id,
|
|
605
|
+
lastTurnId: turns.last.id,
|
|
606
|
+
lastTs: turns.last.ts,
|
|
554
607
|
});
|
|
555
608
|
}
|
|
609
|
+
});
|
|
556
610
|
} catch (err) {
|
|
557
611
|
error = err.message || String(err);
|
|
558
612
|
console.error(`capture pass failed: ${error}`);
|
|
@@ -61,8 +61,9 @@ try {
|
|
|
61
61
|
console.error(`no turn ${args.show}`);
|
|
62
62
|
process.exitCode = 1; return;
|
|
63
63
|
}
|
|
64
|
-
//
|
|
65
|
-
|
|
64
|
+
// Session targets are absent from the bulk union: evaluate the claims
|
|
65
|
+
// against the resolved turn so id lookup respects their tombstones too.
|
|
66
|
+
if (store.claimHides(store.tombstoneClaims(byId), turn)) {
|
|
66
67
|
console.error(`turn ${args.show} is tombstoned`);
|
|
67
68
|
process.exitCode = 1; return;
|
|
68
69
|
}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# Capture algorithms and recovery
|
|
2
|
+
|
|
3
|
+
The record package owns source cursors, journal validation, indexing and capture
|
|
4
|
+
status. This version changes those algorithms; it does not install a bounded
|
|
5
|
+
runner or change service scheduling, hooks, root registration or doctor.
|
|
6
|
+
|
|
7
|
+
## Input contract
|
|
8
|
+
|
|
9
|
+
Input logs are append-only: capture reads added content, rereads on replacement,
|
|
10
|
+
shrink or checkpoint mismatch, cannot see arbitrary edits to an already-read
|
|
11
|
+
prefix that leave those checks intact, and `capture --reset-cursors` is the
|
|
12
|
+
remedy. This assumption applies to aweb logs and cc/pi/codex transcripts. It
|
|
13
|
+
relies on the native writers appending to existing transcript files.
|
|
14
|
+
|
|
15
|
+
The background cursor qualifies file identity, size, modification/change times
|
|
16
|
+
and a SHA-256 witness of at most 4 KiB before its byte boundary. An unchanged
|
|
17
|
+
version needs no historical content scan. Growth with the same identity and
|
|
18
|
+
witness resumes from the boundary. The witness does **not** prove that the
|
|
19
|
+
unobserved prefix is untouched. Explicit `captureCommLog` and
|
|
20
|
+
`captureInteractionLog` library calls retain their full-reprojection default.
|
|
21
|
+
|
|
22
|
+
A complete aweb entry refused by projection is counted when encountered and
|
|
23
|
+
consumed; later unchanged passes do not retry it. Use
|
|
24
|
+
`capture --reset-cursors --source aw` to request another projection attempt.
|
|
25
|
+
Malformed complete lines are similarly skipped once per scan. Unfinished final
|
|
26
|
+
lines keep their boundary for later growth; source read errors and failed durable
|
|
27
|
+
appends do not become successful consumed checkpoints.
|
|
28
|
+
|
|
29
|
+
Detected changes have two outcomes:
|
|
30
|
+
|
|
31
|
+
- Aweb logs reproject from the beginning. Persistent journal IDs suppress
|
|
32
|
+
already captured turns; changed projected content adds a different turn once.
|
|
33
|
+
Replay does not retract an earlier turn.
|
|
34
|
+
- Native transcripts compare their captured positional prefix against a private
|
|
35
|
+
staged snapshot before any append from that input. Equal content, including
|
|
36
|
+
under a new file identity, resumes at the captured boundary. A conflict sets
|
|
37
|
+
the input aside with source, path, line, byte offset and reason. Other inputs
|
|
38
|
+
can continue. This preserves rollback: older writers recover their offsets
|
|
39
|
+
from the physical last transcript turn, so appending a revised earlier line
|
|
40
|
+
would make those writers duplicate the remaining history.
|
|
41
|
+
|
|
42
|
+
## Operator recovery
|
|
43
|
+
|
|
44
|
+
`turn-record capture --reset-cursors [--source aw|cc|pi|codex]` uses the selected
|
|
45
|
+
`--root` and `--owner` (or their existing environment defaults). Omit `--source`
|
|
46
|
+
to select all four. It takes the capture lock nonblockingly, atomically commits
|
|
47
|
+
source reset generations, then exits **without a capture pass**. The next pass
|
|
48
|
+
reprocesses those sources, including when a source cursor was lost. Sources,
|
|
49
|
+
journals, journal-ID lookup, journal-validation checkpoints, the search index,
|
|
50
|
+
ignore rules, tombstones and active set-aside entries are preserved. Held or
|
|
51
|
+
unknown locks, invalid arguments and failed writes refuse success.
|
|
52
|
+
|
|
53
|
+
For a conflicting native prefix, either:
|
|
54
|
+
|
|
55
|
+
1. Restore the captured prefix and run `turn-record capture --retry-set-aside`.
|
|
56
|
+
Retry rechecks the prefix; it never authorizes a changed one.
|
|
57
|
+
2. Add the source path or session ID to `<root>/ignore`. **Later turns from that
|
|
58
|
+
session will not be captured.** Reconciliation applies ignore before opening
|
|
59
|
+
the source, including missing or unreadable sources, and removes the ignored
|
|
60
|
+
input from active set-aside entries. Removing the ignore rule later does not
|
|
61
|
+
authorize the conflict.
|
|
62
|
+
|
|
63
|
+
Retry takes the same nonblocking capture lock, marks all current entries eligible
|
|
64
|
+
for revalidation, then performs one ordinary reconciliation while retaining the
|
|
65
|
+
lock. An interruption after marking leaves the warning active. It is not yet a
|
|
66
|
+
policy-bounded pass. Resetting cursors does not replace retry or clear a refusal.
|
|
67
|
+
|
|
68
|
+
`capture --status` keeps its existing store summary. `capture --health [--json]`
|
|
69
|
+
reads current active set-aside rows from `index/capture.db`, without capturing,
|
|
70
|
+
rebuilding, creating an absent root/database, or contacting a service manager.
|
|
71
|
+
The shared `capture-status.json` is the atomic **last command snapshot**: direct
|
|
72
|
+
synchronous library calls update active issues but do not rewrite that snapshot
|
|
73
|
+
or acquire an implicit root capture lock. Health gets historical run fields from
|
|
74
|
+
the snapshot and never advances them from current issues. A database with no
|
|
75
|
+
snapshot still exposes its active issues.
|
|
76
|
+
|
|
77
|
+
Both command snapshots and health use the same transactional count/sample
|
|
78
|
+
projection: at most 100 deterministic entries, total count and `truncated`.
|
|
79
|
+
Text output has the same bound and explains both remedies. Absent state on a
|
|
80
|
+
never-installed root is not a warning or a reason to create files. A missing
|
|
81
|
+
current database cannot erase a known nonempty command snapshot. Malformed or
|
|
82
|
+
unreadable status is reported as `capture-status-invalid`; an unavailable current
|
|
83
|
+
issue reader fails the command rather than returning successful stale/empty
|
|
84
|
+
health JSON.
|
|
85
|
+
|
|
86
|
+
Reading an existing database requires Node's read-only SQLite support: 22.12+
|
|
87
|
+
on the 22.x line (22.13+ avoids needing `--experimental-sqlite`), 23.2+ on 23.x,
|
|
88
|
+
or a newer major. Normal capture retains its declared runtime floor. Missing
|
|
89
|
+
SQLite support, unsupported schemas, lock/recovery errors, and WAL databases
|
|
90
|
+
fail health without migration or repair. Supported `capture.db` writers retain
|
|
91
|
+
rollback journaling. The health reader rejects WAL from its fixed header before
|
|
92
|
+
opening SQLite; it never creates WAL sidecars, switches mode, recovers a hot
|
|
93
|
+
journal or uses immutable/no-lock shortcuts. This does not protect against an
|
|
94
|
+
external actor racing a mode change or file replacement.
|
|
95
|
+
|
|
96
|
+
Header validation and the read-only transaction run in one private child using
|
|
97
|
+
the current Node executable, only the explicit SQLite runtime flag, and a clean
|
|
98
|
+
environment. It has a ten-second deadline, a 1 MiB output cap and no retries.
|
|
99
|
+
This costs one short process per health read and keeps a raw descriptor close
|
|
100
|
+
from releasing POSIX locks belonging to another SQLite connection in the caller.
|
|
101
|
+
Command snapshots reuse their owned connection for the shared projection.
|
|
102
|
+
|
|
103
|
+
The shared schema is [capture-status.schema.json](capture-status.schema.json).
|
|
104
|
+
In this algorithm-only phase `policy`, `lastRun`, `lastSuccess`, `lastFailure`
|
|
105
|
+
and `lastProgressAt` are null: evidence has not been supplied. They never mean
|
|
106
|
+
that a bounded pass, service installation or successful sweep occurred. Health
|
|
107
|
+
`status: "ok"` with empty rows means no reported set-aside fault, not service
|
|
108
|
+
liveness. Service supervision will extend this same mechanism with actual policy
|
|
109
|
+
and run evidence. Status replacement is atomic with private file permissions;
|
|
110
|
+
persistence failure is an operation failure, not a fabricated success.
|
|
111
|
+
|
|
112
|
+
## Resource model
|
|
113
|
+
|
|
114
|
+
Source and journal reads use 64 KiB chunks. Batches commit at 2,000 records or
|
|
115
|
+
8 MiB of accumulated raw bytes; a single larger record is still supported and
|
|
116
|
+
can exceed that batch target. SQLite connections use a 2 MiB page-cache target
|
|
117
|
+
and disk-backed temporary tables. Metadata query pages hold at most 128 rows;
|
|
118
|
+
keyset queries support the declared Node 22.5 SQLite API without rescanning
|
|
119
|
+
prior pages. Content-processing memory therefore depends on fixed buffers,
|
|
120
|
+
bounded batches/caches and the largest processable individual record, rather
|
|
121
|
+
than total source bytes or all historical IDs. These
|
|
122
|
+
are algorithm settings, **not** a memory/deadline policy or record-size limit.
|
|
123
|
+
|
|
124
|
+
Native root and path discovery retain in-memory metadata inventories, sorted by
|
|
125
|
+
the existing per-directory `localeCompare` order with stable ties and depth-first
|
|
126
|
+
traversal. Aweb account logs retain their code-unit sorted basename order. Their memory grows with file/profile counts, separately from streamed
|
|
127
|
+
content processing; metadata enumeration and sorting still run on unchanged
|
|
128
|
+
passes. Public list APIs retain their full sorted-array output. Per-input
|
|
129
|
+
cursors in `index/capture.db` replace whole-map JSON loads and rewrites. The
|
|
130
|
+
background caller requests capture counts without collecting a receipt inventory;
|
|
131
|
+
it does not run the explicit full-index count query merely to print progress.
|
|
132
|
+
Explicit API and receipt defaults retain complete issue/source/session
|
|
133
|
+
inventories, so their metadata memory may grow with the output they must return.
|
|
134
|
+
They do not retain whole source contents or journals to compute counts and
|
|
135
|
+
first/last visible IDs; receipt summaries use file-backed tombstone claims.
|
|
136
|
+
|
|
137
|
+
Final/attributed/Pi-custody capture and changed-transcript comparison can require
|
|
138
|
+
a complete verified snapshot before mutation. Those paths stream the source to
|
|
139
|
+
a private temporary file under `<root>/.capture-tmp`, hash it while reading,
|
|
140
|
+
and append from the same pinned bytes. Receipt claims databases use that same
|
|
141
|
+
record-root scratch area. Staging costs one source-sized temporary copy and is
|
|
142
|
+
removed on ordinary success or failure. These files are colocated on the record
|
|
143
|
+
root volume, not the system temp volume; the chosen record volume may itself
|
|
144
|
+
be memory-backed. Full receipt verification and recovery may read historical bytes;
|
|
145
|
+
this necessary proof work is distinct from unchanged/small-growth accounting.
|
|
146
|
+
|
|
147
|
+
Journal IDs and validation checkpoints are accelerators, never authority.
|
|
148
|
+
A validation checkpoint qualifies an exact device/inode/size/mtime/ctime version
|
|
149
|
+
under the stream lock. Unknown replacement, shrink or mutation forces bounded
|
|
150
|
+
validation before append. Journal fsync precedes the derived commit, so an
|
|
151
|
+
interruption leaves an old checkpoint that triggers validation. Validation
|
|
152
|
+
commits bounded progress; search rows and byte/count cursors commit together in
|
|
153
|
+
bounded transactions. Ordinary passes seek new journal bytes without parsing
|
|
154
|
+
consumed prefixes. Inconsistent identities or cursors invalidate derived state;
|
|
155
|
+
recovery can cost a historical scan. No journal format or turn ID changes.
|
|
156
|
+
|
|
157
|
+
Scratch directories are private per synchronous caller (0700 directories, 0600
|
|
158
|
+
owner/payload files), with a same-host PID and nonce marker. No supported path
|
|
159
|
+
transfers their ownership or open descriptors to a subprocess. A command holding
|
|
160
|
+
the capture lock removes recognized allocations only after positively proving
|
|
161
|
+
the local allocator dead and rechecking directory and marker identity. Public
|
|
162
|
+
synchronous callers still do not acquire an implicit capture lock; their live
|
|
163
|
+
allocations are preserved by command cleanup. Live/reused PIDs, unknown or foreign
|
|
164
|
+
owners, permission failures, links, special files and incomplete/missing markers
|
|
165
|
+
are preserved. A kill before ownership publication can leave an unowned directory.
|
|
166
|
+
For such leftovers, stop all capture/library users, establish that no caller is
|
|
167
|
+
using the specific allocation, and remove only that confirmed orphan; do not
|
|
168
|
+
blanket-delete the scratch area while callers may be active. No age-only cleanup.
|
|
169
|
+
|
|
170
|
+
## First-upgrade cost
|
|
171
|
+
|
|
172
|
+
The first capture after upgrade must reconcile existing journals and rebuild the
|
|
173
|
+
search index. Per-input capture checkpoints and per-stream index transactions
|
|
174
|
+
still incur substantial synchronous commit costs on storage where sync is costly.
|
|
175
|
+
Connection/transaction amortization is unresolved by these algorithm changes;
|
|
176
|
+
bounded memory and incremental content work do not imply a cheap first pass.
|
|
177
|
+
The bounded service must separately qualify first-upgrade and unchanged passes
|
|
178
|
+
beside actual v0.53.0 on a Linux runner, including filesystem/runtime details,
|
|
179
|
+
connection/transaction/sync counts, resumable prefix proof and deadline progress.
|
|
180
|
+
|
|
181
|
+
## Downgrade and re-upgrade
|
|
182
|
+
|
|
183
|
+
The supported path is v0.53.0 → this version → v0.53.0 → this version, including
|
|
184
|
+
appends in each phase. Known legacy `index/capture-offsets.json` and
|
|
185
|
+
`index/capture-aw-seen.json` maps are invalidated before appends can make them
|
|
186
|
+
unsafe; invalidation failure refuses the append. Unknown root files are left
|
|
187
|
+
alone. Old capture rebuilds its offsets and IDs from the unchanged journals.
|
|
188
|
+
The separate new byte-cursor table leaves the old search-index count schema and
|
|
189
|
+
old named-column writes compatible. Old appends invalidate exact journal
|
|
190
|
+
versions and mismatched index counts on re-upgrade, requiring derived rebuilds.
|
|
191
|
+
Deleting the derived caches has the same recovery cost. No journal-byte migration
|
|
192
|
+
or rollback is required. Native-prefix set-aside is essential to that guarantee;
|
|
193
|
+
it does not rewrite history or append a sentinel or duplicate terminal turn.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"title": "Capture status (algorithm phase)",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"additionalProperties": false,
|
|
6
|
+
"required": [
|
|
7
|
+
"schemaVersion",
|
|
8
|
+
"policy",
|
|
9
|
+
"lastRun",
|
|
10
|
+
"lastSuccess",
|
|
11
|
+
"lastFailure",
|
|
12
|
+
"lastProgressAt",
|
|
13
|
+
"backlog"
|
|
14
|
+
],
|
|
15
|
+
"properties": {
|
|
16
|
+
"schemaVersion": {
|
|
17
|
+
"const": 1
|
|
18
|
+
},
|
|
19
|
+
"policy": {
|
|
20
|
+
"type": "null",
|
|
21
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
22
|
+
},
|
|
23
|
+
"lastRun": {
|
|
24
|
+
"type": "null",
|
|
25
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
26
|
+
},
|
|
27
|
+
"lastSuccess": {
|
|
28
|
+
"type": "null",
|
|
29
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
30
|
+
},
|
|
31
|
+
"lastFailure": {
|
|
32
|
+
"type": "null",
|
|
33
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
34
|
+
},
|
|
35
|
+
"lastProgressAt": {
|
|
36
|
+
"type": "null",
|
|
37
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
38
|
+
},
|
|
39
|
+
"backlog": {
|
|
40
|
+
"type": "object",
|
|
41
|
+
"additionalProperties": false,
|
|
42
|
+
"required": [
|
|
43
|
+
"pending",
|
|
44
|
+
"setAsideCount",
|
|
45
|
+
"setAside",
|
|
46
|
+
"truncated"
|
|
47
|
+
],
|
|
48
|
+
"properties": {
|
|
49
|
+
"pending": {
|
|
50
|
+
"type": "boolean"
|
|
51
|
+
},
|
|
52
|
+
"setAsideCount": {
|
|
53
|
+
"type": "integer",
|
|
54
|
+
"minimum": 0
|
|
55
|
+
},
|
|
56
|
+
"setAside": {
|
|
57
|
+
"type": "array",
|
|
58
|
+
"maxItems": 100,
|
|
59
|
+
"items": {
|
|
60
|
+
"type": "object",
|
|
61
|
+
"additionalProperties": false,
|
|
62
|
+
"required": [
|
|
63
|
+
"source",
|
|
64
|
+
"path",
|
|
65
|
+
"offset",
|
|
66
|
+
"reason"
|
|
67
|
+
],
|
|
68
|
+
"properties": {
|
|
69
|
+
"source": {
|
|
70
|
+
"enum": [
|
|
71
|
+
"aw",
|
|
72
|
+
"cc",
|
|
73
|
+
"pi",
|
|
74
|
+
"codex"
|
|
75
|
+
]
|
|
76
|
+
},
|
|
77
|
+
"path": {
|
|
78
|
+
"type": "string"
|
|
79
|
+
},
|
|
80
|
+
"offset": {
|
|
81
|
+
"type": "integer",
|
|
82
|
+
"minimum": 0
|
|
83
|
+
},
|
|
84
|
+
"reason": {
|
|
85
|
+
"type": "string"
|
|
86
|
+
},
|
|
87
|
+
"line": {
|
|
88
|
+
"type": "integer",
|
|
89
|
+
"minimum": 1
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
"truncated": {
|
|
95
|
+
"type": "boolean"
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|