@awebai/oats 0.54.0 → 0.55.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/oats.mjs +5 -5
  2. package/docs/desktop-cli-api.md +15 -4
  3. package/docs/integrations.md +2 -2
  4. package/docs/knowledge.md +14 -2
  5. package/docs/official-catalog.md +4 -4
  6. package/docs/packages.md +10 -10
  7. package/docs/release-notes/v0.55.0.md +158 -0
  8. package/docs/release-notes/v0.55.1.md +45 -0
  9. package/docs/servers.md +13 -3
  10. package/docs/workspaces.md +1 -1
  11. package/lib/servers.mjs +82 -34
  12. package/package-catalog.json +3 -3
  13. package/package.json +1 -1
  14. package/packages/client/cli-locator.mjs +2 -2
  15. package/packages/client/workspace-views.mjs +4 -3
  16. package/packages/record/README.md +58 -13
  17. package/packages/record/bin/capture.mjs +107 -53
  18. package/packages/record/bin/recall.mjs +3 -2
  19. package/packages/record/docs/capture-service.md +193 -0
  20. package/packages/record/docs/capture-status.schema.json +100 -0
  21. package/packages/record/lib/capture-aw.mjs +96 -94
  22. package/packages/record/lib/capture-cc.mjs +109 -187
  23. package/packages/record/lib/capture-db-reader.mjs +39 -0
  24. package/packages/record/lib/capture-issues.mjs +64 -0
  25. package/packages/record/lib/capture-scratch.mjs +111 -0
  26. package/packages/record/lib/capture-state.mjs +105 -0
  27. package/packages/record/lib/derived-state.mjs +60 -0
  28. package/packages/record/lib/formats.mjs +31 -31
  29. package/packages/record/lib/index-db.mjs +74 -34
  30. package/packages/record/lib/receipt-summary.mjs +62 -0
  31. package/packages/record/lib/session-snapshot.mjs +56 -1
  32. package/packages/record/lib/sqlite-rows.mjs +14 -0
  33. package/packages/record/lib/store.mjs +122 -43
  34. package/packages/record/lib/transcript-prefix.mjs +40 -0
  35. package/skills/oats-getting-started/SKILL.md +1 -1
@@ -2,7 +2,10 @@
2
2
  // are not attribution: a renamed/replaced source must never donate bytes to
3
3
  // the previous source's stream. Reads and validation use the SAME descriptor.
4
4
  import { createHash } from "node:crypto";
5
- import { fstatSync, lstatSync, readSync, statSync } from "node:fs";
5
+ import { closeSync, fstatSync, lstatSync, openSync, readSync, statSync, writeSync } from "node:fs";
6
+ import { createCaptureScratch } from "./capture-scratch.mjs";
7
+ import { join } from "node:path";
8
+ import { fileChunks } from "./file-lines.mjs";
6
9
  import { guardCapturedPath } from "./native-history.mjs";
7
10
 
8
11
  export function identity(stat) {
@@ -41,6 +44,58 @@ export function readRange(fd, start, size, path, capturedPi) {
41
44
  return buf;
42
45
  }
43
46
 
47
+ // A fixed descriptor snapshot boundary: EOF before end is failure, never a
48
+ // successful partial capture. Protected custody is checked around each read.
49
+ export function* sourceChunks(fd, start, end, path, capturedPi) {
50
+ assertProtectedDescriptor(fd, path, capturedPi);
51
+ for (let position = start; position < end;) {
52
+ if (capturedPi) assertProtectedDescriptor(fd, path, capturedPi);
53
+ const bytes = Buffer.allocUnsafe(Math.min(65536, end - position));
54
+ const count = readSync(fd, bytes, 0, bytes.length, position);
55
+ if (!count) throw new Error(`short read of session source: ${path}`);
56
+ position += count;
57
+ yield bytes.subarray(0, count);
58
+ }
59
+ if (capturedPi) assertProtectedDescriptor(fd, path, capturedPi);
60
+ }
61
+
62
+ // Attribution must be verified before the first journal mutation. Keep its
63
+ // immutable bytes in private record-root scratch, not in a source-sized Buffer.
64
+ // The caller owns close(), including when header checks or appends fail.
65
+ export function stageSnapshot(fd, stat, path, capturedPi, root) {
66
+ const scratch = createCaptureScratch(root, "snapshot"), dir = scratch.path;
67
+ let stagedFd;
68
+ try {
69
+ stagedFd = openSync(join(dir, "snapshot"), "wx+", 0o600);
70
+ const hash = createHash("sha256");
71
+ for (const bytes of sourceChunks(fd, 0, stat.size, path, capturedPi)) {
72
+ hash.update(bytes);
73
+ for (let written = 0; written < bytes.length;) {
74
+ const n = writeSync(stagedFd, bytes, written, bytes.length - written);
75
+ if (!n) throw new Error("short write of staged session snapshot");
76
+ written += n;
77
+ }
78
+ }
79
+ return {
80
+ stat, hash: hash.digest("hex"),
81
+ chunks: (start = 0, end = stat.size) => fileChunks(stagedFd, { start, end, size: 65536 }),
82
+ prefixHash(end) {
83
+ const prefix = createHash("sha256");
84
+ for (const bytes of this.chunks(0, end)) prefix.update(bytes);
85
+ return prefix.digest("hex");
86
+ },
87
+ close() {
88
+ try { if (stagedFd !== undefined) closeSync(stagedFd); }
89
+ finally { stagedFd = undefined; scratch.close(); }
90
+ },
91
+ };
92
+ } catch (err) {
93
+ try { if (stagedFd !== undefined) closeSync(stagedFd); }
94
+ finally { scratch.close(); }
95
+ throw err;
96
+ }
97
+ }
98
+
44
99
  export function hashPrefix(fd, size, path, capturedPi) {
45
100
  assertProtectedDescriptor(fd, path, capturedPi);
46
101
  const hash = createHash("sha256"), buf = Buffer.alloc(64 * 1024);
@@ -0,0 +1,14 @@
1
+ // Bounded keyset enumeration using the SQLite API available at Node 22.5.
2
+ // Statement.iterate() requires 22.13; OFFSET paging would rescan old prefixes.
3
+ // Callers supply a SELECT with one unique, non-null text ordering key.
4
+ export function* keysetRows(db, sql, key, parameters = []) {
5
+ if (!/^[a-z_]+$/.test(key)) throw new Error("invalid internal SQLite ordering key");
6
+ const order = `"${key}"`;
7
+ const first = db.prepare(`SELECT * FROM (${sql}) ORDER BY ${order} LIMIT 128`);
8
+ const next = db.prepare(`SELECT * FROM (${sql}) WHERE ${order} > ? ORDER BY ${order} LIMIT 128`);
9
+ let rows = first.all(...parameters);
10
+ while (rows.length) {
11
+ yield* rows;
12
+ rows = next.all(...parameters, rows[rows.length - 1][key]);
13
+ }
14
+ }
@@ -23,6 +23,7 @@ import {
23
23
  linkSync,
24
24
  mkdirSync,
25
25
  openSync,
26
+ opendirSync,
26
27
  readdirSync,
27
28
  readFileSync,
28
29
  readSync,
@@ -31,6 +32,7 @@ import {
31
32
  unlinkSync,
32
33
  writeFileSync,
33
34
  } from "node:fs";
35
+ import { DerivedState, fileVersion, sameVersion, invalidateLegacyMaps } from "./derived-state.mjs";
34
36
  import { randomBytes } from "node:crypto";
35
37
  import { hostname } from "node:os";
36
38
  import { join } from "node:path";
@@ -328,11 +330,6 @@ export class RecordStore {
328
330
  this.owner = owner ?? null;
329
331
  this.lockTimeoutMs = lockTimeoutMs;
330
332
  this.lockStaleMs = lockStaleMs;
331
- // Streams whose full journal this instance has parsed without error.
332
- // The write path validates each journal once before its first append
333
- // (interior corruption must fail loud, never accept appends after the
334
- // damage); its own appends are trusted afterwards.
335
- this.validatedStreams = new Set();
336
333
  }
337
334
 
338
335
  // Serialize every read-repair-write sequence on one stream across
@@ -399,6 +396,17 @@ export class RecordStore {
399
396
  .sort();
400
397
  }
401
398
 
399
+ *iterateStreams() {
400
+ let dir;
401
+ try { dir = opendirSync(this.streamsDir()); }
402
+ catch (err) { if (err.code === "ENOENT") return; throw err; }
403
+ try {
404
+ for (let entry; (entry = dir.readSync()) !== null;) {
405
+ if (validStreamId(entry.name) && existsSync(join(this.streamsDir(), entry.name, "journal.jsonl"))) yield entry.name;
406
+ }
407
+ } finally { dir.closeSync(); }
408
+ }
409
+
402
410
  readStream(streamId) {
403
411
  const path = this.journalPath(streamId);
404
412
  if (!existsSync(path)) return [];
@@ -411,27 +419,86 @@ export class RecordStore {
411
419
  // rules as readStream. Joining fragments only at a newline avoids
412
420
  // repeatedly copying long lines as each chunk arrives.
413
421
  *iterateStream(streamId) {
422
+ for (const entry of this.iterateJournal(streamId)) if (entry.turn !== null) yield entry.turn;
423
+ }
424
+
425
+ // Absolute byte boundaries, including blank lines, permit durable seeks.
426
+ // The final fragment never advances a cursor: it is the owner's torn tail.
427
+ *iterateJournal(streamId, { start = 0, end = Infinity } = {}) {
414
428
  let fd;
415
429
  try { fd = openSync(this.journalPath(streamId), "r"); }
416
430
  catch (e) { if (e.code === "ENOENT") return; throw e; }
417
- let lineStart = 0;
431
+ let position = start;
418
432
  try {
419
- for (const line of bufferLines(fileChunks(fd, { size: 65536 }))) {
420
- // A final fragment without a newline is a torn tail, even if its
421
- // JSON is valid. Readers leave it for the owner's append repair.
433
+ for (const line of bufferLines(fileChunks(fd, { start, end, size: 65536 }))) {
422
434
  if (line[line.length - 1] !== 10) return;
423
435
  let parsed;
424
436
  try { parsed = parseJournal(line); }
425
437
  catch (e) {
426
- if (e instanceof StoreError) throw new StoreError(`corrupt interior journal line at byte ${lineStart}`);
438
+ if (e instanceof StoreError) throw new StoreError(`corrupt interior journal line at byte ${position}`);
427
439
  throw e;
428
440
  }
429
- lineStart += line.length;
430
- yield* parsed.turns;
441
+ position += line.length;
442
+ yield { turn: parsed.turns[0] ?? null, end: position, bytes: line.length };
431
443
  }
432
444
  } finally { closeSync(fd); }
433
445
  }
434
446
 
447
+ withDerived(fn) {
448
+ const state = new DerivedState(this.root);
449
+ try { return fn(state); } finally { state.close(); }
450
+ }
451
+
452
+ // Caller holds the stream lock. Only a version recorded after validation or
453
+ // our fsynced append can skip history. Unknown writes (including old releases),
454
+ // replacement and shrink start a new epoch and invalidate dependent cursors.
455
+ // Partial validation is resumable, but cannot authorize an append.
456
+ validateJournal(streamId, state) {
457
+ const path = this.journalPath(streamId), version = fileVersion(path);
458
+ let checkpoint = state.get("journal", streamId);
459
+ if (checkpoint && (!Number.isSafeInteger(checkpoint.bytes) || checkpoint.bytes < 0 || checkpoint.bytes > (version?.size ?? 0) ||
460
+ !Number.isSafeInteger(checkpoint.count) || checkpoint.count < 0 || checkpoint.count > checkpoint.bytes ||
461
+ typeof checkpoint.epoch !== "string" || typeof checkpoint.complete !== "boolean")) checkpoint = null;
462
+ if (checkpoint && sameVersion(checkpoint.version, version) && checkpoint.complete) return checkpoint;
463
+ if (!checkpoint || !sameVersion(checkpoint.version, version)) {
464
+ checkpoint = { version, epoch: randomBytes(16).toString("hex"), bytes: 0, count: 0, complete: false };
465
+ state.transaction(() => {
466
+ state.db.prepare("DELETE FROM journal_ids WHERE stream = ?").run(streamId);
467
+ state.set("journal", streamId, checkpoint);
468
+ });
469
+ }
470
+ const addId = state.db.prepare("INSERT OR IGNORE INTO journal_ids VALUES (?, ?)");
471
+ const iterator = this.iterateJournal(streamId, { start: checkpoint.bytes, end: version?.size ?? 0 });
472
+ let done = false;
473
+ try {
474
+ while (!done) {
475
+ state.transaction(() => {
476
+ let batch = 0, bytes = 0;
477
+ while (batch < 2000 && bytes < 8 * 1024 * 1024) {
478
+ const next = iterator.next();
479
+ if (next.done) { done = true; break; }
480
+ const entry = next.value;
481
+ if (entry.turn !== null) {
482
+ checkpoint.count++;
483
+ if (typeof entry.turn.id === "string") addId.run(streamId, entry.turn.id);
484
+ }
485
+ checkpoint.bytes = entry.end;
486
+ bytes += entry.bytes; batch++;
487
+ }
488
+ if (!sameVersion(version, fileVersion(path))) throw new StoreError(`journal changed during validation: ${path}`);
489
+ checkpoint.complete = done;
490
+ state.set("journal", streamId, checkpoint);
491
+ });
492
+ }
493
+ return checkpoint;
494
+ } finally { iterator.return(); }
495
+ }
496
+
497
+ journalCheckpoint(streamId) {
498
+ this.journalPath(streamId);
499
+ return this.withStreamLock(streamId, () => this.withDerived(state => this.validateJournal(streamId, state)));
500
+ }
501
+
435
502
  // The ids among `candidates` that the stream holds, in one streamed read of
436
503
  // its journal: a dedupe for a few ids never builds the whole stream's set.
437
504
  idsAmong(streamId, candidates) {
@@ -522,11 +589,17 @@ export class RecordStore {
522
589
  // Returns { turn, appended } — appended=false when the id already exists.
523
590
  appendCore(streamId, core, knownIds = null) {
524
591
  const turn = finishTurn(core);
525
- const ids = knownIds ?? new Set(this.readStream(streamId).map((t) => t.id));
526
- if (ids.has(turn.id)) return { turn, appended: false };
527
- this.append(streamId, turn);
528
- ids.add(turn.id);
529
- return { turn, appended: true };
592
+ if (knownIds) {
593
+ if (knownIds.has(turn.id)) return { turn, appended: false };
594
+ this.append(streamId, turn);
595
+ knownIds.add(turn.id);
596
+ return { turn, appended: true };
597
+ }
598
+ return { turn, appended: this.appendUniqueBatch(streamId, [turn]) === 1 };
599
+ }
600
+
601
+ appendUniqueBatch(streamId, turns) {
602
+ return this.appendBatch(streamId, turns, 8 * 1024 * 1024, true);
530
603
  }
531
604
 
532
605
  // Append many finished turns with bounded writes and one fsync. Same
@@ -535,8 +608,8 @@ export class RecordStore {
535
608
  // chunked by bytes, never one join of the whole batch: a large session
536
609
  // captured in one pass is a single batch, and joining it can exceed
537
610
  // V8's maximum string length (this crashed the first live recapture).
538
- appendBatch(streamId, turns, maxChunkBytes = 32 * 1024 * 1024) {
539
- if (turns.length === 0) return;
611
+ appendBatch(streamId, turns, maxChunkBytes = 32 * 1024 * 1024, unique = false) {
612
+ if (turns.length === 0) return 0;
540
613
  if (!this.owner) throw new StoreError("store has no owner; cannot append");
541
614
  if (streamOwner(streamId) !== this.owner) {
542
615
  throw new StoreError(
@@ -547,34 +620,40 @@ export class RecordStore {
547
620
  if (!verifyTurnId(turn)) throw new StoreError("turn id missing or does not match content");
548
621
  }
549
622
  const path = this.journalPath(streamId);
550
- this.withStreamLock(streamId, () => {
551
- // First append to this stream in this instance: read the whole
552
- // journal, so interior corruption throws here instead of silently
553
- // collecting appends behind the damage. A torn tail is tolerated
554
- // (iterateStream leaves it, as parseJournal treats it as
555
- // final-line-torn) and repaired below. Streamed, one line at a time:
556
- // a journal can be far larger than the heap (awebai/oats#456).
557
- if (!this.validatedStreams.has(streamId) && existsSync(path)) {
558
- for (const _turn of this.iterateStream(streamId)) { /* validated as read */ }
559
- }
560
- this.validatedStreams.add(streamId);
623
+ return this.withStreamLock(streamId, () => {
624
+ invalidateLegacyMaps(this.root);
625
+ return this.withDerived(state => {
626
+ const checkpoint = this.validateJournal(streamId, state);
561
627
  repairTail(path);
562
- let chunk = [];
563
- let chunkBytes = 0;
628
+ // Node 22.5 returns a null-valued row on no match; object truthiness
629
+ // is not an existence test across the supported SQLite runtimes.
630
+ const hasId = state.db.prepare("SELECT 1 AS found FROM journal_ids WHERE stream = ? AND id = ?");
631
+ const addId = state.db.prepare("INSERT OR IGNORE INTO journal_ids VALUES (?, ?)");
632
+ let chunk = [], chunkBytes = 0, appended = 0;
564
633
  const flush = () => {
565
- if (chunk.length === 0) return;
634
+ if (!chunk.length) return;
566
635
  appendFileSync(path, chunk.join("\n") + "\n");
567
- chunk = [];
568
- chunkBytes = 0;
636
+ chunk = []; chunkBytes = 0;
569
637
  };
570
- for (const turn of turns) {
571
- const line = JSON.stringify(turn);
572
- if (chunkBytes > 0 && chunkBytes + line.length > maxChunkBytes) flush();
573
- chunk.push(line);
574
- chunkBytes += line.length + 1;
575
- }
576
- flush();
577
- syncPath(path);
638
+ // A failed transaction can leave appended journal bytes. Its old version
639
+ // checkpoint then fails qualification and a later pass validates/rebuilds.
640
+ return state.transaction(() => {
641
+ for (const turn of turns) {
642
+ if (unique && hasId.get(streamId, turn.id)?.found === 1) continue;
643
+ const line = JSON.stringify(turn), bytes = Buffer.byteLength(line) + 1;
644
+ if (chunkBytes && chunkBytes + bytes > maxChunkBytes) flush();
645
+ chunk.push(line); chunkBytes += bytes;
646
+ addId.run(streamId, turn.id); appended++;
647
+ }
648
+ flush();
649
+ if (existsSync(path)) syncPath(path);
650
+ checkpoint.version = fileVersion(path);
651
+ checkpoint.bytes = checkpoint.version?.size ?? 0;
652
+ checkpoint.count += appended;
653
+ state.set("journal", streamId, checkpoint);
654
+ return appended;
655
+ });
656
+ });
578
657
  });
579
658
  }
580
659
 
@@ -0,0 +1,40 @@
1
+ // Recovery proof for positional transcript journals. The temporary lookup is
2
+ // disk-backed: comparing a long prefix retains one native record at a time.
3
+ import { digest } from "./session-snapshot.mjs";
4
+
5
+ export function compareTranscriptPrefix(store, stream, state, lines) {
6
+ const db = state.db;
7
+ db.exec("DROP TABLE IF EXISTS temp.transcript_prefix; CREATE TEMP TABLE transcript_prefix (line INTEGER PRIMARY KEY, hash TEXT NOT NULL)");
8
+ const put = db.prepare("INSERT OR IGNORE INTO transcript_prefix VALUES (?, ?)");
9
+ const get = db.prepare("SELECT hash FROM transcript_prefix WHERE line = ?");
10
+ let lastLine = 0, lastTs = "", batch = [];
11
+ const flush = () => state.transaction(() => {
12
+ for (const turn of batch) {
13
+ const line = turn.provenance?.origin?.line;
14
+ if (!Number.isSafeInteger(line) || line < 1 || typeof turn.body?.line !== "string") throw new Error(`invalid positional transcript journal: ${stream}`);
15
+ const hash = digest(Buffer.from(turn.body.line));
16
+ const old = get.get(line);
17
+ if (old?.hash != null && old.hash !== hash) throw new Error(`conflicting positional transcript journal: ${stream}, line ${line}`);
18
+ put.run(line, hash);
19
+ if (line >= lastLine) { lastLine = line; lastTs = turn.ts; }
20
+ }
21
+ batch = [];
22
+ });
23
+ let bytes = 0;
24
+ for (const turn of store.iterateStream(stream)) {
25
+ if (turn.kind !== "session") continue;
26
+ batch.push(turn); bytes += Buffer.byteLength(turn.body?.line ?? "");
27
+ if (batch.length >= 2000 || bytes >= 8 * 1024 * 1024) { flush(); bytes = 0; }
28
+ }
29
+ flush();
30
+ let line = 0, offset = 0;
31
+ if (lastLine) for (const record of lines) {
32
+ line++;
33
+ const expected = get.get(line);
34
+ if (expected?.hash != null && expected.hash !== digest(Buffer.from(record.text))) return { conflict: { line, offset, reason: "captured-prefix-changed" } };
35
+ offset += record.bytes;
36
+ if (line === lastLine) break;
37
+ }
38
+ if (line < lastLine) return { conflict: { line: line + 1, offset, reason: "captured-prefix-missing" } };
39
+ return { bytes: offset, line, lastTs };
40
+ }
@@ -61,7 +61,7 @@ members:
61
61
  - git:github.com/acme/platform
62
62
  packages:
63
63
  oats.framework: v1.10.0 # bare versions resolve through the official catalog
64
- oats.okf: v5.0.3
64
+ oats.okf: v5.0.5
65
65
  defaults:
66
66
  capabilities: { oats.core: { from: package } }
67
67
  knowledge: { oats.okf: { from: package } }