@awebai/oats 0.54.0 → 0.55.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/oats.mjs +5 -5
  2. package/docs/desktop-cli-api.md +15 -4
  3. package/docs/integrations.md +2 -2
  4. package/docs/knowledge.md +14 -2
  5. package/docs/official-catalog.md +4 -4
  6. package/docs/packages.md +10 -10
  7. package/docs/release-notes/v0.55.0.md +158 -0
  8. package/docs/release-notes/v0.55.1.md +45 -0
  9. package/docs/servers.md +13 -3
  10. package/docs/workspaces.md +1 -1
  11. package/lib/servers.mjs +82 -34
  12. package/package-catalog.json +3 -3
  13. package/package.json +1 -1
  14. package/packages/client/cli-locator.mjs +2 -2
  15. package/packages/client/workspace-views.mjs +4 -3
  16. package/packages/record/README.md +58 -13
  17. package/packages/record/bin/capture.mjs +107 -53
  18. package/packages/record/bin/recall.mjs +3 -2
  19. package/packages/record/docs/capture-service.md +193 -0
  20. package/packages/record/docs/capture-status.schema.json +100 -0
  21. package/packages/record/lib/capture-aw.mjs +96 -94
  22. package/packages/record/lib/capture-cc.mjs +109 -187
  23. package/packages/record/lib/capture-db-reader.mjs +39 -0
  24. package/packages/record/lib/capture-issues.mjs +64 -0
  25. package/packages/record/lib/capture-scratch.mjs +111 -0
  26. package/packages/record/lib/capture-state.mjs +105 -0
  27. package/packages/record/lib/derived-state.mjs +60 -0
  28. package/packages/record/lib/formats.mjs +31 -31
  29. package/packages/record/lib/index-db.mjs +74 -34
  30. package/packages/record/lib/receipt-summary.mjs +62 -0
  31. package/packages/record/lib/session-snapshot.mjs +56 -1
  32. package/packages/record/lib/sqlite-rows.mjs +14 -0
  33. package/packages/record/lib/store.mjs +122 -43
  34. package/packages/record/lib/transcript-prefix.mjs +40 -0
  35. package/skills/oats-getting-started/SKILL.md +1 -1
@@ -11,14 +11,15 @@
11
11
  // dedupes by id. Reconciliation is the capture: scan, project, append what
12
12
  // is new, in bounded batches.
13
13
  //
14
- // Memory is bounded by one changed log's ids, never by the record or a log's
15
- // bytes (awebai/oats#456): a log is read in chunks, and what the stream
16
- // already holds is looked up for that log's ids alone, in one streamed read
17
- // of the journal, never as the whole stream's id set.
14
+ // Inputs are append-only logs. Background growth uses a bounded boundary
15
+ // witness and byte cursor; explicit per-file calls retain full reprojection.
16
+ // Persistent journal ID lookup makes either path use bounded memory.
18
17
 
19
- import { closeSync, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
20
- import { basename, dirname, join } from "node:path";
18
+ import { closeSync, constants, existsSync, fstatSync, openSync, readdirSync, readSync } from "node:fs";
19
+ import { basename, join } from "node:path";
21
20
  import { homedir } from "node:os";
21
+ import { createHash } from "node:crypto";
22
+ import { DerivedState, statVersion, sameVersion } from "./derived-state.mjs";
22
23
 
23
24
  import {
24
25
  projectCommLogEntry,
@@ -48,9 +49,15 @@ export function listCommLogs(dir = defaultCommLogDir()) {
48
49
  // A final line without its newline is still an entry when it parses; when it
49
50
  // does not, it is a torn tail the client is still writing, never counted. An
50
51
  // interior bad line is skipped and counted (`onSkipped`), never fatal.
51
- function* logEntries(fd, end, onSkipped) {
52
- for (const line of bufferLines(fileChunks(fd, { end }))) {
52
+ function* logEntries(fd, start, end, onSkipped, onConsumed) {
53
+ let offset = start;
54
+ for (const line of bufferLines(fileChunks(fd, { start, end, size: 65536 }))) {
55
+ const lineStart = offset;
56
+ offset += line.length;
53
57
  const final = line[line.length - 1] !== 10;
58
+ // Even ignored whitespace is scanned progress when newline-terminated.
59
+ // The caller persists it only after its pending append batch is durable.
60
+ if (!final) onConsumed(offset);
54
61
  let entry;
55
62
  try {
56
63
  // Inside the try: a line beyond V8's string limit is a bad line like any other.
@@ -61,55 +68,77 @@ function* logEntries(fd, end, onSkipped) {
61
68
  if (!final) onSkipped();
62
69
  continue;
63
70
  }
64
- yield entry;
71
+ yield { entry, start: lineStart, end: final ? lineStart : offset, bytes: line.length };
65
72
  }
66
73
  }
67
74
 
68
75
  const APPEND_BATCH = 2000;
69
76
 
70
- // One log file into `streamId`: project every entry, append the turns the
71
- // stream lacks. With `knownIds` (a caller's own set, kept up to date), that
72
- // set decides; without, the log is read twice over the same prefix (its size
73
- // when opened: the client may be appending): once for its turn ids, which one
74
- // streamed read of the journal narrows to those already there, and once to
75
- // append the rest in batches.
76
- function captureLogFile(store, { streamId, path, project, knownIds }) {
77
- const fd = openSync(path, "r");
77
+ // Check only a bounded checkpoint boundary under the append-only source contract.
78
+ function boundaryWitness(fd, offset) {
79
+ const size = Math.min(offset, 4096), bytes = Buffer.alloc(size);
80
+ let done = 0;
81
+ while (done < size) {
82
+ const n = readSync(fd, bytes, done, size - done, offset - size + done);
83
+ if (!n) throw new Error("short read of log checkpoint boundary");
84
+ done += n;
85
+ }
86
+ return createHash("sha256").update(bytes).digest("hex");
87
+ }
88
+
89
+ function captureLogFile(store, { streamId, path, project, knownIds, incremental = false }) {
90
+ const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
91
+ let state;
78
92
  try {
79
- const end = fstatSync(fd).size;
80
- let entries = 0, skipped = 0, failed = 0;
81
- function* turns() {
82
- entries = 0; skipped = 0; failed = 0;
83
- for (const entry of logEntries(fd, end, () => skipped++)) {
84
- entries++;
85
- let turn;
86
- try { turn = project(entry); }
87
- catch {
88
- // One unprojectable entry must not abort the pass, but it fails
89
- // visibly: counted here and reported by the caller.
90
- failed++;
91
- continue;
92
- }
93
- yield turn;
94
- }
95
- }
96
- let known = knownIds;
97
- if (!known) {
98
- const candidates = new Set();
99
- for (const turn of turns()) candidates.add(turn.id);
100
- known = store.idsAmong(streamId, candidates);
93
+ const stat = fstatSync(fd, { bigint: true });
94
+ if (!stat.isFile()) throw new Error(`log source is not a regular file: ${path}`);
95
+ const version = statVersion(stat), end = version.size;
96
+ state = new DerivedState(store.root);
97
+ const generation = state.get("reset", streamId.split("~")[0] + ":aw") ?? "";
98
+ const key = `${streamId}:${path}`;
99
+ let previous = incremental ? state.get("aw", key) : null;
100
+ if (previous && (!Number.isSafeInteger(previous.bytes) || previous.bytes < 0 || previous.bytes > end || !previous.version)) previous = null;
101
+ const journal = store.journalCheckpoint(streamId);
102
+ let start = 0;
103
+ if (previous && previous.generation === generation && previous.epoch === journal.epoch) {
104
+ if (previous.complete && sameVersion(previous.version, version)) return { entries: 0, appended: 0, skipped: 0, failed: 0, unchanged: true };
105
+ const old = previous.version;
106
+ if (old.dev === version.dev && old.ino === version.ino && (version.size > old.size || sameVersion(old, version)) &&
107
+ previous.bytes <= end && boundaryWitness(fd, previous.bytes) === previous.witness) start = previous.bytes;
101
108
  }
102
- let appended = 0, batch = [];
103
- const flush = () => { store.appendBatch(streamId, batch); appended += batch.length; batch = []; };
104
- for (const turn of turns()) {
105
- if (known.has(turn.id)) continue;
106
- known.add(turn.id);
107
- batch.push(turn);
108
- if (batch.length >= APPEND_BATCH) flush();
109
+ let entries = 0, skipped = 0, failed = 0, appended = 0, batch = [], batchBytes = 0, consumed = start, complete = false;
110
+ const checkpoint = () => {
111
+ if (!incremental) return;
112
+ const bytes = consumed;
113
+ state.set("aw", key, { bytes, version, witness: boundaryWitness(fd, bytes), epoch: journal.epoch, generation, complete });
114
+ };
115
+ const flush = () => {
116
+ if (batch.length) {
117
+ if (knownIds) { store.appendBatch(streamId, batch); appended += batch.length; }
118
+ else appended += store.appendUniqueBatch(streamId, batch);
119
+ batch = []; batchBytes = 0;
120
+ }
121
+ checkpoint();
122
+ };
123
+ for (const record of logEntries(fd, start, end, () => skipped++, (offset) => { consumed = offset; })) {
124
+ entries++;
125
+ let turn;
126
+ // Projection refusal consumes this complete entry just like a skipped
127
+ // malformed line. Resetting source cursors is the explicit retry path;
128
+ // an old refusal must not pin every background pass to historical bytes.
129
+ consumed = record.end;
130
+ try { turn = project(record.entry); }
131
+ catch { failed++; continue; }
132
+ if (knownIds) { if (knownIds.has(turn.id)) continue; knownIds.add(turn.id); }
133
+ batch.push(turn); batchBytes += record.bytes;
134
+ if (batch.length >= APPEND_BATCH || batchBytes >= 8 * 1024 * 1024) flush();
109
135
  }
110
- if (batch.length) flush();
136
+ // Complete refused/malformed/blank lines are consumed; an unfinished final
137
+ // fragment keeps its starting boundary so later growth can finish it.
138
+ complete = true;
139
+ flush();
111
140
  return { entries, appended, skipped, failed };
112
- } finally { closeSync(fd); }
141
+ } finally { try { state?.close(); } finally { closeSync(fd); } }
113
142
  }
114
143
 
115
144
  // Capture one comm-log file into `<owner>~aw`. The account name is the
@@ -136,54 +165,27 @@ export function captureInteractionLog(store, { owner, path, selfName, workspace,
136
165
  return { stream: streamId, ...r };
137
166
  }
138
167
 
139
- // Derived seen-files cache (same pattern as capture-cc): skip a log file
140
- // whose size+mtime match the last pass without reading or re-projecting it.
141
- function seenCachePath(store) {
142
- return join(store.root, "index", "capture-aw-seen.json");
143
- }
144
-
145
- function loadSeenCache(store) {
146
- try {
147
- return JSON.parse(readFileSync(seenCachePath(store), "utf8"));
148
- } catch {
149
- return {};
150
- }
151
- }
152
-
153
- function saveSeenCache(store, cache) {
154
- const path = seenCachePath(store);
155
- mkdirSync(dirname(path), { recursive: true });
156
- writeFileSync(path, JSON.stringify(cache));
168
+ // Retain sorted basename metadata, preserving listCommLogs' code-unit account
169
+ // order in both background capture and public results. Content is streamed;
170
+ // discovery metadata still grows with the number of log files.
171
+ function* iterateCommLogs(dir) {
172
+ let names;
173
+ try { names = readdirSync(dir); }
174
+ catch (err) { if (err.code === "ENOENT") return; throw err; }
175
+ for (const name of names.sort()) if (name.endsWith(".jsonl")) yield join(dir, name);
157
176
  }
158
-
159
- // One reconciliation pass over every default comm log. Unchanged files are
160
- // skipped; each changed one looks up its own ids in the stream.
161
- //
162
- // Files matching the record's ignore list (`<root>/ignore`, see ignore.mjs)
163
- // are skipped before being opened — no turns, no seen-cache entry — and
164
- // reported as `{ account, path, ignored: true }` so a pass stays visible
165
- // about what it refused to read.
166
- export function captureAwLogs(store, { owner, commLogDir, ignore = null } = {}) {
177
+ function* captureAwEntries(store, { owner, commLogDir, ignore }) {
167
178
  const ign = ignore ?? loadIgnore(store.root);
168
- const seen = loadSeenCache(store);
169
- const results = [];
170
- for (const path of listCommLogs(commLogDir ?? defaultCommLogDir())) {
179
+ for (const path of iterateCommLogs(commLogDir ?? defaultCommLogDir())) {
171
180
  const account = basename(path, ".jsonl");
172
- if (ign.ignores(path, [basename(path), account])) {
173
- results.push({ account, path, ignored: true });
174
- continue; // never opened: nothing stored, nothing remembered
175
- }
176
- let stat;
177
- try {
178
- stat = statSync(path);
179
- } catch {
180
- continue;
181
- }
182
- const prev = seen[path];
183
- if (prev && prev.size === stat.size && prev.mtimeMs === stat.mtimeMs) continue;
184
- results.push(captureCommLog(store, { owner, path }));
185
- seen[path] = { size: stat.size, mtimeMs: stat.mtimeMs };
181
+ if (ign.ignores(path, [basename(path), account])) { yield { account, path, ignored: true }; continue; }
182
+ const stream = awStream(owner);
183
+ const result = captureLogFile(store, { streamId: stream, path, incremental: true,
184
+ project: entry => projectCommLogEntry(entry, { selfName: account }) });
185
+ if (!result.unchanged) yield { account, stream, ...result };
186
186
  }
187
- saveSeenCache(store, seen);
188
- return results;
187
+ }
188
+ export function captureAwLogs(store, { owner, commLogDir, ignore = null, collect = true } = {}) {
189
+ const results = captureAwEntries(store, { owner, commLogDir, ignore });
190
+ return collect ? [...results] : results;
189
191
  }