@awebai/oats 0.54.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oats.mjs +5 -5
- package/docs/desktop-cli-api.md +15 -4
- package/docs/integrations.md +2 -2
- package/docs/knowledge.md +14 -2
- package/docs/official-catalog.md +4 -4
- package/docs/packages.md +10 -10
- package/docs/release-notes/v0.55.0.md +158 -0
- package/docs/servers.md +13 -3
- package/docs/workspaces.md +1 -1
- package/lib/servers.mjs +82 -34
- package/package-catalog.json +3 -3
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/client/workspace-views.mjs +4 -3
- package/packages/record/README.md +54 -13
- package/packages/record/bin/capture.mjs +107 -53
- package/packages/record/docs/capture-service.md +193 -0
- package/packages/record/docs/capture-status.schema.json +100 -0
- package/packages/record/lib/capture-aw.mjs +96 -94
- package/packages/record/lib/capture-cc.mjs +109 -187
- package/packages/record/lib/capture-db-reader.mjs +39 -0
- package/packages/record/lib/capture-issues.mjs +64 -0
- package/packages/record/lib/capture-scratch.mjs +111 -0
- package/packages/record/lib/capture-state.mjs +105 -0
- package/packages/record/lib/derived-state.mjs +60 -0
- package/packages/record/lib/formats.mjs +31 -31
- package/packages/record/lib/index-db.mjs +74 -34
- package/packages/record/lib/receipt-summary.mjs +62 -0
- package/packages/record/lib/session-snapshot.mjs +56 -1
- package/packages/record/lib/sqlite-rows.mjs +14 -0
- package/packages/record/lib/store.mjs +122 -43
- package/packages/record/lib/transcript-prefix.mjs +40 -0
- package/skills/oats-getting-started/SKILL.md +1 -1
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"title": "Capture status (algorithm phase)",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"additionalProperties": false,
|
|
6
|
+
"required": [
|
|
7
|
+
"schemaVersion",
|
|
8
|
+
"policy",
|
|
9
|
+
"lastRun",
|
|
10
|
+
"lastSuccess",
|
|
11
|
+
"lastFailure",
|
|
12
|
+
"lastProgressAt",
|
|
13
|
+
"backlog"
|
|
14
|
+
],
|
|
15
|
+
"properties": {
|
|
16
|
+
"schemaVersion": {
|
|
17
|
+
"const": 1
|
|
18
|
+
},
|
|
19
|
+
"policy": {
|
|
20
|
+
"type": "null",
|
|
21
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
22
|
+
},
|
|
23
|
+
"lastRun": {
|
|
24
|
+
"type": "null",
|
|
25
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
26
|
+
},
|
|
27
|
+
"lastSuccess": {
|
|
28
|
+
"type": "null",
|
|
29
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
30
|
+
},
|
|
31
|
+
"lastFailure": {
|
|
32
|
+
"type": "null",
|
|
33
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
34
|
+
},
|
|
35
|
+
"lastProgressAt": {
|
|
36
|
+
"type": "null",
|
|
37
|
+
"description": "No policy or run evidence supplied in the algorithm-only phase."
|
|
38
|
+
},
|
|
39
|
+
"backlog": {
|
|
40
|
+
"type": "object",
|
|
41
|
+
"additionalProperties": false,
|
|
42
|
+
"required": [
|
|
43
|
+
"pending",
|
|
44
|
+
"setAsideCount",
|
|
45
|
+
"setAside",
|
|
46
|
+
"truncated"
|
|
47
|
+
],
|
|
48
|
+
"properties": {
|
|
49
|
+
"pending": {
|
|
50
|
+
"type": "boolean"
|
|
51
|
+
},
|
|
52
|
+
"setAsideCount": {
|
|
53
|
+
"type": "integer",
|
|
54
|
+
"minimum": 0
|
|
55
|
+
},
|
|
56
|
+
"setAside": {
|
|
57
|
+
"type": "array",
|
|
58
|
+
"maxItems": 100,
|
|
59
|
+
"items": {
|
|
60
|
+
"type": "object",
|
|
61
|
+
"additionalProperties": false,
|
|
62
|
+
"required": [
|
|
63
|
+
"source",
|
|
64
|
+
"path",
|
|
65
|
+
"offset",
|
|
66
|
+
"reason"
|
|
67
|
+
],
|
|
68
|
+
"properties": {
|
|
69
|
+
"source": {
|
|
70
|
+
"enum": [
|
|
71
|
+
"aw",
|
|
72
|
+
"cc",
|
|
73
|
+
"pi",
|
|
74
|
+
"codex"
|
|
75
|
+
]
|
|
76
|
+
},
|
|
77
|
+
"path": {
|
|
78
|
+
"type": "string"
|
|
79
|
+
},
|
|
80
|
+
"offset": {
|
|
81
|
+
"type": "integer",
|
|
82
|
+
"minimum": 0
|
|
83
|
+
},
|
|
84
|
+
"reason": {
|
|
85
|
+
"type": "string"
|
|
86
|
+
},
|
|
87
|
+
"line": {
|
|
88
|
+
"type": "integer",
|
|
89
|
+
"minimum": 1
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
"truncated": {
|
|
95
|
+
"type": "boolean"
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
@@ -11,14 +11,15 @@
|
|
|
11
11
|
// dedupes by id. Reconciliation is the capture: scan, project, append what
|
|
12
12
|
// is new, in bounded batches.
|
|
13
13
|
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
// of the journal, never as the whole stream's id set.
|
|
14
|
+
// Inputs are append-only logs. Background growth uses a bounded boundary
|
|
15
|
+
// witness and byte cursor; explicit per-file calls retain full reprojection.
|
|
16
|
+
// Persistent journal ID lookup makes either path use bounded memory.
|
|
18
17
|
|
|
19
|
-
import { closeSync, existsSync, fstatSync,
|
|
20
|
-
import { basename,
|
|
18
|
+
import { closeSync, constants, existsSync, fstatSync, openSync, readdirSync, readSync } from "node:fs";
|
|
19
|
+
import { basename, join } from "node:path";
|
|
21
20
|
import { homedir } from "node:os";
|
|
21
|
+
import { createHash } from "node:crypto";
|
|
22
|
+
import { DerivedState, statVersion, sameVersion } from "./derived-state.mjs";
|
|
22
23
|
|
|
23
24
|
import {
|
|
24
25
|
projectCommLogEntry,
|
|
@@ -48,9 +49,15 @@ export function listCommLogs(dir = defaultCommLogDir()) {
|
|
|
48
49
|
// A final line without its newline is still an entry when it parses; when it
|
|
49
50
|
// does not, it is a torn tail the client is still writing, never counted. An
|
|
50
51
|
// interior bad line is skipped and counted (`onSkipped`), never fatal.
|
|
51
|
-
function* logEntries(fd, end, onSkipped) {
|
|
52
|
-
|
|
52
|
+
function* logEntries(fd, start, end, onSkipped, onConsumed) {
|
|
53
|
+
let offset = start;
|
|
54
|
+
for (const line of bufferLines(fileChunks(fd, { start, end, size: 65536 }))) {
|
|
55
|
+
const lineStart = offset;
|
|
56
|
+
offset += line.length;
|
|
53
57
|
const final = line[line.length - 1] !== 10;
|
|
58
|
+
// Even ignored whitespace is scanned progress when newline-terminated.
|
|
59
|
+
// The caller persists it only after its pending append batch is durable.
|
|
60
|
+
if (!final) onConsumed(offset);
|
|
54
61
|
let entry;
|
|
55
62
|
try {
|
|
56
63
|
// Inside the try: a line beyond V8's string limit is a bad line like any other.
|
|
@@ -61,55 +68,77 @@ function* logEntries(fd, end, onSkipped) {
|
|
|
61
68
|
if (!final) onSkipped();
|
|
62
69
|
continue;
|
|
63
70
|
}
|
|
64
|
-
yield entry;
|
|
71
|
+
yield { entry, start: lineStart, end: final ? lineStart : offset, bytes: line.length };
|
|
65
72
|
}
|
|
66
73
|
}
|
|
67
74
|
|
|
68
75
|
const APPEND_BATCH = 2000;
|
|
69
76
|
|
|
70
|
-
//
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
77
|
+
// Check only a bounded checkpoint boundary under the append-only source contract.
|
|
78
|
+
function boundaryWitness(fd, offset) {
|
|
79
|
+
const size = Math.min(offset, 4096), bytes = Buffer.alloc(size);
|
|
80
|
+
let done = 0;
|
|
81
|
+
while (done < size) {
|
|
82
|
+
const n = readSync(fd, bytes, done, size - done, offset - size + done);
|
|
83
|
+
if (!n) throw new Error("short read of log checkpoint boundary");
|
|
84
|
+
done += n;
|
|
85
|
+
}
|
|
86
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function captureLogFile(store, { streamId, path, project, knownIds, incremental = false }) {
|
|
90
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
|
|
91
|
+
let state;
|
|
78
92
|
try {
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
let known = knownIds;
|
|
97
|
-
if (!known) {
|
|
98
|
-
const candidates = new Set();
|
|
99
|
-
for (const turn of turns()) candidates.add(turn.id);
|
|
100
|
-
known = store.idsAmong(streamId, candidates);
|
|
93
|
+
const stat = fstatSync(fd, { bigint: true });
|
|
94
|
+
if (!stat.isFile()) throw new Error(`log source is not a regular file: ${path}`);
|
|
95
|
+
const version = statVersion(stat), end = version.size;
|
|
96
|
+
state = new DerivedState(store.root);
|
|
97
|
+
const generation = state.get("reset", streamId.split("~")[0] + ":aw") ?? "";
|
|
98
|
+
const key = `${streamId}:${path}`;
|
|
99
|
+
let previous = incremental ? state.get("aw", key) : null;
|
|
100
|
+
if (previous && (!Number.isSafeInteger(previous.bytes) || previous.bytes < 0 || previous.bytes > end || !previous.version)) previous = null;
|
|
101
|
+
const journal = store.journalCheckpoint(streamId);
|
|
102
|
+
let start = 0;
|
|
103
|
+
if (previous && previous.generation === generation && previous.epoch === journal.epoch) {
|
|
104
|
+
if (previous.complete && sameVersion(previous.version, version)) return { entries: 0, appended: 0, skipped: 0, failed: 0, unchanged: true };
|
|
105
|
+
const old = previous.version;
|
|
106
|
+
if (old.dev === version.dev && old.ino === version.ino && (version.size > old.size || sameVersion(old, version)) &&
|
|
107
|
+
previous.bytes <= end && boundaryWitness(fd, previous.bytes) === previous.witness) start = previous.bytes;
|
|
101
108
|
}
|
|
102
|
-
let appended = 0, batch = [];
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
+
let entries = 0, skipped = 0, failed = 0, appended = 0, batch = [], batchBytes = 0, consumed = start, complete = false;
|
|
110
|
+
const checkpoint = () => {
|
|
111
|
+
if (!incremental) return;
|
|
112
|
+
const bytes = consumed;
|
|
113
|
+
state.set("aw", key, { bytes, version, witness: boundaryWitness(fd, bytes), epoch: journal.epoch, generation, complete });
|
|
114
|
+
};
|
|
115
|
+
const flush = () => {
|
|
116
|
+
if (batch.length) {
|
|
117
|
+
if (knownIds) { store.appendBatch(streamId, batch); appended += batch.length; }
|
|
118
|
+
else appended += store.appendUniqueBatch(streamId, batch);
|
|
119
|
+
batch = []; batchBytes = 0;
|
|
120
|
+
}
|
|
121
|
+
checkpoint();
|
|
122
|
+
};
|
|
123
|
+
for (const record of logEntries(fd, start, end, () => skipped++, (offset) => { consumed = offset; })) {
|
|
124
|
+
entries++;
|
|
125
|
+
let turn;
|
|
126
|
+
// Projection refusal consumes this complete entry just like a skipped
|
|
127
|
+
// malformed line. Resetting source cursors is the explicit retry path;
|
|
128
|
+
// an old refusal must not pin every background pass to historical bytes.
|
|
129
|
+
consumed = record.end;
|
|
130
|
+
try { turn = project(record.entry); }
|
|
131
|
+
catch { failed++; continue; }
|
|
132
|
+
if (knownIds) { if (knownIds.has(turn.id)) continue; knownIds.add(turn.id); }
|
|
133
|
+
batch.push(turn); batchBytes += record.bytes;
|
|
134
|
+
if (batch.length >= APPEND_BATCH || batchBytes >= 8 * 1024 * 1024) flush();
|
|
109
135
|
}
|
|
110
|
-
|
|
136
|
+
// Complete refused/malformed/blank lines are consumed; an unfinished final
|
|
137
|
+
// fragment keeps its starting boundary so later growth can finish it.
|
|
138
|
+
complete = true;
|
|
139
|
+
flush();
|
|
111
140
|
return { entries, appended, skipped, failed };
|
|
112
|
-
} finally { closeSync(fd); }
|
|
141
|
+
} finally { try { state?.close(); } finally { closeSync(fd); } }
|
|
113
142
|
}
|
|
114
143
|
|
|
115
144
|
// Capture one comm-log file into `<owner>~aw`. The account name is the
|
|
@@ -136,54 +165,27 @@ export function captureInteractionLog(store, { owner, path, selfName, workspace,
|
|
|
136
165
|
return { stream: streamId, ...r };
|
|
137
166
|
}
|
|
138
167
|
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
return JSON.parse(readFileSync(seenCachePath(store), "utf8"));
|
|
148
|
-
} catch {
|
|
149
|
-
return {};
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
function saveSeenCache(store, cache) {
|
|
154
|
-
const path = seenCachePath(store);
|
|
155
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
156
|
-
writeFileSync(path, JSON.stringify(cache));
|
|
168
|
+
// Retain sorted basename metadata, preserving listCommLogs' code-unit account
|
|
169
|
+
// order in both background capture and public results. Content is streamed;
|
|
170
|
+
// discovery metadata still grows with the number of log files.
|
|
171
|
+
function* iterateCommLogs(dir) {
|
|
172
|
+
let names;
|
|
173
|
+
try { names = readdirSync(dir); }
|
|
174
|
+
catch (err) { if (err.code === "ENOENT") return; throw err; }
|
|
175
|
+
for (const name of names.sort()) if (name.endsWith(".jsonl")) yield join(dir, name);
|
|
157
176
|
}
|
|
158
|
-
|
|
159
|
-
// One reconciliation pass over every default comm log. Unchanged files are
|
|
160
|
-
// skipped; each changed one looks up its own ids in the stream.
|
|
161
|
-
//
|
|
162
|
-
// Files matching the record's ignore list (`<root>/ignore`, see ignore.mjs)
|
|
163
|
-
// are skipped before being opened — no turns, no seen-cache entry — and
|
|
164
|
-
// reported as `{ account, path, ignored: true }` so a pass stays visible
|
|
165
|
-
// about what it refused to read.
|
|
166
|
-
export function captureAwLogs(store, { owner, commLogDir, ignore = null } = {}) {
|
|
177
|
+
function* captureAwEntries(store, { owner, commLogDir, ignore }) {
|
|
167
178
|
const ign = ignore ?? loadIgnore(store.root);
|
|
168
|
-
const
|
|
169
|
-
const results = [];
|
|
170
|
-
for (const path of listCommLogs(commLogDir ?? defaultCommLogDir())) {
|
|
179
|
+
for (const path of iterateCommLogs(commLogDir ?? defaultCommLogDir())) {
|
|
171
180
|
const account = basename(path, ".jsonl");
|
|
172
|
-
if (ign.ignores(path, [basename(path), account])) {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
try {
|
|
178
|
-
stat = statSync(path);
|
|
179
|
-
} catch {
|
|
180
|
-
continue;
|
|
181
|
-
}
|
|
182
|
-
const prev = seen[path];
|
|
183
|
-
if (prev && prev.size === stat.size && prev.mtimeMs === stat.mtimeMs) continue;
|
|
184
|
-
results.push(captureCommLog(store, { owner, path }));
|
|
185
|
-
seen[path] = { size: stat.size, mtimeMs: stat.mtimeMs };
|
|
181
|
+
if (ign.ignores(path, [basename(path), account])) { yield { account, path, ignored: true }; continue; }
|
|
182
|
+
const stream = awStream(owner);
|
|
183
|
+
const result = captureLogFile(store, { streamId: stream, path, incremental: true,
|
|
184
|
+
project: entry => projectCommLogEntry(entry, { selfName: account }) });
|
|
185
|
+
if (!result.unchanged) yield { account, stream, ...result };
|
|
186
186
|
}
|
|
187
|
-
|
|
188
|
-
|
|
187
|
+
}
|
|
188
|
+
export function captureAwLogs(store, { owner, commLogDir, ignore = null, collect = true } = {}) {
|
|
189
|
+
const results = captureAwEntries(store, { owner, commLogDir, ignore });
|
|
190
|
+
return collect ? [...results] : results;
|
|
189
191
|
}
|