@tangle-network/harness-sessions 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +70 -2
- package/dist/builder.d.ts +116 -0
- package/dist/builder.js +332 -0
- package/dist/capture.d.ts +44 -0
- package/dist/capture.js +210 -0
- package/dist/catalog.d.ts +163 -0
- package/dist/catalog.js +163 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +115 -0
- package/dist/formats/claude-code.d.ts +9 -0
- package/dist/formats/claude-code.js +293 -0
- package/dist/formats/codex.d.ts +8 -0
- package/dist/formats/codex.js +379 -0
- package/dist/formats/factory.d.ts +13 -0
- package/dist/formats/factory.js +155 -0
- package/dist/formats/fold.d.ts +13 -0
- package/dist/formats/fold.js +5 -0
- package/dist/formats/head.d.ts +5 -0
- package/dist/formats/head.js +23 -0
- package/dist/formats/kimi.d.ts +9 -0
- package/dist/formats/kimi.js +205 -0
- package/dist/formats/opencode.d.ts +4 -0
- package/dist/formats/opencode.js +241 -0
- package/dist/formats/pi.d.ts +8 -0
- package/dist/formats/pi.js +201 -0
- package/dist/index.d.ts +27 -0
- package/dist/index.js +16 -0
- package/dist/input.d.ts +18 -0
- package/dist/input.js +46 -0
- package/dist/projections/chat.d.ts +31 -0
- package/dist/projections/chat.js +79 -0
- package/dist/projections/otlp.d.ts +12 -0
- package/dist/projections/otlp.js +91 -0
- package/dist/projections/turns.d.ts +51 -0
- package/dist/projections/turns.js +136 -0
- package/dist/registry.d.ts +11 -0
- package/dist/registry.js +39 -0
- package/dist/schema.d.ts +270 -0
- package/dist/schema.js +10 -0
- package/dist/source.d.ts +67 -0
- package/dist/source.js +254 -0
- package/dist/stores.d.ts +7 -0
- package/dist/stores.js +46 -0
- package/package.json +53 -3
package/dist/schema.d.ts
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The normalized form of one harness session, and the store layouts every harness declares.
|
|
3
|
+
*
|
|
4
|
+
* Each coding-agent harness persists its own native session: Claude Code and Pi append JSONL,
|
|
5
|
+
* Codex appends a rollout JSONL, OpenCode writes SQLite. Every consumer that needs to know what
|
|
6
|
+
* happened in a turn reads it through this one shape, so a count, a served model or an ending
|
|
7
|
+
* error means the same thing in the sidecar, the Discovery gate, agent-eval, agent-record and the
|
|
8
|
+
* blog.
|
|
9
|
+
*/
|
|
10
|
+
/** One native session format. The id names the harness family and the on-disk layout. */
|
|
11
|
+
export type SessionFormatId = 'claude-code.projects-jsonl' | 'codex.rollout-jsonl' | 'opencode.sqlite' | 'pi.session-jsonl' | 'kimi.session-dir' | 'factory.session-jsonl' | 'gemini.chat-json' | 'amp.thread-json' | 'forge.sqlite' | 'hermes.state-sqlite' | 'prime.session-jsonl' | 'openclaw.session-jsonl';
|
|
12
|
+
/**
|
|
13
|
+
* Where a harness keeps its sessions, relative to the HOME it ran with. The sidecar's harness
|
|
14
|
+
* session contract imports these declarations; it does not keep a second copy.
|
|
15
|
+
*/
|
|
16
|
+
export interface NativeSessionStore {
|
|
17
|
+
/** Relative to the execution's private runtime HOME. Normalized, never absolute, never `..`. */
|
|
18
|
+
root: string;
|
|
19
|
+
/** Globs under `root` that hold one session. `{id}` is replaced by the native session id. */
|
|
20
|
+
files: readonly string[];
|
|
21
|
+
/** How the harness writes. This decides how a copy is taken while the harness runs. */
|
|
22
|
+
write: 'jsonl-append' | 'json-rewrite' | 'sqlite-wal';
|
|
23
|
+
/** True when one store holds every session in the HOME (opencode.db). Readers select by id. */
|
|
24
|
+
shared: boolean;
|
|
25
|
+
format: SessionFormatId;
|
|
26
|
+
}
|
|
27
|
+
/** Where a format's served-model, usage and ending-error evidence comes from. */
|
|
28
|
+
export interface SessionEvidenceSources {
|
|
29
|
+
/** `none`: the harness records no model at all (Kimi's wire log); served models stay empty. */
|
|
30
|
+
servedModel: 'provider-response' | 'session-record' | 'turn-context' | 'none';
|
|
31
|
+
usage: 'per-response' | 'cumulative' | 'session-total';
|
|
32
|
+
endingError: 'session-record' | 'none';
|
|
33
|
+
}
|
|
34
|
+
export interface TokenUsage {
|
|
35
|
+
/**
|
|
36
|
+
* Prompt tokens that were neither read from nor written to a provider cache. Cache reads and
|
|
37
|
+
* cache writes are counted in their own fields and never inside `input`, so every field can be
|
|
38
|
+
* priced at its own rate. A harness that reports an inclusive input (Codex) is converted.
|
|
39
|
+
*/
|
|
40
|
+
input: number | null;
|
|
41
|
+
output: number | null;
|
|
42
|
+
/** Reasoning tokens as the harness recorded them; providers count them inside `output`. */
|
|
43
|
+
reasoning: number | null;
|
|
44
|
+
cacheRead: number | null;
|
|
45
|
+
cacheWrite: number | null;
|
|
46
|
+
}
|
|
47
|
+
export interface SessionError {
|
|
48
|
+
/** The harness's own classification: `api-error`, `stream_error`, a provider error name. */
|
|
49
|
+
kind: string;
|
|
50
|
+
message: string;
|
|
51
|
+
/** The provider's HTTP status when the harness recorded it. */
|
|
52
|
+
httpStatus: number | null;
|
|
53
|
+
at: string | null;
|
|
54
|
+
}
|
|
55
|
+
export type Part = {
|
|
56
|
+
type: 'text';
|
|
57
|
+
text: string;
|
|
58
|
+
} | {
|
|
59
|
+
type: 'reasoning';
|
|
60
|
+
text: string;
|
|
61
|
+
redacted: boolean;
|
|
62
|
+
} | {
|
|
63
|
+
type: 'tool-call';
|
|
64
|
+
toolCallId: string;
|
|
65
|
+
} | {
|
|
66
|
+
type: 'tool-result';
|
|
67
|
+
toolCallId: string;
|
|
68
|
+
} | {
|
|
69
|
+
type: 'attachment';
|
|
70
|
+
mediaType: string;
|
|
71
|
+
ref: string;
|
|
72
|
+
};
|
|
73
|
+
export interface SessionMessage {
|
|
74
|
+
id: string;
|
|
75
|
+
role: 'system' | 'user' | 'assistant' | 'tool';
|
|
76
|
+
/**
|
|
77
|
+
* Who produced the message. `injected` is text the harness or the runtime added to the
|
|
78
|
+
* conversation (environment context, instructions, error notices), not the human's prompt.
|
|
79
|
+
*/
|
|
80
|
+
actor: 'human' | 'agent' | 'subagent-spawn' | 'injected' | 'tool-result';
|
|
81
|
+
at: string | null;
|
|
82
|
+
/** The model call that produced an assistant message; null for every other message. */
|
|
83
|
+
modelCallId: string | null;
|
|
84
|
+
parts: Part[];
|
|
85
|
+
}
|
|
86
|
+
export interface SessionToolCall {
|
|
87
|
+
id: string;
|
|
88
|
+
name: string;
|
|
89
|
+
/** The arguments as structured data when the harness recorded JSON, otherwise the raw text. */
|
|
90
|
+
input: unknown;
|
|
91
|
+
/** The arguments exactly as the harness recorded them when it recorded text. */
|
|
92
|
+
inputText: string | null;
|
|
93
|
+
result: {
|
|
94
|
+
output: unknown;
|
|
95
|
+
text: string | null;
|
|
96
|
+
isError: boolean;
|
|
97
|
+
at: string | null;
|
|
98
|
+
/** The harness's own structured result (Claude Code `toolUseResult`, Pi `details`, OpenCode `metadata`). */
|
|
99
|
+
details: unknown;
|
|
100
|
+
} | null;
|
|
101
|
+
status: 'completed' | 'error' | 'pending' | 'interrupted';
|
|
102
|
+
startedAt: string | null;
|
|
103
|
+
endedAt: string | null;
|
|
104
|
+
messageId: string;
|
|
105
|
+
}
|
|
106
|
+
export interface ModelCall {
|
|
107
|
+
id: string;
|
|
108
|
+
at: string | null;
|
|
109
|
+
provider: string | null;
|
|
110
|
+
/** The model that answered, from the provider response or the session record. Never the request. */
|
|
111
|
+
servedModel: string | null;
|
|
112
|
+
/**
|
|
113
|
+
* The model the harness sent the request to, when the session records it. A failed request has
|
|
114
|
+
* no served model; this says which model it was meant for. Never counted as served.
|
|
115
|
+
*/
|
|
116
|
+
requestedModel: string | null;
|
|
117
|
+
usage: TokenUsage | null;
|
|
118
|
+
costUsd: number | null;
|
|
119
|
+
stopReason: string | null;
|
|
120
|
+
error: SessionError | null;
|
|
121
|
+
}
|
|
122
|
+
export interface SessionEnding {
|
|
123
|
+
status: 'completed' | 'error' | 'aborted' | 'open';
|
|
124
|
+
error: SessionError | null;
|
|
125
|
+
at: string | null;
|
|
126
|
+
}
|
|
127
|
+
export interface SessionIntegrity {
|
|
128
|
+
sourceFiles: Array<{
|
|
129
|
+
path: string;
|
|
130
|
+
sha256: string;
|
|
131
|
+
bytes: number;
|
|
132
|
+
}>;
|
|
133
|
+
/** Records the reader could not parse. Never fatal in recover mode. */
|
|
134
|
+
unparsedRecords: number;
|
|
135
|
+
/** True when a source copy was cut short (a bounded capture kept only a prefix). */
|
|
136
|
+
truncated: boolean;
|
|
137
|
+
gaps: string[];
|
|
138
|
+
}
|
|
139
|
+
export interface HarnessSession {
|
|
140
|
+
schema: 'tangle.harness-session.v1';
|
|
141
|
+
harness: string;
|
|
142
|
+
format: SessionFormatId;
|
|
143
|
+
nativeSessionId: string;
|
|
144
|
+
parentNativeSessionId: string | null;
|
|
145
|
+
cwd: string | null;
|
|
146
|
+
startedAt: string | null;
|
|
147
|
+
endedAt: string | null;
|
|
148
|
+
messages: SessionMessage[];
|
|
149
|
+
toolCalls: SessionToolCall[];
|
|
150
|
+
modelCalls: ModelCall[];
|
|
151
|
+
/** `provider/model` (or the bare model when no provider is recorded) -> calls it answered. */
|
|
152
|
+
servedModels: Record<string, number>;
|
|
153
|
+
modelSource: SessionEvidenceSources['servedModel'];
|
|
154
|
+
/** Null when the harness recorded no usage at all. Never reported as zero. */
|
|
155
|
+
usage: TokenUsage | null;
|
|
156
|
+
ending: SessionEnding;
|
|
157
|
+
children: Array<{
|
|
158
|
+
nativeSessionId: string;
|
|
159
|
+
toolCallId: string | null;
|
|
160
|
+
}>;
|
|
161
|
+
integrity: SessionIntegrity;
|
|
162
|
+
}
|
|
163
|
+
/** What `summarize` keeps: counts and the facts a gate needs, in constant memory. */
|
|
164
|
+
export interface SessionSummary {
|
|
165
|
+
harness: string;
|
|
166
|
+
format: SessionFormatId;
|
|
167
|
+
nativeSessionId: string;
|
|
168
|
+
parentNativeSessionId: string | null;
|
|
169
|
+
modelCalls: number;
|
|
170
|
+
/** Assistant messages the agent produced. */
|
|
171
|
+
replies: number;
|
|
172
|
+
toolCalls: number;
|
|
173
|
+
servedModels: Record<string, number>;
|
|
174
|
+
/** The served-model key (`provider/model`) of the last model call that a model answered. */
|
|
175
|
+
lastServedModel: string | null;
|
|
176
|
+
modelSource: HarnessSession['modelSource'];
|
|
177
|
+
usage: TokenUsage | null;
|
|
178
|
+
ending: SessionEnding;
|
|
179
|
+
unparsedRecords: number;
|
|
180
|
+
bytes: number;
|
|
181
|
+
truncated: boolean;
|
|
182
|
+
startedAt: string | null;
|
|
183
|
+
endedAt: string | null;
|
|
184
|
+
}
|
|
185
|
+
/** One discovered session, before it is read. */
|
|
186
|
+
export interface SessionRef {
|
|
187
|
+
harness: string;
|
|
188
|
+
format: SessionFormatId;
|
|
189
|
+
nativeSessionId: string;
|
|
190
|
+
/** The session's primary file (the JSONL, or the SQLite store for a shared store). */
|
|
191
|
+
path: string;
|
|
192
|
+
/** Every file whose bytes the read depends on, including subagent files and SQLite siblings. */
|
|
193
|
+
files: readonly string[];
|
|
194
|
+
/** The HOME the session was found under, when it was located through a store. */
|
|
195
|
+
home: string | null;
|
|
196
|
+
cwd: string | null;
|
|
197
|
+
mtimeMs: number;
|
|
198
|
+
parentNativeSessionId: string | null;
|
|
199
|
+
/** True when the copy this ref points at is a bounded prefix of the session. */
|
|
200
|
+
truncated?: boolean;
|
|
201
|
+
}
|
|
202
|
+
export interface LocateOptions {
|
|
203
|
+
nativeSessionId?: string;
|
|
204
|
+
/** Sessions whose recorded cwd equals or starts with this path. */
|
|
205
|
+
cwd?: string;
|
|
206
|
+
/** Sessions modified at or after this epoch ms. */
|
|
207
|
+
sinceMs?: number;
|
|
208
|
+
}
|
|
209
|
+
export interface ReadOptions {
|
|
210
|
+
signal?: AbortSignal;
|
|
211
|
+
/** `recover` (default) counts unparsable records; `strict` throws on the first one. */
|
|
212
|
+
corruption?: 'recover' | 'strict';
|
|
213
|
+
}
|
|
214
|
+
export interface HarnessSessionReader {
|
|
215
|
+
harness: string;
|
|
216
|
+
aliases: readonly string[];
|
|
217
|
+
formats: readonly SessionFormatId[];
|
|
218
|
+
stores: readonly NativeSessionStore[];
|
|
219
|
+
/**
|
|
220
|
+
* Globs under the first store's root: the file this reader folds for one session, and for a
|
|
221
|
+
* harness that writes each subagent's transcript separately, those files.
|
|
222
|
+
*/
|
|
223
|
+
globs: {
|
|
224
|
+
session: string;
|
|
225
|
+
children?: string;
|
|
226
|
+
};
|
|
227
|
+
evidence: SessionEvidenceSources;
|
|
228
|
+
locate(home: string, opts?: LocateOptions): Promise<SessionRef[]>;
|
|
229
|
+
read(ref: SessionRef, opts?: ReadOptions): Promise<HarnessSession>;
|
|
230
|
+
/** Streams the session in constant memory with respect to message and tool-output size. */
|
|
231
|
+
summarize(ref: SessionRef, opts?: Pick<ReadOptions, 'signal'>): Promise<SessionSummary>;
|
|
232
|
+
}
|
|
233
|
+
/** One copy of a native session that the sidecar captured for an execution. */
|
|
234
|
+
export interface NativeSessionCopy {
|
|
235
|
+
harness: string;
|
|
236
|
+
format: SessionFormatId;
|
|
237
|
+
executionId: string;
|
|
238
|
+
attempt: number;
|
|
239
|
+
nativeSessionId: string;
|
|
240
|
+
/** `snapshot`: taken while the harness ran. `settled`: taken after it exited, however it ended. */
|
|
241
|
+
copy: 'snapshot' | 'settled';
|
|
242
|
+
capturedAt: string;
|
|
243
|
+
/** Null where the sidecar did not observe the value at the copy point. */
|
|
244
|
+
process: {
|
|
245
|
+
pid: number | null;
|
|
246
|
+
exitedAt: string | null;
|
|
247
|
+
exitCode: number | null;
|
|
248
|
+
signal: string | null;
|
|
249
|
+
};
|
|
250
|
+
files: Array<{
|
|
251
|
+
/** Path inside the archive, relative to the archive root. */
|
|
252
|
+
path: string;
|
|
253
|
+
/** The store root the file belongs to, relative to the runtime HOME. */
|
|
254
|
+
store: string;
|
|
255
|
+
/** The file's own path relative to the runtime HOME; the sidecar archive's payload names are flat. */
|
|
256
|
+
storePath?: string;
|
|
257
|
+
sizeBytes: number;
|
|
258
|
+
sha256: string;
|
|
259
|
+
method: 'byte-prefix' | 'atomic-file' | 'sqlite-backup';
|
|
260
|
+
truncated: boolean;
|
|
261
|
+
}>;
|
|
262
|
+
complete: boolean;
|
|
263
|
+
missingReasons: string[];
|
|
264
|
+
}
|
|
265
|
+
/** The part of the sidecar's raw-evidence archive manifest this package reads. */
|
|
266
|
+
export interface RawEvidenceArchiveManifestV2 {
|
|
267
|
+
schema: 'tangle.raw-evidence-archive.v2';
|
|
268
|
+
nativeSessions: NativeSessionCopy[];
|
|
269
|
+
}
|
|
270
|
+
export declare const HARNESS_SESSION_SCHEMA: 'tangle.harness-session.v1';
|
package/dist/schema.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The normalized form of one harness session, and the store layouts every harness declares.
|
|
3
|
+
*
|
|
4
|
+
* Each coding-agent harness persists its own native session: Claude Code and Pi append JSONL,
|
|
5
|
+
* Codex appends a rollout JSONL, OpenCode writes SQLite. Every consumer that needs to know what
|
|
6
|
+
* happened in a turn reads it through this one shape, so a count, a served model or an ending
|
|
7
|
+
* error means the same thing in the sidecar, the Discovery gate, agent-eval, agent-record and the
|
|
8
|
+
* blog.
|
|
9
|
+
*/
|
|
10
|
+
export const HARNESS_SESSION_SCHEMA = 'tangle.harness-session.v1';
|
package/dist/source.d.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
/** What one source file contributed to a read: its identity and what could not be parsed. */
|
|
3
|
+
export interface SourceStats {
|
|
4
|
+
path: string;
|
|
5
|
+
sha256: string;
|
|
6
|
+
bytes: number;
|
|
7
|
+
unparsed: number;
|
|
8
|
+
/** The file ended without a newline and its last bytes were not a complete record. */
|
|
9
|
+
tornTail: boolean;
|
|
10
|
+
/** The line number (1-based) of the record most recently yielded. */
|
|
11
|
+
line: number;
|
|
12
|
+
/** Line numbers of the first unparsable lines (at most 50), for the gaps a reader reports. */
|
|
13
|
+
unparsedLines?: number[];
|
|
14
|
+
}
|
|
15
|
+
/** Records to fold, with the stats of the bytes they came from. */
|
|
16
|
+
export interface RecordSource {
|
|
17
|
+
stats: SourceStats;
|
|
18
|
+
records: AsyncIterable<Record<string, unknown>>;
|
|
19
|
+
}
|
|
20
|
+
export declare class SessionParseError extends Error {
|
|
21
|
+
readonly path: string;
|
|
22
|
+
readonly line: number;
|
|
23
|
+
constructor(path: string, line: number, reason: string);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Streams the JSON records of one append-only JSONL file. Each record is parsed and yielded in
|
|
27
|
+
* order; the whole file is hashed as it is read. An unparsable line is counted (or thrown in
|
|
28
|
+
* strict mode). A final line without a newline that does not parse is a record the harness was
|
|
29
|
+
* still writing when the copy was taken, so it is reported as a torn tail rather than corruption.
|
|
30
|
+
*/
|
|
31
|
+
export declare function readJsonlRecords(path: string, stats: SourceStats, options?: {
|
|
32
|
+
strict?: boolean;
|
|
33
|
+
signal?: AbortSignal;
|
|
34
|
+
}): AsyncGenerator<Record<string, unknown>>;
|
|
35
|
+
export declare function newSourceStats(path: string): SourceStats;
|
|
36
|
+
/** The records of a JSONL file, streamed. */
|
|
37
|
+
export declare function fileRecords(path: string, options?: {
|
|
38
|
+
strict?: boolean;
|
|
39
|
+
signal?: AbortSignal;
|
|
40
|
+
}): RecordSource;
|
|
41
|
+
/** Stats for bytes already in memory. */
|
|
42
|
+
export declare function memoryStats(label: string, text?: string): SourceStats;
|
|
43
|
+
/** The records of JSONL text already in memory (a retained copy, a blob), read synchronously. */
|
|
44
|
+
export declare function textRecordsSync(text: string, stats: SourceStats, options?: {
|
|
45
|
+
strict?: boolean;
|
|
46
|
+
}): Generator<Record<string, unknown>>;
|
|
47
|
+
/** Records a caller already parsed (one per line, in order). Non-objects count as unparsed. */
|
|
48
|
+
export declare function arrayRecordsSync(values: readonly unknown[], stats: SourceStats): Generator<Record<string, unknown>>;
|
|
49
|
+
/** Hash a file without parsing it. */
|
|
50
|
+
export declare function hashFile(path: string): Promise<{
|
|
51
|
+
sha256: string;
|
|
52
|
+
bytes: number;
|
|
53
|
+
}>;
|
|
54
|
+
/**
|
|
55
|
+
* Run `use` against a private copy of a SQLite store. Opening a live store can checkpoint its
|
|
56
|
+
* write-ahead log and change the bytes a capture is keeping, so the store and its -wal and -shm
|
|
57
|
+
* siblings are copied first and only the copy is opened. `use` is synchronous so no other read can
|
|
58
|
+
* replace the copy while it runs.
|
|
59
|
+
*/
|
|
60
|
+
export declare function withSqliteCopy<T>(path: string, use: (db: DatabaseSync, sources: Array<{
|
|
61
|
+
path: string;
|
|
62
|
+
sha256: string;
|
|
63
|
+
bytes: number;
|
|
64
|
+
}>) => T): Promise<T>;
|
|
65
|
+
/** Remove the private store copy now instead of at process exit. */
|
|
66
|
+
export declare function releaseSqliteCopy(): void;
|
|
67
|
+
export declare function isMissing(error: unknown): boolean;
|
package/dist/source.js
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { createReadStream, rmSync } from 'node:fs';
|
|
3
|
+
import { copyFile, mkdtemp, stat } from 'node:fs/promises';
|
|
4
|
+
import { createRequire } from 'node:module';
|
|
5
|
+
import { tmpdir } from 'node:os';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
const MAX_UNPARSED_LINES = 50;
|
|
8
|
+
function noteUnparsed(stats, line) {
|
|
9
|
+
stats.unparsed += 1;
|
|
10
|
+
stats.unparsedLines ??= [];
|
|
11
|
+
if (stats.unparsedLines.length < MAX_UNPARSED_LINES)
|
|
12
|
+
stats.unparsedLines.push(line);
|
|
13
|
+
}
|
|
14
|
+
export class SessionParseError extends Error {
|
|
15
|
+
path;
|
|
16
|
+
line;
|
|
17
|
+
constructor(path, line, reason) {
|
|
18
|
+
super(`${path}:${line}: ${reason}`);
|
|
19
|
+
this.path = path;
|
|
20
|
+
this.line = line;
|
|
21
|
+
this.name = 'SessionParseError';
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
const NEWLINE = 0x0a;
|
|
25
|
+
/**
|
|
26
|
+
* Streams the JSON records of one append-only JSONL file. Each record is parsed and yielded in
|
|
27
|
+
* order; the whole file is hashed as it is read. An unparsable line is counted (or thrown in
|
|
28
|
+
* strict mode). A final line without a newline that does not parse is a record the harness was
|
|
29
|
+
* still writing when the copy was taken, so it is reported as a torn tail rather than corruption.
|
|
30
|
+
*/
|
|
31
|
+
export async function* readJsonlRecords(path, stats, options = {}) {
|
|
32
|
+
const hash = createHash('sha256');
|
|
33
|
+
let pending = [];
|
|
34
|
+
let pendingBytes = 0;
|
|
35
|
+
let lineNumber = 0;
|
|
36
|
+
const parse = (bytes, terminated) => {
|
|
37
|
+
lineNumber += 1;
|
|
38
|
+
const text = bytes.toString('utf8').replace(/\r$/u, '');
|
|
39
|
+
if (text.trim().length === 0)
|
|
40
|
+
return undefined;
|
|
41
|
+
let value;
|
|
42
|
+
try {
|
|
43
|
+
value = JSON.parse(text);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
if (!terminated) {
|
|
47
|
+
stats.tornTail = true;
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
if (options.strict)
|
|
51
|
+
throw new SessionParseError(path, lineNumber, 'malformed JSON');
|
|
52
|
+
noteUnparsed(stats, lineNumber);
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
56
|
+
if (options.strict)
|
|
57
|
+
throw new SessionParseError(path, lineNumber, 'record is not a JSON object');
|
|
58
|
+
noteUnparsed(stats, lineNumber);
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
return value;
|
|
62
|
+
};
|
|
63
|
+
const stream = createReadStream(path, { highWaterMark: 1 << 20, signal: options.signal });
|
|
64
|
+
for await (const chunk of stream) {
|
|
65
|
+
hash.update(chunk);
|
|
66
|
+
stats.bytes += chunk.length;
|
|
67
|
+
let start = 0;
|
|
68
|
+
for (;;) {
|
|
69
|
+
const index = chunk.indexOf(NEWLINE, start);
|
|
70
|
+
if (index === -1)
|
|
71
|
+
break;
|
|
72
|
+
let line;
|
|
73
|
+
if (pendingBytes > 0) {
|
|
74
|
+
pending.push(chunk.subarray(start, index));
|
|
75
|
+
line = Buffer.concat(pending);
|
|
76
|
+
pending = [];
|
|
77
|
+
pendingBytes = 0;
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
line = chunk.subarray(start, index);
|
|
81
|
+
}
|
|
82
|
+
const record = parse(line, true);
|
|
83
|
+
if (record !== undefined) {
|
|
84
|
+
stats.line = lineNumber;
|
|
85
|
+
yield record;
|
|
86
|
+
}
|
|
87
|
+
start = index + 1;
|
|
88
|
+
}
|
|
89
|
+
if (start < chunk.length) {
|
|
90
|
+
pending.push(chunk.subarray(start));
|
|
91
|
+
pendingBytes += chunk.length - start;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (pendingBytes > 0) {
|
|
95
|
+
const record = parse(Buffer.concat(pending), false);
|
|
96
|
+
if (record !== undefined) {
|
|
97
|
+
stats.line = lineNumber;
|
|
98
|
+
yield record;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
stats.sha256 = `sha256:${hash.digest('hex')}`;
|
|
102
|
+
}
|
|
103
|
+
export function newSourceStats(path) {
|
|
104
|
+
return { path, sha256: '', bytes: 0, unparsed: 0, tornTail: false, line: 0 };
|
|
105
|
+
}
|
|
106
|
+
/** The records of a JSONL file, streamed. */
|
|
107
|
+
export function fileRecords(path, options = {}) {
|
|
108
|
+
const stats = newSourceStats(path);
|
|
109
|
+
return { stats, records: readJsonlRecords(path, stats, options) };
|
|
110
|
+
}
|
|
111
|
+
/** Stats for bytes already in memory. */
|
|
112
|
+
export function memoryStats(label, text) {
|
|
113
|
+
if (text === undefined)
|
|
114
|
+
return { path: label, sha256: '', bytes: 0, unparsed: 0, tornTail: false, line: 0 };
|
|
115
|
+
const bytes = Buffer.from(text, 'utf8');
|
|
116
|
+
return { path: label, sha256: `sha256:${createHash('sha256').update(bytes).digest('hex')}`, bytes: bytes.length, unparsed: 0, tornTail: false, line: 0 };
|
|
117
|
+
}
|
|
118
|
+
/** The records of JSONL text already in memory (a retained copy, a blob), read synchronously. */
|
|
119
|
+
export function* textRecordsSync(text, stats, options = {}) {
|
|
120
|
+
const lines = text.split('\n');
|
|
121
|
+
for (const [index, raw] of lines.entries()) {
|
|
122
|
+
const line = raw.replace(/\r$/u, '');
|
|
123
|
+
if (line.trim().length === 0)
|
|
124
|
+
continue;
|
|
125
|
+
let value;
|
|
126
|
+
try {
|
|
127
|
+
value = JSON.parse(line);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
if (index === lines.length - 1) {
|
|
131
|
+
stats.tornTail = true;
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
if (options.strict)
|
|
135
|
+
throw new SessionParseError(stats.path, index + 1, 'malformed JSON');
|
|
136
|
+
noteUnparsed(stats, index + 1);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
140
|
+
if (options.strict)
|
|
141
|
+
throw new SessionParseError(stats.path, index + 1, 'record is not a JSON object');
|
|
142
|
+
noteUnparsed(stats, index + 1);
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
stats.line = index + 1;
|
|
146
|
+
yield value;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/** Records a caller already parsed (one per line, in order). Non-objects count as unparsed. */
|
|
150
|
+
export function* arrayRecordsSync(values, stats) {
|
|
151
|
+
for (const [index, value] of values.entries()) {
|
|
152
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
153
|
+
noteUnparsed(stats, index + 1);
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
stats.line = index + 1;
|
|
157
|
+
yield value;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/** Hash a file without parsing it. */
|
|
161
|
+
export async function hashFile(path) {
|
|
162
|
+
const hash = createHash('sha256');
|
|
163
|
+
let bytes = 0;
|
|
164
|
+
for await (const chunk of createReadStream(path)) {
|
|
165
|
+
hash.update(chunk);
|
|
166
|
+
bytes += chunk.length;
|
|
167
|
+
}
|
|
168
|
+
return { sha256: `sha256:${hash.digest('hex')}`, bytes };
|
|
169
|
+
}
|
|
170
|
+
// `node:sqlite` is loaded through `createRequire`: bundlers rewrite a dynamic `import()` of a
|
|
171
|
+
// `node:` builtin under some targets, and a require obtained this way is not analyzable by them.
|
|
172
|
+
function loadSqlite() {
|
|
173
|
+
const nodeRequire = createRequire(import.meta.url);
|
|
174
|
+
return nodeRequire('node:sqlite');
|
|
175
|
+
}
|
|
176
|
+
// One private copy is kept per process: reading every session of one store (a 4 GB opencode.db
|
|
177
|
+
// holding 26 sessions) copies it once, not once per session. It is replaced when a different
|
|
178
|
+
// store, or a store that changed, is read, and removed when the process exits.
|
|
179
|
+
let current = null;
|
|
180
|
+
function dropCopy() {
|
|
181
|
+
if (current === null)
|
|
182
|
+
return;
|
|
183
|
+
try {
|
|
184
|
+
current.db.close();
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
// already closed
|
|
188
|
+
}
|
|
189
|
+
rmSync(current.dir, { recursive: true, force: true });
|
|
190
|
+
current = null;
|
|
191
|
+
}
|
|
192
|
+
process.once('exit', dropCopy);
|
|
193
|
+
async function storeKey(path) {
|
|
194
|
+
const parts = [];
|
|
195
|
+
for (const suffix of ['', '-wal']) {
|
|
196
|
+
try {
|
|
197
|
+
const s = await stat(`${path}${suffix}`);
|
|
198
|
+
parts.push(`${s.dev}:${s.ino}:${s.size}:${s.mtimeMs}`);
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
if (suffix === '' || !isMissing(error))
|
|
202
|
+
throw error;
|
|
203
|
+
parts.push('-');
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return `${path}|${parts.join('|')}`;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Run `use` against a private copy of a SQLite store. Opening a live store can checkpoint its
|
|
210
|
+
* write-ahead log and change the bytes a capture is keeping, so the store and its -wal and -shm
|
|
211
|
+
* siblings are copied first and only the copy is opened. `use` is synchronous so no other read can
|
|
212
|
+
* replace the copy while it runs.
|
|
213
|
+
*/
|
|
214
|
+
export async function withSqliteCopy(path, use) {
|
|
215
|
+
const key = await storeKey(path);
|
|
216
|
+
if (current?.key !== key) {
|
|
217
|
+
const { DatabaseSync } = loadSqlite();
|
|
218
|
+
const dir = await mkdtemp(join(tmpdir(), 'harness-sessions-sqlite-'));
|
|
219
|
+
try {
|
|
220
|
+
const sources = [];
|
|
221
|
+
for (const suffix of ['', '-wal', '-shm']) {
|
|
222
|
+
const copy = join(dir, `store.db${suffix}`);
|
|
223
|
+
try {
|
|
224
|
+
await copyFile(`${path}${suffix}`, copy);
|
|
225
|
+
}
|
|
226
|
+
catch (error) {
|
|
227
|
+
if (suffix === '' || !isMissing(error))
|
|
228
|
+
throw error;
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
// The shared-memory index is rebuilt from the WAL; only the store and its log carry data.
|
|
232
|
+
if (suffix !== '-shm')
|
|
233
|
+
sources.push({ path: `${path}${suffix}`, ...(await hashFile(copy)) });
|
|
234
|
+
}
|
|
235
|
+
// Opened writable so SQLite replays the copied write-ahead log into the private copy.
|
|
236
|
+
const db = new DatabaseSync(join(dir, 'store.db'));
|
|
237
|
+
dropCopy();
|
|
238
|
+
current = { key, dir, db, sources };
|
|
239
|
+
}
|
|
240
|
+
catch (error) {
|
|
241
|
+
rmSync(dir, { recursive: true, force: true });
|
|
242
|
+
throw error;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
return use(current.db, current.sources);
|
|
246
|
+
}
|
|
247
|
+
/** Remove the private store copy now instead of at process exit. */
|
|
248
|
+
export function releaseSqliteCopy() {
|
|
249
|
+
dropCopy();
|
|
250
|
+
}
|
|
251
|
+
export function isMissing(error) {
|
|
252
|
+
const code = error?.code;
|
|
253
|
+
return code === 'ENOENT' || code === 'ENOTDIR';
|
|
254
|
+
}
|
package/dist/stores.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { NativeSessionStore } from './schema.js';
|
|
2
|
+
export { assertStoreRoot, EVIDENCE, SESSION_FORMAT_IDS, STORES, storeGlob } from './catalog.js';
|
|
3
|
+
/** Every file under `dir`, as paths relative to it with `/` separators. Missing dirs yield none. */
|
|
4
|
+
export declare function walkFiles(dir: string, signal?: AbortSignal): Promise<string[]>;
|
|
5
|
+
/** Files under `home` that one store glob matches, as absolute paths. */
|
|
6
|
+
export declare function storeFiles(home: string, store: NativeSessionStore, glob: string, nativeSessionId?: string): Promise<string[]>;
|
|
7
|
+
export declare function mtimeMs(path: string): Promise<number>;
|
package/dist/stores.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { readdir, stat } from 'node:fs/promises';
|
|
2
|
+
import { join, relative, sep } from 'node:path';
|
|
3
|
+
import { storeGlob } from './catalog.js';
|
|
4
|
+
import { isMissing } from './source.js';
|
|
5
|
+
export { assertStoreRoot, EVIDENCE, SESSION_FORMAT_IDS, STORES, storeGlob } from './catalog.js';
|
|
6
|
+
/** Every file under `dir`, as paths relative to it with `/` separators. Missing dirs yield none. */
|
|
7
|
+
export async function walkFiles(dir, signal) {
|
|
8
|
+
const out = [];
|
|
9
|
+
const visit = async (current) => {
|
|
10
|
+
signal?.throwIfAborted();
|
|
11
|
+
let entries;
|
|
12
|
+
try {
|
|
13
|
+
entries = await readdir(current, { withFileTypes: true });
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
if (isMissing(error))
|
|
17
|
+
return;
|
|
18
|
+
throw error;
|
|
19
|
+
}
|
|
20
|
+
for (const entry of entries) {
|
|
21
|
+
const path = join(current, entry.name);
|
|
22
|
+
if (entry.isDirectory())
|
|
23
|
+
await visit(path);
|
|
24
|
+
else if (entry.isFile())
|
|
25
|
+
out.push(relative(dir, path).split(sep).join('/'));
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
await visit(dir);
|
|
29
|
+
return out.sort();
|
|
30
|
+
}
|
|
31
|
+
/** Files under `home` that one store glob matches, as absolute paths. */
|
|
32
|
+
export async function storeFiles(home, store, glob, nativeSessionId) {
|
|
33
|
+
const root = join(home, store.root);
|
|
34
|
+
const pattern = storeGlob(glob, nativeSessionId);
|
|
35
|
+
return (await walkFiles(root)).filter((path) => pattern.test(path)).map((path) => join(root, path));
|
|
36
|
+
}
|
|
37
|
+
export async function mtimeMs(path) {
|
|
38
|
+
try {
|
|
39
|
+
return (await stat(path)).mtimeMs;
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
if (isMissing(error))
|
|
43
|
+
return 0;
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
}
|