@inixiative/archive 0.12.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/package.json +2 -2
- package/prisma/migrations/20261012120000_runtime/migration.sql +10 -0
- package/prisma/schema.prisma +5 -0
- package/src/agents.ts +98 -38
- package/src/cli.ts +108 -29
- package/src/client.ts +55 -2
- package/src/collector.ts +270 -27
- package/src/config.ts +6 -1
- package/src/import-file.ts +29 -2
- package/src/import.ts +134 -13
- package/src/index.ts +26 -0
- package/src/schemas.ts +2 -0
- package/src/server.ts +4 -0
- package/src/store.ts +4 -0
package/src/collector.ts
CHANGED
|
@@ -1,10 +1,22 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
closeSync,
|
|
3
|
+
existsSync,
|
|
4
|
+
lstatSync,
|
|
5
|
+
openSync,
|
|
6
|
+
readdirSync,
|
|
7
|
+
readFileSync,
|
|
8
|
+
readSync,
|
|
9
|
+
realpathSync,
|
|
10
|
+
} from 'node:fs';
|
|
11
|
+
import { homedir, tmpdir } from 'node:os';
|
|
12
|
+
import { dirname, isAbsolute, join, resolve } from 'node:path';
|
|
3
13
|
import { importTranscriptFile } from './import-file';
|
|
4
|
-
import { archiveKey } from './index';
|
|
14
|
+
import { archiveKey, archiveSnapshotSchema, snapshotDigest, UNROUTED } from './index';
|
|
5
15
|
import type { ArchiveClient } from './remote';
|
|
6
16
|
import { type AtlasFileConcepts, atlasTags, provenanceReferences } from './tags';
|
|
7
17
|
|
|
18
|
+
export { UNROUTED } from './index';
|
|
19
|
+
|
|
8
20
|
export interface CollectionSource {
|
|
9
21
|
directory: string;
|
|
10
22
|
source: 'codex' | 'claude-code';
|
|
@@ -12,12 +24,42 @@ export interface CollectionSource {
|
|
|
12
24
|
projectId: string;
|
|
13
25
|
/** Also match the git worktrees of each root; nested checkouts stay separate. */
|
|
14
26
|
worktrees?: boolean;
|
|
27
|
+
/** Also match directories below each root, unless a more specific root claims them. */
|
|
28
|
+
subdirectories?: boolean;
|
|
29
|
+
/** Take every session no other collector of this source claims, as projectId `unrouted`. */
|
|
30
|
+
catchAll?: boolean;
|
|
15
31
|
/** Tag sessions with the Atlas concepts of files they touched, from `atlas graph` in the session's checkout. */
|
|
16
32
|
atlas?: boolean;
|
|
17
33
|
tags?: string[];
|
|
18
34
|
}
|
|
19
35
|
|
|
20
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Where sessions are never swept in by a subdirectory or catch-all match: temp directories (test
|
|
38
|
+
* fixtures, agent scratchpads) and Foundry's decision receipts. An exact root is explicit and wins.
|
|
39
|
+
* Absolute entries match the directory and below; relative ones match those path segments anywhere.
|
|
40
|
+
*/
|
|
41
|
+
export const defaultExclusions = () => [
|
|
42
|
+
...new Set(
|
|
43
|
+
['/tmp', '/private/tmp', '/var/folders', '/private/var/folders', tmpdir()].flatMap((path) => {
|
|
44
|
+
try {
|
|
45
|
+
return [resolve(path), realpathSync(path)];
|
|
46
|
+
} catch {
|
|
47
|
+
return [resolve(path)];
|
|
48
|
+
}
|
|
49
|
+
}),
|
|
50
|
+
),
|
|
51
|
+
'.foundry/decision-receipts',
|
|
52
|
+
];
|
|
53
|
+
export function excluded(cwd: string, exclusions: string[]) {
|
|
54
|
+
return exclusions.some((entry) =>
|
|
55
|
+
isAbsolute(entry)
|
|
56
|
+
? cwd === resolve(entry) || cwd.startsWith(`${resolve(entry)}/`)
|
|
57
|
+
: `${cwd}/`.includes(`/${entry.replace(/^\/+|\/+$/g, '')}/`),
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Every transcript (`.jsonl`) under a directory, in order, without following links. */
|
|
62
|
+
export function transcriptFiles(directory: string): string[] {
|
|
21
63
|
const result: string[] = [];
|
|
22
64
|
const visit = (path: string) => {
|
|
23
65
|
const stat = lstatSync(path);
|
|
@@ -29,10 +71,10 @@ function files(directory: string): string[] {
|
|
|
29
71
|
return result;
|
|
30
72
|
}
|
|
31
73
|
|
|
32
|
-
type SessionMetadata = { cwd: string; branch?: string; repository?: string };
|
|
74
|
+
export type SessionMetadata = { cwd: string; branch?: string; repository?: string };
|
|
33
75
|
|
|
34
|
-
/**
|
|
35
|
-
function sessionMetadata(
|
|
76
|
+
/** Provider metadata, never text/tags/model suggestions; `routeSession` decides where it goes. */
|
|
77
|
+
export function sessionMetadata(
|
|
36
78
|
file: string,
|
|
37
79
|
source: CollectionSource['source'],
|
|
38
80
|
): SessionMetadata | undefined {
|
|
@@ -96,42 +138,204 @@ function worktreesOf(root: string): string[] {
|
|
|
96
138
|
|
|
97
139
|
/** Exact directories this collection accepts, re-read each scan so new worktrees join. */
|
|
98
140
|
export function collectionRoots(config: CollectionSource): Set<string> {
|
|
99
|
-
|
|
141
|
+
// Providers record the real path (macOS /tmp is /private/tmp), so roots match by it too.
|
|
142
|
+
const roots = config.projectRoots.flatMap((root) => {
|
|
143
|
+
try {
|
|
144
|
+
return [...new Set([resolve(root), realpathSync(root)])];
|
|
145
|
+
} catch {
|
|
146
|
+
return [resolve(root)];
|
|
147
|
+
}
|
|
148
|
+
});
|
|
100
149
|
return new Set(config.worktrees ? [...roots, ...roots.flatMap(worktreesOf)] : roots);
|
|
101
150
|
}
|
|
102
151
|
|
|
152
|
+
/**
|
|
153
|
+
* The collector a session's cwd belongs to. The longest matching root wins: a root matches its own
|
|
154
|
+
* directory (and worktrees), and with `subdirectories` the directories below it. Ties go to the
|
|
155
|
+
* first declared. Excluded directories match only exact roots. Unclaimed sessions go to the first
|
|
156
|
+
* catch-all.
|
|
157
|
+
*/
|
|
158
|
+
export function routeSession(
|
|
159
|
+
cwd: string,
|
|
160
|
+
collectors: CollectionSource[],
|
|
161
|
+
options: { exclude?: string[]; roots?: (collector: CollectionSource) => Set<string> } = {},
|
|
162
|
+
) {
|
|
163
|
+
const rootsOf = options.roots ?? collectionRoots;
|
|
164
|
+
const broad = !excluded(cwd, options.exclude ?? defaultExclusions());
|
|
165
|
+
let best: { collector: CollectionSource; length: number } | undefined;
|
|
166
|
+
for (const collector of collectors) {
|
|
167
|
+
if (collector.catchAll) continue;
|
|
168
|
+
for (const root of rootsOf(collector)) {
|
|
169
|
+
const matches =
|
|
170
|
+
cwd === root || (broad && collector.subdirectories && cwd.startsWith(`${root}/`));
|
|
171
|
+
if (matches && (!best || root.length > best.length))
|
|
172
|
+
best = { collector, length: root.length };
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
if (best) return best.collector;
|
|
176
|
+
return broad ? collectors.find((c) => c.catchAll) : undefined;
|
|
177
|
+
}
|
|
178
|
+
|
|
103
179
|
/** Where collected sessions go: an Archive server, or a store in-process. */
|
|
104
180
|
export type ArchiveWriter = Pick<ArchiveClient, 'sourceId' | 'head' | 'capture'>;
|
|
105
181
|
|
|
106
|
-
|
|
182
|
+
/**
|
|
183
|
+
* The access path the harness's login implies when a transcript is silent, read from the harness
|
|
184
|
+
* home that owns the history directory (`<claude config>/projects`, `<codex home>/sessions`):
|
|
185
|
+
* Claude Code's subscription login (`oauthAccount`) unless its settings supply an API key, and
|
|
186
|
+
* Codex's `auth.json` mode. Undefined when neither says; throws when a file exists but cannot be
|
|
187
|
+
* read, so a bad read never relabels sessions.
|
|
188
|
+
*/
|
|
189
|
+
export function harnessAccess(
|
|
190
|
+
source: CollectionSource['source'],
|
|
191
|
+
directory: string,
|
|
192
|
+
): 'subscription' | 'api' | undefined {
|
|
193
|
+
const json = (path: string) =>
|
|
194
|
+
existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : undefined;
|
|
195
|
+
const harnessHome = dirname(resolve(directory));
|
|
196
|
+
if (source === 'claude-code') {
|
|
197
|
+
const settings = json(join(harnessHome, 'settings.json'));
|
|
198
|
+
if (settings?.apiKeyHelper || settings?.env?.ANTHROPIC_API_KEY) return 'api';
|
|
199
|
+
// The default config directory keeps its login beside it, in ~/.claude.json.
|
|
200
|
+
const login =
|
|
201
|
+
harnessHome === join(homedir(), '.claude')
|
|
202
|
+
? join(homedir(), '.claude.json')
|
|
203
|
+
: join(harnessHome, '.claude.json');
|
|
204
|
+
const account = json(login)?.oauthAccount;
|
|
205
|
+
return account && typeof account === 'object' ? 'subscription' : undefined;
|
|
206
|
+
}
|
|
207
|
+
const auth = json(join(harnessHome, 'auth.json'));
|
|
208
|
+
if (auth?.auth_mode === 'chatgpt' || auth?.tokens) return 'subscription';
|
|
209
|
+
if (auth?.auth_mode === 'apikey' || auth?.OPENAI_API_KEY) return 'api';
|
|
210
|
+
return undefined;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* What a watching collector remembers between scans: each file's size and mtime with the outcome,
|
|
215
|
+
* so an untouched file is neither re-read nor re-sent. Keyed by the routing and archive it was
|
|
216
|
+
* decided under; a stored session is re-checked against the server now and then, so one deleted
|
|
217
|
+
* there is captured again.
|
|
218
|
+
*/
|
|
219
|
+
export type CollectorCache = Map<
|
|
220
|
+
string,
|
|
221
|
+
{
|
|
222
|
+
size: number;
|
|
223
|
+
mtimeMs: number;
|
|
224
|
+
routing: string;
|
|
225
|
+
outcome: 'unchanged' | 'skipped' | 'failed';
|
|
226
|
+
meta?: SessionMetadata;
|
|
227
|
+
stored?: { id: string; digest: string; checkedAt: number };
|
|
228
|
+
}
|
|
229
|
+
>;
|
|
230
|
+
const REVALIDATE_MS = 10 * 60_000;
|
|
231
|
+
|
|
232
|
+
export interface CollectOptions {
|
|
233
|
+
/** Every collector of this source and directory, this one included; the most specific wins. */
|
|
234
|
+
peers?: CollectionSource[];
|
|
235
|
+
exclude?: string[];
|
|
236
|
+
cache?: CollectorCache;
|
|
237
|
+
/** The harness login's access path; defaults to reading the history directory's harness home. */
|
|
238
|
+
harness?: (
|
|
239
|
+
source: CollectionSource['source'],
|
|
240
|
+
directory: string,
|
|
241
|
+
) => 'subscription' | 'api' | undefined;
|
|
242
|
+
now?: () => number;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** A rejection the same snapshot will get again: invalid input or a refused write, not an outage. */
|
|
246
|
+
const deterministic = (error: unknown) => {
|
|
247
|
+
const status = (error as { status?: unknown } | null)?.status;
|
|
248
|
+
return (
|
|
249
|
+
(error instanceof Error && error.name === 'ZodError') ||
|
|
250
|
+
(typeof status === 'number' && status >= 400 && status < 500 && status !== 409)
|
|
251
|
+
);
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
export async function collectSessions(
|
|
255
|
+
archive: ArchiveWriter,
|
|
256
|
+
config: CollectionSource,
|
|
257
|
+
options: CollectOptions = {},
|
|
258
|
+
) {
|
|
107
259
|
const sourceId = await archive.sourceId();
|
|
108
|
-
const
|
|
260
|
+
const now = options.now ?? Date.now;
|
|
261
|
+
const peers = options.peers ?? [config];
|
|
262
|
+
const exclude = options.exclude ?? defaultExclusions();
|
|
263
|
+
const cache: CollectorCache = options.cache ?? new Map();
|
|
264
|
+
const routing = JSON.stringify([sourceId, config, peers, exclude]);
|
|
109
265
|
const result = { imported: 0, unchanged: 0, skipped: 0, failed: 0 };
|
|
110
266
|
// Per scan: checkouts change between scans.
|
|
267
|
+
const rootSets = new Map<CollectionSource, Set<string>>();
|
|
268
|
+
const roots = (collector: CollectionSource) => {
|
|
269
|
+
if (!rootSets.has(collector)) rootSets.set(collector, collectionRoots(collector));
|
|
270
|
+
return rootSets.get(collector)!;
|
|
271
|
+
};
|
|
272
|
+
const routes = new Map<string, CollectionSource | undefined>();
|
|
111
273
|
const repositories = new Map<string, string | undefined>();
|
|
112
274
|
const graphs = new Map<string, AtlasFileConcepts | undefined>();
|
|
113
|
-
|
|
275
|
+
let harness: 'subscription' | 'api' | undefined | null = null;
|
|
276
|
+
for (const file of transcriptFiles(config.directory)) {
|
|
114
277
|
try {
|
|
115
|
-
const
|
|
116
|
-
|
|
117
|
-
if (
|
|
278
|
+
const before = lstatSync(file);
|
|
279
|
+
const known = cache.get(file);
|
|
280
|
+
if (
|
|
281
|
+
known?.size === before.size &&
|
|
282
|
+
known.mtimeMs === before.mtimeMs &&
|
|
283
|
+
known.routing === routing
|
|
284
|
+
) {
|
|
285
|
+
const stored = known.stored;
|
|
286
|
+
if (!stored || now() - stored.checkedAt < REVALIDATE_MS) {
|
|
287
|
+
result[known.outcome]++;
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
if ((await archive.head(stored.id))?.digest === stored.digest) {
|
|
291
|
+
stored.checkedAt = now();
|
|
292
|
+
result.unchanged++;
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
const remember = (
|
|
297
|
+
outcome: 'unchanged' | 'skipped' | 'failed',
|
|
298
|
+
meta?: SessionMetadata,
|
|
299
|
+
stored?: { id: string; digest: string },
|
|
300
|
+
) =>
|
|
301
|
+
cache.set(file, {
|
|
302
|
+
size: before.size,
|
|
303
|
+
mtimeMs: before.mtimeMs,
|
|
304
|
+
routing,
|
|
305
|
+
outcome,
|
|
306
|
+
meta,
|
|
307
|
+
...(stored ? { stored: { ...stored, checkedAt: now() } } : {}),
|
|
308
|
+
});
|
|
309
|
+
// A transcript's first cwd never changes, so it is read once per file.
|
|
310
|
+
const meta = known?.meta ?? sessionMetadata(file, config.source);
|
|
311
|
+
if (!meta) {
|
|
118
312
|
result.skipped++;
|
|
313
|
+
remember('skipped');
|
|
119
314
|
continue;
|
|
120
315
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
sourceId,
|
|
125
|
-
projectId: config.projectId,
|
|
126
|
-
});
|
|
127
|
-
const after = lstatSync(file);
|
|
128
|
-
if (before.size !== after.size || before.mtimeMs !== after.mtimeMs) {
|
|
316
|
+
if (!routes.has(meta.cwd))
|
|
317
|
+
routes.set(meta.cwd, routeSession(meta.cwd, peers, { exclude, roots }));
|
|
318
|
+
if (routes.get(meta.cwd) !== config) {
|
|
129
319
|
result.skipped++;
|
|
320
|
+
remember('skipped', meta);
|
|
130
321
|
continue;
|
|
131
322
|
}
|
|
132
|
-
|
|
133
|
-
|
|
323
|
+
let snapshot: ReturnType<typeof importTranscriptFile>;
|
|
324
|
+
try {
|
|
325
|
+
snapshot = importTranscriptFile(file, {
|
|
326
|
+
source: config.source,
|
|
327
|
+
sourceId,
|
|
328
|
+
projectId: config.projectId,
|
|
329
|
+
});
|
|
330
|
+
} catch {
|
|
331
|
+
// The same bytes fail the same way; a partial file is retried once it changes.
|
|
134
332
|
result.failed++;
|
|
333
|
+
remember('failed', meta);
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
const after = lstatSync(file);
|
|
337
|
+
if (before.size !== after.size || before.mtimeMs !== after.mtimeMs) {
|
|
338
|
+
result.skipped++;
|
|
135
339
|
continue;
|
|
136
340
|
}
|
|
137
341
|
const { cwd } = meta;
|
|
@@ -139,15 +343,54 @@ export async function collectSessions(archive: ArchiveWriter, config: Collection
|
|
|
139
343
|
if (config.atlas && !graphs.has(cwd)) graphs.set(cwd, atlasGraph(cwd));
|
|
140
344
|
const graph = graphs.get(cwd);
|
|
141
345
|
snapshot.tags = [
|
|
142
|
-
...new Set([
|
|
346
|
+
...new Set([
|
|
347
|
+
...(config.tags ?? []),
|
|
348
|
+
...(graph ? atlasTags(snapshot, graph, roots(config)) : []),
|
|
349
|
+
]),
|
|
143
350
|
].slice(0, 100);
|
|
144
351
|
const references = provenanceReferences({
|
|
145
352
|
branch: meta.branch,
|
|
146
353
|
repository: meta.repository ?? repositories.get(cwd),
|
|
147
354
|
});
|
|
148
355
|
if (references.length) snapshot.references = references;
|
|
149
|
-
if (
|
|
150
|
-
|
|
356
|
+
if (snapshot.runtime?.accessPath === 'unknown') {
|
|
357
|
+
if (harness === null)
|
|
358
|
+
harness = (options.harness ?? harnessAccess)(config.source, config.directory);
|
|
359
|
+
if (harness)
|
|
360
|
+
snapshot.runtime = { ...snapshot.runtime, accessPath: harness, evidence: 'harness' };
|
|
361
|
+
}
|
|
362
|
+
const id = archiveKey(snapshot);
|
|
363
|
+
const old = await archive.head(id);
|
|
364
|
+
// A session belongs to one project; only one the catch-all held moves to its routed project.
|
|
365
|
+
if (old && old.projectId !== config.projectId && old.projectId !== UNROUTED) {
|
|
366
|
+
result.failed++;
|
|
367
|
+
remember('failed', meta);
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
// An unchanged session is not sent again: the server already holds this digest.
|
|
371
|
+
// Digest what the server will: the schema orders keys, and fields set above were appended.
|
|
372
|
+
let digest: string;
|
|
373
|
+
try {
|
|
374
|
+
digest = snapshotDigest(archiveSnapshotSchema.parse(snapshot));
|
|
375
|
+
} catch {
|
|
376
|
+
// Over the archive's limits: the same bytes are refused again.
|
|
377
|
+
result.failed++;
|
|
378
|
+
remember('failed', meta);
|
|
379
|
+
continue;
|
|
380
|
+
}
|
|
381
|
+
if (old?.digest === digest) result.unchanged++;
|
|
382
|
+
else {
|
|
383
|
+
try {
|
|
384
|
+
const captured = await archive.capture(snapshot);
|
|
385
|
+
digest = captured.digest;
|
|
386
|
+
result[captured.changed ? 'imported' : 'unchanged']++;
|
|
387
|
+
} catch (error) {
|
|
388
|
+
result.failed++;
|
|
389
|
+
if (deterministic(error)) remember('failed', meta);
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
remember('unchanged', meta, { id, digest });
|
|
151
394
|
} catch {
|
|
152
395
|
result.failed++;
|
|
153
396
|
} // Partial/malformed files remain intact and retry on the next scan.
|
package/src/config.ts
CHANGED
|
@@ -13,8 +13,13 @@ import {
|
|
|
13
13
|
} from 'node:fs';
|
|
14
14
|
import { dirname, isAbsolute, join } from 'node:path';
|
|
15
15
|
import { z } from 'zod';
|
|
16
|
+
import { UNROUTED } from './index';
|
|
16
17
|
|
|
17
|
-
const projectId = z
|
|
18
|
+
const projectId = z
|
|
19
|
+
.string()
|
|
20
|
+
.min(1)
|
|
21
|
+
.max(256)
|
|
22
|
+
.refine((id) => id !== UNROUTED, `${UNROUTED} sessions stay on this machine`);
|
|
18
23
|
/** A remote Archive this one syncs to directly with its server token. */
|
|
19
24
|
const archiveDestination = z
|
|
20
25
|
.strictObject({
|
package/src/import-file.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { closeSync, fstatSync, openSync, readSync } from 'node:fs';
|
|
1
|
+
import { closeSync, existsSync, fstatSync, openSync, readFileSync, readSync } from 'node:fs';
|
|
2
2
|
import { type ImportOptions, importTranscriptLines } from './import';
|
|
3
|
+
import { archiveLineageSchema } from './index';
|
|
3
4
|
|
|
4
5
|
/** Bound raw record memory while skipping unsupported payloads from long native histories. */
|
|
5
6
|
export function importTranscriptFile(file: string, options: ImportOptions) {
|
|
@@ -41,5 +42,31 @@ export function importTranscriptFile(file: string, options: ImportOptions) {
|
|
|
41
42
|
closeSync(descriptor);
|
|
42
43
|
}
|
|
43
44
|
}
|
|
44
|
-
|
|
45
|
+
const snapshot = importTranscriptLines(lines(), options);
|
|
46
|
+
if (options.source === 'claude-code' && snapshot.lineage) {
|
|
47
|
+
const meta = subagentMeta(file);
|
|
48
|
+
if (meta) snapshot.lineage = archiveLineageSchema.parse({ ...snapshot.lineage, ...meta });
|
|
49
|
+
}
|
|
50
|
+
return snapshot;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Claude Code writes `agent-<id>.meta.json` beside a subagent transcript: its type, task and depth. */
|
|
54
|
+
function subagentMeta(file: string) {
|
|
55
|
+
const path = file.replace(/\.jsonl$/, '.meta.json');
|
|
56
|
+
if (path === file || !existsSync(path)) return undefined;
|
|
57
|
+
let meta: Record<string, unknown>;
|
|
58
|
+
try {
|
|
59
|
+
meta = JSON.parse(readFileSync(path, 'utf8'));
|
|
60
|
+
} catch {
|
|
61
|
+
return undefined;
|
|
62
|
+
}
|
|
63
|
+
const text = (value: unknown, max: number) =>
|
|
64
|
+
typeof value === 'string' && value ? value.slice(0, max) : undefined;
|
|
65
|
+
const fields = {
|
|
66
|
+
agentType: text(meta.agentType, 200),
|
|
67
|
+
description: text(meta.description, 500),
|
|
68
|
+
toolUseId: text(meta.toolUseId, 256),
|
|
69
|
+
depth: Number.isInteger(meta.spawnDepth) ? (meta.spawnDepth as number) : undefined,
|
|
70
|
+
};
|
|
71
|
+
return Object.fromEntries(Object.entries(fields).filter(([, value]) => value !== undefined));
|
|
45
72
|
}
|
package/src/import.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
type ArchiveEntry,
|
|
3
|
+
type ArchiveLineage,
|
|
4
|
+
type ArchiveSnapshot,
|
|
5
|
+
archiveSnapshotSchema,
|
|
6
|
+
} from './index';
|
|
2
7
|
|
|
3
8
|
export interface ImportOptions {
|
|
4
9
|
source: 'codex' | 'claude-code';
|
|
@@ -27,7 +32,35 @@ const injectedTags = [
|
|
|
27
32
|
'user_instructions',
|
|
28
33
|
'permissions',
|
|
29
34
|
'INSTRUCTIONS',
|
|
35
|
+
'codex_internal_context',
|
|
36
|
+
'heartbeat',
|
|
37
|
+
'in-app-browser-context',
|
|
38
|
+
'external_codex_apps_open_page',
|
|
39
|
+
'send_user_message_question_reply',
|
|
40
|
+
'turn_aborted',
|
|
30
41
|
].join('|');
|
|
42
|
+
/** Prompts Codex writes in the user's role. */
|
|
43
|
+
const injectedPreamble = /^(?:# In app browser:|The following is the Codex agent history)/;
|
|
44
|
+
/**
|
|
45
|
+
* Nothing but blocks of harness-style tags such as `<codex_internal_context>`: `snake_case` or
|
|
46
|
+
* `kebab-case` names, unlike markup a person pastes. A linear scan, since a backtracking pattern is
|
|
47
|
+
* exponential on many sibling blocks.
|
|
48
|
+
*/
|
|
49
|
+
function onlyTagBlocks(text: string) {
|
|
50
|
+
const open = /\s*<([a-z][a-z0-9]*(?:[_-][a-z0-9]+)+)(?:\s[^>]*)?>/y;
|
|
51
|
+
let at = 0;
|
|
52
|
+
let blocks = 0;
|
|
53
|
+
while (at < text.length) {
|
|
54
|
+
open.lastIndex = at;
|
|
55
|
+
const match = open.exec(text);
|
|
56
|
+
if (!match) return blocks > 0 && !text.slice(at).trim();
|
|
57
|
+
const close = text.indexOf(`</${match[1]}>`, open.lastIndex);
|
|
58
|
+
if (close < 0) return false;
|
|
59
|
+
at = close + match[1]!.length + 3;
|
|
60
|
+
blocks++;
|
|
61
|
+
}
|
|
62
|
+
return blocks > 0;
|
|
63
|
+
}
|
|
31
64
|
const injectedBlock = new RegExp(`<(${injectedTags})(?:\\s[^>]*)?>[\\s\\S]*?</\\1>`, 'g');
|
|
32
65
|
const injectedOpening = new RegExp(`^<(?:${injectedTags})(?:>|\\s)`);
|
|
33
66
|
/** Foundry's native-session envelope; the person's text is the first User Message section. */
|
|
@@ -35,7 +68,7 @@ const foundryUserMessage =
|
|
|
35
68
|
/^# User Message[ \t]*\n([\s\S]*?)(?=\n# (?:User Message|Assistant Message|Foundry Injection)[ \t]*\n|(?![\s\S]))/m;
|
|
36
69
|
|
|
37
70
|
/** What the person typed in a user message, without injected content; undefined when nothing is left. */
|
|
38
|
-
function typedPrompt(text: string) {
|
|
71
|
+
export function typedPrompt(text: string) {
|
|
39
72
|
let prompt = text
|
|
40
73
|
.replace(injectedBlock, '')
|
|
41
74
|
.replace(/<\/?pasted_content(?:\s[^>]*)?>/g, '')
|
|
@@ -43,7 +76,13 @@ function typedPrompt(text: string) {
|
|
|
43
76
|
.trim();
|
|
44
77
|
if (/^# (?:System Context|User Message)[ \t]*\n/.test(prompt))
|
|
45
78
|
prompt = prompt.match(foundryUserMessage)?.[1].trim() ?? '';
|
|
46
|
-
if (
|
|
79
|
+
if (
|
|
80
|
+
!prompt ||
|
|
81
|
+
injectedOpening.test(prompt) ||
|
|
82
|
+
injectedPreamble.test(prompt) ||
|
|
83
|
+
onlyTagBlocks(prompt) ||
|
|
84
|
+
/^\[Request interrupted by user/.test(prompt)
|
|
85
|
+
)
|
|
47
86
|
return undefined;
|
|
48
87
|
return prompt;
|
|
49
88
|
}
|
|
@@ -66,6 +105,16 @@ export function importTranscriptLines(
|
|
|
66
105
|
let contentBytes = 0;
|
|
67
106
|
// Codex states the model per turn; Claude Code per assistant record.
|
|
68
107
|
let turn: { model?: string; effort?: string } = {};
|
|
108
|
+
// A Codex session started by a script (codex exec, a custom originator) has no typed prompts.
|
|
109
|
+
let scripted = false;
|
|
110
|
+
let metaSeen = false;
|
|
111
|
+
let lineage: ArchiveLineage | undefined;
|
|
112
|
+
let provider: string | undefined = options.source === 'claude-code' ? 'anthropic' : undefined;
|
|
113
|
+
// Only a subscription's rate-limit windows prove the access path; an API key leaves no mark.
|
|
114
|
+
let subscription = false;
|
|
115
|
+
const text = (value: unknown) => (typeof value === 'string' && value ? value : undefined);
|
|
116
|
+
const optional = <K extends string>(key: K, value: string | undefined) =>
|
|
117
|
+
(value ? { [key]: value } : {}) as Partial<Record<K, string>>;
|
|
69
118
|
for (const rawLine of lines) {
|
|
70
119
|
index++;
|
|
71
120
|
const line = rawLine.trim();
|
|
@@ -79,11 +128,52 @@ export function importTranscriptLines(
|
|
|
79
128
|
if (!row || typeof row !== 'object' || Array.isArray(row))
|
|
80
129
|
throw new Error(`Invalid transcript record at line ${index + 1}`);
|
|
81
130
|
if (options.source === 'codex' && row.type === 'session_meta') {
|
|
82
|
-
const
|
|
131
|
+
const meta = row.payload ?? {};
|
|
132
|
+
const id = meta.id;
|
|
133
|
+
// A subagent or forked rollout starts with its own meta, then replays its parent's history,
|
|
134
|
+
// parent meta included. The first meta is the session; a later one is lineage, not identity.
|
|
135
|
+
if (metaSeen && lineage) continue;
|
|
83
136
|
if (sessionId && id && id !== sessionId)
|
|
84
137
|
throw new Error('Transcript session identity mismatch');
|
|
85
138
|
sessionId ??= id;
|
|
139
|
+
metaSeen = true;
|
|
140
|
+
// Subagent and guardian threads, and `codex exec`, are driven by an agent or a script.
|
|
141
|
+
const { originator, source } = meta;
|
|
142
|
+
if (typeof originator === 'string')
|
|
143
|
+
scripted = !/^codex/i.test(originator) || /^codex_(?:exec|sdk|mcp)/.test(originator);
|
|
144
|
+
const spawn =
|
|
145
|
+
source && typeof source === 'object' && 'subagent' in source
|
|
146
|
+
? source.subagent?.thread_spawn
|
|
147
|
+
: undefined;
|
|
148
|
+
if (source === 'exec' || (source && typeof source === 'object' && 'subagent' in source))
|
|
149
|
+
scripted = true;
|
|
150
|
+
const parent = text(spawn?.parent_thread_id) ?? text(meta.parent_thread_id);
|
|
151
|
+
const forked = text(meta.forked_from_id);
|
|
152
|
+
if (parent || forked)
|
|
153
|
+
lineage = {
|
|
154
|
+
kind: source && typeof source === 'object' && 'subagent' in source ? 'subagent' : 'fork',
|
|
155
|
+
parentSessionId: (parent ?? forked)!,
|
|
156
|
+
...optional('agentType', text(spawn?.agent_role ?? meta.agent_role)),
|
|
157
|
+
...optional('agentName', text(spawn?.agent_nickname ?? meta.agent_nickname)),
|
|
158
|
+
...(Number.isInteger(spawn?.depth) ? { depth: spawn.depth } : {}),
|
|
159
|
+
};
|
|
160
|
+
const named = text(meta.model_provider)?.toLowerCase();
|
|
161
|
+
provider = named && /^[a-z0-9][a-z0-9_-]{0,63}$/.test(named) ? named : 'openai';
|
|
86
162
|
}
|
|
163
|
+
if (
|
|
164
|
+
options.source === 'codex' &&
|
|
165
|
+
row.type === 'event_msg' &&
|
|
166
|
+
row.payload?.type === 'token_count' &&
|
|
167
|
+
text(row.payload.rate_limits?.plan_type)
|
|
168
|
+
)
|
|
169
|
+
subscription = true;
|
|
170
|
+
if (
|
|
171
|
+
options.source === 'claude-code' &&
|
|
172
|
+
row.quotaLimits &&
|
|
173
|
+
typeof row.quotaLimits === 'object' &&
|
|
174
|
+
text(row.quotaLimits.rateLimitType)
|
|
175
|
+
)
|
|
176
|
+
subscription = true;
|
|
87
177
|
if (options.source === 'claude-code' && row.sessionId) {
|
|
88
178
|
// Subagent transcripts reuse their parent's sessionId; each is its own archive.
|
|
89
179
|
const id =
|
|
@@ -92,8 +182,9 @@ export function importTranscriptLines(
|
|
|
92
182
|
: row.sessionId;
|
|
93
183
|
if (sessionId && sessionId !== id) throw new Error('Mixed transcript sessions');
|
|
94
184
|
sessionId = id;
|
|
185
|
+
if (row.isSidechain && typeof row.agentId === 'string')
|
|
186
|
+
lineage ??= { kind: 'subagent', parentSessionId: row.sessionId };
|
|
95
187
|
}
|
|
96
|
-
const text = (value: unknown) => (typeof value === 'string' && value ? value : undefined);
|
|
97
188
|
if (options.source === 'codex' && row.type === 'turn_context')
|
|
98
189
|
turn = { model: text(row.payload?.model), effort: text(row.payload?.effort) };
|
|
99
190
|
const producer =
|
|
@@ -102,6 +193,26 @@ export function importTranscriptLines(
|
|
|
102
193
|
: row.type === 'assistant'
|
|
103
194
|
? { model: text(row.message?.model), effort: text(row.effort) }
|
|
104
195
|
: {};
|
|
196
|
+
// A user record the person did not type: an expansion, a compaction summary, a subagent's prompt,
|
|
197
|
+
// a message from the harness, another agent or an SDK caller, or a scripted Codex session's prompt.
|
|
198
|
+
// It is kept as an event, not a user turn.
|
|
199
|
+
const written =
|
|
200
|
+
options.source === 'claude-code'
|
|
201
|
+
? row.type === 'user' &&
|
|
202
|
+
Boolean(
|
|
203
|
+
row.isMeta ||
|
|
204
|
+
row.isCompactSummary ||
|
|
205
|
+
row.isSidechain ||
|
|
206
|
+
(row.origin && typeof row.origin === 'object'
|
|
207
|
+
? row.origin.kind !== 'human'
|
|
208
|
+
: typeof row.entrypoint === 'string' && row.entrypoint.startsWith('sdk')),
|
|
209
|
+
)
|
|
210
|
+
: scripted && row.type === 'response_item' && row.payload?.role === 'user';
|
|
211
|
+
// User text is a user entry only when the person typed it; markers and injected blocks are events.
|
|
212
|
+
const textKind = (role: 'user' | 'assistant', content: unknown) =>
|
|
213
|
+
role === 'user' && (written || typeof content !== 'string' || !typedPrompt(content))
|
|
214
|
+
? 'event'
|
|
215
|
+
: role;
|
|
105
216
|
const timestamp = typeof row.timestamp === 'number' ? row.timestamp : Date.parse(row.timestamp);
|
|
106
217
|
const add = (kind: ArchiveEntry['kind'], content: unknown, suffix: string, callId?: string) => {
|
|
107
218
|
if (typeof content !== 'string' || !content.length) return;
|
|
@@ -118,10 +229,10 @@ export function importTranscriptLines(
|
|
|
118
229
|
'Supported transcript content exceeds the archive size limit; no archive imported',
|
|
119
230
|
);
|
|
120
231
|
seen.set(id, identity);
|
|
121
|
-
//
|
|
122
|
-
if (
|
|
123
|
-
if (
|
|
124
|
-
else metaPrompt ??= typedPrompt(content);
|
|
232
|
+
// A prompt someone else wrote (an expansion, a subagent's task) titles the session only when nothing was typed.
|
|
233
|
+
if (!prompt && !row.isCompactSummary) {
|
|
234
|
+
if (kind === 'user') prompt = typedPrompt(content);
|
|
235
|
+
else if (kind === 'event' && written) metaPrompt ??= typedPrompt(content);
|
|
125
236
|
}
|
|
126
237
|
entries.push({
|
|
127
238
|
id,
|
|
@@ -139,12 +250,16 @@ export function importTranscriptLines(
|
|
|
139
250
|
});
|
|
140
251
|
};
|
|
141
252
|
if (options.source === 'codex') {
|
|
142
|
-
|
|
253
|
+
// Codex records the person stopping a turn as its own event; Claude Code writes this marker.
|
|
254
|
+
if (row.type === 'event_msg' && row.payload?.type === 'turn_aborted') {
|
|
255
|
+
if (row.payload.reason === 'interrupted')
|
|
256
|
+
add('event', '[Request interrupted by user]', 'aborted');
|
|
257
|
+
} else if (row.type === 'response_item') {
|
|
143
258
|
const p = row.payload ?? {};
|
|
144
259
|
if (p.type === 'message' && ['user', 'assistant'].includes(p.role)) {
|
|
145
260
|
for (const [i, block] of (p.content ?? []).entries()) {
|
|
146
261
|
if (['input_text', 'output_text', 'text'].includes(block.type))
|
|
147
|
-
add(p.role, block.text, `text-${i}`);
|
|
262
|
+
add(textKind(p.role, block.text), block.text, `text-${i}`);
|
|
148
263
|
else omissions.add('Non-text message content is not included');
|
|
149
264
|
}
|
|
150
265
|
} else if (['function_call', 'custom_tool_call'].includes(p.type)) {
|
|
@@ -181,10 +296,10 @@ export function importTranscriptLines(
|
|
|
181
296
|
}
|
|
182
297
|
} else if (['user', 'assistant'].includes(row.type) && row.message) {
|
|
183
298
|
const blocks = row.message.content;
|
|
184
|
-
if (typeof blocks === 'string') add(row.type, blocks, 'text');
|
|
299
|
+
if (typeof blocks === 'string') add(textKind(row.type, blocks), blocks, 'text');
|
|
185
300
|
else if (Array.isArray(blocks))
|
|
186
301
|
for (const [i, block] of blocks.entries()) {
|
|
187
|
-
if (block.type === 'text') add(row.type, block.text, `text-${i}`);
|
|
302
|
+
if (block.type === 'text') add(textKind(row.type, block.text), block.text, `text-${i}`);
|
|
188
303
|
else if (block.type === 'tool_use')
|
|
189
304
|
add(
|
|
190
305
|
'tool-call',
|
|
@@ -221,6 +336,12 @@ export function importTranscriptLines(
|
|
|
221
336
|
sessionId,
|
|
222
337
|
title,
|
|
223
338
|
projectId: options.projectId,
|
|
339
|
+
runtime: {
|
|
340
|
+
provider: provider ?? 'openai',
|
|
341
|
+
accessPath: subscription ? 'subscription' : 'unknown',
|
|
342
|
+
...(subscription ? { evidence: 'transcript' } : {}),
|
|
343
|
+
},
|
|
344
|
+
...(lineage ? { lineage } : {}),
|
|
224
345
|
tags: [],
|
|
225
346
|
capturedAt: Date.now(),
|
|
226
347
|
coverage: {
|