@inixiative/archive 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -47
- package/compose.yaml +37 -0
- package/package.json +12 -6
- package/prisma/migrations/20261001022030_init/migration.sql +119 -0
- package/prisma/migrations/migration_lock.toml +3 -0
- package/prisma/schema.prisma +125 -0
- package/src/agents.ts +5 -15
- package/src/cli.ts +204 -149
- package/src/client.ts +15 -15
- package/src/collector.ts +17 -16
- package/src/import.ts +17 -0
- package/src/index.ts +21 -0
- package/src/remote.ts +166 -0
- package/src/schemas.ts +40 -0
- package/src/server.ts +104 -58
- package/src/store.ts +438 -0
- package/src/tags.ts +96 -36
- package/src/local.ts +0 -149
package/src/cli.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { randomBytes } from 'node:crypto';
|
|
3
3
|
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
4
4
|
import { join, resolve } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
5
6
|
import { parseArgs } from 'node:util';
|
|
6
7
|
import {
|
|
7
8
|
agentStatus,
|
|
@@ -17,7 +18,6 @@ import { importChatGPTThread } from './chatgpt';
|
|
|
17
18
|
import {
|
|
18
19
|
archiveRequest,
|
|
19
20
|
kingdomFields,
|
|
20
|
-
publishArchive,
|
|
21
21
|
routingPreview,
|
|
22
22
|
searchRemotes,
|
|
23
23
|
syncArchives,
|
|
@@ -25,10 +25,20 @@ import {
|
|
|
25
25
|
import { collectSessions } from './collector';
|
|
26
26
|
import { archiveDestinationSchema, connectDestination, readDestinations } from './config';
|
|
27
27
|
import { importTranscriptFile } from './import-file';
|
|
28
|
-
import { archiveKey } from './index';
|
|
29
|
-
import { LocalArchiveStore } from './local';
|
|
30
28
|
import { previewImports } from './preview';
|
|
29
|
+
import { ArchiveClient, DEFAULT_URL } from './remote';
|
|
31
30
|
import { startArchiveServer } from './server';
|
|
31
|
+
import { ArchiveStore } from './store';
|
|
32
|
+
|
|
33
|
+
/** Applies pending schema migrations to the archive database. */
|
|
34
|
+
export function migrate(databaseUrl: string) {
|
|
35
|
+
const schema = fileURLToPath(new URL('../prisma/schema.prisma', import.meta.url));
|
|
36
|
+
const run = Bun.spawnSync(
|
|
37
|
+
[process.execPath, 'x', 'prisma', 'migrate', 'deploy', '--schema', schema],
|
|
38
|
+
{ env: { ...process.env, DATABASE_URL: databaseUrl }, stdout: 'ignore', stderr: 'pipe' },
|
|
39
|
+
);
|
|
40
|
+
if (!run.success) throw new Error(`Archive migration failed: ${run.stderr.toString().trim()}`);
|
|
41
|
+
}
|
|
32
42
|
|
|
33
43
|
export async function runCli(args = Bun.argv.slice(2)) {
|
|
34
44
|
const { values: v, positionals } = parseArgs({
|
|
@@ -36,7 +46,6 @@ export async function runCli(args = Bun.argv.slice(2)) {
|
|
|
36
46
|
allowPositionals: true,
|
|
37
47
|
options: {
|
|
38
48
|
home: { type: 'string', default: defaultHome() },
|
|
39
|
-
store: { type: 'string' },
|
|
40
49
|
config: { type: 'string' },
|
|
41
50
|
url: { type: 'string' },
|
|
42
51
|
kind: { type: 'string', default: 'archive' },
|
|
@@ -58,59 +67,112 @@ export async function runCli(args = Bun.argv.slice(2)) {
|
|
|
58
67
|
worktrees: { type: 'boolean', default: false },
|
|
59
68
|
atlas: { type: 'boolean', default: false },
|
|
60
69
|
tag: { type: 'string', multiple: true },
|
|
70
|
+
untag: { type: 'string', multiple: true },
|
|
61
71
|
limit: { type: 'string', default: '100' },
|
|
62
72
|
port: { type: 'string' },
|
|
63
73
|
name: { type: 'string' },
|
|
64
74
|
hostname: { type: 'string', default: '127.0.0.1' },
|
|
65
75
|
watch: { type: 'boolean', default: false },
|
|
76
|
+
sync: { type: 'boolean', default: false },
|
|
66
77
|
remote: { type: 'boolean', default: false },
|
|
67
|
-
destination: { type: 'string' },
|
|
68
78
|
help: { type: 'boolean' },
|
|
69
79
|
},
|
|
70
80
|
});
|
|
71
81
|
const home = resolve(v.home!),
|
|
72
|
-
config = v.config ?? join(home, 'destinations.json')
|
|
73
|
-
|
|
82
|
+
config = v.config ?? join(home, 'destinations.json'),
|
|
83
|
+
tokenFile = v['token-file'] ? resolve(v['token-file']) : join(home, 'server.token');
|
|
74
84
|
const output = (value: unknown) => console.log(JSON.stringify(value, null, 2));
|
|
75
85
|
const command = positionals[0];
|
|
76
86
|
if (v.help || !command) {
|
|
77
87
|
console.log(
|
|
78
|
-
'Archive: init | serve | connect (setup) | preview | import | collect | list | export | tag | routes | sync | search | agents\n' +
|
|
79
|
-
'
|
|
80
|
-
'
|
|
88
|
+
'Archive: init | up | down | serve | connect (setup) | preview | import | collect | list | export | tag | routes | sync | search | agents\n' +
|
|
89
|
+
'up | down (the local Archive: bundled compose.yaml with Postgres, data in --home)\n' +
|
|
90
|
+
'serve [--port 4700 --hostname 127.0.0.1] [--sync] (DATABASE_URL; token from ARCHIVE_SERVER_TOKEN or --token-file)\n' +
|
|
91
|
+
'import --file PATH --source codex|claude-code|chatgpt --project-id ID [--tag TAG]\n' +
|
|
81
92
|
'collect --directory HISTORY --source codex|claude-code --project-root EXACT_CWD [--project-root ...] [--worktrees] [--atlas] --project-id ID [--watch]\n' +
|
|
82
|
-
'
|
|
83
|
-
'
|
|
93
|
+
'list | export --id ID | tag --id ID [--tag TAG ...] [--untag TAG ...] | search --query TEXT [--remote]\n' +
|
|
94
|
+
'connect --url HTTPS_URL --project-id ID (--token-env ENV | --token-file PATH) [--kind kingdom ...] (destinations for serve --sync)\n' +
|
|
95
|
+
'sync | routes (DATABASE_URL: publish to, or preview, destinations once)\n' +
|
|
84
96
|
'agents add-collector --name N --source codex|claude-code --project-id ID --project-root DIR [--project-root ...] [--directory HISTORY] [--worktrees] [--atlas]\n' +
|
|
85
|
-
'agents remove-collector --name N |
|
|
86
|
-
|
|
97
|
+
'agents remove-collector --name N | server --url URL | install | uninstall | status\n' +
|
|
98
|
+
`Data commands talk to the Archive server at --url (default ${DEFAULT_URL}) with the token in --token-file (default <home>/server.token) or ARCHIVE_TOKEN.`,
|
|
87
99
|
);
|
|
88
100
|
return;
|
|
89
101
|
}
|
|
90
|
-
|
|
102
|
+
const readToken = () =>
|
|
103
|
+
existsSync(tokenFile) ? readFileSync(tokenFile, 'utf8').trim() : undefined;
|
|
104
|
+
const init = () => {
|
|
91
105
|
mkdirSync(home, { recursive: true, mode: 0o700 });
|
|
92
|
-
const tokenFile = join(home, 'server.token');
|
|
93
106
|
if (!existsSync(tokenFile))
|
|
94
|
-
writeFileSync(tokenFile, randomBytes(32).toString('hex')
|
|
107
|
+
writeFileSync(tokenFile, `${randomBytes(32).toString('hex')}\n`, { mode: 0o600, flag: 'wx' });
|
|
95
108
|
chmodSync(tokenFile, 0o600);
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
109
|
+
};
|
|
110
|
+
if (command === 'init') {
|
|
111
|
+
init();
|
|
112
|
+
output({ initialized: true, home, tokenFile });
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
if (command === 'up' || command === 'down') {
|
|
116
|
+
// The machine's one local Archive: the bundled compose file, with its data in the home.
|
|
117
|
+
if (command === 'up') init();
|
|
118
|
+
const compose = fileURLToPath(new URL('../compose.yaml', import.meta.url));
|
|
119
|
+
const run = Bun.spawnSync(
|
|
120
|
+
[
|
|
121
|
+
'docker',
|
|
122
|
+
'compose',
|
|
123
|
+
'--project-name',
|
|
124
|
+
'archive',
|
|
125
|
+
'--file',
|
|
126
|
+
compose,
|
|
127
|
+
...(command === 'up' ? ['up', '--detach', '--wait'] : ['down']),
|
|
128
|
+
],
|
|
129
|
+
{
|
|
130
|
+
env: {
|
|
131
|
+
...process.env,
|
|
132
|
+
ARCHIVE_DATA_DIR: home,
|
|
133
|
+
// Compose requires the variable even to stop; down never needs the real token.
|
|
134
|
+
ARCHIVE_SERVER_TOKEN: readToken() ?? 'x'.repeat(32),
|
|
135
|
+
},
|
|
136
|
+
stdout: 'inherit',
|
|
137
|
+
stderr: 'inherit',
|
|
138
|
+
},
|
|
139
|
+
);
|
|
140
|
+
if (!run.success) throw new Error(`docker compose ${command} failed`);
|
|
141
|
+
output(command === 'up' ? { running: DEFAULT_URL, home } : { stopped: true });
|
|
99
142
|
return;
|
|
100
143
|
}
|
|
144
|
+
const databaseUrl = () => {
|
|
145
|
+
const url = process.env.DATABASE_URL;
|
|
146
|
+
if (!url) throw new Error('DATABASE_URL is required');
|
|
147
|
+
return url;
|
|
148
|
+
};
|
|
101
149
|
if (command === 'serve') {
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
token,
|
|
109
|
-
port: Number(v.port ?? process.env.PORT ?? 4411),
|
|
150
|
+
const url = databaseUrl();
|
|
151
|
+
migrate(url);
|
|
152
|
+
const instance = await startArchiveServer({
|
|
153
|
+
databaseUrl: url,
|
|
154
|
+
token: process.env.ARCHIVE_SERVER_TOKEN ?? readToken() ?? '',
|
|
155
|
+
port: Number(v.port ?? process.env.PORT ?? 4700),
|
|
110
156
|
hostname: v.hostname,
|
|
111
157
|
});
|
|
112
|
-
output({ listening: instance.server.url.href,
|
|
158
|
+
output({ listening: instance.server.url.href, sync: v.sync });
|
|
159
|
+
let stopped = false;
|
|
160
|
+
// One process owns the database connection: the server and its sync share it.
|
|
161
|
+
const sync = (async () => {
|
|
162
|
+
while (v.sync && !stopped) {
|
|
163
|
+
try {
|
|
164
|
+
const results = await syncArchives(instance.store, readDestinations(config));
|
|
165
|
+
const failed = results.filter((r) => r.status.startsWith('failed')).length;
|
|
166
|
+
if (failed) console.error(JSON.stringify({ sync: 'failed', archives: failed }));
|
|
167
|
+
} catch {
|
|
168
|
+
console.error(JSON.stringify({ sync: 'unavailable' }));
|
|
169
|
+
}
|
|
170
|
+
for (let i = 0; i < 30 && !stopped; i++) await Bun.sleep(1000);
|
|
171
|
+
}
|
|
172
|
+
})();
|
|
113
173
|
const stop = async () => {
|
|
174
|
+
stopped = true;
|
|
175
|
+
await sync;
|
|
114
176
|
await instance.close();
|
|
115
177
|
process.exit(0);
|
|
116
178
|
};
|
|
@@ -119,12 +181,12 @@ export async function runCli(args = Bun.argv.slice(2)) {
|
|
|
119
181
|
return;
|
|
120
182
|
}
|
|
121
183
|
if (command === 'connect' || command === 'setup') {
|
|
122
|
-
const
|
|
184
|
+
const destinationTokenFile = v['token-file'] && resolve(v['token-file']);
|
|
123
185
|
const destination = archiveDestinationSchema.parse({
|
|
124
186
|
projectId: v['project-id'],
|
|
125
187
|
url: v.url,
|
|
126
|
-
tokenEnv: v['token-env'] ?? (
|
|
127
|
-
tokenFile,
|
|
188
|
+
tokenEnv: v['token-env'] ?? (destinationTokenFile ? undefined : 'ARCHIVE_TOKEN'),
|
|
189
|
+
tokenFile: destinationTokenFile,
|
|
128
190
|
kind: v.kind,
|
|
129
191
|
...(v.kind === 'kingdom'
|
|
130
192
|
? {
|
|
@@ -152,11 +214,7 @@ export async function runCli(args = Bun.argv.slice(2)) {
|
|
|
152
214
|
if (command === 'agents') {
|
|
153
215
|
const file = join(home, 'agents.json');
|
|
154
216
|
const current = readAgents(file);
|
|
155
|
-
const
|
|
156
|
-
const toggle = () => {
|
|
157
|
-
if (state !== 'on' && state !== 'off') throw new Error(`agents ${action} requires on|off`);
|
|
158
|
-
return state === 'on';
|
|
159
|
-
};
|
|
217
|
+
const action = positionals[1];
|
|
160
218
|
const base = { collectors: [], ...current };
|
|
161
219
|
if (action === 'add-collector') {
|
|
162
220
|
if (!v.name || !v['project-id'] || !v['project-root'] || !v.source)
|
|
@@ -184,16 +242,10 @@ export async function runCli(args = Bun.argv.slice(2)) {
|
|
|
184
242
|
collectors: base.collectors.filter((c) => c.name !== v.name),
|
|
185
243
|
}),
|
|
186
244
|
);
|
|
187
|
-
} else if (action === '
|
|
188
|
-
|
|
189
|
-
output(
|
|
190
|
-
|
|
191
|
-
file,
|
|
192
|
-
toggle() ? { ...rest, serve: v.port ? { port: Number(v.port) } : {} } : rest,
|
|
193
|
-
),
|
|
194
|
-
);
|
|
195
|
-
} else if (action === 'sync') output(writeAgents(file, { ...base, sync: toggle() }));
|
|
196
|
-
else if (action === 'install') {
|
|
245
|
+
} else if (action === 'server') {
|
|
246
|
+
if (!v.url) throw new Error('agents server requires --url');
|
|
247
|
+
output(writeAgents(file, { ...base, server: { url: v.url } }));
|
|
248
|
+
} else if (action === 'install') {
|
|
197
249
|
if (!current) throw new Error('No agents.json; declare agents first');
|
|
198
250
|
output(installAgents(current, { home }));
|
|
199
251
|
} else if (action === 'uninstall') output(uninstallAgents({ home }));
|
|
@@ -218,115 +270,118 @@ export async function runCli(args = Bun.argv.slice(2)) {
|
|
|
218
270
|
if (results.some((r) => 'error' in r)) process.exitCode = 1;
|
|
219
271
|
return;
|
|
220
272
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
if (
|
|
225
|
-
!v.file ||
|
|
226
|
-
!['codex', 'claude-code', 'chatgpt'].includes(v.source ?? '') ||
|
|
227
|
-
!v['project-id']
|
|
228
|
-
)
|
|
229
|
-
throw new Error(
|
|
230
|
-
'Import requires --file, --source codex|claude-code|chatgpt and --project-id',
|
|
231
|
-
);
|
|
232
|
-
const snapshot =
|
|
233
|
-
v.source === 'chatgpt'
|
|
234
|
-
? importChatGPTThread(JSON.parse(readFileSync(v.file, 'utf8')), {
|
|
235
|
-
sourceId: store.sourceId,
|
|
236
|
-
projectId: v['project-id'],
|
|
237
|
-
title: v.title,
|
|
238
|
-
})
|
|
239
|
-
: importTranscriptFile(v.file, {
|
|
240
|
-
source: v.source as 'codex' | 'claude-code',
|
|
241
|
-
sourceId: store.sourceId,
|
|
242
|
-
projectId: v['project-id'],
|
|
243
|
-
sessionId: v['session-id'],
|
|
244
|
-
title: v.title,
|
|
245
|
-
});
|
|
246
|
-
snapshot.tags = [
|
|
247
|
-
...new Set([...(store.read(archiveKey(snapshot))?.snapshot.tags ?? []), ...(v.tag ?? [])]),
|
|
248
|
-
];
|
|
249
|
-
output({
|
|
250
|
-
...store.capture(snapshot),
|
|
251
|
-
entries: snapshot.entries.length,
|
|
252
|
-
coverage: snapshot.coverage,
|
|
253
|
-
});
|
|
254
|
-
} else if (command === 'list') output(store.list());
|
|
255
|
-
else if (command === 'routes') output(routingPreview(store, readDestinations(config)));
|
|
256
|
-
else if (command === 'sync' || command === 'collect') {
|
|
257
|
-
if (
|
|
258
|
-
command === 'collect' &&
|
|
259
|
-
(!v.directory ||
|
|
260
|
-
!v['project-root'] ||
|
|
261
|
-
!v['project-id'] ||
|
|
262
|
-
!['codex', 'claude-code'].includes(v.source ?? ''))
|
|
263
|
-
)
|
|
264
|
-
throw new Error('Collect requires --directory, --source, --project-root and --project-id');
|
|
265
|
-
let stopped = false;
|
|
266
|
-
const stop = () => {
|
|
267
|
-
stopped = true;
|
|
268
|
-
};
|
|
269
|
-
process.once('SIGINT', stop);
|
|
270
|
-
process.once('SIGTERM', stop);
|
|
271
|
-
do {
|
|
272
|
-
if (command === 'collect') {
|
|
273
|
-
const result = await collectSessions(store, {
|
|
274
|
-
directory: v.directory!,
|
|
275
|
-
source: v.source as 'codex' | 'claude-code',
|
|
276
|
-
projectRoots: v['project-root']!,
|
|
277
|
-
worktrees: v.worktrees,
|
|
278
|
-
atlas: v.atlas,
|
|
279
|
-
projectId: v['project-id']!,
|
|
280
|
-
tags: v.tag,
|
|
281
|
-
});
|
|
282
|
-
output(result);
|
|
283
|
-
if (!v.watch && result.failed) process.exitCode = 1;
|
|
284
|
-
} else {
|
|
285
|
-
const results = await syncArchives(store, readDestinations(config));
|
|
286
|
-
output(results);
|
|
287
|
-
if (!v.watch && results.some((r) => r.status.startsWith('failed'))) process.exitCode = 1;
|
|
288
|
-
}
|
|
289
|
-
if (v.watch && !stopped) await Bun.sleep(1000);
|
|
290
|
-
// Short waits allow prompt shutdown; network requests have their own timeout.
|
|
291
|
-
for (let i = 1; v.watch && !stopped && i < 30; i++) await Bun.sleep(1000);
|
|
292
|
-
} while (v.watch && !stopped);
|
|
293
|
-
process.removeListener('SIGINT', stop);
|
|
294
|
-
process.removeListener('SIGTERM', stop);
|
|
295
|
-
} else if (command === 'publish') {
|
|
296
|
-
if (!v.id || !v.destination) throw new Error('Publish requires --id and --destination');
|
|
297
|
-
output(
|
|
298
|
-
await publishArchive(
|
|
299
|
-
store,
|
|
300
|
-
v.id,
|
|
301
|
-
archiveDestinationSchema.parse(JSON.parse(readFileSync(v.destination, 'utf8'))),
|
|
302
|
-
),
|
|
303
|
-
);
|
|
304
|
-
} else if (command === 'search')
|
|
305
|
-
output(store.search(v.id ? [v.id] : store.list().map((a) => a.id), v.query!));
|
|
306
|
-
else if (command === 'export' || command === 'tag') {
|
|
307
|
-
const archive = v.id ? store.read(v.id) : undefined;
|
|
308
|
-
if (!archive) throw new Error('Valid --id required');
|
|
309
|
-
if (command === 'export') output(archive.snapshot);
|
|
273
|
+
if (command === 'sync' || command === 'routes') {
|
|
274
|
+
const store = new ArchiveStore(databaseUrl());
|
|
275
|
+
try {
|
|
276
|
+
if (command === 'routes') output(await routingPreview(store, readDestinations(config)));
|
|
310
277
|
else {
|
|
311
|
-
|
|
312
|
-
output(
|
|
313
|
-
|
|
314
|
-
...archive.snapshot,
|
|
315
|
-
capturedAt: Date.now(),
|
|
316
|
-
tags: [...new Set([...archive.snapshot.tags, ...v.tag])],
|
|
317
|
-
}),
|
|
318
|
-
);
|
|
278
|
+
const results = await syncArchives(store, readDestinations(config));
|
|
279
|
+
output(results);
|
|
280
|
+
if (results.some((r) => r.status.startsWith('failed'))) process.exitCode = 1;
|
|
319
281
|
}
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
|
|
282
|
+
} finally {
|
|
283
|
+
await store.close();
|
|
284
|
+
}
|
|
285
|
+
return;
|
|
323
286
|
}
|
|
287
|
+
const token = process.env.ARCHIVE_TOKEN ?? readToken();
|
|
288
|
+
if (!token) throw new Error('No Archive token: run `archive init` or set ARCHIVE_TOKEN');
|
|
289
|
+
const archive = new ArchiveClient({ url: v.url ?? DEFAULT_URL, token });
|
|
290
|
+
if (command === 'import') {
|
|
291
|
+
if (
|
|
292
|
+
!v.file ||
|
|
293
|
+
!['codex', 'claude-code', 'chatgpt'].includes(v.source ?? '') ||
|
|
294
|
+
!v['project-id']
|
|
295
|
+
)
|
|
296
|
+
throw new Error(
|
|
297
|
+
'Import requires --file, --source codex|claude-code|chatgpt and --project-id',
|
|
298
|
+
);
|
|
299
|
+
const sourceId = await archive.sourceId();
|
|
300
|
+
const snapshot =
|
|
301
|
+
v.source === 'chatgpt'
|
|
302
|
+
? importChatGPTThread(JSON.parse(readFileSync(v.file, 'utf8')), {
|
|
303
|
+
sourceId,
|
|
304
|
+
projectId: v['project-id'],
|
|
305
|
+
title: v.title,
|
|
306
|
+
})
|
|
307
|
+
: importTranscriptFile(v.file, {
|
|
308
|
+
source: v.source as 'codex' | 'claude-code',
|
|
309
|
+
sourceId,
|
|
310
|
+
projectId: v['project-id'],
|
|
311
|
+
sessionId: v['session-id'],
|
|
312
|
+
title: v.title,
|
|
313
|
+
});
|
|
314
|
+
snapshot.tags = [...new Set(v.tag ?? [])];
|
|
315
|
+
output({
|
|
316
|
+
...(await archive.capture(snapshot)),
|
|
317
|
+
entries: snapshot.entries.length,
|
|
318
|
+
coverage: snapshot.coverage,
|
|
319
|
+
});
|
|
320
|
+
} else if (command === 'list') output(await archive.list());
|
|
321
|
+
else if (command === 'collect') {
|
|
322
|
+
if (
|
|
323
|
+
!v.directory ||
|
|
324
|
+
!v['project-root'] ||
|
|
325
|
+
!v['project-id'] ||
|
|
326
|
+
!['codex', 'claude-code'].includes(v.source ?? '')
|
|
327
|
+
)
|
|
328
|
+
throw new Error('Collect requires --directory, --source, --project-root and --project-id');
|
|
329
|
+
let stopped = false;
|
|
330
|
+
const stop = () => {
|
|
331
|
+
stopped = true;
|
|
332
|
+
};
|
|
333
|
+
process.once('SIGINT', stop);
|
|
334
|
+
process.once('SIGTERM', stop);
|
|
335
|
+
do {
|
|
336
|
+
try {
|
|
337
|
+
const result = await collectSessions(archive, {
|
|
338
|
+
directory: v.directory,
|
|
339
|
+
source: v.source as 'codex' | 'claude-code',
|
|
340
|
+
projectRoots: v['project-root'],
|
|
341
|
+
worktrees: v.worktrees,
|
|
342
|
+
atlas: v.atlas,
|
|
343
|
+
projectId: v['project-id'],
|
|
344
|
+
tags: v.tag,
|
|
345
|
+
});
|
|
346
|
+
output(result);
|
|
347
|
+
if (!v.watch && result.failed) process.exitCode = 1;
|
|
348
|
+
} catch (error) {
|
|
349
|
+
// The server may be starting; a watching collector retries.
|
|
350
|
+
if (!v.watch) throw error;
|
|
351
|
+
console.error(JSON.stringify({ collect: 'server unavailable' }));
|
|
352
|
+
}
|
|
353
|
+
// Short waits allow prompt shutdown; network requests have their own timeout.
|
|
354
|
+
for (let i = 0; v.watch && !stopped && i < 30; i++) await Bun.sleep(1000);
|
|
355
|
+
} while (v.watch && !stopped);
|
|
356
|
+
process.removeListener('SIGINT', stop);
|
|
357
|
+
process.removeListener('SIGTERM', stop);
|
|
358
|
+
} else if (command === 'search')
|
|
359
|
+
output(
|
|
360
|
+
await archive.search({
|
|
361
|
+
query: v.query,
|
|
362
|
+
limit: Number(v.limit) > 100 ? 100 : Number(v.limit),
|
|
363
|
+
}),
|
|
364
|
+
);
|
|
365
|
+
else if (command === 'export' || command === 'tag') {
|
|
366
|
+
if (!v.id) throw new Error('Valid --id required');
|
|
367
|
+
if (command === 'export') {
|
|
368
|
+
const read = await archive.read(v.id);
|
|
369
|
+
if (!read) throw new Error('Valid --id required');
|
|
370
|
+
output(read.snapshot);
|
|
371
|
+
} else {
|
|
372
|
+
if (!v.tag?.length && !v.untag?.length)
|
|
373
|
+
throw new Error('Tag requires at least one --tag or --untag');
|
|
374
|
+
output(await archive.tag(v.id, { add: v.tag, remove: v.untag }));
|
|
375
|
+
}
|
|
376
|
+
} else throw new Error('Unknown command; use --help');
|
|
324
377
|
}
|
|
325
378
|
|
|
326
379
|
if (import.meta.main)
|
|
327
|
-
runCli().catch(() => {
|
|
380
|
+
runCli().catch((error) => {
|
|
328
381
|
console.error(
|
|
329
|
-
|
|
382
|
+
process.env.ARCHIVE_DEBUG
|
|
383
|
+
? error
|
|
384
|
+
: 'Archive command failed. Check arguments, server and destination access, and local paths. Use --help for usage. Set ARCHIVE_DEBUG=1 for details.',
|
|
330
385
|
);
|
|
331
386
|
process.exitCode = 1;
|
|
332
387
|
});
|
package/src/client.ts
CHANGED
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
destinationToken,
|
|
6
6
|
destinationUrl,
|
|
7
7
|
} from './config';
|
|
8
|
-
import type {
|
|
8
|
+
import type { ArchiveStore } from './store';
|
|
9
9
|
|
|
10
10
|
export { type ArchiveDestination, archiveDestinationSchema } from './config';
|
|
11
11
|
|
|
@@ -45,27 +45,27 @@ export async function archiveRequest(
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
export async function publishArchive(
|
|
48
|
-
store:
|
|
48
|
+
store: ArchiveStore,
|
|
49
49
|
id: string,
|
|
50
50
|
input: ArchiveDestination,
|
|
51
51
|
transport: typeof fetch = fetch,
|
|
52
52
|
) {
|
|
53
53
|
const destination = archiveDestinationSchema.parse(input);
|
|
54
|
-
const archive = store.
|
|
55
|
-
if (!archive || archive.
|
|
54
|
+
const archive = await store.head(id);
|
|
55
|
+
if (!archive || archive.projectId !== destination.projectId)
|
|
56
56
|
throw new Error('Archive is outside destination project');
|
|
57
57
|
const receiptKey = destinationIdentity(destination);
|
|
58
58
|
let sent = false;
|
|
59
59
|
for (let attempt = 0; attempt < 8; attempt++) {
|
|
60
|
-
const latest = store.
|
|
61
|
-
const previousDigest = store.receipt(id, receiptKey);
|
|
62
|
-
let pending = store.pending(id, receiptKey);
|
|
60
|
+
const latest = (await store.head(id))!;
|
|
61
|
+
const previousDigest = await store.receipt(id, receiptKey);
|
|
62
|
+
let pending = await store.pending(id, receiptKey);
|
|
63
63
|
if (!pending) {
|
|
64
64
|
if (previousDigest === latest.digest) return { unchanged: !sent };
|
|
65
|
-
store.enqueue(id, receiptKey, { revision: latest.revision });
|
|
66
|
-
pending = store.pending(id, receiptKey)!;
|
|
65
|
+
await store.enqueue(id, receiptKey, { revision: latest.revision });
|
|
66
|
+
pending = (await store.pending(id, receiptKey))!;
|
|
67
67
|
}
|
|
68
|
-
const queued = store.read(id, pending.revision);
|
|
68
|
+
const queued = await store.read(id, pending.revision);
|
|
69
69
|
if (!queued || queued.snapshot.projectId !== destination.projectId)
|
|
70
70
|
throw new Error('Pending archive is outside destination project');
|
|
71
71
|
const body = await archiveRequest(
|
|
@@ -79,14 +79,14 @@ export async function publishArchive(
|
|
|
79
79
|
transport,
|
|
80
80
|
);
|
|
81
81
|
if (body.data?.digest !== queued.digest) throw new Error('Archive acknowledgement mismatch');
|
|
82
|
-
store.delivered(id, receiptKey, queued.digest);
|
|
82
|
+
await store.delivered(id, receiptKey, queued.digest);
|
|
83
83
|
sent = true;
|
|
84
84
|
}
|
|
85
85
|
throw new Error('Archive changed repeatedly during publication; retry sync');
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
-
export function routingPreview(store:
|
|
89
|
-
return store.list().map((archive) => ({
|
|
88
|
+
export async function routingPreview(store: ArchiveStore, destinations: ArchiveDestination[]) {
|
|
89
|
+
return (await store.list()).map((archive) => ({
|
|
90
90
|
id: archive.id,
|
|
91
91
|
projectId: archive.projectId,
|
|
92
92
|
tags: archive.tags,
|
|
@@ -101,9 +101,9 @@ export function routingPreview(store: LocalArchiveStore, destinations: ArchiveDe
|
|
|
101
101
|
}));
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
-
export async function syncArchives(store:
|
|
104
|
+
export async function syncArchives(store: ArchiveStore, destinations: ArchiveDestination[]) {
|
|
105
105
|
const results: { id: string; destination: string; status: string }[] = [];
|
|
106
|
-
for (const archive of store.list())
|
|
106
|
+
for (const archive of await store.list())
|
|
107
107
|
for (const destination of destinations.filter((d) => d.projectId === archive.projectId)) {
|
|
108
108
|
try {
|
|
109
109
|
const result = await publishArchive(store, archive.id, destination);
|
package/src/collector.ts
CHANGED
|
@@ -2,8 +2,8 @@ import { closeSync, existsSync, lstatSync, openSync, readdirSync, readSync } fro
|
|
|
2
2
|
import { isAbsolute, join, resolve } from 'node:path';
|
|
3
3
|
import { importTranscriptFile } from './import-file';
|
|
4
4
|
import { archiveKey } from './index';
|
|
5
|
-
import type {
|
|
6
|
-
import { type AtlasFileConcepts, atlasTags,
|
|
5
|
+
import type { ArchiveClient } from './remote';
|
|
6
|
+
import { type AtlasFileConcepts, atlasTags, provenanceReferences } from './tags';
|
|
7
7
|
|
|
8
8
|
export interface CollectionSource {
|
|
9
9
|
directory: string;
|
|
@@ -100,7 +100,11 @@ export function collectionRoots(config: CollectionSource): Set<string> {
|
|
|
100
100
|
return new Set(config.worktrees ? [...roots, ...roots.flatMap(worktreesOf)] : roots);
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
-
|
|
103
|
+
/** Where collected sessions go: an Archive server, or a store in-process. */
|
|
104
|
+
export type ArchiveWriter = Pick<ArchiveClient, 'sourceId' | 'head' | 'capture'>;
|
|
105
|
+
|
|
106
|
+
export async function collectSessions(archive: ArchiveWriter, config: CollectionSource) {
|
|
107
|
+
const sourceId = await archive.sourceId();
|
|
104
108
|
const roots = collectionRoots(config);
|
|
105
109
|
const result = { imported: 0, unchanged: 0, skipped: 0, failed: 0 };
|
|
106
110
|
// Per scan: checkouts change between scans.
|
|
@@ -117,7 +121,7 @@ export async function collectSessions(store: LocalArchiveStore, config: Collecti
|
|
|
117
121
|
const before = lstatSync(file);
|
|
118
122
|
const snapshot = importTranscriptFile(file, {
|
|
119
123
|
source: config.source,
|
|
120
|
-
sourceId
|
|
124
|
+
sourceId,
|
|
121
125
|
projectId: config.projectId,
|
|
122
126
|
});
|
|
123
127
|
const after = lstatSync(file);
|
|
@@ -125,8 +129,8 @@ export async function collectSessions(store: LocalArchiveStore, config: Collecti
|
|
|
125
129
|
result.skipped++;
|
|
126
130
|
continue;
|
|
127
131
|
}
|
|
128
|
-
const old =
|
|
129
|
-
if (old && old.
|
|
132
|
+
const old = await archive.head(archiveKey(snapshot));
|
|
133
|
+
if (old && old.projectId !== config.projectId) {
|
|
130
134
|
result.failed++;
|
|
131
135
|
continue;
|
|
132
136
|
}
|
|
@@ -135,17 +139,14 @@ export async function collectSessions(store: LocalArchiveStore, config: Collecti
|
|
|
135
139
|
if (config.atlas && !graphs.has(cwd)) graphs.set(cwd, atlasGraph(cwd));
|
|
136
140
|
const graph = graphs.get(cwd);
|
|
137
141
|
snapshot.tags = [
|
|
138
|
-
...new Set([
|
|
139
|
-
...(old?.snapshot.tags ?? []),
|
|
140
|
-
...(config.tags ?? []),
|
|
141
|
-
...provenanceTags({
|
|
142
|
-
branch: meta.branch,
|
|
143
|
-
repository: meta.repository ?? repositories.get(cwd),
|
|
144
|
-
}),
|
|
145
|
-
...(graph ? atlasTags(snapshot, graph, roots) : []),
|
|
146
|
-
]),
|
|
142
|
+
...new Set([...(config.tags ?? []), ...(graph ? atlasTags(snapshot, graph, roots) : [])]),
|
|
147
143
|
].slice(0, 100);
|
|
148
|
-
|
|
144
|
+
const references = provenanceReferences({
|
|
145
|
+
branch: meta.branch,
|
|
146
|
+
repository: meta.repository ?? repositories.get(cwd),
|
|
147
|
+
});
|
|
148
|
+
if (references.length) snapshot.references = references;
|
|
149
|
+
if ((await archive.capture(snapshot)).changed) result.imported++;
|
|
149
150
|
else result.unchanged++;
|
|
150
151
|
} catch {
|
|
151
152
|
result.failed++;
|
package/src/import.ts
CHANGED
|
@@ -21,6 +21,8 @@ export function importTranscriptLines(
|
|
|
21
21
|
const seen = new Map<string, string>();
|
|
22
22
|
let index = -1;
|
|
23
23
|
let contentBytes = 0;
|
|
24
|
+
// Codex states the model per turn; Claude Code per assistant record.
|
|
25
|
+
let turn: { model?: string; effort?: string } = {};
|
|
24
26
|
for (const rawLine of lines) {
|
|
25
27
|
index++;
|
|
26
28
|
const line = rawLine.trim();
|
|
@@ -48,6 +50,15 @@ export function importTranscriptLines(
|
|
|
48
50
|
if (sessionId && sessionId !== id) throw new Error('Mixed transcript sessions');
|
|
49
51
|
sessionId = id;
|
|
50
52
|
}
|
|
53
|
+
const text = (value: unknown) => (typeof value === 'string' && value ? value : undefined);
|
|
54
|
+
if (options.source === 'codex' && row.type === 'turn_context')
|
|
55
|
+
turn = { model: text(row.payload?.model), effort: text(row.payload?.effort) };
|
|
56
|
+
const producer =
|
|
57
|
+
options.source === 'codex'
|
|
58
|
+
? turn
|
|
59
|
+
: row.type === 'assistant'
|
|
60
|
+
? { model: text(row.message?.model), effort: text(row.effort) }
|
|
61
|
+
: {};
|
|
51
62
|
const timestamp = typeof row.timestamp === 'number' ? row.timestamp : Date.parse(row.timestamp);
|
|
52
63
|
const add = (kind: ArchiveEntry['kind'], content: unknown, suffix: string, callId?: string) => {
|
|
53
64
|
if (typeof content !== 'string' || !content.length) return;
|
|
@@ -71,6 +82,12 @@ export function importTranscriptLines(
|
|
|
71
82
|
timestamp: Number.isFinite(timestamp) ? timestamp : null,
|
|
72
83
|
sourceRef: `line:${index + 1}`,
|
|
73
84
|
...(callId ? { callId } : {}),
|
|
85
|
+
...(['assistant', 'tool-call', 'reasoning-summary'].includes(kind)
|
|
86
|
+
? {
|
|
87
|
+
...(producer.model ? { model: producer.model } : {}),
|
|
88
|
+
...(producer.effort ? { effort: producer.effort } : {}),
|
|
89
|
+
}
|
|
90
|
+
: {}),
|
|
74
91
|
});
|
|
75
92
|
};
|
|
76
93
|
if (options.source === 'codex') {
|