@aiwg/cli 2026.10.4 → 2026.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/artifacts/cli.js +15 -3
- package/dist/src/artifacts/index-builder.js +3 -2
- package/dist/src/artifacts/index-preparation.js +91 -26
- package/dist/src/artifacts/index-refresh.js +17 -2
- package/dist/src/artifacts/index-worker.mjs +9 -7
- package/dist/src/cli/handlers/sessions.js +39 -6
- package/dist/src/sessions/adapters/codex.js +23 -10
- package/dist/src/sessions/batch-contracts.js +1 -1
- package/dist/src/sessions/batch-import.js +202 -2
- package/dist/src/sessions/contracts.js +10 -1
- package/dist/src/sessions/coverage-preflight.js +26 -1
- package/dist/src/sessions/policy.js +18 -7
- package/dist/src/sessions/refresh.js +2 -3
- package/dist/src/sessions/repository.js +4 -2
- package/dist/src/sessions/workspace-discovery.js +89 -22
- package/package.json +1 -1
|
@@ -20,6 +20,7 @@ import path from 'node:path';
|
|
|
20
20
|
import { readFileSync } from 'node:fs';
|
|
21
21
|
import { fileURLToPath } from 'node:url';
|
|
22
22
|
import { GRAPH_CONFIGS, OPERATIONAL_DISCOVERY_TYPES, OPERATIONAL_SHOW_TYPES, isOperationalShowType, loadUserGraphConfigs, loadGlobalGraphConfigs, orderedGraphEntries, } from './types.js';
|
|
23
|
+
import { resolveIndexWorkers } from './index-preparation.js';
|
|
23
24
|
import { SUPPORTED_VIEWS } from './corpus-views/renderers.js';
|
|
24
25
|
import { parseResourceSelector, readVerifiedRegularFile, } from '../resources/web-release.js';
|
|
25
26
|
import { findPackageRoot } from '../cli/find-package-root.js';
|
|
@@ -332,6 +333,7 @@ export function printIndexUsage() {
|
|
|
332
333
|
console.log(' Session import defaults: 67108864 bytes per record, 1073741824 bytes (1 GiB) per source; source 0 is unlimited.');
|
|
333
334
|
console.log(' --exclude <id-or-glob> (repeatable): skip session IDs or absolute path globs; adds to sessions.import.exclude and AIWG_SESSIONS_IMPORT_EXCLUDE (comma-separated).');
|
|
334
335
|
console.log(' Import limits: CLI flag > AIWG_SESSIONS_IMPORT_MAX_RECORD_BYTES / AIWG_SESSIONS_IMPORT_MAX_SOURCE_BYTES > workspace config > default.');
|
|
336
|
+
console.log(' build/refresh --index-workers <n>: preparation processes (0 = serial); flag > AIWG_INDEX_WORKERS > automatic CPU/memory limits.');
|
|
335
337
|
console.log(' sync [--backend fortemi-core] [--graph <name> | --all] [--repo <name>]');
|
|
336
338
|
console.log(' [--privacy private|sanitized|public] [--generated-at <iso>] [--json]');
|
|
337
339
|
console.log(' --graph <name> Target a specific graph (framework, project, codebase, source, user, or user-defined)');
|
|
@@ -503,6 +505,7 @@ async function handleBuild(args) {
|
|
|
503
505
|
console.log('Usage: aiwg index build [options]');
|
|
504
506
|
console.log('');
|
|
505
507
|
console.log('Options:');
|
|
508
|
+
console.log(' --index-workers <n> Preparation processes (0 = serial); overrides AIWG_INDEX_WORKERS');
|
|
506
509
|
console.log(' --force Full rebuild (ignore checksums, re-index everything)');
|
|
507
510
|
console.log(' --verbose Show detailed progress during indexing');
|
|
508
511
|
console.log(' --all Build all known graphs (including user-defined)');
|
|
@@ -530,6 +533,15 @@ async function handleBuild(args) {
|
|
|
530
533
|
console.log(' aiwg index sync --all');
|
|
531
534
|
return;
|
|
532
535
|
}
|
|
536
|
+
let workers;
|
|
537
|
+
try {
|
|
538
|
+
workers = resolveIndexWorkers(parseFlagValue(args, '--index-workers', 'Error: --index-workers requires a value'));
|
|
539
|
+
}
|
|
540
|
+
catch (error) {
|
|
541
|
+
console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
|
|
542
|
+
process.exitCode = 1;
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
533
545
|
// Dynamic import to keep the CLI router lightweight
|
|
534
546
|
const { buildIndex } = await import('./index-builder.js');
|
|
535
547
|
const cwd = process.cwd();
|
|
@@ -554,7 +566,7 @@ async function handleBuild(args) {
|
|
|
554
566
|
// --graph X: build the JSON graph if X is one; otherwise X may be a
|
|
555
567
|
// research-corpus markdown view, rendered below.
|
|
556
568
|
if (graph in GRAPH_CONFIGS) {
|
|
557
|
-
await buildIndex(cwd, { force, verbose, scope, includeIgnored, graph, explicit: true });
|
|
569
|
+
await buildIndex(cwd, { force, verbose, scope, includeIgnored, workers, graph, explicit: true });
|
|
558
570
|
jsonBuilt = true;
|
|
559
571
|
builtGraphs.add(graph);
|
|
560
572
|
}
|
|
@@ -562,7 +574,7 @@ async function handleBuild(args) {
|
|
|
562
574
|
else if (all) {
|
|
563
575
|
// Build all known graphs — user asked for everything, but don't hard-error on missing dirs
|
|
564
576
|
for (const [name] of orderedGraphEntries(Object.entries(GRAPH_CONFIGS))) {
|
|
565
|
-
await buildIndex(cwd, { force, verbose, includeIgnored, graph: name, explicit: false });
|
|
577
|
+
await buildIndex(cwd, { force, verbose, includeIgnored, workers, graph: name, explicit: false });
|
|
566
578
|
builtGraphs.add(name);
|
|
567
579
|
}
|
|
568
580
|
jsonBuilt = true;
|
|
@@ -571,7 +583,7 @@ async function handleBuild(args) {
|
|
|
571
583
|
// Default: build graphs with defaultBuild=true; skip gracefully if their dirs don't exist
|
|
572
584
|
for (const [name, config] of orderedGraphEntries(Object.entries(GRAPH_CONFIGS))) {
|
|
573
585
|
if (config.defaultBuild) {
|
|
574
|
-
await buildIndex(cwd, { force, verbose, includeIgnored, scope: name === Object.keys(GRAPH_CONFIGS)[0] ? scope : undefined, graph: name, explicit: false });
|
|
586
|
+
await buildIndex(cwd, { force, verbose, includeIgnored, workers, scope: name === Object.keys(GRAPH_CONFIGS)[0] ? scope : undefined, graph: name, explicit: false });
|
|
575
587
|
builtGraphs.add(name);
|
|
576
588
|
}
|
|
577
589
|
}
|
|
@@ -24,7 +24,7 @@ import { FRAMEWORK_RELATIONS_REVISION } from './framework-source-digest.js';
|
|
|
24
24
|
import { normalizeOperationalState } from './operational-state.js';
|
|
25
25
|
import { DEFAULT_PROJECT_AIWG_DIR, resolveProjectAiwgDir, } from '../config/project-artifacts.js';
|
|
26
26
|
import { normalizeStateTransferProjection } from './state-transfer.js';
|
|
27
|
-
import { runIndexPreparation } from './index-preparation.js';
|
|
27
|
+
import { resolveIndexWorkers, runIndexPreparation } from './index-preparation.js';
|
|
28
28
|
import { collectGraphIndexFiles, findArtifactFiles, indexPathFor } from './index-files.js';
|
|
29
29
|
function absoluteEntryPath(cwd, entryPath, graph) {
|
|
30
30
|
if ((!graph || graph === 'project') && entryPath.startsWith(`${DEFAULT_PROJECT_AIWG_DIR}/`)) {
|
|
@@ -911,6 +911,7 @@ export function prepareIndexArtifact(job) {
|
|
|
911
911
|
}
|
|
912
912
|
export async function buildIndex(cwd, options = {}) {
|
|
913
913
|
const { force = false, verbose = false, scope, includeIgnored = false, outputDir, graph, explicit = true } = options;
|
|
914
|
+
const workers = resolveIndexWorkers(options.workers);
|
|
914
915
|
const startTime = Date.now();
|
|
915
916
|
const log = options.quiet ? (..._args) => { } : console.log;
|
|
916
917
|
// Ensure user-defined graphs are loaded
|
|
@@ -1119,7 +1120,7 @@ export async function buildIndex(cwd, options = {}) {
|
|
|
1119
1120
|
jobs.push({ fullPath, relativePath, mtimeMs: stat.mtimeMs, birthtimeMs: stat.birthtimeMs, sizeBytes: stat.size,
|
|
1120
1121
|
...(!force && existingEntries[relativePath] ? { existingChecksum: existingEntries[relativePath].checksum } : {}) });
|
|
1121
1122
|
}
|
|
1122
|
-
const preparation = await runIndexPreparation(jobs, prepareIndexArtifact, import.meta.url, options.
|
|
1123
|
+
const preparation = await runIndexPreparation(jobs, prepareIndexArtifact, import.meta.url, workers, options.onDiagnostic ?? (message => console.warn(`Index preparation: ${message}`)));
|
|
1123
1124
|
const preparedArtifacts = new Map(jobs.map((job, index) => [job.fullPath, preparation.results[index]]));
|
|
1124
1125
|
for (const fullPath of files) {
|
|
1125
1126
|
const relativePath = indexPathFor(cwd, fullPath, graph);
|
|
@@ -1,65 +1,130 @@
|
|
|
1
1
|
/** Bounded artifact-local preparation; ordered graph reduction stays in index-builder. */
|
|
2
2
|
import os from 'node:os';
|
|
3
|
-
import {
|
|
3
|
+
import { fork } from 'node:child_process';
|
|
4
|
+
import { signalChild } from '../process/signal-child.mjs';
|
|
5
|
+
const MIB = 1024 ** 2;
|
|
6
|
+
const WORKER_HEAP_MB = 256;
|
|
7
|
+
// Leave room for V8's young generation, executable code, IPC and native allocations.
|
|
8
|
+
const WORKER_BUDGET_BYTES = 512 * MIB;
|
|
9
|
+
// Parsing and metadata extraction amplify input size; do large files only after the pool exits.
|
|
10
|
+
const MAX_WORKER_ARTIFACT_BYTES = MIB;
|
|
11
|
+
export function resolveIndexWorkers(requested, env = process.env) {
|
|
12
|
+
const value = requested ?? env.AIWG_INDEX_WORKERS;
|
|
13
|
+
if (requested === undefined && (value === undefined || value === ''))
|
|
14
|
+
return undefined;
|
|
15
|
+
if ((typeof value === 'string' && !/^\d+$/.test(value))
|
|
16
|
+
|| !Number.isSafeInteger(Number(value)) || Number(value) < 0) {
|
|
17
|
+
throw new Error(`${requested === undefined ? 'AIWG_INDEX_WORKERS' : '--index-workers'} must be a nonnegative safe integer (0 = serial)`);
|
|
18
|
+
}
|
|
19
|
+
return Number(value);
|
|
20
|
+
}
|
|
4
21
|
export function indexPreparationWorkers(jobs, requested, cores = os.availableParallelism(), memory = Math.min(os.freemem(), process.availableMemory?.() ?? Infinity), totalBytes = 0) {
|
|
5
|
-
|
|
6
|
-
throw new Error('Index workers must be a nonnegative integer');
|
|
22
|
+
requested = resolveIndexWorkers(requested);
|
|
7
23
|
if (jobs < 64 || cores < 3 || requested === 0 || requested === 1)
|
|
8
24
|
return 0;
|
|
9
|
-
//
|
|
10
|
-
// Explicit
|
|
11
|
-
if (requested === undefined && (jobs < 1000 || !Number.isFinite(totalBytes) || totalBytes < 16 *
|
|
25
|
+
// Process startup costs dominate lightweight scans even at thousands of files.
|
|
26
|
+
// Explicit requests bypass only this conservative automatic workload gate.
|
|
27
|
+
if (requested === undefined && (jobs < 1000 || !Number.isFinite(totalBytes) || totalBytes < 16 * MIB))
|
|
12
28
|
return 0;
|
|
13
|
-
const byMemory = Math.floor((memory - 512 *
|
|
29
|
+
const byMemory = Math.floor((memory - 512 * MIB) / WORKER_BUDGET_BYTES);
|
|
14
30
|
const bounded = Math.max(0, Math.min(requested ?? 4, 4, cores - 1, byMemory, jobs));
|
|
15
31
|
return bounded >= 2 ? bounded : 0;
|
|
16
32
|
}
|
|
17
|
-
export async function runIndexPreparation(jobs, serial, moduleUrl, requested) {
|
|
18
|
-
const
|
|
33
|
+
export async function runIndexPreparation(jobs, serial, moduleUrl, requested, onDiagnostic) {
|
|
34
|
+
const diagnostics = [];
|
|
35
|
+
const diagnose = (message) => { diagnostics.push(message); onDiagnostic?.(message); };
|
|
36
|
+
const eligible = jobs.map((job, index) => ({ job, index })).filter(({ job }) => Number.isFinite(job.sizeBytes) && job.sizeBytes >= 0 && job.sizeBytes <= MAX_WORKER_ARTIFACT_BYTES);
|
|
37
|
+
const workerCount = indexPreparationWorkers(eligible.length, requested, undefined, undefined, eligible.reduce((total, { job }) => total + job.sizeBytes, 0));
|
|
19
38
|
if (!workerCount)
|
|
20
|
-
return { results: jobs.map(serial), workerCount: 0 };
|
|
39
|
+
return { results: jobs.map(serial), workerCount: 0, diagnostics };
|
|
21
40
|
const results = new Array(jobs.length), workers = [];
|
|
41
|
+
const closed = [];
|
|
22
42
|
let next = 0, completed = 0, stopped = false;
|
|
23
43
|
try {
|
|
24
44
|
await new Promise((resolve, reject) => {
|
|
25
|
-
|
|
45
|
+
// Stop dispatch on a process failure. Reap the entire pool before retrying
|
|
46
|
+
// unfinished work in the main isolate, avoiding overlapping memory peaks.
|
|
47
|
+
const recover = (message) => {
|
|
48
|
+
if (stopped)
|
|
49
|
+
return;
|
|
50
|
+
stopped = true;
|
|
51
|
+
diagnose(`${message}; retrying unfinished preparation serially`);
|
|
52
|
+
resolve();
|
|
53
|
+
};
|
|
26
54
|
const dispatch = (worker) => {
|
|
27
|
-
if (
|
|
28
|
-
|
|
29
|
-
|
|
55
|
+
if (stopped || next >= eligible.length)
|
|
56
|
+
return;
|
|
57
|
+
const { index, job } = eligible[next++];
|
|
58
|
+
try {
|
|
59
|
+
worker.send({ index, job }, error => { if (error)
|
|
60
|
+
recover(`Index preparation IPC failed for ${job.relativePath}: ${error.message}`); });
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
recover(`Index preparation IPC failed for ${job.relativePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
30
64
|
}
|
|
31
65
|
};
|
|
32
|
-
for (let index = 0; index < workerCount; index++) {
|
|
33
|
-
|
|
34
|
-
|
|
66
|
+
for (let index = 0; index < workerCount && !stopped; index++) {
|
|
67
|
+
// A worker_threads resourceLimit is not a process boundary: V8 can abort
|
|
68
|
+
// Node during allocation before its isolate-limit error is delivered.
|
|
69
|
+
// A child heap fatal/SIGABRT is instead an ordinary, recoverable exit.
|
|
70
|
+
const worker = fork(new URL('./index-worker.mjs', import.meta.url), [moduleUrl], {
|
|
71
|
+
execArgv: [`--max-old-space-size=${WORKER_HEAP_MB}`, '--max-semi-space-size=16'],
|
|
72
|
+
stdio: ['ignore', 'ignore', 'pipe', 'ipc'], serialization: 'advanced',
|
|
35
73
|
});
|
|
36
74
|
workers.push(worker);
|
|
37
|
-
worker.
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
worker.on('
|
|
75
|
+
closed.push(new Promise(done => worker.once('close', () => done())));
|
|
76
|
+
let stderr = '';
|
|
77
|
+
let activePath = '<startup>';
|
|
78
|
+
worker.stderr?.on('data', chunk => { stderr = (stderr + String(chunk)).slice(-8192); });
|
|
79
|
+
worker.on('error', error => recover(`Index preparation worker failed for ${activePath}: ${error.message}`));
|
|
80
|
+
worker.on('exit', (code, signal) => {
|
|
81
|
+
const oom = /heap out of memory|ERR_WORKER_OUT_OF_MEMORY/i.test(stderr) ? ' (heap out of memory)' : '';
|
|
82
|
+
recover(`Index preparation worker exited early for ${activePath} (${signal ?? code})${oom}`);
|
|
83
|
+
});
|
|
84
|
+
worker.on('message', (message) => {
|
|
41
85
|
if (stopped)
|
|
42
86
|
return;
|
|
43
87
|
if (message.error) {
|
|
44
|
-
|
|
88
|
+
stopped = true;
|
|
89
|
+
reject(new Error(message.error));
|
|
45
90
|
return;
|
|
46
91
|
}
|
|
47
92
|
results[message.index] = message.result;
|
|
48
93
|
completed++;
|
|
49
|
-
if (completed ===
|
|
94
|
+
if (completed === eligible.length) {
|
|
50
95
|
stopped = true;
|
|
51
96
|
resolve();
|
|
52
97
|
}
|
|
53
|
-
else
|
|
98
|
+
else {
|
|
99
|
+
activePath = eligible[next]?.job.relativePath ?? '<idle>';
|
|
54
100
|
dispatch(worker);
|
|
101
|
+
}
|
|
55
102
|
});
|
|
103
|
+
activePath = eligible[next]?.job.relativePath ?? '<idle>';
|
|
56
104
|
dispatch(worker);
|
|
57
105
|
}
|
|
58
106
|
});
|
|
59
|
-
return { results, workerCount };
|
|
60
107
|
}
|
|
61
108
|
finally {
|
|
62
|
-
|
|
109
|
+
stopped = true;
|
|
110
|
+
for (const worker of workers)
|
|
111
|
+
if (worker.exitCode === null && worker.signalCode === null)
|
|
112
|
+
signalChild(worker, 'SIGKILL');
|
|
113
|
+
await Promise.all(closed);
|
|
114
|
+
}
|
|
115
|
+
const largeCount = jobs.length - eligible.length;
|
|
116
|
+
if (largeCount)
|
|
117
|
+
diagnose(`${largeCount} artifact(s) above the worker size threshold prepared serially`);
|
|
118
|
+
for (let index = 0; index < jobs.length; index++) {
|
|
119
|
+
if (!results[index]) {
|
|
120
|
+
try {
|
|
121
|
+
results[index] = serial(jobs[index]);
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
throw new Error(`${jobs[index].relativePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
63
127
|
}
|
|
128
|
+
return { results, workerCount, diagnostics };
|
|
64
129
|
}
|
|
65
130
|
//# sourceMappingURL=index-preparation.js.map
|
|
@@ -6,6 +6,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
|
6
6
|
import { findPackageRoot } from '../cli/find-package-root.js';
|
|
7
7
|
import { projectAiwgPath, projectControlPath } from '../config/project-artifacts.js';
|
|
8
8
|
import { getUserStateDir } from '../config/user-state.js';
|
|
9
|
+
import { resolveIndexWorkers } from './index-preparation.js';
|
|
9
10
|
import { collectGraphIndexFiles } from './index-files.js';
|
|
10
11
|
import { collectIndexStatus } from './index-status.js';
|
|
11
12
|
import { getFortemiCoreSyncStatus, syncFortemiCoreIndex } from './fortemi-core-sync.js';
|
|
@@ -258,7 +259,8 @@ async function executeJob(job, options, aliases, runtime) {
|
|
|
258
259
|
return;
|
|
259
260
|
}
|
|
260
261
|
const { buildIndex } = await import('./index-builder.js');
|
|
261
|
-
await buildIndex(workspace, { graph, force: Boolean(options.full), explicit: false, quiet: true
|
|
262
|
+
await buildIndex(workspace, { graph, force: Boolean(options.full), explicit: false, quiet: true, workers: options.workers,
|
|
263
|
+
onDiagnostic: message => (job.diagnostics ??= []).push(message) });
|
|
262
264
|
const built = readJson(path.join(directory, 'metadata.json'));
|
|
263
265
|
if (!built)
|
|
264
266
|
throw new Error(`graph did not produce metadata: ${graph}`);
|
|
@@ -351,6 +353,7 @@ function jobKey(job, aliases) {
|
|
|
351
353
|
...(job.kind === 'sessions' && aliases.length > 1 ? [aliases] : [])]);
|
|
352
354
|
}
|
|
353
355
|
export async function refreshIndexes(options = {}, dependencies = {}) {
|
|
356
|
+
options = { ...options, workers: resolveIndexWorkers(options.workers) };
|
|
354
357
|
const start = Date.now();
|
|
355
358
|
const workspaces = workspacePaths(options);
|
|
356
359
|
const jobs = planWorkspaceRefresh(options, workspaces);
|
|
@@ -360,6 +363,13 @@ export async function refreshIndexes(options = {}, dependencies = {}) {
|
|
|
360
363
|
totals: { jobs: jobs.length, ok: 0, skipped: 0, deferred: 0, errors: 0, noManifest: 0, excluded: 0, wallMs: 0 },
|
|
361
364
|
};
|
|
362
365
|
const finish = () => {
|
|
366
|
+
for (const job of jobs) {
|
|
367
|
+
for (const receipt of job.sessionImports ?? (job.sessionImport ? [job.sessionImport] : [])) {
|
|
368
|
+
for (const warning of receipt.run.preflight?.scopeWarnings ?? []) {
|
|
369
|
+
report.messages.push(`WARNING ${job.workspace}: ${warning.message}`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
363
373
|
for (const job of jobs.filter(job => job.noManifest)) {
|
|
364
374
|
const command = `aiwg sessions discover --workspace '${job.workspace.replace(/'/g, "'\\''")}'`;
|
|
365
375
|
report.messages.push(`Session refresh skipped for ${job.workspace}: no authorized discovery manifest. Run \`${command}\` or configure sessions.discovery_manifests in .aiwg/aiwg.config.`);
|
|
@@ -507,7 +517,7 @@ export async function runIndexRefresh(args, cwd = process.cwd(), signal) {
|
|
|
507
517
|
options.dryRun = true;
|
|
508
518
|
else if (flag === '--json')
|
|
509
519
|
json = true;
|
|
510
|
-
else if (['--workspace', '--root', '--only', '--lock-wait-ms', '--max-record-bytes', '--max-source-bytes', '--exclude'].includes(flag)) {
|
|
520
|
+
else if (['--workspace', '--root', '--only', '--lock-wait-ms', '--max-record-bytes', '--max-source-bytes', '--exclude', '--index-workers'].includes(flag)) {
|
|
511
521
|
const value = args[++i];
|
|
512
522
|
if (!value || value.startsWith('--'))
|
|
513
523
|
throw new Error(`${flag} requires a value`);
|
|
@@ -519,6 +529,8 @@ export async function runIndexRefresh(args, cwd = process.cwd(), signal) {
|
|
|
519
529
|
options.maxRecordBytes = value;
|
|
520
530
|
else if (flag === '--max-source-bytes')
|
|
521
531
|
options.maxSourceBytes = value;
|
|
532
|
+
else if (flag === '--index-workers')
|
|
533
|
+
options.workers = resolveIndexWorkers(value);
|
|
522
534
|
else if (flag === '--exclude')
|
|
523
535
|
(options.exclude ??= []).push(value);
|
|
524
536
|
else if (flag === '--only') {
|
|
@@ -537,6 +549,7 @@ export async function runIndexRefresh(args, cwd = process.cwd(), signal) {
|
|
|
537
549
|
else
|
|
538
550
|
throw new Error(`unknown refresh option: ${flag}`);
|
|
539
551
|
}
|
|
552
|
+
options.workers = resolveIndexWorkers(options.workers);
|
|
540
553
|
resolveImportLimits(undefined, { maxRecordBytes: options.maxRecordBytes, maxSourceBytes: options.maxSourceBytes });
|
|
541
554
|
resolveImportExclusions(undefined, options.exclude);
|
|
542
555
|
const report = await refreshIndexes(options);
|
|
@@ -547,6 +560,8 @@ export async function runIndexRefresh(args, cwd = process.cwd(), signal) {
|
|
|
547
560
|
for (const job of report.jobs)
|
|
548
561
|
console.log(` ${job.status}: ${job.kind} ${job.scope === 'host' ? '[host]' : job.workspace}${job.graph ? ` [${job.graph}]` : ''}${job.error || job.reason ? `: ${job.error ?? job.reason}` : ''}`);
|
|
549
562
|
for (const job of report.jobs) {
|
|
563
|
+
for (const diagnostic of job.diagnostics ?? [])
|
|
564
|
+
console.log(` preparation: ${diagnostic}`);
|
|
550
565
|
for (const receipt of job.sessionImports ?? (job.sessionImport ? [job.sessionImport] : [])) {
|
|
551
566
|
for (const source of receipt.run.sources) {
|
|
552
567
|
if (source.status === 'excluded')
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
/** Artifact worker uses the exact caller module: source never silently loads stale dist. */
|
|
2
|
-
|
|
3
|
-
const module =
|
|
4
|
-
? await (await import('tsx/esm/api')).tsImport(
|
|
5
|
-
: await import(
|
|
6
|
-
|
|
7
|
-
try {
|
|
8
|
-
catch (error) {
|
|
2
|
+
const moduleUrl = process.argv[2];
|
|
3
|
+
const module = moduleUrl.endsWith('.ts')
|
|
4
|
+
? await (await import('tsx/esm/api')).tsImport(moduleUrl, import.meta.url)
|
|
5
|
+
: await import(moduleUrl);
|
|
6
|
+
process.on('message', ({ index, job }) => {
|
|
7
|
+
try { process.send({ index, result: module.prepareIndexArtifact(job) }); }
|
|
8
|
+
catch (error) { process.send({ index, error: `${job.relativePath}: ${error.message}` }); }
|
|
9
9
|
});
|
|
10
|
+
// The parent may itself be interrupted; do not leave orphaned preparation processes.
|
|
11
|
+
process.on('disconnect', () => process.exit(0));
|
|
@@ -2,6 +2,7 @@ import { existsSync, linkSync, mkdirSync, readFileSync, realpathSync, renameSync
|
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
3
|
import { basename, dirname, isAbsolute, join, resolve, } from 'node:path';
|
|
4
4
|
import { pathToFileURL } from 'node:url';
|
|
5
|
+
import { providerScopeWarnings } from '../../sessions/coverage-preflight.js';
|
|
5
6
|
import { CLAUDE_ADAPTER_VERSION, ClaudeSessionAdapter, CODEX_ADAPTER_VERSION, CodexSessionAdapter, COPILOT_ADAPTER_VERSION, CopilotSessionAdapter, CURSOR_ADAPTER_VERSION, CursorSessionAdapter, FACTORY_ADAPTER_VERSION, FactorySessionAdapter, HERMES_ADAPTER_VERSION, HermesSessionAdapter, OPENCODE_ADAPTER_VERSION, OpenCodeSessionAdapter, OPENCLAW_ADAPTER_VERSION, OpenClawSessionAdapter, OPENHUMAN_ADAPTER_VERSION, OpenHumanSessionAdapter, GROKBOT_ADAPTER_VERSION, GROKBOT_NATIVE_REPLICA_LOCATOR_CLASS, GrokbotSessionAdapter, GROK_BUILD_ADAPTER_VERSION, GROK_BUILD_CLI_EXPORT_LOCATOR_CLASS, GrokBuildSessionAdapter, grokBuildList, grokBuildSearch, grokBuildExport, grokHeadlessSessionIdFromFile, MUSE_ADAPTER_VERSION, MuseSessionAdapter, PI_ADAPTER_VERSION, PiSessionAdapter, DEEPSEEK_HARNESS_ADAPTER_VERSION, DeepSeekHarnessSessionAdapter, OmpSessionAdapter, OMP_ADAPTER_VERSION, WARP_ADAPTER_VERSION, WarpSessionAdapter, DEVIN_DESKTOP_ADAPTER_VERSION, DevinDesktopSessionAdapter, CandidateExtractionService, GENERIC_ADAPTER_VERSION, GenericSessionInterchangeAdapter, IncrementalSessionImporter, ImportLeaseContentionError, FilesystemMemoryDestination, FilesystemPromotionDispositionCoordinator, MemoryPromotionGateway, SESSION_CONTRACT_VERSION, SessionContractError, SessionRepository, SessionSourceSchema, StructuralCandidateExtractor, resolveMemoryConsumerManifest, assertSessionProviderId, acquireImportLease, defaultDiscoveryManifestPath, discoverWorkspaceHistories, createSessionAdapterRegistry, coveragePreflight, expectedSessionProviders, deriveSessionTimeline, importDiscoveryManifest, previewDiscoveryImport, publicDiscoveryManifest, readDiscoveryManifest, redactSourceLocator, sha256, parseTimelineGap, writeDiscoveryManifest, buildSessionExportPlan, reverifySessionExportPlan, buildSessionAiwgFortemiIndexExport, recoverAiwgFortemiIndexFromFullV1Shard, embedAttachmentsInFullV1Shard, recoverEmbeddedAttachments, SESSION_EXPORT_PLAN_SCHEMA_VERSION, } from '../../sessions/index.js';
|
|
6
7
|
import { resolveImportExclusions } from '../../sessions/import-exclusions.js';
|
|
7
8
|
import { resolveImportLimits } from '../../sessions/import-limits.js';
|
|
@@ -58,7 +59,7 @@ Commands:
|
|
|
58
59
|
audit --workspace <id> Read content-free mutation events
|
|
59
60
|
analytics <view> --workspace <id> Summary, tool-calls, escalations, or HITL
|
|
60
61
|
forensics <view> --workspace <id> Authorized timeline, indicators, or evidence
|
|
61
|
-
doctor Check catalog
|
|
62
|
+
doctor Check catalog integrity and configured provider scope
|
|
62
63
|
|
|
63
64
|
Options:
|
|
64
65
|
--json Emit the versioned JSON contract
|
|
@@ -70,7 +71,8 @@ Options:
|
|
|
70
71
|
Catalog defaults to <project-or-cwd>/.aiwg/sessions/catalog.sqlite
|
|
71
72
|
Imports for other workspaces stay in the selected catalog;
|
|
72
73
|
use --db to read that catalog from another directory
|
|
73
|
-
--expected-providers <ids>
|
|
74
|
+
--expected-providers <ids> Explicit scope for this invocation; refresh re-reads workspace config
|
|
75
|
+
--carry-forward <catalog> Rebuild from sources plus re-redacted stored history (import-discovered)
|
|
74
76
|
--manifest <path> Override the discovery manifest path
|
|
75
77
|
--provider-home <path> Override the provider home root (testing/portable homes)
|
|
76
78
|
--codex-root <path> Authorize a shared Codex sessions/export root (repeatable)
|
|
@@ -725,12 +727,22 @@ async function executeCommand(ctx, args) {
|
|
|
725
727
|
providerLogsModified: false,
|
|
726
728
|
});
|
|
727
729
|
}
|
|
728
|
-
case 'doctor':
|
|
730
|
+
case 'doctor': {
|
|
731
|
+
const workspacePath = resolve(ctx.cwd, args.values.get('--workspace') ?? ctx.cwd);
|
|
732
|
+
const workspace = existsSync(workspacePath) ? realpathSync(workspacePath) : workspacePath;
|
|
733
|
+
const config = await readAiwgConfig(workspace);
|
|
734
|
+
const manifestPath = resolve(ctx.cwd, args.values.get('--manifest') ?? defaultDiscoveryManifestPath(workspace));
|
|
735
|
+
const preflight = existsSync(manifestPath)
|
|
736
|
+
? (await readDiscoveryManifest(manifestPath)).preflight
|
|
737
|
+
: repository.getCoverage(workspace).preflight;
|
|
738
|
+
const warnings = providerScopeWarnings(config?.providers ?? [], preflight?.expectedProviders ?? [], preflight?.scopeWarnings);
|
|
729
739
|
return ok(command, {
|
|
730
740
|
database: redactSourceLocator(databasePath(ctx, args)),
|
|
731
741
|
health: repository.doctor(),
|
|
742
|
+
warnings,
|
|
732
743
|
jsonContractVersion: JSON_CONTRACT_VERSION,
|
|
733
744
|
});
|
|
745
|
+
}
|
|
734
746
|
default:
|
|
735
747
|
throw new CliError('UNKNOWN_COMMAND', `unknown sessions command: ${command}`, EXIT.usage);
|
|
736
748
|
}
|
|
@@ -1283,6 +1295,7 @@ async function discoverWorkspace(ctx, args) {
|
|
|
1283
1295
|
],
|
|
1284
1296
|
});
|
|
1285
1297
|
const data = {
|
|
1298
|
+
warnings: manifest.preflight?.scopeWarnings ?? [],
|
|
1286
1299
|
manifest: publicDiscoveryManifest(manifest),
|
|
1287
1300
|
manifestPath: redactSourceLocator(manifestPath),
|
|
1288
1301
|
exactManifestRequiredForImport: true,
|
|
@@ -1304,7 +1317,20 @@ async function importDiscovered(ctx, args) {
|
|
|
1304
1317
|
if (manifest.workspaceId !== canonicalWorkspace) {
|
|
1305
1318
|
throw new CliError('MANIFEST_WORKSPACE_MISMATCH', 'discovery manifest does not belong to the explicitly authorized workspace', EXIT.contract);
|
|
1306
1319
|
}
|
|
1320
|
+
const carryPath = args.values.has('--carry-forward')
|
|
1321
|
+
? resolve(ctx.cwd, args.values.get('--carry-forward')) : undefined;
|
|
1322
|
+
if (carryPath) {
|
|
1323
|
+
const sourceStat = statSync(carryPath);
|
|
1324
|
+
const target = databasePath(ctx, args);
|
|
1325
|
+
if (existsSync(target)) {
|
|
1326
|
+
const targetStat = statSync(target);
|
|
1327
|
+
if (sourceStat.dev === targetStat.dev && sourceStat.ino === targetStat.ino) {
|
|
1328
|
+
throw new CliError('INVALID_ARGUMENT', 'carry-forward requires a separate destination catalog', EXIT.usage);
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1307
1332
|
const expected = requestedProviders(args);
|
|
1333
|
+
const warnings = providerScopeWarnings(config?.providers ?? [], manifest.preflight?.expectedProviders ?? [], manifest.preflight?.scopeWarnings);
|
|
1308
1334
|
if (expected && JSON.stringify(expected) !== JSON.stringify(manifest.preflight?.expectedProviders)) {
|
|
1309
1335
|
throw new SessionContractError('INVALID_ARGUMENT', 'expected provider scope differs from the saved manifest; rediscover with --expected-providers');
|
|
1310
1336
|
}
|
|
@@ -1320,22 +1346,27 @@ async function importDiscovered(ctx, args) {
|
|
|
1320
1346
|
// A preview can still describe the exact manifest before a catalog exists.
|
|
1321
1347
|
}
|
|
1322
1348
|
return preview('import-discovered', {
|
|
1349
|
+
warnings,
|
|
1323
1350
|
manifest: publicDiscoveryManifest(manifest),
|
|
1324
1351
|
receipt: previewDiscoveryImport(manifest, existing, new Date(), exclude),
|
|
1325
1352
|
confirmationRequired: !isDryRun(ctx, args),
|
|
1326
1353
|
wouldPersist: false,
|
|
1354
|
+
...(carryPath ? { carryForward: redactSourceLocator(carryPath) } : {}),
|
|
1327
1355
|
});
|
|
1328
1356
|
}
|
|
1329
1357
|
const lease = await acquireImportLease(databasePath(ctx, args), sha256(['discovered-import', manifest.manifestId, manifest.workspaceId].join('\0')), {
|
|
1330
1358
|
waitMs: boundedInteger(args.values.get('--lock-wait-ms'), 5_000, 0, 300_000, '--lock-wait-ms'),
|
|
1331
1359
|
});
|
|
1332
1360
|
try {
|
|
1333
|
-
const
|
|
1361
|
+
const previous = carryPath ? new SessionRepository(carryPath, { readonly: true }) : undefined;
|
|
1362
|
+
let repository;
|
|
1334
1363
|
try {
|
|
1364
|
+
repository = openRepository(ctx, args);
|
|
1335
1365
|
const interruptedRuns = repository.interruptRunningBatchImports();
|
|
1336
1366
|
const receipt = await importDiscoveryManifest({
|
|
1337
1367
|
manifest,
|
|
1338
1368
|
repository,
|
|
1369
|
+
carryForward: previous,
|
|
1339
1370
|
limits,
|
|
1340
1371
|
exclude,
|
|
1341
1372
|
signal: ctx.signal,
|
|
@@ -1343,6 +1374,7 @@ async function importDiscovered(ctx, args) {
|
|
|
1343
1374
|
});
|
|
1344
1375
|
receipt.interruptedRuns = interruptedRuns;
|
|
1345
1376
|
return ok('import-discovered', {
|
|
1377
|
+
warnings,
|
|
1346
1378
|
manifestId: manifest.manifestId,
|
|
1347
1379
|
receipt,
|
|
1348
1380
|
resumable: receipt.run.status !== 'complete',
|
|
@@ -1354,7 +1386,8 @@ async function importDiscovered(ctx, args) {
|
|
|
1354
1386
|
});
|
|
1355
1387
|
}
|
|
1356
1388
|
finally {
|
|
1357
|
-
repository
|
|
1389
|
+
repository?.close();
|
|
1390
|
+
previous?.close();
|
|
1358
1391
|
}
|
|
1359
1392
|
}
|
|
1360
1393
|
finally {
|
|
@@ -1710,7 +1743,7 @@ function parseArgs(argv) {
|
|
|
1710
1743
|
'--entity', '--sensitivity', '--extraction-state', '--page-size', '--max-documents',
|
|
1711
1744
|
'--state', '--reviewer', '--reason', '--policy-version', '--min-confidence',
|
|
1712
1745
|
'--consumer', '--actor-class', '--reason-code', '--dependent-action', '--basis',
|
|
1713
|
-
'--manifest', '--provider-home', '--codex-root', '--omp-root', '--muse-root', '--dsh-root', '--grokbot-root', '--lock-wait-ms', '--min-coverage', '--gap',
|
|
1746
|
+
'--carry-forward', '--manifest', '--provider-home', '--codex-root', '--omp-root', '--muse-root', '--dsh-root', '--grokbot-root', '--lock-wait-ms', '--min-coverage', '--gap',
|
|
1714
1747
|
'--inactivity-threshold', '--expected-providers', '--exclude', '--max-record-bytes', '--max-source-bytes',
|
|
1715
1748
|
'--control-events',
|
|
1716
1749
|
'--session', '--status', '--actor', '--group-by',
|
|
@@ -202,16 +202,7 @@ function normalizeRolloutRecord(state, envelope, line, sourceArtifactSessionId)
|
|
|
202
202
|
firstMetadata = true;
|
|
203
203
|
state.metadataId = declaredId;
|
|
204
204
|
state.nativeSessionId = declaredId;
|
|
205
|
-
const
|
|
206
|
-
const lineage = Object.fromEntries(Object.entries({
|
|
207
|
-
forkedFromId: stringValue(payload.forked_from_id),
|
|
208
|
-
parentThreadId: stringValue(spawn.parent_thread_id),
|
|
209
|
-
depth: nonnegativeInteger(spawn.depth ?? payload.depth),
|
|
210
|
-
agentPath: stringValue(payload.agent_path ?? spawn.agent_path),
|
|
211
|
-
agentNickname: stringValue(payload.agent_nickname ?? spawn.agent_nickname),
|
|
212
|
-
subagentHistoryStartOrdinal: nonnegativeInteger(payload.subagent_history_start_ordinal),
|
|
213
|
-
threadSource: stringValue(payload.thread_source),
|
|
214
|
-
}).filter(([, value]) => value !== undefined));
|
|
205
|
+
const lineage = rolloutLineage(payload);
|
|
215
206
|
if (Object.keys(lineage).length > 0)
|
|
216
207
|
state.lineage = lineage;
|
|
217
208
|
}
|
|
@@ -243,6 +234,28 @@ function normalizeRolloutRecord(state, envelope, line, sourceArtifactSessionId)
|
|
|
243
234
|
}
|
|
244
235
|
return record;
|
|
245
236
|
}
|
|
237
|
+
/** Apply the rollout identity rule to legacy catalog rows, including split parent replay. */
|
|
238
|
+
export function recoverCodexRolloutIdentity(events) {
|
|
239
|
+
const metadata = [...events].sort((a, b) => a.sequence - b.sequence || a.eventId.localeCompare(b.eventId))
|
|
240
|
+
.find(event => asObject(event.extensions['native.codex']).rolloutType === 'session_meta');
|
|
241
|
+
if (!metadata?.nativeId)
|
|
242
|
+
return undefined; // App Server sources can contain multiple independent threads.
|
|
243
|
+
const native = asObject(metadata.extensions['native.codex']);
|
|
244
|
+
return { nativeSessionId: metadata.nativeId,
|
|
245
|
+
lineage: { ...rolloutLineage(asObject(native.unknownFields)), ...asObject(native.lineage) } };
|
|
246
|
+
}
|
|
247
|
+
function rolloutLineage(payload) {
|
|
248
|
+
const spawn = asObject(asObject(asObject(payload.source).subagent).thread_spawn);
|
|
249
|
+
return Object.fromEntries(Object.entries({
|
|
250
|
+
forkedFromId: stringValue(payload.forked_from_id),
|
|
251
|
+
parentThreadId: stringValue(spawn.parent_thread_id),
|
|
252
|
+
depth: nonnegativeInteger(spawn.depth ?? payload.depth),
|
|
253
|
+
agentPath: stringValue(payload.agent_path ?? spawn.agent_path),
|
|
254
|
+
agentNickname: stringValue(payload.agent_nickname ?? spawn.agent_nickname),
|
|
255
|
+
subagentHistoryStartOrdinal: nonnegativeInteger(payload.subagent_history_start_ordinal),
|
|
256
|
+
threadSource: stringValue(payload.thread_source),
|
|
257
|
+
}).filter(([, value]) => value !== undefined));
|
|
258
|
+
}
|
|
246
259
|
function nonnegativeInteger(value) {
|
|
247
260
|
return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
|
|
248
261
|
? value : undefined;
|
|
@@ -24,7 +24,7 @@ export function coverageFromBatchRun(run, now = new Date(), staleAfterMs = 24 *
|
|
|
24
24
|
const unresolvedProviders = preflight?.unresolvedProviders
|
|
25
25
|
?? [...new Set([...unavailable, ...exportRequired, ...notChecked])].sort();
|
|
26
26
|
const acquisitionIncomplete = rejected > 0 || pending > 0 || skipped > 0
|
|
27
|
-
|| (preflight
|
|
27
|
+
|| (preflight && ['explicit', 'workspace-config'].includes(preflight.expectedProviderBasis) && unresolvedProviders.length > 0);
|
|
28
28
|
const incomplete = acquisitionIncomplete || !preflight || preflight.status !== 'complete'
|
|
29
29
|
|| unavailable.length > 0 || accepted + duplicated + previouslyCommitted + excluded === 0
|
|
30
30
|
|| exportRequired.length > 0 || notChecked.length > 0;
|
|
@@ -1,13 +1,21 @@
|
|
|
1
|
-
import { stat } from 'node:fs/promises';
|
|
1
|
+
import { mkdtemp, rm, stat } from 'node:fs/promises';
|
|
2
|
+
import { tmpdir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
2
4
|
import { createSessionAdapterRegistry } from './adapter-registry.js';
|
|
3
5
|
import { coveragePreflight, sessionRuntimeIdentity } from './coverage-preflight.js';
|
|
4
|
-
import { SESSION_CONTRACT_VERSION, SessionContractError, schemaIssueDetail, SessionSourceSchema, sha256, } from './contracts.js';
|
|
6
|
+
import { SESSION_CONTRACT_VERSION, SessionContractError, schemaIssueDetail, SessionSourceSchema, sha256, stableSessionId, stableEventId, } from './contracts.js';
|
|
5
7
|
import { BATCH_IMPORT_VERSION, coverageFromBatchRun, } from './batch-contracts.js';
|
|
6
8
|
import { IncrementalSessionImporter, SessionImportFailure } from './importer.js';
|
|
7
9
|
import { matchImportExclusion, resolveImportExclusions } from './import-exclusions.js';
|
|
8
10
|
import { resolveImportLimits } from './import-limits.js';
|
|
9
11
|
import { fingerprintSourceFile } from './readers.js';
|
|
12
|
+
import { SessionRepository } from './repository.js';
|
|
13
|
+
import { redactSessionValue, sanitizeSessionEvent } from './policy.js';
|
|
14
|
+
import { deriveSessionIntent } from './origin.js';
|
|
15
|
+
import { recoverCodexRolloutIdentity } from './adapters/codex.js';
|
|
10
16
|
export async function importDiscoveryManifest(options) {
|
|
17
|
+
if (options.carryForward)
|
|
18
|
+
return rebuildFromCatalog(options);
|
|
11
19
|
const exclusions = resolveImportExclusions(undefined, options.exclude);
|
|
12
20
|
const limits = { ...resolveImportLimits(undefined, {
|
|
13
21
|
maxRecordBytes: options.limits?.maxRecordBytes, maxSourceBytes: options.limits?.maxTotalBytes,
|
|
@@ -25,6 +33,8 @@ export async function importDiscoveryManifest(options) {
|
|
|
25
33
|
}
|
|
26
34
|
run.schemaVersion = BATCH_IMPORT_VERSION;
|
|
27
35
|
run.preflight = coveragePreflight({
|
|
36
|
+
configuredProviders: options.manifest.preflight?.configuredProviders,
|
|
37
|
+
scopeWarnings: options.manifest.preflight?.scopeWarnings,
|
|
28
38
|
expectedProviders: options.manifest.preflight?.expectedProviders,
|
|
29
39
|
expectedProviderBasis: options.manifest.preflight?.expectedProviderBasis ?? 'legacy',
|
|
30
40
|
runtime: options.runtime ?? sessionRuntimeIdentity(),
|
|
@@ -160,6 +170,8 @@ export function previewDiscoveryImport(manifest, existing, now = new Date(), exc
|
|
|
160
170
|
return disposition;
|
|
161
171
|
}),
|
|
162
172
|
preflight: coveragePreflight({
|
|
173
|
+
configuredProviders: manifest.preflight?.configuredProviders,
|
|
174
|
+
scopeWarnings: manifest.preflight?.scopeWarnings,
|
|
163
175
|
expectedProviders: manifest.preflight?.expectedProviders,
|
|
164
176
|
expectedProviderBasis: manifest.preflight?.expectedProviderBasis ?? 'legacy',
|
|
165
177
|
reports: manifest.providers,
|
|
@@ -202,6 +214,7 @@ function receiptFor(run, now) {
|
|
|
202
214
|
run,
|
|
203
215
|
coverage,
|
|
204
216
|
interruptedRuns: [],
|
|
217
|
+
...(run.carryForward ? { carryForward: run.carryForward } : {}),
|
|
205
218
|
totals: {
|
|
206
219
|
discovered: coverage.sources.discovered,
|
|
207
220
|
accepted: coverage.sources.accepted,
|
|
@@ -304,4 +317,191 @@ function isAccepted(status) {
|
|
|
304
317
|
|| status === 'duplicate'
|
|
305
318
|
|| status === 'previously-committed';
|
|
306
319
|
}
|
|
320
|
+
/** Reuse source import and normalized repository gates, without copying FTS/analytics. */
|
|
321
|
+
async function rebuildFromCatalog(options) {
|
|
322
|
+
const previous = options.carryForward;
|
|
323
|
+
if (previous === options.repository) {
|
|
324
|
+
throw new SessionContractError('INVALID_ARGUMENT', 'carry-forward requires a separate destination catalog');
|
|
325
|
+
}
|
|
326
|
+
const workspaceId = options.manifest.workspaceId;
|
|
327
|
+
const exclusions = resolveImportExclusions(undefined, options.exclude);
|
|
328
|
+
const byProvider = {};
|
|
329
|
+
const counts = (provider) => byProvider[provider] ??= {
|
|
330
|
+
reimported: 0, carried: 0, droppedAsDuplicate: 0, excluded: 0,
|
|
331
|
+
};
|
|
332
|
+
const oldSessions = catalogSessions(previous, workspaceId);
|
|
333
|
+
const available = [];
|
|
334
|
+
const present = new Set();
|
|
335
|
+
const excludedSources = new Set();
|
|
336
|
+
for (const source of options.manifest.sources) {
|
|
337
|
+
if (matchImportExclusion(source, exclusions) ?? source.exclusion) {
|
|
338
|
+
excludedSources.add(source.sourceId);
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
let details;
|
|
342
|
+
try {
|
|
343
|
+
details = await stat(source.locator);
|
|
344
|
+
}
|
|
345
|
+
catch (error) {
|
|
346
|
+
if (['ENOENT', 'ENOTDIR'].includes(error.code ?? ''))
|
|
347
|
+
continue;
|
|
348
|
+
throw error;
|
|
349
|
+
}
|
|
350
|
+
// Refresh the snapshot, but retain the explicitly saved authorization and source identity.
|
|
351
|
+
const fingerprint = await fingerprintSourceFile({ selectedPath: source.locator,
|
|
352
|
+
allowedRoots: [source.authorizedRoot] });
|
|
353
|
+
available.push({ ...source, sizeBytes: details.size, modifiedAt: details.mtime.toISOString(),
|
|
354
|
+
digest: fingerprint.digest });
|
|
355
|
+
present.add(source.sourceId);
|
|
356
|
+
}
|
|
357
|
+
const manifest = { ...options.manifest, sources: available,
|
|
358
|
+
manifestId: sha256(JSON.stringify(['carry-forward-v1', options.manifest.manifestId, available])) };
|
|
359
|
+
const stagingDirectory = await mkdtemp(join(tmpdir(), 'aiwg-session-rebuild-'));
|
|
360
|
+
let staged;
|
|
361
|
+
try {
|
|
362
|
+
staged = new SessionRepository(join(stagingDirectory, 'staging.sqlite'));
|
|
363
|
+
const receipt = await importDiscoveryManifest({ ...options, manifest, repository: staged,
|
|
364
|
+
carryForward: undefined, fault: undefined });
|
|
365
|
+
const identities = new Set();
|
|
366
|
+
const destinationCodex = new Map(catalogSessions(options.repository, workspaceId)
|
|
367
|
+
.filter(session => session.provider === 'codex').map(session => [session.nativeSessionId, session.sourceId]));
|
|
368
|
+
const transfer = (repository, sessions, category) => {
|
|
369
|
+
// A legacy Codex rollout may have split replayed parent events into phantom rows.
|
|
370
|
+
// Recover the first session_meta ID for the whole source before grouping events.
|
|
371
|
+
const sourceGroups = new Map();
|
|
372
|
+
for (const session of sessions) {
|
|
373
|
+
const group = sourceGroups.get(session.sourceId) ?? [];
|
|
374
|
+
group.push(session);
|
|
375
|
+
sourceGroups.set(session.sourceId, group);
|
|
376
|
+
}
|
|
377
|
+
for (const [sourceId, group] of sourceGroups) {
|
|
378
|
+
if (options.signal?.aborted)
|
|
379
|
+
throw new SessionContractError('IMPORT_INTERRUPTED', 'catalog rebuild interrupted');
|
|
380
|
+
const source = repository.getSource(sourceId, workspaceId);
|
|
381
|
+
if (!source)
|
|
382
|
+
throw new SessionContractError('IMPORT_CONFLICT', 'catalog session has no authorized source');
|
|
383
|
+
const allEvents = group.flatMap(session => repository.listEvents(session.sessionId, workspaceId))
|
|
384
|
+
.sort((a, b) => a.sequence - b.sequence || a.eventId.localeCompare(b.eventId));
|
|
385
|
+
const rollout = source.provider === 'codex' ? recoverCodexRolloutIdentity(allEvents) : undefined;
|
|
386
|
+
const childId = rollout?.nativeSessionId;
|
|
387
|
+
// Prefer the child's aggregate when the legacy source also has phantom parent aggregates.
|
|
388
|
+
group.sort((a, b) => Number(b.nativeSessionId === childId) - Number(a.nativeSessionId === childId));
|
|
389
|
+
const canonical = new Map();
|
|
390
|
+
for (const session of group) {
|
|
391
|
+
const nativeId = childId ?? session.nativeSessionId;
|
|
392
|
+
const locator = options.manifest.sources.find(item => item.sourceId === sourceId)?.locator
|
|
393
|
+
?? source.redactedLocator;
|
|
394
|
+
if (excludedSources.has(sourceId) || matchImportExclusion({ provider: session.provider,
|
|
395
|
+
locator, locatorClass: source.locatorClass, nativeSessionId: nativeId }, exclusions)) {
|
|
396
|
+
counts(session.provider).excluded++;
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
const key = session.provider === 'codex' ? `${session.provider}\0${nativeId}` : session.sessionId;
|
|
400
|
+
if (identities.has(key)) {
|
|
401
|
+
counts(session.provider).droppedAsDuplicate++;
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
const existing = canonical.get(key);
|
|
405
|
+
if (existing) {
|
|
406
|
+
counts(session.provider).droppedAsDuplicate++;
|
|
407
|
+
existing.events.push(...repository.listEvents(session.sessionId, workspaceId));
|
|
408
|
+
}
|
|
409
|
+
else
|
|
410
|
+
canonical.set(key, { session: { ...session, nativeSessionId: nativeId },
|
|
411
|
+
events: repository.listEvents(session.sessionId, workspaceId) });
|
|
412
|
+
}
|
|
413
|
+
for (const [key, entry] of canonical) {
|
|
414
|
+
if (source.provider === 'codex' && destinationCodex.has(entry.session.nativeSessionId)
|
|
415
|
+
&& destinationCodex.get(entry.session.nativeSessionId) !== sourceId) {
|
|
416
|
+
throw new SessionContractError('IMPORT_CONFLICT', 'destination has a different source for this Codex identity; select a fresh catalog');
|
|
417
|
+
}
|
|
418
|
+
const session = redactSessionValue({ ...entry.session,
|
|
419
|
+
sessionId: stableSessionId(source.provider, sourceId, entry.session.nativeSessionId),
|
|
420
|
+
}).value;
|
|
421
|
+
// Hash only sanitized content; receipts and IDs must never retain old plaintext.
|
|
422
|
+
const sanitized = entry.events.map(sanitizeSessionEvent);
|
|
423
|
+
const runId = sha256(JSON.stringify(['catalog-carry-v1', session.sessionId, sanitized]));
|
|
424
|
+
const events = sanitized.map(event => {
|
|
425
|
+
const digest = event.digest;
|
|
426
|
+
return { ...event, sessionId: session.sessionId, importRunId: runId, digest,
|
|
427
|
+
eventId: stableEventId(source.provider, sourceId, {
|
|
428
|
+
nativeSessionId: session.nativeSessionId, nativeEventId: event.nativeId ?? event.eventId,
|
|
429
|
+
sequence: event.sequence, kind: event.kind, text: event.searchableText,
|
|
430
|
+
rawReference: event.rawReference,
|
|
431
|
+
}, digest) };
|
|
432
|
+
});
|
|
433
|
+
if (rollout && Object.keys(rollout.lineage).length > 0) {
|
|
434
|
+
session.extensions['native.codex'] = { lineage: rollout.lineage };
|
|
435
|
+
}
|
|
436
|
+
session.intent = deriveSessionIntent(events);
|
|
437
|
+
const checkpoint = repository.getCheckpoint(sourceId, source.adapterVersion)
|
|
438
|
+
?? { cursor: '', recordsRead: events.length, bytesRead: Buffer.byteLength(JSON.stringify(events)) };
|
|
439
|
+
options.repository.applyImport({ source, sessions: [session], events,
|
|
440
|
+
run: { contractVersion: SESSION_CONTRACT_VERSION, importRunId: runId, sourceId,
|
|
441
|
+
parserVersion: source.adapterVersion, policyVersion: '1.0.0',
|
|
442
|
+
sourceSchemaVersion: source.sourceSchemaVersion, consistency: source.consistency,
|
|
443
|
+
status: 'running', checkpoint, startedAt: (options.now?.() ?? new Date()).toISOString(),
|
|
444
|
+
completedAt: null, errorCode: null } }, checkpoint);
|
|
445
|
+
for (const old of group.filter(item => (childId ?? item.nativeSessionId) === entry.session.nativeSessionId)) {
|
|
446
|
+
for (const tag of repository.listTags(old.sessionId, workspaceId)) {
|
|
447
|
+
options.repository.tagSession(session.sessionId, tag, workspaceId);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
identities.add(key);
|
|
451
|
+
counts(source.provider)[category]++;
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
};
|
|
455
|
+
const reimported = catalogSessions(staged, workspaceId);
|
|
456
|
+
transfer(staged, reimported, 'reimported');
|
|
457
|
+
for (const sourceId of present) {
|
|
458
|
+
const old = oldSessions.filter(session => session.sourceId === sourceId && session.provider === 'codex');
|
|
459
|
+
const imported = reimported.filter(session => session.sourceId === sourceId);
|
|
460
|
+
// Account for phantom rows eliminated by current source parsing, separately from copied duplicates.
|
|
461
|
+
if (imported.length > 0 && old.length > imported.length) {
|
|
462
|
+
counts('codex').droppedAsDuplicate += old.length - imported.length;
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
transfer(previous, oldSessions.filter(session => !present.has(session.sourceId)), 'carried');
|
|
466
|
+
options.repository.reindex(workspaceId);
|
|
467
|
+
const finalSessions = catalogSessions(options.repository, workspaceId);
|
|
468
|
+
const sourceIds = new Set([...receipt.run.sources.map(item => item.sourceId),
|
|
469
|
+
...oldSessions.map(session => session.sourceId)]);
|
|
470
|
+
for (const sourceId of sourceIds) {
|
|
471
|
+
const accepted = finalSessions.filter(session => session.sourceId === sourceId);
|
|
472
|
+
let disposition = receipt.run.sources.find(item => item.sourceId === sourceId);
|
|
473
|
+
if (!disposition) {
|
|
474
|
+
const old = oldSessions.find(session => session.sourceId === sourceId);
|
|
475
|
+
const excluded = excludedSources.has(sourceId) || matchImportExclusion({
|
|
476
|
+
provider: old.provider, nativeSessionId: old.nativeSessionId,
|
|
477
|
+
locator: previous.getSource(sourceId, workspaceId)?.redactedLocator ?? '', locatorClass: 'catalog',
|
|
478
|
+
}, exclusions);
|
|
479
|
+
disposition = { sourceId, provider: old.provider,
|
|
480
|
+
status: excluded ? 'excluded' : accepted.length > 0 ? 'committed' : 'duplicate',
|
|
481
|
+
attempts: 1, sessionsAccepted: 0, eventsAccepted: 0,
|
|
482
|
+
errorCode: null, diagnostic: null, updatedAt: receipt.run.updatedAt };
|
|
483
|
+
receipt.run.sources.push(disposition);
|
|
484
|
+
}
|
|
485
|
+
disposition.sessionsAccepted = accepted.length;
|
|
486
|
+
disposition.eventsAccepted = accepted.reduce((sum, session) => sum + options.repository.listEvents(session.sessionId, workspaceId).length, 0);
|
|
487
|
+
}
|
|
488
|
+
receipt.run.carryForward = { byProvider };
|
|
489
|
+
options.repository.saveBatchImportRun(receipt.run);
|
|
490
|
+
return redactSessionValue(receiptFor(receipt.run, options.now?.() ?? new Date())).value;
|
|
491
|
+
}
|
|
492
|
+
finally {
|
|
493
|
+
staged?.close();
|
|
494
|
+
await rm(stagingDirectory, { recursive: true, force: true });
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
function catalogSessions(repository, workspaceId) {
|
|
498
|
+
const sessions = [];
|
|
499
|
+
let cursor;
|
|
500
|
+
do {
|
|
501
|
+
const page = repository.listSessions({ workspaceId, limit: 500, cursor });
|
|
502
|
+
sessions.push(...page.items);
|
|
503
|
+
cursor = page.nextCursor ?? undefined;
|
|
504
|
+
} while (cursor);
|
|
505
|
+
return sessions;
|
|
506
|
+
}
|
|
307
507
|
//# sourceMappingURL=batch-import.js.map
|
|
@@ -20,9 +20,18 @@ export const SessionRuntimeIdentitySchema = z.object({
|
|
|
20
20
|
canonicalRoot: z.string().nullable(),
|
|
21
21
|
diagnostics: z.array(z.string()),
|
|
22
22
|
});
|
|
23
|
+
export const SessionProviderScopeWarningSchema = z.object({
|
|
24
|
+
code: z.literal('CONFIGURED_PROVIDERS_OUTSIDE_SCOPE'),
|
|
25
|
+
configuredProviders: z.array(ExpectedSessionProviderSchema),
|
|
26
|
+
expectedProviders: z.array(ExpectedSessionProviderSchema),
|
|
27
|
+
missingProviders: z.array(ExpectedSessionProviderSchema),
|
|
28
|
+
message: z.string(),
|
|
29
|
+
});
|
|
23
30
|
export const SessionCoveragePreflightSchema = z.object({
|
|
24
31
|
expectedProviders: z.array(ExpectedSessionProviderSchema).min(1),
|
|
25
|
-
expectedProviderBasis: z.enum(['explicit', 'declared-default', 'legacy']),
|
|
32
|
+
expectedProviderBasis: z.enum(['explicit', 'workspace-config', 'declared-default', 'legacy']),
|
|
33
|
+
configuredProviders: z.array(ExpectedSessionProviderSchema).optional(),
|
|
34
|
+
scopeWarnings: z.array(SessionProviderScopeWarningSchema).optional(),
|
|
26
35
|
supportedProviders: z.array(ExpectedSessionProviderSchema),
|
|
27
36
|
runtime: SessionRuntimeIdentitySchema,
|
|
28
37
|
status: z.enum(['partial', 'complete']),
|
|
@@ -19,6 +19,29 @@ export function expectedSessionProviders(values) {
|
|
|
19
19
|
}
|
|
20
20
|
return [...new Set(providers)].sort();
|
|
21
21
|
}
|
|
22
|
+
export function providerScopeWarnings(configured, expected, recorded = []) {
|
|
23
|
+
const configuredProviders = configured.length ? expectedSessionProviders(configured) : [];
|
|
24
|
+
const expectedProviders = expected.length ? expectedSessionProviders(expected) : [];
|
|
25
|
+
const missingProviders = configuredProviders.filter(provider => !expectedProviders.includes(provider));
|
|
26
|
+
const current = missingProviders.length ? [{
|
|
27
|
+
code: 'CONFIGURED_PROVIDERS_OUTSIDE_SCOPE', configuredProviders, expectedProviders, missingProviders,
|
|
28
|
+
message: `Configured providers outside discovery scope: ${missingProviders.join(', ')}. Rediscover or refresh with current workspace config; verify authorized roots before rebuilding.`,
|
|
29
|
+
}] : [];
|
|
30
|
+
return [...recorded, ...current].filter((warning, index, all) => all.findIndex(item => JSON.stringify(item) === JSON.stringify(warning)) === index);
|
|
31
|
+
}
|
|
32
|
+
/** Explicit scope applies once; maintenance combines current config with saved root grants. */
|
|
33
|
+
export function resolveSessionProviderScope(configured, explicit, authorized) {
|
|
34
|
+
const configuredProviders = configured.length ? expectedSessionProviders(configured) : [];
|
|
35
|
+
const values = [...configuredProviders, ...(authorized ?? [])];
|
|
36
|
+
const expectedProviders = expectedSessionProviders(explicit ?? (values.length ? values : SESSION_PROVIDER_IDS));
|
|
37
|
+
return {
|
|
38
|
+
expectedProviders,
|
|
39
|
+
expectedProviderBasis: explicit ? 'explicit' : configuredProviders.length ? 'workspace-config'
|
|
40
|
+
: authorized?.length ? 'explicit' : 'declared-default',
|
|
41
|
+
configuredProviders,
|
|
42
|
+
scopeWarnings: providerScopeWarnings(configuredProviders, expectedProviders),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
22
45
|
/** Resolve the code actually loaded, independently of cwd, AIWG_ROOT and corpus redirects. */
|
|
23
46
|
export function sessionRuntimeIdentity() {
|
|
24
47
|
const packageRoot = findPackageRoot(dirname(fileURLToPath(import.meta.url)));
|
|
@@ -103,6 +126,8 @@ export function coveragePreflight(options = {}) {
|
|
|
103
126
|
return SessionCoveragePreflightSchema.parse({
|
|
104
127
|
expectedProviders,
|
|
105
128
|
expectedProviderBasis: options.expectedProviderBasis ?? (options.expectedProviders ? 'explicit' : 'declared-default'),
|
|
129
|
+
...(options.configuredProviders ? { configuredProviders: options.configuredProviders } : {}),
|
|
130
|
+
...(options.scopeWarnings ? { scopeWarnings: options.scopeWarnings } : {}),
|
|
106
131
|
supportedProviders: adapters.map((adapter) => adapter.provider), runtime,
|
|
107
132
|
status: 'partial', unresolvedProviders: expectedProviders, ledger,
|
|
108
133
|
});
|
|
@@ -127,7 +152,7 @@ export function reconcileCoveragePreflight(preflight, sources) {
|
|
|
127
152
|
const runtimeGap = ['mismatch', 'stale', 'unknown'].includes(preflight.runtime.installationState);
|
|
128
153
|
return {
|
|
129
154
|
...preflight, ledger, unresolvedProviders,
|
|
130
|
-
status: unresolvedProviders.length > 0 || runtimeGap || preflight.expectedProviderBasis
|
|
155
|
+
status: unresolvedProviders.length > 0 || runtimeGap || !['explicit', 'workspace-config'].includes(preflight.expectedProviderBasis)
|
|
131
156
|
? 'partial' : 'complete',
|
|
132
157
|
};
|
|
133
158
|
}
|
|
@@ -56,20 +56,31 @@ async function safeRealpath(path, root = false) {
|
|
|
56
56
|
throw new SessionContractError('SOURCE_NOT_AUTHORIZED', root ? 'an authorized source root is inaccessible' : 'selected session source is inaccessible');
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
|
+
// Newline gaps are bounded and each repetition consumes a line break, keeping
|
|
60
|
+
// indentation scans disjoint even on large malformed records (#3273, #3404).
|
|
61
|
+
const VALUE_GAP = /[ \t]*(?:\r?\n[ \t]*){0,2}/.source;
|
|
62
|
+
const NEXT_LINE_GAP = /[ \t]*(?:\r?\n[ \t]*){1,2}/.source;
|
|
63
|
+
const QUOTED_VALUE = /"(?:\\[^\r\n]|[^"\\\r\n])+"|'(?:\\[^\r\n]|[^'\\\r\n])+'|`(?!`)(?:\\[^\r\n]|[^`\\\r\n])+`(?!`)/.source;
|
|
64
|
+
const GENERATED_VALUE = /(?=[A-Za-z0-9._~+/=-]*[0-9])(?=[A-Za-z0-9._~+/=-]*[A-Za-z])[A-Za-z0-9._~+/=-]{20,}/.source;
|
|
65
|
+
function assignmentPattern(key, separator, sameLineValue) {
|
|
66
|
+
// Only the next-line branch needs stronger evidence: quoted text or a long
|
|
67
|
+
// generated value. In particular, YAML lists and Markdown fences are prose.
|
|
68
|
+
return new RegExp(`\\b${key}[ \\t]*${separator}(?:${NEXT_LINE_GAP}(?:${QUOTED_VALUE}|${GENERATED_VALUE})|[ \\t]*(?:${QUOTED_VALUE}|${sameLineValue}))`, 'gi');
|
|
69
|
+
}
|
|
59
70
|
// Ambiguous token components are bounded; whitespace and key blocks use disjoint scans.
|
|
60
71
|
const SECRET_PATTERNS = [
|
|
61
|
-
['bearer-token',
|
|
62
|
-
['authorization',
|
|
63
|
-
['cookie',
|
|
64
|
-
['environment-assignment',
|
|
65
|
-
['credential',
|
|
72
|
+
['bearer-token', new RegExp(`\\bBearer(?:[ \\t]+|${NEXT_LINE_GAP})(?:${QUOTED_VALUE}|\\\\"[A-Za-z0-9._~+/=-]{12,}\\\\"|[A-Za-z0-9._~+/=-]{12,})`, 'gi')],
|
|
73
|
+
['authorization', new RegExp(`\\bAuthorization[ \\t]*:${VALUE_GAP}[^\\s][^\\r\\n]*`, 'gi')],
|
|
74
|
+
['cookie', new RegExp(`\\b(?:Cookie|Set-Cookie)[ \\t]*:${VALUE_GAP}[^\\s][^\\r\\n]*`, 'gi')],
|
|
75
|
+
['environment-assignment', assignmentPattern('(?:AWS_SECRET_ACCESS_KEY|[A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY))', '=', /[^\s]+/.source)],
|
|
76
|
+
['credential', assignmentPattern('(?:api[_-]?key|token|password|secret)', '[:=]', /["'`]?[^\s"'`,;]{6,}/.source)],
|
|
66
77
|
['credential', /^[ \t]*password[ \t]+[^\s]+[ \t]*$/gim],
|
|
67
78
|
// Space-separated values only when they look generated (20+ chars, letters and digits), so prose stays.
|
|
68
|
-
['credential',
|
|
79
|
+
['credential', new RegExp(`\\b(?:api[_-]?key|token|secret)[ \\t]+${GENERATED_VALUE}`, 'gi')],
|
|
69
80
|
['provider-token', /\b(?:sk-(?:ant-|proj-)?|[sr]k_(?:live|test)_|gh[pousr]_|github_pat_|glpat-|xai-|npm_|hf_|xox[baprs]-|AKIA|AIza)[A-Za-z0-9._~+/=-]{8,}/g],
|
|
70
81
|
['provider-token', /https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9_-]+\/[A-Za-z0-9_-]+\/[A-Za-z0-9_-]+/g],
|
|
71
82
|
['connection-string', /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis):\/\/[^\s]+/gi],
|
|
72
|
-
['credential-url',
|
|
83
|
+
['credential-url', new RegExp(`\\bhttps?://${VALUE_GAP}[^/\\s:@]+:${VALUE_GAP}(?:${QUOTED_VALUE}|[^/\\s@]+)@${VALUE_GAP}[^\\s"'\x60<>\\\\]+`, 'gi')],
|
|
73
84
|
['fixture-credential', /\bredaction-canary-[A-Za-z0-9_-]+\b/gi],
|
|
74
85
|
['email', /\b[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}\b/gi],
|
|
75
86
|
];
|
|
@@ -20,7 +20,7 @@ export function sessionRefreshAvailable() {
|
|
|
20
20
|
return false;
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
/** Refresh
|
|
23
|
+
/** Refresh current configured scope within saved provider root grants. */
|
|
24
24
|
export async function refreshSessionCatalog(workspace, full, waitMs, signal, workspaceAliases = [], flags = {}) {
|
|
25
25
|
const canonicalWorkspace = realpathSync(workspace);
|
|
26
26
|
const config = await readAiwgConfig(canonicalWorkspace);
|
|
@@ -56,8 +56,7 @@ export async function refreshSessionCatalog(workspace, full, waitMs, signal, wor
|
|
|
56
56
|
const manifest = await discoverWorkspaceHistories({
|
|
57
57
|
workspace: canonicalWorkspace,
|
|
58
58
|
workspaceAliases: [workspace, ...workspaceAliases],
|
|
59
|
-
|
|
60
|
-
authorizedRoots,
|
|
59
|
+
refreshFrom: { ...previous, authorizedRoots },
|
|
61
60
|
registry: createSessionAdapterRegistry(limits),
|
|
62
61
|
exclude,
|
|
63
62
|
knownSources: previous.sources,
|
|
@@ -13,7 +13,7 @@ const VISIBLE_SESSION_PREDICATE = `s.lifecycle != 'tombstoned' AND EXISTS (
|
|
|
13
13
|
)`;
|
|
14
14
|
export class SessionRepository {
|
|
15
15
|
db;
|
|
16
|
-
constructor(path = ':memory:') {
|
|
16
|
+
constructor(path = ':memory:', options = {}) {
|
|
17
17
|
let Database;
|
|
18
18
|
try {
|
|
19
19
|
Database = requireFeaturePackage('better-sqlite3');
|
|
@@ -23,7 +23,9 @@ export class SessionRepository {
|
|
|
23
23
|
'run `aiwg features install sqlite`, then retry. If the native build fails, install ' +
|
|
24
24
|
'Python 3, make, and a C/C++ compiler supported by node-gyp.', { cause });
|
|
25
25
|
}
|
|
26
|
-
this.db = new Database(path);
|
|
26
|
+
this.db = new Database(path, options.readonly ? { readonly: true, fileMustExist: true } : undefined);
|
|
27
|
+
if (options.readonly)
|
|
28
|
+
return;
|
|
27
29
|
this.db.pragma('foreign_keys = ON');
|
|
28
30
|
this.db.pragma('journal_mode = WAL');
|
|
29
31
|
this.db.pragma('busy_timeout = 5000');
|
|
@@ -5,10 +5,11 @@ import { inspectMuseNativeWorkspace } from './adapters/muse.js';
|
|
|
5
5
|
import { readOmpSessionHeader } from './adapters/omp.js';
|
|
6
6
|
import { grokbotConversationIdFromLocator } from './adapters/grokbot-locator.js';
|
|
7
7
|
import { resolveOmpPaths } from '../providers/omp-paths.mjs';
|
|
8
|
-
import {
|
|
8
|
+
import { readAiwgConfig } from '../config/aiwg-config.js';
|
|
9
|
+
import { SessionCoveragePreflightSchema, SessionContractError, sha256, } from './contracts.js';
|
|
9
10
|
import { redactSourceLocator } from './discovery.js';
|
|
10
11
|
import { createSessionAdapterRegistry } from './adapter-registry.js';
|
|
11
|
-
import { coveragePreflight,
|
|
12
|
+
import { coveragePreflight, providerScopeWarnings, resolveSessionProviderScope } from './coverage-preflight.js';
|
|
12
13
|
import { fingerprintSourceFile, streamJsonLines } from './readers.js';
|
|
13
14
|
import { matchSessionWorkspace } from './workspace-matching.js';
|
|
14
15
|
import { matchImportExclusion, nativeSessionIdFromLocator, resolveImportExclusions } from './import-exclusions.js';
|
|
@@ -20,9 +21,26 @@ const MANUAL_EXPORT_PROVIDERS = new Set([
|
|
|
20
21
|
export async function discoverWorkspaceHistories(options) {
|
|
21
22
|
const registry = options.registry ?? createSessionAdapterRegistry();
|
|
22
23
|
const exclusions = resolveImportExclusions(undefined, options.exclude);
|
|
23
|
-
const expectedProviders = expectedSessionProviders(options.expectedProviders ?? SESSION_PROVIDER_IDS);
|
|
24
24
|
const supportedProviders = new Set(registry.list().map((adapter) => adapter.provider));
|
|
25
25
|
const workspacePath = await canonicalPath(options.workspace);
|
|
26
|
+
const config = await readAiwgConfig(workspacePath);
|
|
27
|
+
const configuredProviders = config?.providers ?? [];
|
|
28
|
+
const explicitlyRootedProviders = [
|
|
29
|
+
...(options.codexRoot || options.codexRoots?.length || config?.sessions?.codex_roots?.length ? ['codex'] : []),
|
|
30
|
+
...(options.ompRoot ? ['omp'] : []),
|
|
31
|
+
...(options.museRoot ? ['muse'] : []),
|
|
32
|
+
...(options.dshRoot ? ['deepseek-harness'] : []),
|
|
33
|
+
...(options.grokbotRoot ? ['grokbot'] : []),
|
|
34
|
+
];
|
|
35
|
+
const savedRoots = options.authorizedRoots ?? options.refreshFrom?.authorizedRoots;
|
|
36
|
+
const authorizedProviders = options.refreshFrom
|
|
37
|
+
? Object.keys(savedRoots ?? {}).filter(provider => savedRoots?.[provider]?.length)
|
|
38
|
+
: configuredProviders.length ? explicitlyRootedProviders : undefined;
|
|
39
|
+
const providerScope = resolveSessionProviderScope(configuredProviders, options.expectedProviders, authorizedProviders);
|
|
40
|
+
if (options.refreshFrom) {
|
|
41
|
+
providerScope.scopeWarnings = providerScopeWarnings(configuredProviders, options.refreshFrom.preflight?.expectedProviders ?? []);
|
|
42
|
+
}
|
|
43
|
+
const { expectedProviders } = providerScope;
|
|
26
44
|
const workspaceId = workspacePath;
|
|
27
45
|
const workspacePaths = [...new Set([workspacePath, resolve(options.workspace),
|
|
28
46
|
...(options.workspaceAliases ?? []).map((alias) => resolve(alias))])];
|
|
@@ -52,12 +70,19 @@ export async function discoverWorkspaceHistories(options) {
|
|
|
52
70
|
? [resolve(options.grokbotRoot)] : [] },
|
|
53
71
|
{
|
|
54
72
|
provider: 'claude',
|
|
55
|
-
|
|
73
|
+
// Same scope rule as codex: configured providers keep their root grant
|
|
74
|
+
// even when a narrow discovery does not scan them (#3402).
|
|
75
|
+
roots: savedRoots || (!expectedProviders.includes('claude') && !providerScope.configuredProviders?.includes('claude'))
|
|
76
|
+
? [] : await claudeDiscoveryRoots(providerHomes, workspacePaths, workspacePath, registry, exclusions),
|
|
56
77
|
},
|
|
57
78
|
{
|
|
58
79
|
provider: 'codex',
|
|
59
|
-
roots:
|
|
60
|
-
|
|
80
|
+
roots: savedRoots || (!expectedProviders.includes('codex') && !providerScope.configuredProviders?.includes('codex')
|
|
81
|
+
&& !explicitlyRootedProviders.includes('codex'))
|
|
82
|
+
? [] : await codexDiscoveryRoots({ ...options, codexRoots: [
|
|
83
|
+
...(config?.sessions?.codex_roots ?? []).map(root => resolve(workspacePath, root)),
|
|
84
|
+
...(options.codexRoots ?? []),
|
|
85
|
+
] }),
|
|
61
86
|
},
|
|
62
87
|
{
|
|
63
88
|
provider: 'cursor',
|
|
@@ -85,11 +110,22 @@ export async function discoverWorkspaceHistories(options) {
|
|
|
85
110
|
: providerHomes.map((providerHome) => join(providerHome, '.pi', 'agent', 'sessions'))).flatMap((sessionsRoot) => workspacePaths.map((workspace) => join(sessionsRoot, `--${workspaceKey(workspace, false)}--`))),
|
|
86
111
|
},
|
|
87
112
|
];
|
|
88
|
-
const authorizedRoots = { ...
|
|
113
|
+
const authorizedRoots = { ...savedRoots };
|
|
89
114
|
const reports = new Map();
|
|
90
115
|
const candidates = [];
|
|
91
116
|
const seen = new Set();
|
|
92
117
|
for (const entry of discoverable) {
|
|
118
|
+
// A hand-run narrow scope must not discard explicit root grants, including
|
|
119
|
+
// configured providers under an authorized home. Do not scan their histories.
|
|
120
|
+
if (!savedRoots && !expectedProviders.includes(entry.provider)
|
|
121
|
+
&& (explicitlyRootedProviders.includes(entry.provider)
|
|
122
|
+
|| (providerScope.configuredProviders?.includes(entry.provider) && options.providerHome))) {
|
|
123
|
+
const roots = [];
|
|
124
|
+
for (const root of entry.roots)
|
|
125
|
+
if (await pathExists(root))
|
|
126
|
+
roots.push(await canonicalPath(root));
|
|
127
|
+
authorizedRoots[entry.provider] = [...new Set(roots)];
|
|
128
|
+
}
|
|
93
129
|
if (!expectedProviders.includes(entry.provider) || !supportedProviders.has(entry.provider))
|
|
94
130
|
continue;
|
|
95
131
|
const adapter = registry.get(entry.provider);
|
|
@@ -97,10 +133,10 @@ export async function discoverWorkspaceHistories(options) {
|
|
|
97
133
|
continue;
|
|
98
134
|
const availableRoots = [];
|
|
99
135
|
const rootCounts = new Map();
|
|
100
|
-
for (const root of
|
|
136
|
+
for (const root of savedRoots ? savedRoots[entry.provider] ?? [] : entry.roots) {
|
|
101
137
|
if (await pathExists(root)) {
|
|
102
138
|
const canonical = await canonicalPath(root);
|
|
103
|
-
if (
|
|
139
|
+
if (savedRoots && canonical !== root) {
|
|
104
140
|
throw new SessionContractError('INVALID_ARGUMENT', 'previously authorized provider root changed; rediscover with explicit authorization');
|
|
105
141
|
}
|
|
106
142
|
if (!availableRoots.includes(canonical)) {
|
|
@@ -198,9 +234,12 @@ export async function discoverWorkspaceHistories(options) {
|
|
|
198
234
|
const authorizedRoot = scope.allowedRoots.find((root) => locator === root || locator.startsWith(`${root}/`));
|
|
199
235
|
if (!authorizedRoot)
|
|
200
236
|
continue;
|
|
201
|
-
if (entry.provider === 'claude'
|
|
202
|
-
|
|
203
|
-
|
|
237
|
+
if (entry.provider === 'claude') {
|
|
238
|
+
const exactProject = workspacePaths.some(path => basename(authorizedRoot) === claudeProjectKey(path));
|
|
239
|
+
workspaceMatch = await claudeSourceWorkspaceMatch(locator, workspacePath, exactProject);
|
|
240
|
+
if (!workspaceMatch)
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
204
243
|
const fingerprint = await fingerprintSourceFile({
|
|
205
244
|
selectedPath: locator,
|
|
206
245
|
allowedRoots: [authorizedRoot],
|
|
@@ -274,7 +313,7 @@ export async function discoverWorkspaceHistories(options) {
|
|
|
274
313
|
const sources = candidates.sort(compareSources);
|
|
275
314
|
const providers = [...reports.values()].sort((a, b) => a.provider.localeCompare(b.provider));
|
|
276
315
|
const preflight = coveragePreflight({
|
|
277
|
-
|
|
316
|
+
...providerScope,
|
|
278
317
|
registry, runtime: options.runtime, reports: providers,
|
|
279
318
|
});
|
|
280
319
|
const identity = {
|
|
@@ -512,7 +551,40 @@ function asObject(value) {
|
|
|
512
551
|
async function canonicalPath(path) {
|
|
513
552
|
return realpath(resolve(path));
|
|
514
553
|
}
|
|
515
|
-
|
|
554
|
+
/** Claude keys are lossy: use transcript cwd, never decode hyphens into paths. */
|
|
555
|
+
async function claudeDiscoveryRoots(homes, paths, workspace, registry, exclusions) {
|
|
556
|
+
const roots = homes.flatMap(home => paths.map(path => join(home, '.claude', 'projects', claudeProjectKey(path))));
|
|
557
|
+
const adapter = registry.get('claude');
|
|
558
|
+
for (const home of homes) {
|
|
559
|
+
const projects = join(home, '.claude', 'projects');
|
|
560
|
+
let entries;
|
|
561
|
+
try {
|
|
562
|
+
entries = await readdir(projects, { withFileTypes: true });
|
|
563
|
+
}
|
|
564
|
+
catch (error) {
|
|
565
|
+
if (['ENOENT', 'ENOTDIR'].includes(error.code ?? ''))
|
|
566
|
+
continue;
|
|
567
|
+
throw error;
|
|
568
|
+
}
|
|
569
|
+
for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
570
|
+
if (!entry.isDirectory())
|
|
571
|
+
continue;
|
|
572
|
+
const root = join(projects, entry.name);
|
|
573
|
+
if (roots.includes(root))
|
|
574
|
+
continue;
|
|
575
|
+
for await (const descriptor of adapter.discover({ workspaceId: workspace, allowedRoots: [root] })) {
|
|
576
|
+
if (matchImportExclusion(descriptor, exclusions))
|
|
577
|
+
continue;
|
|
578
|
+
if (await claudeSourceWorkspaceMatch(descriptor.locator, workspace, false)) {
|
|
579
|
+
roots.push(root);
|
|
580
|
+
break;
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
return [...new Set(roots)];
|
|
586
|
+
}
|
|
587
|
+
async function claudeSourceWorkspaceMatch(locator, workspacePath, exactProject) {
|
|
516
588
|
let count = 0;
|
|
517
589
|
for await (const { line } of streamJsonLines(locator, { end: 1024 * 1024 - 1 })) {
|
|
518
590
|
if (++count > 50)
|
|
@@ -527,15 +599,10 @@ async function claudeSourceMatchesWorkspace(locator, workspacePath) {
|
|
|
527
599
|
const cwd = asObject(value).cwd;
|
|
528
600
|
if (typeof cwd !== 'string' || cwd.length === 0)
|
|
529
601
|
continue;
|
|
530
|
-
|
|
531
|
-
return await canonicalPath(cwd) === workspacePath;
|
|
532
|
-
}
|
|
533
|
-
catch {
|
|
534
|
-
return resolve(cwd) === workspacePath;
|
|
535
|
-
}
|
|
602
|
+
return matchSessionWorkspace(cwd, workspacePath);
|
|
536
603
|
}
|
|
537
|
-
//
|
|
538
|
-
return
|
|
604
|
+
// Only exact project keys can establish ownership without cwd.
|
|
605
|
+
return exactProject ? 'exact' : null;
|
|
539
606
|
}
|
|
540
607
|
export function claudeProjectKey(path) {
|
|
541
608
|
// Claude Code 2.1.302 replaces each character, then caps keys at 200
|