@gobing-ai/ts-llm-jsonl-importer 0.4.59 → 0.4.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/cancellation.d.ts +9 -0
- package/dist/cancellation.d.ts.map +1 -0
- package/dist/cancellation.js +13 -0
- package/dist/errors.d.ts +10 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +14 -0
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +42 -22
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/opencode-importer.d.ts +8 -0
- package/dist/opencode-importer.d.ts.map +1 -1
- package/dist/opencode-importer.js +12 -0
- package/dist/schema-sql.d.ts +1 -1
- package/dist/schema-sql.js +1 -1
- package/dist/types.d.ts +11 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/cancellation.ts +14 -0
- package/src/errors.ts +16 -0
- package/src/importer.ts +43 -23
- package/src/index.ts +1 -1
- package/src/opencode-importer.ts +20 -0
- package/src/schema-sql.ts +1 -1
- package/src/types.ts +11 -0
package/README.md
CHANGED
|
@@ -69,6 +69,32 @@ values, same `ImportResult`).
|
|
|
69
69
|
When `readFileStream` is unavailable (e.g. Cloudflare Workers), the importer falls back to
|
|
70
70
|
`readFile` + split transparently.
|
|
71
71
|
|
|
72
|
+
## Cancellation
|
|
73
|
+
|
|
74
|
+
Both entry points accept a cooperative `signal` (`AbortSignal`, feature A21 / ADR-112):
|
|
75
|
+
|
|
76
|
+
```ts
|
|
77
|
+
const controller = new AbortController();
|
|
78
|
+
const resultPromise = runJsonlImport('codex', {
|
|
79
|
+
db,
|
|
80
|
+
roots: ['./agent-history'],
|
|
81
|
+
mode: 'incremental',
|
|
82
|
+
signal: controller.signal,
|
|
83
|
+
});
|
|
84
|
+
controller.abort(); // cancel from a watchdog, shutdown hook, or deadline
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Cancellation is cooperative: the signal is checked at safe boundaries — before schema and
|
|
88
|
+
checkpoint writes, before each source file, and between bounded batches — never inside a
|
|
89
|
+
transaction. A batch already in flight settles first (its record, ledger, and checkpoint writes
|
|
90
|
+
commit atomically), then the run rejects with `ImportCancelledError` (a `HistoryImportError`).
|
|
91
|
+
No invocation-owned write happens after settlement, and incremental resume continues from the
|
|
92
|
+
last committed checkpoint; a cancelled mid-file run never arms the file-identity short-circuit.
|
|
93
|
+
`runOpenCodeImport` discards queued-but-unissued operations when cancelled, so the OpenCode
|
|
94
|
+
store stays authoritative for the next run. A cancellation cannot interrupt a synchronous SQLite
|
|
95
|
+
call mid-flight — the containing process remains the hard fallback for blocking work. Omitting
|
|
96
|
+
`signal` preserves existing behavior.
|
|
97
|
+
|
|
72
98
|
## Import Specific Files
|
|
73
99
|
|
|
74
100
|
```ts
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cooperative cancellation boundary (feature A21 / ADR-112). Call it only at safe
|
|
3
|
+
* boundaries — before starting further asynchronous work or writes — so any in-flight
|
|
4
|
+
* batch settles first and the public promise can never reject ahead of its own writes.
|
|
5
|
+
* Synchronous SQLite work cannot be interrupted here; the containing process remains
|
|
6
|
+
* the hard fallback for blocking work.
|
|
7
|
+
*/
|
|
8
|
+
export declare function throwIfImportAborted(signal: AbortSignal | undefined): void;
|
|
9
|
+
//# sourceMappingURL=cancellation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cancellation.d.ts","sourceRoot":"","sources":["../src/cancellation.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,IAAI,CAI1E"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { ImportCancelledError } from './errors.js';
|
|
2
|
+
/**
|
|
3
|
+
* Cooperative cancellation boundary (feature A21 / ADR-112). Call it only at safe
|
|
4
|
+
* boundaries — before starting further asynchronous work or writes — so any in-flight
|
|
5
|
+
* batch settles first and the public promise can never reject ahead of its own writes.
|
|
6
|
+
* Synchronous SQLite work cannot be interrupted here; the containing process remains
|
|
7
|
+
* the hard fallback for blocking work.
|
|
8
|
+
*/
|
|
9
|
+
export function throwIfImportAborted(signal) {
|
|
10
|
+
if (signal?.aborted) {
|
|
11
|
+
throw new ImportCancelledError('Import cancelled by abort signal', signal.reason);
|
|
12
|
+
}
|
|
13
|
+
}
|
package/dist/errors.d.ts
CHANGED
|
@@ -3,4 +3,14 @@ export declare class HistoryImportError extends Error {
|
|
|
3
3
|
readonly details?: unknown | undefined;
|
|
4
4
|
constructor(message: string, details?: unknown | undefined);
|
|
5
5
|
}
|
|
6
|
+
/**
|
|
7
|
+
* Error raised when a run is cancelled through its {@link ImportOptions.signal} abort signal
|
|
8
|
+
* (feature A21 / ADR-112). Thrown only after in-flight transaction and checkpoint work has
|
|
9
|
+
* settled, so catching it means "cancellation settled — no further writes from this
|
|
10
|
+
* invocation". Incremental resume continues from the last committed checkpoint.
|
|
11
|
+
*/
|
|
12
|
+
export declare class ImportCancelledError extends HistoryImportError {
|
|
13
|
+
readonly abortReason?: unknown | undefined;
|
|
14
|
+
constructor(message?: string, abortReason?: unknown | undefined);
|
|
15
|
+
}
|
|
6
16
|
//# sourceMappingURL=errors.d.ts.map
|
package/dist/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,qBAAa,kBAAmB,SAAQ,KAAK;IAGrC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO;gBAD1B,OAAO,EAAE,MAAM,EACN,OAAO,CAAC,EAAE,OAAO,YAAA;CAKjC"}
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,qBAAa,kBAAmB,SAAQ,KAAK;IAGrC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO;gBAD1B,OAAO,EAAE,MAAM,EACN,OAAO,CAAC,EAAE,OAAO,YAAA;CAKjC;AAED;;;;;GAKG;AACH,qBAAa,oBAAqB,SAAQ,kBAAkB;IAGpD,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO;gBAD9B,OAAO,SAAqC,EACnC,WAAW,CAAC,EAAE,OAAO,YAAA;CAKrC"}
|
package/dist/errors.js
CHANGED
|
@@ -7,3 +7,17 @@ export class HistoryImportError extends Error {
|
|
|
7
7
|
this.name = 'HistoryImportError';
|
|
8
8
|
}
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* Error raised when a run is cancelled through its {@link ImportOptions.signal} abort signal
|
|
12
|
+
* (feature A21 / ADR-112). Thrown only after in-flight transaction and checkpoint work has
|
|
13
|
+
* settled, so catching it means "cancellation settled — no further writes from this
|
|
14
|
+
* invocation". Incremental resume continues from the last committed checkpoint.
|
|
15
|
+
*/
|
|
16
|
+
export class ImportCancelledError extends HistoryImportError {
|
|
17
|
+
abortReason;
|
|
18
|
+
constructor(message = 'Import cancelled by abort signal', abortReason) {
|
|
19
|
+
super(message, abortReason);
|
|
20
|
+
this.abortReason = abortReason;
|
|
21
|
+
this.name = 'ImportCancelledError';
|
|
22
|
+
}
|
|
23
|
+
}
|
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA8ZrH"}
|
package/dist/importer.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
|
|
2
|
+
import { throwIfImportAborted } from './cancellation.js';
|
|
2
3
|
import { sha256 } from './hash.js';
|
|
3
4
|
import { applyHistoryImportSchema, checkpointUpsertOp, codexUsageAttributionUpdateOp, ensureTargetTable, ledgerExistingHashes, ledgerInsertOp, loadSourceCheckpoints, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, } from './jsonl-importer-dao.js';
|
|
4
5
|
import { claudeToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers.js';
|
|
@@ -48,12 +49,18 @@ function attachableDuration(timing, messageTsMs) {
|
|
|
48
49
|
* @param options - Importer options (database, roots/files, mode, etc.).
|
|
49
50
|
*/
|
|
50
51
|
export async function runJsonlImport(source, options) {
|
|
52
|
+
// Cancellation boundary (feature A21): before ANY invocation-owned write, including the
|
|
53
|
+
// schema setup below. An already-aborted signal must leave the database untouched.
|
|
54
|
+
throwIfImportAborted(options.signal);
|
|
51
55
|
const definition = resolveSourceDefinition(source);
|
|
52
56
|
const resolvedSource = definition.source;
|
|
53
57
|
const fileSystem = options.fileSystem ?? createNodeFileSystem();
|
|
54
58
|
await applyHistoryImportSchema(options.db);
|
|
55
59
|
const mode = options.mode ?? 'incremental';
|
|
56
60
|
const files = await discoverFiles(definition, options.roots, options.files, fileSystem, options.paths);
|
|
61
|
+
// Cancellation boundary: file discovery can walk many roots; a signal firing during it
|
|
62
|
+
// must stop before the full-mode checkpoint reset (an invocation-owned write).
|
|
63
|
+
throwIfImportAborted(options.signal);
|
|
57
64
|
if (mode === 'full' && !options.dryRun) {
|
|
58
65
|
await resetCheckpoints(options.db, resolvedSource, files);
|
|
59
66
|
}
|
|
@@ -80,6 +87,8 @@ export async function runJsonlImport(source, options) {
|
|
|
80
87
|
// Only incremental mode consults checkpoints; full/force-file read everything.
|
|
81
88
|
const checkpoints = mode === 'incremental' ? await loadSourceCheckpoints(options.db, resolvedSource) : undefined;
|
|
82
89
|
for (const file of files) {
|
|
90
|
+
// Cancellation boundary: stop before starting another file's work.
|
|
91
|
+
throwIfImportAborted(options.signal);
|
|
83
92
|
const entry = checkpoints?.get(file);
|
|
84
93
|
const checkpoint = entry?.line ?? 0;
|
|
85
94
|
if (checkpoints !== undefined && entry !== undefined && entry.size !== null && entry.mtimeMs !== null) {
|
|
@@ -93,6 +102,10 @@ export async function runJsonlImport(source, options) {
|
|
|
93
102
|
}
|
|
94
103
|
let lineNumber = 0;
|
|
95
104
|
for await (const rawLine of readLines(fileSystem, file)) {
|
|
105
|
+
// Cancellation boundary between bounded per-line batches: a batch already in
|
|
106
|
+
// flight settled above (its record + ledger + checkpoint writes commit
|
|
107
|
+
// atomically), so rejecting here never leaves a partial line behind.
|
|
108
|
+
throwIfImportAborted(options.signal);
|
|
96
109
|
lineNumber += 1;
|
|
97
110
|
const line = rawLine.trim();
|
|
98
111
|
if (line.length === 0 || lineNumber <= checkpoint)
|
|
@@ -310,31 +323,38 @@ export async function runJsonlImport(source, options) {
|
|
|
310
323
|
checkpointUpdates += 1;
|
|
311
324
|
}
|
|
312
325
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
326
|
+
}
|
|
327
|
+
// 0675 R1/R4 (relocated from the per-line loop for feature A21): stamp file identity
|
|
328
|
+
// once per READ file, only after the file's line loop COMPLETED. A cancelled run
|
|
329
|
+
// throws at the per-line boundary above and must not arm the identity short-circuit:
|
|
330
|
+
// a mid-file checkpoint carrying full-file identity would make the next incremental
|
|
331
|
+
// run skip the whole file and silently lose the un-imported tail. "Once per read file
|
|
332
|
+
// (never for skipped ones)" is also what self-heals legacy rows whose stored identity
|
|
333
|
+
// is NULL — without a stamp a fully-imported database never arms the short-circuit.
|
|
334
|
+
// Skipped when nothing changed since the stored checkpoint to keep no-op runs
|
|
335
|
+
// write-free. Degraded entries (pre-migration database without identity columns,
|
|
336
|
+
// 0678 fail-open) keep the line-only write shape via checkpointUpsertOp's optional
|
|
337
|
+
// identity.
|
|
338
|
+
if (!options.dryRun && mode === 'incremental') {
|
|
339
|
+
const statAfter = await safeStat(fileSystem, file);
|
|
340
|
+
// Degraded = a STORED entry without identity (pre-migration database, 0678
|
|
341
|
+
// fail-open). No stored entry means first-ever import: stamp normally.
|
|
342
|
+
const identityDegraded = entry !== undefined && entry.size === null && entry.mtimeMs === null;
|
|
343
|
+
const identityChanged = entry !== undefined &&
|
|
344
|
+
(entry.size !== (statAfter?.size ?? null) || entry.mtimeMs !== (statAfter?.mtimeMs ?? null));
|
|
345
|
+
if (statAfter !== null && !identityDegraded && (entry === undefined || identityChanged)) {
|
|
346
|
+
await options.db.batch([
|
|
347
|
+
checkpointUpsertOp(resolvedSource, file, Math.max(lineNumber, checkpoint), options.now, {
|
|
348
|
+
size: statAfter.size,
|
|
349
|
+
mtimeMs: statAfter.mtimeMs,
|
|
350
|
+
}),
|
|
351
|
+
]);
|
|
352
|
+
checkpointUpdates += 1;
|
|
335
353
|
}
|
|
336
354
|
}
|
|
337
355
|
}
|
|
356
|
+
// Cancellation boundary: stop before the full-mode reconciliation deletes.
|
|
357
|
+
throwIfImportAborted(options.signal);
|
|
338
358
|
let reconciliation;
|
|
339
359
|
if (mode === 'full') {
|
|
340
360
|
reconciliation = await reconcileFullImport(options.db, resolvedSource, desiredHashes, files, options.dryRun ?? false);
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { DERIVED_DURATION_CEILING_MS, type DeriveAssistantDurationsResult, DURATION_SOURCE_DERIVED, deriveAssistantDurations, } from './assistant-duration';
|
|
2
|
-
export { HistoryImportError } from './errors';
|
|
2
|
+
export { HistoryImportError, ImportCancelledError } from './errors';
|
|
3
3
|
export { sha256, stableJson } from './hash';
|
|
4
4
|
export { runJsonlImport } from './importer';
|
|
5
5
|
export { applyHistoryImportSchema, normalizeSourceFilePaths, TYPED_HISTORY_TABLES } from './jsonl-importer-dao';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,2BAA2B,EAC3B,KAAK,8BAA8B,EACnC,uBAAuB,EACvB,wBAAwB,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,2BAA2B,EAC3B,KAAK,8BAA8B,EACnC,uBAAuB,EACvB,wBAAwB,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACpE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAChH,OAAO,EACH,qBAAqB,EACrB,iBAAiB,EACjB,KAAK,iBAAiB,EACtB,uBAAuB,EACvB,cAAc,GACjB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,6BAA6B,EAAE,MAAM,cAAc,CAAC;AACxF,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAC7E,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,oBAAoB,EACpB,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { DERIVED_DURATION_CEILING_MS, DURATION_SOURCE_DERIVED, deriveAssistantDurations, } from './assistant-duration.js';
|
|
2
|
-
export { HistoryImportError } from './errors.js';
|
|
2
|
+
export { HistoryImportError, ImportCancelledError } from './errors.js';
|
|
3
3
|
export { sha256, stableJson } from './hash.js';
|
|
4
4
|
export { runJsonlImport } from './importer.js';
|
|
5
5
|
export { applyHistoryImportSchema, normalizeSourceFilePaths, TYPED_HISTORY_TABLES } from './jsonl-importer-dao.js';
|
|
@@ -11,6 +11,14 @@ export interface OpenCodeImportOptions {
|
|
|
11
11
|
dryRun?: boolean;
|
|
12
12
|
redactionRules?: readonly RedactionRule[];
|
|
13
13
|
now?: () => Date;
|
|
14
|
+
/**
|
|
15
|
+
* Cooperative cancellation signal (feature A21 / ADR-112). Checked before the schema
|
|
16
|
+
* write, between source pages, and before the single settlement batch. Work already
|
|
17
|
+
* queued but not yet issued is discarded (the OpenCode store stays authoritative and a
|
|
18
|
+
* later run re-reads it); a batch already in flight settles before the run rejects with
|
|
19
|
+
* {@link ImportCancelledError}. Omitting the signal preserves existing behavior.
|
|
20
|
+
*/
|
|
21
|
+
signal?: AbortSignal;
|
|
14
22
|
}
|
|
15
23
|
/** Import OpenCode's current SQLite history store into the forensic contract tables. */
|
|
16
24
|
export declare function runOpenCodeImport(options: OpenCodeImportOptions): Promise<ImportResult>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAiB/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACxB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAiJ7F"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createDbAdapter } from '@gobing-ai/ts-db';
|
|
2
2
|
import { ambientRuntimePaths, createNodeFileSystem, joinPath, } from '@gobing-ai/ts-runtime';
|
|
3
|
+
import { throwIfImportAborted } from './cancellation.js';
|
|
3
4
|
import { sha256 } from './hash.js';
|
|
4
5
|
import { applyHistoryImportSchema, deleteCheckpointOperation, openCodeBulkWriteOperations, openCodeDeleteOperations, readOpenCodeExistingEntries, readOpenCodeMessages, readOpenCodeParts, } from './jsonl-importer-dao.js';
|
|
5
6
|
import { maybeArgsRaw } from './mappers.js';
|
|
@@ -8,6 +9,9 @@ const SOURCE = 'opencode';
|
|
|
8
9
|
const PAGE_SIZE = 250;
|
|
9
10
|
/** Import OpenCode's current SQLite history store into the forensic contract tables. */
|
|
10
11
|
export async function runOpenCodeImport(options) {
|
|
12
|
+
// Cancellation boundary (feature A21): before ANY invocation-owned write, including the
|
|
13
|
+
// schema setup below. An already-aborted signal must leave both databases untouched.
|
|
14
|
+
throwIfImportAborted(options.signal);
|
|
11
15
|
const paths = options.paths ?? ambientRuntimePaths();
|
|
12
16
|
const sourceDatabase = options.sourceDatabase ?? joinPath(paths.home, '.local/share/opencode/opencode.db');
|
|
13
17
|
const fileSystem = options.fileSystem ?? createNodeFileSystem();
|
|
@@ -41,6 +45,10 @@ export async function runOpenCodeImport(options) {
|
|
|
41
45
|
let reconciliation;
|
|
42
46
|
try {
|
|
43
47
|
while (true) {
|
|
48
|
+
// Cancellation boundary between bounded pages: stop before reading/queueing
|
|
49
|
+
// further work. Operations queued but not yet issued are discarded below —
|
|
50
|
+
// nothing was written, so the OpenCode store stays authoritative for resume.
|
|
51
|
+
throwIfImportAborted(options.signal);
|
|
44
52
|
const messages = await readOpenCodeMessages(sourceDb, lastTime, lastId, PAGE_SIZE);
|
|
45
53
|
if (messages.length === 0)
|
|
46
54
|
break;
|
|
@@ -111,6 +119,10 @@ export async function runOpenCodeImport(options) {
|
|
|
111
119
|
}
|
|
112
120
|
reconciliation = { staleTargetRows, staleLedgerRows, staleCheckpointRows };
|
|
113
121
|
}
|
|
122
|
+
// Cancellation boundary before the single settlement batch: everything queued above
|
|
123
|
+
// is discarded (never issued), so a cancelled run writes nothing and the checkpoint
|
|
124
|
+
// stays where the previous run left it.
|
|
125
|
+
throwIfImportAborted(options.signal);
|
|
114
126
|
if (operations.length > 0)
|
|
115
127
|
await options.db.batch(operations);
|
|
116
128
|
}
|
package/dist/schema-sql.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
5
|
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
6
|
*/
|
|
7
|
-
export declare const HISTORY_IMPORT_SCHEMA_VERSION = "0.4.
|
|
7
|
+
export declare const HISTORY_IMPORT_SCHEMA_VERSION = "0.4.60";
|
|
8
8
|
/**
|
|
9
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
10
10
|
*
|
package/dist/schema-sql.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
5
|
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
6
|
*/
|
|
7
|
-
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.60';
|
|
8
8
|
/**
|
|
9
9
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
|
10
10
|
*
|
package/dist/types.d.ts
CHANGED
|
@@ -70,6 +70,17 @@ export interface ImportOptions {
|
|
|
70
70
|
readonly dryRun?: boolean;
|
|
71
71
|
readonly redactionRules?: readonly RedactionRule[];
|
|
72
72
|
readonly now?: () => Date;
|
|
73
|
+
/**
|
|
74
|
+
* Cooperative cancellation signal (feature A21 / ADR-112). Checked at safe boundaries —
|
|
75
|
+
* before schema and checkpoint writes, before each source file, and between bounded
|
|
76
|
+
* per-line batches — never inside a transaction: a batch already in flight settles first
|
|
77
|
+
* (its record, ledger, and checkpoint writes commit atomically), then the run rejects
|
|
78
|
+
* with {@link ImportCancelledError}. No invocation-owned write happens after settlement,
|
|
79
|
+
* and incremental resume continues from the last committed checkpoint. A cancellation
|
|
80
|
+
* cannot interrupt a synchronous SQLite call mid-flight — the containing process remains
|
|
81
|
+
* the hard fallback for blocking work. Omitting the signal preserves existing behavior.
|
|
82
|
+
*/
|
|
83
|
+
readonly signal?: AbortSignal;
|
|
73
84
|
/**
|
|
74
85
|
* Optional cwd/home anchor (ADR-023 A1 / task 0042). When set, registry `defaultRoots`
|
|
75
86
|
* resolve against `paths.home` instead of the ambient working directory; explicit
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,4FAA4F;IAC5F,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,qEAAqE;AACrE,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;;;;;;OASG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,4FAA4F;IAC5F,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,qEAAqE;AACrE,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.60",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.60",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.60",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.60",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { ImportCancelledError } from './errors';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Cooperative cancellation boundary (feature A21 / ADR-112). Call it only at safe
|
|
5
|
+
* boundaries — before starting further asynchronous work or writes — so any in-flight
|
|
6
|
+
* batch settles first and the public promise can never reject ahead of its own writes.
|
|
7
|
+
* Synchronous SQLite work cannot be interrupted here; the containing process remains
|
|
8
|
+
* the hard fallback for blocking work.
|
|
9
|
+
*/
|
|
10
|
+
export function throwIfImportAborted(signal: AbortSignal | undefined): void {
|
|
11
|
+
if (signal?.aborted) {
|
|
12
|
+
throw new ImportCancelledError('Import cancelled by abort signal', signal.reason);
|
|
13
|
+
}
|
|
14
|
+
}
|
package/src/errors.ts
CHANGED
|
@@ -8,3 +8,19 @@ export class HistoryImportError extends Error {
|
|
|
8
8
|
this.name = 'HistoryImportError';
|
|
9
9
|
}
|
|
10
10
|
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Error raised when a run is cancelled through its {@link ImportOptions.signal} abort signal
|
|
14
|
+
* (feature A21 / ADR-112). Thrown only after in-flight transaction and checkpoint work has
|
|
15
|
+
* settled, so catching it means "cancellation settled — no further writes from this
|
|
16
|
+
* invocation". Incremental resume continues from the last committed checkpoint.
|
|
17
|
+
*/
|
|
18
|
+
export class ImportCancelledError extends HistoryImportError {
|
|
19
|
+
constructor(
|
|
20
|
+
message = 'Import cancelled by abort signal',
|
|
21
|
+
readonly abortReason?: unknown,
|
|
22
|
+
) {
|
|
23
|
+
super(message, abortReason);
|
|
24
|
+
this.name = 'ImportCancelledError';
|
|
25
|
+
}
|
|
26
|
+
}
|
package/src/importer.ts
CHANGED
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
resolvePath,
|
|
8
8
|
walkDir,
|
|
9
9
|
} from '@gobing-ai/ts-runtime';
|
|
10
|
+
import { throwIfImportAborted } from './cancellation';
|
|
10
11
|
import { sha256 } from './hash';
|
|
11
12
|
import {
|
|
12
13
|
applyHistoryImportSchema,
|
|
@@ -127,6 +128,9 @@ function attachableDuration(
|
|
|
127
128
|
* @param options - Importer options (database, roots/files, mode, etc.).
|
|
128
129
|
*/
|
|
129
130
|
export async function runJsonlImport(source: string | SourceDefinition, options: ImportOptions): Promise<ImportResult> {
|
|
131
|
+
// Cancellation boundary (feature A21): before ANY invocation-owned write, including the
|
|
132
|
+
// schema setup below. An already-aborted signal must leave the database untouched.
|
|
133
|
+
throwIfImportAborted(options.signal);
|
|
130
134
|
const definition = resolveSourceDefinition(source);
|
|
131
135
|
const resolvedSource = definition.source;
|
|
132
136
|
const fileSystem = options.fileSystem ?? createNodeFileSystem();
|
|
@@ -134,6 +138,9 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
134
138
|
|
|
135
139
|
const mode = options.mode ?? 'incremental';
|
|
136
140
|
const files = await discoverFiles(definition, options.roots, options.files, fileSystem, options.paths);
|
|
141
|
+
// Cancellation boundary: file discovery can walk many roots; a signal firing during it
|
|
142
|
+
// must stop before the full-mode checkpoint reset (an invocation-owned write).
|
|
143
|
+
throwIfImportAborted(options.signal);
|
|
137
144
|
if (mode === 'full' && !options.dryRun) {
|
|
138
145
|
await resetCheckpoints(options.db, resolvedSource, files);
|
|
139
146
|
}
|
|
@@ -165,6 +172,8 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
165
172
|
const checkpoints = mode === 'incremental' ? await loadSourceCheckpoints(options.db, resolvedSource) : undefined;
|
|
166
173
|
|
|
167
174
|
for (const file of files) {
|
|
175
|
+
// Cancellation boundary: stop before starting another file's work.
|
|
176
|
+
throwIfImportAborted(options.signal);
|
|
168
177
|
const entry = checkpoints?.get(file);
|
|
169
178
|
const checkpoint = entry?.line ?? 0;
|
|
170
179
|
if (checkpoints !== undefined && entry !== undefined && entry.size !== null && entry.mtimeMs !== null) {
|
|
@@ -178,6 +187,10 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
178
187
|
}
|
|
179
188
|
let lineNumber = 0;
|
|
180
189
|
for await (const rawLine of readLines(fileSystem, file)) {
|
|
190
|
+
// Cancellation boundary between bounded per-line batches: a batch already in
|
|
191
|
+
// flight settled above (its record + ledger + checkpoint writes commit
|
|
192
|
+
// atomically), so rejecting here never leaves a partial line behind.
|
|
193
|
+
throwIfImportAborted(options.signal);
|
|
181
194
|
lineNumber += 1;
|
|
182
195
|
const line = rawLine.trim();
|
|
183
196
|
if (line.length === 0 || lineNumber <= checkpoint) continue;
|
|
@@ -469,32 +482,39 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
469
482
|
checkpointUpdates += 1;
|
|
470
483
|
}
|
|
471
484
|
}
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
485
|
+
}
|
|
486
|
+
// 0675 R1/R4 (relocated from the per-line loop for feature A21): stamp file identity
|
|
487
|
+
// once per READ file, only after the file's line loop COMPLETED. A cancelled run
|
|
488
|
+
// throws at the per-line boundary above and must not arm the identity short-circuit:
|
|
489
|
+
// a mid-file checkpoint carrying full-file identity would make the next incremental
|
|
490
|
+
// run skip the whole file and silently lose the un-imported tail. "Once per read file
|
|
491
|
+
// (never for skipped ones)" is also what self-heals legacy rows whose stored identity
|
|
492
|
+
// is NULL — without a stamp a fully-imported database never arms the short-circuit.
|
|
493
|
+
// Skipped when nothing changed since the stored checkpoint to keep no-op runs
|
|
494
|
+
// write-free. Degraded entries (pre-migration database without identity columns,
|
|
495
|
+
// 0678 fail-open) keep the line-only write shape via checkpointUpsertOp's optional
|
|
496
|
+
// identity.
|
|
497
|
+
if (!options.dryRun && mode === 'incremental') {
|
|
498
|
+
const statAfter = await safeStat(fileSystem, file);
|
|
499
|
+
// Degraded = a STORED entry without identity (pre-migration database, 0678
|
|
500
|
+
// fail-open). No stored entry means first-ever import: stamp normally.
|
|
501
|
+
const identityDegraded = entry !== undefined && entry.size === null && entry.mtimeMs === null;
|
|
502
|
+
const identityChanged =
|
|
503
|
+
entry !== undefined &&
|
|
504
|
+
(entry.size !== (statAfter?.size ?? null) || entry.mtimeMs !== (statAfter?.mtimeMs ?? null));
|
|
505
|
+
if (statAfter !== null && !identityDegraded && (entry === undefined || identityChanged)) {
|
|
506
|
+
await options.db.batch([
|
|
507
|
+
checkpointUpsertOp(resolvedSource, file, Math.max(lineNumber, checkpoint), options.now, {
|
|
508
|
+
size: statAfter.size,
|
|
509
|
+
mtimeMs: statAfter.mtimeMs,
|
|
510
|
+
}),
|
|
511
|
+
]);
|
|
512
|
+
checkpointUpdates += 1;
|
|
495
513
|
}
|
|
496
514
|
}
|
|
497
515
|
}
|
|
516
|
+
// Cancellation boundary: stop before the full-mode reconciliation deletes.
|
|
517
|
+
throwIfImportAborted(options.signal);
|
|
498
518
|
let reconciliation: ReconcileSummary | undefined;
|
|
499
519
|
if (mode === 'full') {
|
|
500
520
|
reconciliation = await reconcileFullImport(
|
package/src/index.ts
CHANGED
|
@@ -4,7 +4,7 @@ export {
|
|
|
4
4
|
DURATION_SOURCE_DERIVED,
|
|
5
5
|
deriveAssistantDurations,
|
|
6
6
|
} from './assistant-duration';
|
|
7
|
-
export { HistoryImportError } from './errors';
|
|
7
|
+
export { HistoryImportError, ImportCancelledError } from './errors';
|
|
8
8
|
export { sha256, stableJson } from './hash';
|
|
9
9
|
export { runJsonlImport } from './importer';
|
|
10
10
|
export { applyHistoryImportSchema, normalizeSourceFilePaths, TYPED_HISTORY_TABLES } from './jsonl-importer-dao';
|
package/src/opencode-importer.ts
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
joinPath,
|
|
7
7
|
type RuntimePaths,
|
|
8
8
|
} from '@gobing-ai/ts-runtime';
|
|
9
|
+
import { throwIfImportAborted } from './cancellation';
|
|
9
10
|
import { sha256 } from './hash';
|
|
10
11
|
import {
|
|
11
12
|
applyHistoryImportSchema,
|
|
@@ -43,10 +44,21 @@ export interface OpenCodeImportOptions {
|
|
|
43
44
|
dryRun?: boolean;
|
|
44
45
|
redactionRules?: readonly RedactionRule[];
|
|
45
46
|
now?: () => Date;
|
|
47
|
+
/**
|
|
48
|
+
* Cooperative cancellation signal (feature A21 / ADR-112). Checked before the schema
|
|
49
|
+
* write, between source pages, and before the single settlement batch. Work already
|
|
50
|
+
* queued but not yet issued is discarded (the OpenCode store stays authoritative and a
|
|
51
|
+
* later run re-reads it); a batch already in flight settles before the run rejects with
|
|
52
|
+
* {@link ImportCancelledError}. Omitting the signal preserves existing behavior.
|
|
53
|
+
*/
|
|
54
|
+
signal?: AbortSignal;
|
|
46
55
|
}
|
|
47
56
|
|
|
48
57
|
/** Import OpenCode's current SQLite history store into the forensic contract tables. */
|
|
49
58
|
export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise<ImportResult> {
|
|
59
|
+
// Cancellation boundary (feature A21): before ANY invocation-owned write, including the
|
|
60
|
+
// schema setup below. An already-aborted signal must leave both databases untouched.
|
|
61
|
+
throwIfImportAborted(options.signal);
|
|
50
62
|
const paths = options.paths ?? ambientRuntimePaths();
|
|
51
63
|
const sourceDatabase = options.sourceDatabase ?? joinPath(paths.home, '.local/share/opencode/opencode.db');
|
|
52
64
|
const fileSystem = options.fileSystem ?? createNodeFileSystem();
|
|
@@ -82,6 +94,10 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
|
|
|
82
94
|
|
|
83
95
|
try {
|
|
84
96
|
while (true) {
|
|
97
|
+
// Cancellation boundary between bounded pages: stop before reading/queueing
|
|
98
|
+
// further work. Operations queued but not yet issued are discarded below —
|
|
99
|
+
// nothing was written, so the OpenCode store stays authoritative for resume.
|
|
100
|
+
throwIfImportAborted(options.signal);
|
|
85
101
|
const messages = await readOpenCodeMessages(sourceDb, lastTime, lastId, PAGE_SIZE);
|
|
86
102
|
if (messages.length === 0) break;
|
|
87
103
|
|
|
@@ -160,6 +176,10 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
|
|
|
160
176
|
}
|
|
161
177
|
reconciliation = { staleTargetRows, staleLedgerRows, staleCheckpointRows };
|
|
162
178
|
}
|
|
179
|
+
// Cancellation boundary before the single settlement batch: everything queued above
|
|
180
|
+
// is discarded (never issued), so a cancelled run writes nothing and the checkpoint
|
|
181
|
+
// stays where the previous run left it.
|
|
182
|
+
throwIfImportAborted(options.signal);
|
|
163
183
|
if (operations.length > 0) await options.db.batch(operations);
|
|
164
184
|
} finally {
|
|
165
185
|
sourceDb.close();
|
package/src/schema-sql.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Guaranteed to match the package version; a bump-or-fail test enforces that
|
|
5
5
|
* any modification to HISTORY_IMPORT_SCHEMA_SQL must be accompanied by a version bump.
|
|
6
6
|
*/
|
|
7
|
-
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.
|
|
7
|
+
export const HISTORY_IMPORT_SCHEMA_VERSION = '0.4.60';
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* DDL string that creates the history import checkpoint, ledger, and typed contract tables.
|
package/src/types.ts
CHANGED
|
@@ -93,6 +93,17 @@ export interface ImportOptions {
|
|
|
93
93
|
readonly dryRun?: boolean;
|
|
94
94
|
readonly redactionRules?: readonly RedactionRule[];
|
|
95
95
|
readonly now?: () => Date;
|
|
96
|
+
/**
|
|
97
|
+
* Cooperative cancellation signal (feature A21 / ADR-112). Checked at safe boundaries —
|
|
98
|
+
* before schema and checkpoint writes, before each source file, and between bounded
|
|
99
|
+
* per-line batches — never inside a transaction: a batch already in flight settles first
|
|
100
|
+
* (its record, ledger, and checkpoint writes commit atomically), then the run rejects
|
|
101
|
+
* with {@link ImportCancelledError}. No invocation-owned write happens after settlement,
|
|
102
|
+
* and incremental resume continues from the last committed checkpoint. A cancellation
|
|
103
|
+
* cannot interrupt a synchronous SQLite call mid-flight — the containing process remains
|
|
104
|
+
* the hard fallback for blocking work. Omitting the signal preserves existing behavior.
|
|
105
|
+
*/
|
|
106
|
+
readonly signal?: AbortSignal;
|
|
96
107
|
/**
|
|
97
108
|
* Optional cwd/home anchor (ADR-023 A1 / task 0042). When set, registry `defaultRoots`
|
|
98
109
|
* resolve against `paths.home` instead of the ambient working directory; explicit
|