@gobing-ai/ts-llm-jsonl-importer 0.4.39 → 0.4.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +60 -20
- package/dist/jsonl-importer-dao.d.ts +6 -1
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +14 -1
- package/dist/mappers.d.ts +19 -0
- package/dist/mappers.d.ts.map +1 -1
- package/dist/mappers.js +35 -0
- package/dist/opencode-importer.d.ts.map +1 -1
- package/dist/opencode-importer.js +2 -0
- package/dist/schema-sql.d.ts.map +1 -1
- package/dist/schema-sql.js +1 -0
- package/dist/sources.d.ts.map +1 -1
- package/dist/sources.js +7 -1
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/importer.ts +69 -22
- package/src/jsonl-importer-dao.ts +15 -0
- package/src/mappers.ts +46 -0
- package/src/opencode-importer.ts +2 -0
- package/src/schema-sql.ts +1 -0
- package/src/sources.ts +15 -9
- package/src/types.ts +9 -0
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAsTrH"}
|
package/dist/importer.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
|
|
2
2
|
import { sha256 } from './hash.js';
|
|
3
|
-
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, } from './jsonl-importer-dao.js';
|
|
4
|
-
import { ompToolResultTiming, timestampToEpochMs } from './mappers.js';
|
|
3
|
+
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, } from './jsonl-importer-dao.js';
|
|
4
|
+
import { claudeToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers.js';
|
|
5
5
|
import { redactRecord } from './redaction.js';
|
|
6
6
|
import { resolveSourceDefinition } from './sources.js';
|
|
7
7
|
const TOOL_CALL_KEY_SEP = '\u0000';
|
|
@@ -64,6 +64,7 @@ export async function runJsonlImport(source, options) {
|
|
|
64
64
|
let importedRecords = 0;
|
|
65
65
|
let skippedDuplicates = 0;
|
|
66
66
|
let unknownRecords = 0;
|
|
67
|
+
let skippedCorruptLines = 0;
|
|
67
68
|
let checkpointUpdates = 0;
|
|
68
69
|
// Full-mode desired set (R1, task 0504): every record hash the current source produces.
|
|
69
70
|
// ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
|
|
@@ -80,9 +81,12 @@ export async function runJsonlImport(source, options) {
|
|
|
80
81
|
if (line.length === 0 || lineNumber <= checkpoint)
|
|
81
82
|
continue;
|
|
82
83
|
processedLines += 1;
|
|
83
|
-
const raw = parseJsonLine(line, file, lineNumber, parseErrors);
|
|
84
|
-
if (raw === undefined)
|
|
84
|
+
const raw = parseJsonLine(line, file, lineNumber, parseErrors, definition.corruptLinePolicy);
|
|
85
|
+
if (raw === undefined) {
|
|
86
|
+
if (definition.corruptLinePolicy === 'skip')
|
|
87
|
+
skippedCorruptLines += 1;
|
|
85
88
|
continue;
|
|
89
|
+
}
|
|
86
90
|
const splitRecords = splitRawRecord(definition, raw, {
|
|
87
91
|
source: resolvedSource,
|
|
88
92
|
sourceFile: file,
|
|
@@ -136,13 +140,18 @@ export async function runJsonlImport(source, options) {
|
|
|
136
140
|
// re-imported line's row already exists and its duration UPDATE still targets
|
|
137
141
|
// the same deterministic record_hash. The assistant message timestamp is the
|
|
138
142
|
// fallback's start bound; every call in one message shares it.
|
|
139
|
-
if (definition.source === 'omp') {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
143
|
+
if (definition.source === 'omp' || definition.source === 'claude') {
|
|
144
|
+
// omp only: assistant-message timestamps are the fallback duration's
|
|
145
|
+
// start bound. claude attaches result_bytes only — no fallback
|
|
146
|
+
// duration exists natively (never-fabricate, 0624 R2).
|
|
147
|
+
const assistantTs = definition.source === 'omp' ? new Map() : null;
|
|
148
|
+
if (assistantTs !== null) {
|
|
149
|
+
for (const entry of prepared) {
|
|
150
|
+
if (entry.split.targetTable === 'history_message' &&
|
|
151
|
+
entry.normalized.role === 'assistant' &&
|
|
152
|
+
typeof entry.normalized.session_id === 'string') {
|
|
153
|
+
assistantTs.set(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`, timestampToEpochMs(entry.normalized.ts));
|
|
154
|
+
}
|
|
146
155
|
}
|
|
147
156
|
}
|
|
148
157
|
for (const entry of prepared) {
|
|
@@ -153,7 +162,7 @@ export async function runJsonlImport(source, options) {
|
|
|
153
162
|
typeof entry.normalized.session_id === 'string') {
|
|
154
163
|
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
155
164
|
recordHash: entry.recordHash,
|
|
156
|
-
messageTsMs: assistantTs
|
|
165
|
+
messageTsMs: assistantTs?.get(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`),
|
|
157
166
|
});
|
|
158
167
|
}
|
|
159
168
|
}
|
|
@@ -220,6 +229,27 @@ export async function runJsonlImport(source, options) {
|
|
|
220
229
|
}
|
|
221
230
|
}
|
|
222
231
|
}
|
|
232
|
+
// Task 0624 R2 (claude): attach result sizes from this line's
|
|
233
|
+
// tool_result blocks. Pairing is by `tool_use_id` → `call_id`
|
|
234
|
+
// (source ∘ session ∘ call), the same key the omp branch uses.
|
|
235
|
+
// No duration is written — claude has no native wall time and the
|
|
236
|
+
// guarded timestamp fallback does not extend to it (0624 Design).
|
|
237
|
+
if (definition.source === 'claude') {
|
|
238
|
+
const timing = claudeToolResultTiming(raw);
|
|
239
|
+
if (timing !== null) {
|
|
240
|
+
const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message');
|
|
241
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
242
|
+
if (typeof sessionId === 'string') {
|
|
243
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
244
|
+
if (pending === undefined) {
|
|
245
|
+
pending = await resolveToolCallRow(options.db, resolvedSource, sessionId, timing.toolCallId);
|
|
246
|
+
}
|
|
247
|
+
if (pending !== undefined) {
|
|
248
|
+
ops.push(toolCallResultBytesUpdateOp(pending.recordHash, timing.resultBytes));
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
223
253
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
224
254
|
// re-imported after a mid-batch crash.
|
|
225
255
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
|
@@ -244,6 +274,7 @@ export async function runJsonlImport(source, options) {
|
|
|
244
274
|
processedLines,
|
|
245
275
|
importedRecords,
|
|
246
276
|
skippedDuplicates,
|
|
277
|
+
skippedCorruptLines,
|
|
247
278
|
unknownRecords,
|
|
248
279
|
parseErrors,
|
|
249
280
|
validationErrors,
|
|
@@ -251,19 +282,28 @@ export async function runJsonlImport(source, options) {
|
|
|
251
282
|
reconciliation,
|
|
252
283
|
};
|
|
253
284
|
}
|
|
254
|
-
function parseJsonLine(line, sourceFile, sourceLine, parseErrors) {
|
|
285
|
+
function parseJsonLine(line, sourceFile, sourceLine, parseErrors, corruptLinePolicy = 'error') {
|
|
286
|
+
let parsed;
|
|
255
287
|
try {
|
|
256
|
-
|
|
257
|
-
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
258
|
-
parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
|
|
259
|
-
return undefined;
|
|
260
|
-
}
|
|
261
|
-
return value;
|
|
288
|
+
parsed = JSON.parse(line);
|
|
262
289
|
}
|
|
263
290
|
catch (error) {
|
|
264
|
-
|
|
291
|
+
if (corruptLinePolicy !== 'skip') {
|
|
292
|
+
parseErrors.push({
|
|
293
|
+
sourceFile,
|
|
294
|
+
sourceLine,
|
|
295
|
+
reason: error instanceof Error ? error.message : 'Invalid JSON',
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
return undefined;
|
|
299
|
+
}
|
|
300
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
301
|
+
if (corruptLinePolicy !== 'skip') {
|
|
302
|
+
parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
|
|
303
|
+
}
|
|
265
304
|
return undefined;
|
|
266
305
|
}
|
|
306
|
+
return parsed;
|
|
267
307
|
}
|
|
268
308
|
function splitRawRecord(definition, raw, context) {
|
|
269
309
|
const targetTable = targetTableFor(definition.targetTable);
|
|
@@ -54,6 +54,11 @@ export declare function checkpointUpsertOp(source: string, sourceFile: string, l
|
|
|
54
54
|
* two measurement paths stay distinguishable.
|
|
55
55
|
*/
|
|
56
56
|
declare function toolCallDurationUpdateOp(recordHash: string, startedAt: string | null, completedAt: string | null, durationMs: number | null): DbBatchOp;
|
|
57
|
+
/**
|
|
58
|
+
* Targeted UPDATE op attaching a result size to a previously-inserted
|
|
59
|
+
* `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
|
|
60
|
+
*/
|
|
61
|
+
declare function toolCallResultBytesUpdateOp(recordHash: string, resultBytes: number): DbBatchOp;
|
|
57
62
|
/**
|
|
58
63
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
59
64
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -136,5 +141,5 @@ export declare function deleteCheckpointOperation(source: string, sourceFile: st
|
|
|
136
141
|
/** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
|
|
137
142
|
export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
|
|
138
143
|
export type { CheckpointRow };
|
|
139
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
|
144
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };
|
|
140
145
|
//# sourceMappingURL=jsonl-importer-dao.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAwDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,CAOvF;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,2BAA2B,EAC3B,eAAe,GAClB,CAAC"}
|
|
@@ -27,6 +27,7 @@ const TYPED_TABLE_COLUMNS = {
|
|
|
27
27
|
'provenance',
|
|
28
28
|
'run_id',
|
|
29
29
|
'task_wbs',
|
|
30
|
+
'request_id',
|
|
30
31
|
'imported_at',
|
|
31
32
|
],
|
|
32
33
|
history_tool_call: [
|
|
@@ -212,6 +213,18 @@ function toolCallDurationUpdateOp(recordHash, startedAt, completedAt, durationMs
|
|
|
212
213
|
params: [startedAt, completedAt, durationMs, recordHash],
|
|
213
214
|
};
|
|
214
215
|
}
|
|
216
|
+
/**
|
|
217
|
+
* Targeted UPDATE op attaching a result size to a previously-inserted
|
|
218
|
+
* `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
|
|
219
|
+
*/
|
|
220
|
+
function toolCallResultBytesUpdateOp(recordHash, resultBytes) {
|
|
221
|
+
return {
|
|
222
|
+
sql: `UPDATE history_tool_call
|
|
223
|
+
SET result_bytes = ?
|
|
224
|
+
WHERE record_hash = ?`,
|
|
225
|
+
params: [resultBytes, recordHash],
|
|
226
|
+
};
|
|
227
|
+
}
|
|
215
228
|
/**
|
|
216
229
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
217
230
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -486,4 +499,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
|
|
|
486
499
|
}
|
|
487
500
|
return operations;
|
|
488
501
|
}
|
|
489
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
|
502
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };
|
package/dist/mappers.d.ts
CHANGED
|
@@ -33,6 +33,25 @@ export declare function timestampToEpochMs(value: unknown): number | undefined;
|
|
|
33
33
|
* joins the originating `toolCall.id` exactly.
|
|
34
34
|
*/
|
|
35
35
|
export declare function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null;
|
|
36
|
+
/** Signals carried by a Claude tool_result user line (task 0624 R2). */
|
|
37
|
+
export interface ClaudeToolResultTiming {
|
|
38
|
+
/** The `toolu_…` id from the answered tool_use block — joins `tool_use.id` exactly. */
|
|
39
|
+
toolCallId: string;
|
|
40
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
41
|
+
timestampMs: number | undefined;
|
|
42
|
+
/** Serialized size in bytes of the tool result payload. */
|
|
43
|
+
resultBytes: number;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Extract tool_result signals from a raw Claude Code record (task 0624 R2), or
|
|
47
|
+
* null when the record carries no tool_result block. Claude Code emits
|
|
48
|
+
* `type: "user"` lines whose `message.content` holds `{type: "tool_result",
|
|
49
|
+
* tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
|
|
50
|
+
* envelope. `resultBytes` prefers the model-visible block content and falls
|
|
51
|
+
* back to `toolUseResult`; both are null-safe. No wall time exists natively —
|
|
52
|
+
* claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
|
|
53
|
+
*/
|
|
54
|
+
export declare function claudeToolResultTiming(raw: Record<string, unknown>): ClaudeToolResultTiming | null;
|
|
36
55
|
/** Map a Codex JSONL record into one or more ETL split entries. */
|
|
37
56
|
export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
38
57
|
/** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
|
package/dist/mappers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAoExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAwCD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAMhG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkH/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,wEAAwE;AACxE,MAAM,WAAW,sBAAsB;IACnC,uFAAuF;IACvF,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,sBAAsB,GAAG,IAAI,CAkBlG;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAoK9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAiMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAgFD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
|
package/dist/mappers.js
CHANGED
|
@@ -24,6 +24,7 @@ const MESSAGE_MAPPER_KEYS = [
|
|
|
24
24
|
'provenance',
|
|
25
25
|
'run_id',
|
|
26
26
|
'task_wbs',
|
|
27
|
+
'request_id',
|
|
27
28
|
];
|
|
28
29
|
/** Columns the mapper may produce for history_tool_call. */
|
|
29
30
|
const TOOL_CALL_MAPPER_KEYS = [
|
|
@@ -194,6 +195,7 @@ export function claudeSplit(raw) {
|
|
|
194
195
|
content_text: contentText ?? null,
|
|
195
196
|
cwd: cwd ?? null,
|
|
196
197
|
provenance: 'ambient',
|
|
198
|
+
request_id: s(raw.requestId) ?? null,
|
|
197
199
|
},
|
|
198
200
|
});
|
|
199
201
|
// Tool calls from content blocks
|
|
@@ -209,6 +211,7 @@ export function claudeSplit(raw) {
|
|
|
209
211
|
_messageSplitIndex: messageSplitIndex,
|
|
210
212
|
session_id: sessionId,
|
|
211
213
|
seq,
|
|
214
|
+
call_id: s(b.id),
|
|
212
215
|
tool_name: String(b.name ?? ''),
|
|
213
216
|
args_digest: argsDigest(b.input),
|
|
214
217
|
args_raw: maybeArgsRaw('claude', String(b.name ?? ''), b.input),
|
|
@@ -510,6 +513,38 @@ export function ompToolResultTiming(raw) {
|
|
|
510
513
|
const wallTimeMs = typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
|
|
511
514
|
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
512
515
|
}
|
|
516
|
+
/**
|
|
517
|
+
* Extract tool_result signals from a raw Claude Code record (task 0624 R2), or
|
|
518
|
+
* null when the record carries no tool_result block. Claude Code emits
|
|
519
|
+
* `type: "user"` lines whose `message.content` holds `{type: "tool_result",
|
|
520
|
+
* tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
|
|
521
|
+
* envelope. `resultBytes` prefers the model-visible block content and falls
|
|
522
|
+
* back to `toolUseResult`; both are null-safe. No wall time exists natively —
|
|
523
|
+
* claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
|
|
524
|
+
*/
|
|
525
|
+
export function claudeToolResultTiming(raw) {
|
|
526
|
+
const content = o(raw.message).content;
|
|
527
|
+
if (!Array.isArray(content))
|
|
528
|
+
return null;
|
|
529
|
+
for (const block of content) {
|
|
530
|
+
if (typeof block !== 'object' || block === null)
|
|
531
|
+
continue;
|
|
532
|
+
const b = block;
|
|
533
|
+
if (b.type !== 'tool_result')
|
|
534
|
+
continue;
|
|
535
|
+
const toolCallId = s(b.tool_use_id);
|
|
536
|
+
if (toolCallId === undefined)
|
|
537
|
+
continue;
|
|
538
|
+
const payload = b.content ?? raw.toolUseResult;
|
|
539
|
+
const resultBytes = JSON.stringify(payload ?? null)?.length ?? 2;
|
|
540
|
+
return {
|
|
541
|
+
toolCallId,
|
|
542
|
+
timestampMs: timestampToEpochMs(raw.timestamp),
|
|
543
|
+
resultBytes,
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
return null;
|
|
547
|
+
}
|
|
513
548
|
// ---------------------------------------------------------------------------
|
|
514
549
|
// Codex mapper
|
|
515
550
|
// ---------------------------------------------------------------------------
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAgB/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,
|
|
1
|
+
{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAgB/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAqI7F"}
|
|
@@ -124,6 +124,7 @@ export async function runOpenCodeImport(options) {
|
|
|
124
124
|
processedLines,
|
|
125
125
|
importedRecords,
|
|
126
126
|
skippedDuplicates,
|
|
127
|
+
skippedCorruptLines: 0,
|
|
127
128
|
unknownRecords: 0,
|
|
128
129
|
parseErrors,
|
|
129
130
|
validationErrors,
|
|
@@ -263,6 +264,7 @@ function emptyResult(mode) {
|
|
|
263
264
|
processedLines: 0,
|
|
264
265
|
importedRecords: 0,
|
|
265
266
|
skippedDuplicates: 0,
|
|
267
|
+
skippedCorruptLines: 0,
|
|
266
268
|
unknownRecords: 0,
|
|
267
269
|
parseErrors: [],
|
|
268
270
|
validationErrors: [],
|
package/dist/schema-sql.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,
|
|
1
|
+
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA8E9B,CAAC"}
|
package/dist/schema-sql.js
CHANGED
package/dist/sources.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAyBA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAsH9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAAsB,CAAC;AAEpD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,
|
|
1
|
+
{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAyBA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAsH9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAAsB,CAAC;AAEpD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAiEjF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAM5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,GAAG,gBAAgB,CA+BvF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,gBAAgB,CAK1F"}
|
package/dist/sources.js
CHANGED
|
@@ -103,7 +103,13 @@ export const SOURCE_DEFINITIONS = {
|
|
|
103
103
|
codex: customSourceDefinition('codex', 'Codex', ['.codex/sessions'], ['*.jsonl'], codexSplit, CODEX_FIELD_MAP, CODEX_SCHEMA),
|
|
104
104
|
omp: customSourceDefinition('omp', 'OMP', ['.omp/agent/sessions'], ['*.jsonl'], ompSplit, OMP_FIELD_MAP, OMP_SCHEMA),
|
|
105
105
|
grok: customSourceDefinition('grok', 'Grok', ['.grok/sessions'], ['*.jsonl'], grokSplit, GROK_FIELD_MAP, GROK_SCHEMA),
|
|
106
|
-
agy:
|
|
106
|
+
agy: {
|
|
107
|
+
...customSourceDefinition('agy', 'Antigravity CLI', ['.gemini/antigravity-cli/brain'], ['*.jsonl'], agySplit, AGY_FIELD_MAP, AGY_SCHEMA),
|
|
108
|
+
// agy's producer interleaves non-JSON fragments and torn tail writes into its
|
|
109
|
+
// brain jsonl (task 0623: 789/89,818 lines, all unrecoverable). Skipping keeps
|
|
110
|
+
// the 99%+ good records importable without degrading the whole source.
|
|
111
|
+
corruptLinePolicy: 'skip',
|
|
112
|
+
},
|
|
107
113
|
gemini: customSourceDefinition('gemini', 'Gemini CLI', ['.gemini/tmp', '.config/gemini'], ['*.jsonl'], geminiSplit, GEMINI_FIELD_MAP, GEMINI_SCHEMA),
|
|
108
114
|
opencode: sourceDefinition('opencode', 'OpenCode', ['.opencode', '.local/share/opencode'], ['*.jsonl']),
|
|
109
115
|
antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
|
package/dist/types.d.ts
CHANGED
|
@@ -41,6 +41,13 @@ export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
|
|
|
41
41
|
readonly displayName: string;
|
|
42
42
|
readonly defaultRoots: readonly string[];
|
|
43
43
|
readonly filePatterns: readonly string[];
|
|
44
|
+
/**
|
|
45
|
+
* Policy for lines that fail JSON parsing. `'error'` (default) records a parse error and
|
|
46
|
+
* degrades the import; `'skip'` counts the line and moves on — for producers whose logs
|
|
47
|
+
* are known to interleave non-JSON fragments and torn tail writes (agy task 0623:
|
|
48
|
+
* 789/89,818 lines unparseable, all unrecoverable garbage).
|
|
49
|
+
*/
|
|
50
|
+
readonly corruptLinePolicy?: 'error' | 'skip';
|
|
44
51
|
readonly targetTable: string;
|
|
45
52
|
readonly splitConfig: SplitConfig;
|
|
46
53
|
readonly fieldMap: Readonly<Record<string, string>>;
|
|
@@ -100,6 +107,8 @@ export interface ImportResult {
|
|
|
100
107
|
readonly importedRecords: number;
|
|
101
108
|
readonly skippedDuplicates: number;
|
|
102
109
|
readonly unknownRecords: number;
|
|
110
|
+
/** Lines dropped by `corruptLinePolicy: 'skip'` instead of degrading the import. */
|
|
111
|
+
readonly skippedCorruptLines: number;
|
|
103
112
|
readonly parseErrors: readonly ImportIssue[];
|
|
104
113
|
readonly validationErrors: readonly ImportIssue[];
|
|
105
114
|
readonly checkpointUpdates: number;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.41",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.41",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.41",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.41",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
package/src/importer.ts
CHANGED
|
@@ -20,8 +20,9 @@ import {
|
|
|
20
20
|
resetCheckpoints,
|
|
21
21
|
targetTableFor,
|
|
22
22
|
toolCallDurationUpdateOp,
|
|
23
|
+
toolCallResultBytesUpdateOp,
|
|
23
24
|
} from './jsonl-importer-dao';
|
|
24
|
-
import { type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
|
|
25
|
+
import { claudeToolResultTiming, type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
|
|
25
26
|
import { redactRecord } from './redaction';
|
|
26
27
|
import { resolveSourceDefinition } from './sources';
|
|
27
28
|
import type {
|
|
@@ -143,6 +144,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
143
144
|
let importedRecords = 0;
|
|
144
145
|
let skippedDuplicates = 0;
|
|
145
146
|
let unknownRecords = 0;
|
|
147
|
+
let skippedCorruptLines = 0;
|
|
146
148
|
let checkpointUpdates = 0;
|
|
147
149
|
// Full-mode desired set (R1, task 0504): every record hash the current source produces.
|
|
148
150
|
// ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
|
|
@@ -161,8 +163,11 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
161
163
|
if (line.length === 0 || lineNumber <= checkpoint) continue;
|
|
162
164
|
processedLines += 1;
|
|
163
165
|
|
|
164
|
-
const raw = parseJsonLine(line, file, lineNumber, parseErrors);
|
|
165
|
-
if (raw === undefined)
|
|
166
|
+
const raw = parseJsonLine(line, file, lineNumber, parseErrors, definition.corruptLinePolicy);
|
|
167
|
+
if (raw === undefined) {
|
|
168
|
+
if (definition.corruptLinePolicy === 'skip') skippedCorruptLines += 1;
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
166
171
|
|
|
167
172
|
const splitRecords = splitRawRecord(definition, raw, {
|
|
168
173
|
source: resolvedSource,
|
|
@@ -226,18 +231,23 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
226
231
|
// re-imported line's row already exists and its duration UPDATE still targets
|
|
227
232
|
// the same deterministic record_hash. The assistant message timestamp is the
|
|
228
233
|
// fallback's start bound; every call in one message shares it.
|
|
229
|
-
if (definition.source === 'omp') {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
)
|
|
234
|
+
if (definition.source === 'omp' || definition.source === 'claude') {
|
|
235
|
+
// omp only: assistant-message timestamps are the fallback duration's
|
|
236
|
+
// start bound. claude attaches result_bytes only — no fallback
|
|
237
|
+
// duration exists natively (never-fabricate, 0624 R2).
|
|
238
|
+
const assistantTs = definition.source === 'omp' ? new Map<string, number | undefined>() : null;
|
|
239
|
+
if (assistantTs !== null) {
|
|
240
|
+
for (const entry of prepared) {
|
|
241
|
+
if (
|
|
242
|
+
entry.split.targetTable === 'history_message' &&
|
|
243
|
+
entry.normalized.role === 'assistant' &&
|
|
244
|
+
typeof entry.normalized.session_id === 'string'
|
|
245
|
+
) {
|
|
246
|
+
assistantTs.set(
|
|
247
|
+
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
248
|
+
timestampToEpochMs(entry.normalized.ts),
|
|
249
|
+
);
|
|
250
|
+
}
|
|
241
251
|
}
|
|
242
252
|
}
|
|
243
253
|
for (const entry of prepared) {
|
|
@@ -250,7 +260,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
250
260
|
) {
|
|
251
261
|
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
252
262
|
recordHash: entry.recordHash,
|
|
253
|
-
messageTsMs: assistantTs
|
|
263
|
+
messageTsMs: assistantTs?.get(
|
|
254
264
|
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
255
265
|
),
|
|
256
266
|
});
|
|
@@ -360,6 +370,32 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
360
370
|
}
|
|
361
371
|
}
|
|
362
372
|
}
|
|
373
|
+
// Task 0624 R2 (claude): attach result sizes from this line's
|
|
374
|
+
// tool_result blocks. Pairing is by `tool_use_id` → `call_id`
|
|
375
|
+
// (source ∘ session ∘ call), the same key the omp branch uses.
|
|
376
|
+
// No duration is written — claude has no native wall time and the
|
|
377
|
+
// guarded timestamp fallback does not extend to it (0624 Design).
|
|
378
|
+
if (definition.source === 'claude') {
|
|
379
|
+
const timing = claudeToolResultTiming(raw);
|
|
380
|
+
if (timing !== null) {
|
|
381
|
+
const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message');
|
|
382
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
383
|
+
if (typeof sessionId === 'string') {
|
|
384
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
385
|
+
if (pending === undefined) {
|
|
386
|
+
pending = await resolveToolCallRow(
|
|
387
|
+
options.db,
|
|
388
|
+
resolvedSource,
|
|
389
|
+
sessionId,
|
|
390
|
+
timing.toolCallId,
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
if (pending !== undefined) {
|
|
394
|
+
ops.push(toolCallResultBytesUpdateOp(pending.recordHash, timing.resultBytes));
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
363
399
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
364
400
|
// re-imported after a mid-batch crash.
|
|
365
401
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
|
@@ -392,6 +428,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
392
428
|
processedLines,
|
|
393
429
|
importedRecords,
|
|
394
430
|
skippedDuplicates,
|
|
431
|
+
skippedCorruptLines,
|
|
395
432
|
unknownRecords,
|
|
396
433
|
parseErrors,
|
|
397
434
|
validationErrors,
|
|
@@ -405,18 +442,28 @@ function parseJsonLine(
|
|
|
405
442
|
sourceFile: string,
|
|
406
443
|
sourceLine: number,
|
|
407
444
|
parseErrors: ImportIssue[],
|
|
445
|
+
corruptLinePolicy: SourceDefinition['corruptLinePolicy'] = 'error',
|
|
408
446
|
): JsonObject | undefined {
|
|
447
|
+
let parsed: unknown;
|
|
409
448
|
try {
|
|
410
|
-
|
|
411
|
-
|
|
449
|
+
parsed = JSON.parse(line);
|
|
450
|
+
} catch (error) {
|
|
451
|
+
if (corruptLinePolicy !== 'skip') {
|
|
452
|
+
parseErrors.push({
|
|
453
|
+
sourceFile,
|
|
454
|
+
sourceLine,
|
|
455
|
+
reason: error instanceof Error ? error.message : 'Invalid JSON',
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
return undefined;
|
|
459
|
+
}
|
|
460
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
461
|
+
if (corruptLinePolicy !== 'skip') {
|
|
412
462
|
parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
|
|
413
|
-
return undefined;
|
|
414
463
|
}
|
|
415
|
-
return value as JsonObject;
|
|
416
|
-
} catch (error) {
|
|
417
|
-
parseErrors.push({ sourceFile, sourceLine, reason: error instanceof Error ? error.message : 'Invalid JSON' });
|
|
418
464
|
return undefined;
|
|
419
465
|
}
|
|
466
|
+
return parsed as JsonObject;
|
|
420
467
|
}
|
|
421
468
|
|
|
422
469
|
function splitRawRecord(
|
|
@@ -34,6 +34,7 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
|
34
34
|
'provenance',
|
|
35
35
|
'run_id',
|
|
36
36
|
'task_wbs',
|
|
37
|
+
'request_id',
|
|
37
38
|
'imported_at',
|
|
38
39
|
],
|
|
39
40
|
history_tool_call: [
|
|
@@ -279,6 +280,19 @@ function toolCallDurationUpdateOp(
|
|
|
279
280
|
};
|
|
280
281
|
}
|
|
281
282
|
|
|
283
|
+
/**
|
|
284
|
+
* Targeted UPDATE op attaching a result size to a previously-inserted
|
|
285
|
+
* `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
|
|
286
|
+
*/
|
|
287
|
+
function toolCallResultBytesUpdateOp(recordHash: string, resultBytes: number): DbBatchOp {
|
|
288
|
+
return {
|
|
289
|
+
sql: `UPDATE history_tool_call
|
|
290
|
+
SET result_bytes = ?
|
|
291
|
+
WHERE record_hash = ?`,
|
|
292
|
+
params: [resultBytes, recordHash],
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
|
|
282
296
|
/**
|
|
283
297
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
284
298
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -713,5 +727,6 @@ export {
|
|
|
713
727
|
targetTableFor,
|
|
714
728
|
timestamp,
|
|
715
729
|
toolCallDurationUpdateOp,
|
|
730
|
+
toolCallResultBytesUpdateOp,
|
|
716
731
|
writeCheckpoint,
|
|
717
732
|
};
|
package/src/mappers.ts
CHANGED
|
@@ -27,6 +27,7 @@ const MESSAGE_MAPPER_KEYS: readonly string[] = [
|
|
|
27
27
|
'provenance',
|
|
28
28
|
'run_id',
|
|
29
29
|
'task_wbs',
|
|
30
|
+
'request_id',
|
|
30
31
|
];
|
|
31
32
|
|
|
32
33
|
/** Columns the mapper may produce for history_tool_call. */
|
|
@@ -209,6 +210,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
209
210
|
content_text: contentText ?? null,
|
|
210
211
|
cwd: cwd ?? null,
|
|
211
212
|
provenance: 'ambient',
|
|
213
|
+
request_id: s(raw.requestId) ?? null,
|
|
212
214
|
},
|
|
213
215
|
});
|
|
214
216
|
|
|
@@ -224,6 +226,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
224
226
|
_messageSplitIndex: messageSplitIndex,
|
|
225
227
|
session_id: sessionId,
|
|
226
228
|
seq,
|
|
229
|
+
call_id: s(b.id),
|
|
227
230
|
tool_name: String(b.name ?? ''),
|
|
228
231
|
args_digest: argsDigest(b.input),
|
|
229
232
|
args_raw: maybeArgsRaw('claude', String(b.name ?? ''), b.input),
|
|
@@ -554,6 +557,49 @@ export function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResult
|
|
|
554
557
|
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
555
558
|
}
|
|
556
559
|
|
|
560
|
+
// ---------------------------------------------------------------------------
|
|
561
|
+
// Claude toolResult signals (task 0624 R2)
|
|
562
|
+
// ---------------------------------------------------------------------------
|
|
563
|
+
|
|
564
|
+
/** Signals carried by a Claude tool_result user line (task 0624 R2). */
|
|
565
|
+
export interface ClaudeToolResultTiming {
|
|
566
|
+
/** The `toolu_…` id from the answered tool_use block — joins `tool_use.id` exactly. */
|
|
567
|
+
toolCallId: string;
|
|
568
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
569
|
+
timestampMs: number | undefined;
|
|
570
|
+
/** Serialized size in bytes of the tool result payload. */
|
|
571
|
+
resultBytes: number;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Extract tool_result signals from a raw Claude Code record (task 0624 R2), or
|
|
576
|
+
* null when the record carries no tool_result block. Claude Code emits
|
|
577
|
+
* `type: "user"` lines whose `message.content` holds `{type: "tool_result",
|
|
578
|
+
* tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
|
|
579
|
+
* envelope. `resultBytes` prefers the model-visible block content and falls
|
|
580
|
+
* back to `toolUseResult`; both are null-safe. No wall time exists natively —
|
|
581
|
+
* claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
|
|
582
|
+
*/
|
|
583
|
+
export function claudeToolResultTiming(raw: Record<string, unknown>): ClaudeToolResultTiming | null {
|
|
584
|
+
const content = o(raw.message).content;
|
|
585
|
+
if (!Array.isArray(content)) return null;
|
|
586
|
+
for (const block of content) {
|
|
587
|
+
if (typeof block !== 'object' || block === null) continue;
|
|
588
|
+
const b = block as Record<string, unknown>;
|
|
589
|
+
if (b.type !== 'tool_result') continue;
|
|
590
|
+
const toolCallId = s(b.tool_use_id);
|
|
591
|
+
if (toolCallId === undefined) continue;
|
|
592
|
+
const payload = b.content ?? raw.toolUseResult;
|
|
593
|
+
const resultBytes = JSON.stringify(payload ?? null)?.length ?? 2;
|
|
594
|
+
return {
|
|
595
|
+
toolCallId,
|
|
596
|
+
timestampMs: timestampToEpochMs(raw.timestamp),
|
|
597
|
+
resultBytes,
|
|
598
|
+
};
|
|
599
|
+
}
|
|
600
|
+
return null;
|
|
601
|
+
}
|
|
602
|
+
|
|
557
603
|
// ---------------------------------------------------------------------------
|
|
558
604
|
// Codex mapper
|
|
559
605
|
// ---------------------------------------------------------------------------
|
package/src/opencode-importer.ts
CHANGED
|
@@ -172,6 +172,7 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
|
|
|
172
172
|
processedLines,
|
|
173
173
|
importedRecords,
|
|
174
174
|
skippedDuplicates,
|
|
175
|
+
skippedCorruptLines: 0,
|
|
175
176
|
unknownRecords: 0,
|
|
176
177
|
parseErrors,
|
|
177
178
|
validationErrors,
|
|
@@ -331,6 +332,7 @@ function emptyResult(mode: ImportMode): ImportResult {
|
|
|
331
332
|
processedLines: 0,
|
|
332
333
|
importedRecords: 0,
|
|
333
334
|
skippedDuplicates: 0,
|
|
335
|
+
skippedCorruptLines: 0,
|
|
334
336
|
unknownRecords: 0,
|
|
335
337
|
parseErrors: [],
|
|
336
338
|
validationErrors: [],
|
package/src/schema-sql.ts
CHANGED
package/src/sources.ts
CHANGED
|
@@ -190,15 +190,21 @@ export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinitio
|
|
|
190
190
|
GROK_FIELD_MAP,
|
|
191
191
|
GROK_SCHEMA,
|
|
192
192
|
),
|
|
193
|
-
agy:
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
193
|
+
agy: {
|
|
194
|
+
...customSourceDefinition(
|
|
195
|
+
'agy',
|
|
196
|
+
'Antigravity CLI',
|
|
197
|
+
['.gemini/antigravity-cli/brain'],
|
|
198
|
+
['*.jsonl'],
|
|
199
|
+
agySplit,
|
|
200
|
+
AGY_FIELD_MAP,
|
|
201
|
+
AGY_SCHEMA,
|
|
202
|
+
),
|
|
203
|
+
// agy's producer interleaves non-JSON fragments and torn tail writes into its
|
|
204
|
+
// brain jsonl (task 0623: 789/89,818 lines, all unrecoverable). Skipping keeps
|
|
205
|
+
// the 99%+ good records importable without degrading the whole source.
|
|
206
|
+
corruptLinePolicy: 'skip',
|
|
207
|
+
},
|
|
202
208
|
gemini: customSourceDefinition(
|
|
203
209
|
'gemini',
|
|
204
210
|
'Gemini CLI',
|
package/src/types.ts
CHANGED
|
@@ -62,6 +62,13 @@ export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
|
|
|
62
62
|
readonly displayName: string;
|
|
63
63
|
readonly defaultRoots: readonly string[];
|
|
64
64
|
readonly filePatterns: readonly string[];
|
|
65
|
+
/**
|
|
66
|
+
* Policy for lines that fail JSON parsing. `'error'` (default) records a parse error and
|
|
67
|
+
* degrades the import; `'skip'` counts the line and moves on — for producers whose logs
|
|
68
|
+
* are known to interleave non-JSON fragments and torn tail writes (agy task 0623:
|
|
69
|
+
* 789/89,818 lines unparseable, all unrecoverable garbage).
|
|
70
|
+
*/
|
|
71
|
+
readonly corruptLinePolicy?: 'error' | 'skip';
|
|
65
72
|
readonly targetTable: string;
|
|
66
73
|
readonly splitConfig: SplitConfig;
|
|
67
74
|
readonly fieldMap: Readonly<Record<string, string>>;
|
|
@@ -126,6 +133,8 @@ export interface ImportResult {
|
|
|
126
133
|
readonly importedRecords: number;
|
|
127
134
|
readonly skippedDuplicates: number;
|
|
128
135
|
readonly unknownRecords: number;
|
|
136
|
+
/** Lines dropped by `corruptLinePolicy: 'skip'` instead of degrading the import. */
|
|
137
|
+
readonly skippedCorruptLines: number;
|
|
129
138
|
readonly parseErrors: readonly ImportIssue[];
|
|
130
139
|
readonly validationErrors: readonly ImportIssue[];
|
|
131
140
|
readonly checkpointUpdates: number;
|