@gobing-ai/ts-llm-jsonl-importer 0.4.39 → 0.4.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAkRrH"}
1
+ {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAsTrH"}
package/dist/importer.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
2
2
  import { sha256 } from './hash.js';
3
- import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, } from './jsonl-importer-dao.js';
4
- import { ompToolResultTiming, timestampToEpochMs } from './mappers.js';
3
+ import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, } from './jsonl-importer-dao.js';
4
+ import { claudeToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers.js';
5
5
  import { redactRecord } from './redaction.js';
6
6
  import { resolveSourceDefinition } from './sources.js';
7
7
  const TOOL_CALL_KEY_SEP = '\u0000';
@@ -64,6 +64,7 @@ export async function runJsonlImport(source, options) {
64
64
  let importedRecords = 0;
65
65
  let skippedDuplicates = 0;
66
66
  let unknownRecords = 0;
67
+ let skippedCorruptLines = 0;
67
68
  let checkpointUpdates = 0;
68
69
  // Full-mode desired set (R1, task 0504): every record hash the current source produces.
69
70
  // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
@@ -80,9 +81,12 @@ export async function runJsonlImport(source, options) {
80
81
  if (line.length === 0 || lineNumber <= checkpoint)
81
82
  continue;
82
83
  processedLines += 1;
83
- const raw = parseJsonLine(line, file, lineNumber, parseErrors);
84
- if (raw === undefined)
84
+ const raw = parseJsonLine(line, file, lineNumber, parseErrors, definition.corruptLinePolicy);
85
+ if (raw === undefined) {
86
+ if (definition.corruptLinePolicy === 'skip')
87
+ skippedCorruptLines += 1;
85
88
  continue;
89
+ }
86
90
  const splitRecords = splitRawRecord(definition, raw, {
87
91
  source: resolvedSource,
88
92
  sourceFile: file,
@@ -136,13 +140,18 @@ export async function runJsonlImport(source, options) {
136
140
  // re-imported line's row already exists and its duration UPDATE still targets
137
141
  // the same deterministic record_hash. The assistant message timestamp is the
138
142
  // fallback's start bound; every call in one message shares it.
139
- if (definition.source === 'omp') {
140
- const assistantTs = new Map();
141
- for (const entry of prepared) {
142
- if (entry.split.targetTable === 'history_message' &&
143
- entry.normalized.role === 'assistant' &&
144
- typeof entry.normalized.session_id === 'string') {
145
- assistantTs.set(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`, timestampToEpochMs(entry.normalized.ts));
143
+ if (definition.source === 'omp' || definition.source === 'claude') {
144
+ // omp only: assistant-message timestamps are the fallback duration's
145
+ // start bound. claude attaches result_bytes only — no fallback
146
+ // duration exists natively (never-fabricate, 0624 R2).
147
+ const assistantTs = definition.source === 'omp' ? new Map() : null;
148
+ if (assistantTs !== null) {
149
+ for (const entry of prepared) {
150
+ if (entry.split.targetTable === 'history_message' &&
151
+ entry.normalized.role === 'assistant' &&
152
+ typeof entry.normalized.session_id === 'string') {
153
+ assistantTs.set(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`, timestampToEpochMs(entry.normalized.ts));
154
+ }
146
155
  }
147
156
  }
148
157
  for (const entry of prepared) {
@@ -153,7 +162,7 @@ export async function runJsonlImport(source, options) {
153
162
  typeof entry.normalized.session_id === 'string') {
154
163
  toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
155
164
  recordHash: entry.recordHash,
156
- messageTsMs: assistantTs.get(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`),
165
+ messageTsMs: assistantTs?.get(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`),
157
166
  });
158
167
  }
159
168
  }
@@ -220,6 +229,27 @@ export async function runJsonlImport(source, options) {
220
229
  }
221
230
  }
222
231
  }
232
+ // Task 0624 R2 (claude): attach result sizes from this line's
233
+ // tool_result blocks. Pairing is by `tool_use_id` → `call_id`
234
+ // (source ∘ session ∘ call), the same key the omp branch uses.
235
+ // No duration is written — claude has no native wall time and the
236
+ // guarded timestamp fallback does not extend to it (0624 Design).
237
+ if (definition.source === 'claude') {
238
+ const timing = claudeToolResultTiming(raw);
239
+ if (timing !== null) {
240
+ const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message');
241
+ const sessionId = resultEntry?.normalized.session_id;
242
+ if (typeof sessionId === 'string') {
243
+ let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
244
+ if (pending === undefined) {
245
+ pending = await resolveToolCallRow(options.db, resolvedSource, sessionId, timing.toolCallId);
246
+ }
247
+ if (pending !== undefined) {
248
+ ops.push(toolCallResultBytesUpdateOp(pending.recordHash, timing.resultBytes));
249
+ }
250
+ }
251
+ }
252
+ }
223
253
  // Checkpoint upsert joins the same batch so an accepted line can never be
224
254
  // re-imported after a mid-batch crash.
225
255
  ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
@@ -244,6 +274,7 @@ export async function runJsonlImport(source, options) {
244
274
  processedLines,
245
275
  importedRecords,
246
276
  skippedDuplicates,
277
+ skippedCorruptLines,
247
278
  unknownRecords,
248
279
  parseErrors,
249
280
  validationErrors,
@@ -251,19 +282,28 @@ export async function runJsonlImport(source, options) {
251
282
  reconciliation,
252
283
  };
253
284
  }
254
- function parseJsonLine(line, sourceFile, sourceLine, parseErrors) {
285
+ function parseJsonLine(line, sourceFile, sourceLine, parseErrors, corruptLinePolicy = 'error') {
286
+ let parsed;
255
287
  try {
256
- const value = JSON.parse(line);
257
- if (value === null || typeof value !== 'object' || Array.isArray(value)) {
258
- parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
259
- return undefined;
260
- }
261
- return value;
288
+ parsed = JSON.parse(line);
262
289
  }
263
290
  catch (error) {
264
- parseErrors.push({ sourceFile, sourceLine, reason: error instanceof Error ? error.message : 'Invalid JSON' });
291
+ if (corruptLinePolicy !== 'skip') {
292
+ parseErrors.push({
293
+ sourceFile,
294
+ sourceLine,
295
+ reason: error instanceof Error ? error.message : 'Invalid JSON',
296
+ });
297
+ }
298
+ return undefined;
299
+ }
300
+ if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
301
+ if (corruptLinePolicy !== 'skip') {
302
+ parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
303
+ }
265
304
  return undefined;
266
305
  }
306
+ return parsed;
267
307
  }
268
308
  function splitRawRecord(definition, raw, context) {
269
309
  const targetTable = targetTableFor(definition.targetTable);
@@ -54,6 +54,11 @@ export declare function checkpointUpsertOp(source: string, sourceFile: string, l
54
54
  * two measurement paths stay distinguishable.
55
55
  */
56
56
  declare function toolCallDurationUpdateOp(recordHash: string, startedAt: string | null, completedAt: string | null, durationMs: number | null): DbBatchOp;
57
+ /**
58
+ * Targeted UPDATE op attaching a result size to a previously-inserted
59
+ * `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
60
+ */
61
+ declare function toolCallResultBytesUpdateOp(recordHash: string, resultBytes: number): DbBatchOp;
57
62
  /**
58
63
  * Query which of the given hashes already exist in the ledger, in chunks of at
59
64
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -136,5 +141,5 @@ export declare function deleteCheckpointOperation(source: string, sourceFile: st
136
141
  /** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
137
142
  export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
138
143
  export type { CheckpointRow };
139
- export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
144
+ export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };
140
145
  //# sourceMappingURL=jsonl-importer-dao.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAuDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,eAAe,GAClB,CAAC"}
1
+ {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAwDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,CAOvF;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,2BAA2B,EAC3B,eAAe,GAClB,CAAC"}
@@ -27,6 +27,7 @@ const TYPED_TABLE_COLUMNS = {
27
27
  'provenance',
28
28
  'run_id',
29
29
  'task_wbs',
30
+ 'request_id',
30
31
  'imported_at',
31
32
  ],
32
33
  history_tool_call: [
@@ -212,6 +213,18 @@ function toolCallDurationUpdateOp(recordHash, startedAt, completedAt, durationMs
212
213
  params: [startedAt, completedAt, durationMs, recordHash],
213
214
  };
214
215
  }
216
+ /**
217
+ * Targeted UPDATE op attaching a result size to a previously-inserted
218
+ * `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
219
+ */
220
+ function toolCallResultBytesUpdateOp(recordHash, resultBytes) {
221
+ return {
222
+ sql: `UPDATE history_tool_call
223
+ SET result_bytes = ?
224
+ WHERE record_hash = ?`,
225
+ params: [resultBytes, recordHash],
226
+ };
227
+ }
215
228
  /**
216
229
  * Query which of the given hashes already exist in the ledger, in chunks of at
217
230
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -486,4 +499,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
486
499
  }
487
500
  return operations;
488
501
  }
489
- export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
502
+ export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };
package/dist/mappers.d.ts CHANGED
@@ -33,6 +33,25 @@ export declare function timestampToEpochMs(value: unknown): number | undefined;
33
33
  * joins the originating `toolCall.id` exactly.
34
34
  */
35
35
  export declare function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null;
36
+ /** Signals carried by a Claude tool_result user line (task 0624 R2). */
37
+ export interface ClaudeToolResultTiming {
38
+ /** The `toolu_…` id from the answered tool_use block — joins `tool_use.id` exactly. */
39
+ toolCallId: string;
40
+ /** Message timestamp as epoch millis, when parseable. */
41
+ timestampMs: number | undefined;
42
+ /** Serialized size in bytes of the tool result payload. */
43
+ resultBytes: number;
44
+ }
45
+ /**
46
+ * Extract tool_result signals from a raw Claude Code record (task 0624 R2), or
47
+ * null when the record carries no tool_result block. Claude Code emits
48
+ * `type: "user"` lines whose `message.content` holds `{type: "tool_result",
49
+ * tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
50
+ * envelope. `resultBytes` prefers the model-visible block content and falls
51
+ * back to `toolUseResult`; both are null-safe. No wall time exists natively —
52
+ * claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
53
+ */
54
+ export declare function claudeToolResultTiming(raw: Record<string, unknown>): ClaudeToolResultTiming | null;
36
55
  /** Map a Codex JSONL record into one or more ETL split entries. */
37
56
  export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
38
57
  /** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
@@ -1 +1 @@
1
- {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAwCD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAMhG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAgH/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAoK9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAiMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAgFD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
1
+ {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAoExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAwCD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAMhG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkH/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,wEAAwE;AACxE,MAAM,WAAW,sBAAsB;IACnC,uFAAuF;IACvF,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,sBAAsB,GAAG,IAAI,CAkBlG;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAoK9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAiMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAgFD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
package/dist/mappers.js CHANGED
@@ -24,6 +24,7 @@ const MESSAGE_MAPPER_KEYS = [
24
24
  'provenance',
25
25
  'run_id',
26
26
  'task_wbs',
27
+ 'request_id',
27
28
  ];
28
29
  /** Columns the mapper may produce for history_tool_call. */
29
30
  const TOOL_CALL_MAPPER_KEYS = [
@@ -194,6 +195,7 @@ export function claudeSplit(raw) {
194
195
  content_text: contentText ?? null,
195
196
  cwd: cwd ?? null,
196
197
  provenance: 'ambient',
198
+ request_id: s(raw.requestId) ?? null,
197
199
  },
198
200
  });
199
201
  // Tool calls from content blocks
@@ -209,6 +211,7 @@ export function claudeSplit(raw) {
209
211
  _messageSplitIndex: messageSplitIndex,
210
212
  session_id: sessionId,
211
213
  seq,
214
+ call_id: s(b.id),
212
215
  tool_name: String(b.name ?? ''),
213
216
  args_digest: argsDigest(b.input),
214
217
  args_raw: maybeArgsRaw('claude', String(b.name ?? ''), b.input),
@@ -510,6 +513,38 @@ export function ompToolResultTiming(raw) {
510
513
  const wallTimeMs = typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
511
514
  return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
512
515
  }
516
+ /**
517
+ * Extract tool_result signals from a raw Claude Code record (task 0624 R2), or
518
+ * null when the record carries no tool_result block. Claude Code emits
519
+ * `type: "user"` lines whose `message.content` holds `{type: "tool_result",
520
+ * tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
521
+ * envelope. `resultBytes` prefers the model-visible block content and falls
522
+ * back to `toolUseResult`; both are null-safe. No wall time exists natively —
523
+ * claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
524
+ */
525
+ export function claudeToolResultTiming(raw) {
526
+ const content = o(raw.message).content;
527
+ if (!Array.isArray(content))
528
+ return null;
529
+ for (const block of content) {
530
+ if (typeof block !== 'object' || block === null)
531
+ continue;
532
+ const b = block;
533
+ if (b.type !== 'tool_result')
534
+ continue;
535
+ const toolCallId = s(b.tool_use_id);
536
+ if (toolCallId === undefined)
537
+ continue;
538
+ const payload = b.content ?? raw.toolUseResult;
539
+ const resultBytes = JSON.stringify(payload ?? null)?.length ?? 2;
540
+ return {
541
+ toolCallId,
542
+ timestampMs: timestampToEpochMs(raw.timestamp),
543
+ resultBytes,
544
+ };
545
+ }
546
+ return null;
547
+ }
513
548
  // ---------------------------------------------------------------------------
514
549
  // Codex mapper
515
550
  // ---------------------------------------------------------------------------
@@ -1 +1 @@
1
- {"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAgB/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAoI7F"}
1
+ {"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAgB/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAqI7F"}
@@ -124,6 +124,7 @@ export async function runOpenCodeImport(options) {
124
124
  processedLines,
125
125
  importedRecords,
126
126
  skippedDuplicates,
127
+ skippedCorruptLines: 0,
127
128
  unknownRecords: 0,
128
129
  parseErrors,
129
130
  validationErrors,
@@ -263,6 +264,7 @@ function emptyResult(mode) {
263
264
  processedLines: 0,
264
265
  importedRecords: 0,
265
266
  skippedDuplicates: 0,
267
+ skippedCorruptLines: 0,
266
268
  unknownRecords: 0,
267
269
  parseErrors: [],
268
270
  validationErrors: [],
@@ -1 +1 @@
1
- {"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA6E9B,CAAC"}
1
+ {"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA8E9B,CAAC"}
@@ -50,6 +50,7 @@ CREATE TABLE IF NOT EXISTS history_message (
50
50
  provenance TEXT NOT NULL,
51
51
  run_id TEXT,
52
52
  task_wbs TEXT,
53
+ request_id TEXT,
53
54
  imported_at TEXT NOT NULL
54
55
  );
55
56
 
@@ -1 +1 @@
1
- {"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAyBA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAsH9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAAsB,CAAC;AAEpD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CA2DjF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAM5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,GAAG,gBAAgB,CA+BvF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,gBAAgB,CAK1F"}
1
+ {"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAyBA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAsH9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAAsB,CAAC;AAEpD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAiEjF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAM5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,GAAG,gBAAgB,CA+BvF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,gBAAgB,CAK1F"}
package/dist/sources.js CHANGED
@@ -103,7 +103,13 @@ export const SOURCE_DEFINITIONS = {
103
103
  codex: customSourceDefinition('codex', 'Codex', ['.codex/sessions'], ['*.jsonl'], codexSplit, CODEX_FIELD_MAP, CODEX_SCHEMA),
104
104
  omp: customSourceDefinition('omp', 'OMP', ['.omp/agent/sessions'], ['*.jsonl'], ompSplit, OMP_FIELD_MAP, OMP_SCHEMA),
105
105
  grok: customSourceDefinition('grok', 'Grok', ['.grok/sessions'], ['*.jsonl'], grokSplit, GROK_FIELD_MAP, GROK_SCHEMA),
106
- agy: customSourceDefinition('agy', 'Antigravity CLI', ['.gemini/antigravity-cli/brain'], ['*.jsonl'], agySplit, AGY_FIELD_MAP, AGY_SCHEMA),
106
+ agy: {
107
+ ...customSourceDefinition('agy', 'Antigravity CLI', ['.gemini/antigravity-cli/brain'], ['*.jsonl'], agySplit, AGY_FIELD_MAP, AGY_SCHEMA),
108
+ // agy's producer interleaves non-JSON fragments and torn tail writes into its
109
+ // brain jsonl (task 0623: 789/89,818 lines, all unrecoverable). Skipping keeps
110
+ // the 99%+ good records importable without degrading the whole source.
111
+ corruptLinePolicy: 'skip',
112
+ },
107
113
  gemini: customSourceDefinition('gemini', 'Gemini CLI', ['.gemini/tmp', '.config/gemini'], ['*.jsonl'], geminiSplit, GEMINI_FIELD_MAP, GEMINI_SCHEMA),
108
114
  opencode: sourceDefinition('opencode', 'OpenCode', ['.opencode', '.local/share/opencode'], ['*.jsonl']),
109
115
  antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
package/dist/types.d.ts CHANGED
@@ -41,6 +41,13 @@ export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
41
41
  readonly displayName: string;
42
42
  readonly defaultRoots: readonly string[];
43
43
  readonly filePatterns: readonly string[];
44
+ /**
45
+ * Policy for lines that fail JSON parsing. `'error'` (default) records a parse error and
46
+ * degrades the import; `'skip'` counts the line and moves on — for producers whose logs
47
+ * are known to interleave non-JSON fragments and torn tail writes (agy task 0623:
48
+ * 789/89,818 lines unparseable, all unrecoverable garbage).
49
+ */
50
+ readonly corruptLinePolicy?: 'error' | 'skip';
44
51
  readonly targetTable: string;
45
52
  readonly splitConfig: SplitConfig;
46
53
  readonly fieldMap: Readonly<Record<string, string>>;
@@ -100,6 +107,8 @@ export interface ImportResult {
100
107
  readonly importedRecords: number;
101
108
  readonly skippedDuplicates: number;
102
109
  readonly unknownRecords: number;
110
+ /** Lines dropped by `corruptLinePolicy: 'skip'` instead of degrading the import. */
111
+ readonly skippedCorruptLines: number;
103
112
  readonly parseErrors: readonly ImportIssue[];
104
113
  readonly validationErrors: readonly ImportIssue[];
105
114
  readonly checkpointUpdates: number;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gobing-ai/ts-llm-jsonl-importer",
3
- "version": "0.4.39",
3
+ "version": "0.4.41",
4
4
  "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
5
5
  "keywords": [
6
6
  "typescript",
@@ -47,9 +47,9 @@
47
47
  "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
48
48
  },
49
49
  "dependencies": {
50
- "@gobing-ai/ts-db": "^0.4.39",
51
- "@gobing-ai/ts-runtime": "^0.4.39",
52
- "@gobing-ai/ts-utils": "^0.4.39",
50
+ "@gobing-ai/ts-db": "^0.4.41",
51
+ "@gobing-ai/ts-runtime": "^0.4.41",
52
+ "@gobing-ai/ts-utils": "^0.4.41",
53
53
  "zod": "^4.1.0"
54
54
  },
55
55
  "devDependencies": {
package/src/importer.ts CHANGED
@@ -20,8 +20,9 @@ import {
20
20
  resetCheckpoints,
21
21
  targetTableFor,
22
22
  toolCallDurationUpdateOp,
23
+ toolCallResultBytesUpdateOp,
23
24
  } from './jsonl-importer-dao';
24
- import { type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
25
+ import { claudeToolResultTiming, type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
25
26
  import { redactRecord } from './redaction';
26
27
  import { resolveSourceDefinition } from './sources';
27
28
  import type {
@@ -143,6 +144,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
143
144
  let importedRecords = 0;
144
145
  let skippedDuplicates = 0;
145
146
  let unknownRecords = 0;
147
+ let skippedCorruptLines = 0;
146
148
  let checkpointUpdates = 0;
147
149
  // Full-mode desired set (R1, task 0504): every record hash the current source produces.
148
150
  // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
@@ -161,8 +163,11 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
161
163
  if (line.length === 0 || lineNumber <= checkpoint) continue;
162
164
  processedLines += 1;
163
165
 
164
- const raw = parseJsonLine(line, file, lineNumber, parseErrors);
165
- if (raw === undefined) continue;
166
+ const raw = parseJsonLine(line, file, lineNumber, parseErrors, definition.corruptLinePolicy);
167
+ if (raw === undefined) {
168
+ if (definition.corruptLinePolicy === 'skip') skippedCorruptLines += 1;
169
+ continue;
170
+ }
166
171
 
167
172
  const splitRecords = splitRawRecord(definition, raw, {
168
173
  source: resolvedSource,
@@ -226,18 +231,23 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
226
231
  // re-imported line's row already exists and its duration UPDATE still targets
227
232
  // the same deterministic record_hash. The assistant message timestamp is the
228
233
  // fallback's start bound; every call in one message shares it.
229
- if (definition.source === 'omp') {
230
- const assistantTs = new Map<string, number | undefined>();
231
- for (const entry of prepared) {
232
- if (
233
- entry.split.targetTable === 'history_message' &&
234
- entry.normalized.role === 'assistant' &&
235
- typeof entry.normalized.session_id === 'string'
236
- ) {
237
- assistantTs.set(
238
- `${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
239
- timestampToEpochMs(entry.normalized.ts),
240
- );
234
+ if (definition.source === 'omp' || definition.source === 'claude') {
235
+ // omp only: assistant-message timestamps are the fallback duration's
236
+ // start bound. claude attaches result_bytes only — no fallback
237
+ // duration exists natively (never-fabricate, 0624 R2).
238
+ const assistantTs = definition.source === 'omp' ? new Map<string, number | undefined>() : null;
239
+ if (assistantTs !== null) {
240
+ for (const entry of prepared) {
241
+ if (
242
+ entry.split.targetTable === 'history_message' &&
243
+ entry.normalized.role === 'assistant' &&
244
+ typeof entry.normalized.session_id === 'string'
245
+ ) {
246
+ assistantTs.set(
247
+ `${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
248
+ timestampToEpochMs(entry.normalized.ts),
249
+ );
250
+ }
241
251
  }
242
252
  }
243
253
  for (const entry of prepared) {
@@ -250,7 +260,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
250
260
  ) {
251
261
  toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
252
262
  recordHash: entry.recordHash,
253
- messageTsMs: assistantTs.get(
263
+ messageTsMs: assistantTs?.get(
254
264
  `${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
255
265
  ),
256
266
  });
@@ -360,6 +370,32 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
360
370
  }
361
371
  }
362
372
  }
373
+ // Task 0624 R2 (claude): attach result sizes from this line's
374
+ // tool_result blocks. Pairing is by `tool_use_id` → `call_id`
375
+ // (source ∘ session ∘ call), the same key the omp branch uses.
376
+ // No duration is written — claude has no native wall time and the
377
+ // guarded timestamp fallback does not extend to it (0624 Design).
378
+ if (definition.source === 'claude') {
379
+ const timing = claudeToolResultTiming(raw);
380
+ if (timing !== null) {
381
+ const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message');
382
+ const sessionId = resultEntry?.normalized.session_id;
383
+ if (typeof sessionId === 'string') {
384
+ let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
385
+ if (pending === undefined) {
386
+ pending = await resolveToolCallRow(
387
+ options.db,
388
+ resolvedSource,
389
+ sessionId,
390
+ timing.toolCallId,
391
+ );
392
+ }
393
+ if (pending !== undefined) {
394
+ ops.push(toolCallResultBytesUpdateOp(pending.recordHash, timing.resultBytes));
395
+ }
396
+ }
397
+ }
398
+ }
363
399
  // Checkpoint upsert joins the same batch so an accepted line can never be
364
400
  // re-imported after a mid-batch crash.
365
401
  ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
@@ -392,6 +428,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
392
428
  processedLines,
393
429
  importedRecords,
394
430
  skippedDuplicates,
431
+ skippedCorruptLines,
395
432
  unknownRecords,
396
433
  parseErrors,
397
434
  validationErrors,
@@ -405,18 +442,28 @@ function parseJsonLine(
405
442
  sourceFile: string,
406
443
  sourceLine: number,
407
444
  parseErrors: ImportIssue[],
445
+ corruptLinePolicy: SourceDefinition['corruptLinePolicy'] = 'error',
408
446
  ): JsonObject | undefined {
447
+ let parsed: unknown;
409
448
  try {
410
- const value = JSON.parse(line);
411
- if (value === null || typeof value !== 'object' || Array.isArray(value)) {
449
+ parsed = JSON.parse(line);
450
+ } catch (error) {
451
+ if (corruptLinePolicy !== 'skip') {
452
+ parseErrors.push({
453
+ sourceFile,
454
+ sourceLine,
455
+ reason: error instanceof Error ? error.message : 'Invalid JSON',
456
+ });
457
+ }
458
+ return undefined;
459
+ }
460
+ if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
461
+ if (corruptLinePolicy !== 'skip') {
412
462
  parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
413
- return undefined;
414
463
  }
415
- return value as JsonObject;
416
- } catch (error) {
417
- parseErrors.push({ sourceFile, sourceLine, reason: error instanceof Error ? error.message : 'Invalid JSON' });
418
464
  return undefined;
419
465
  }
466
+ return parsed as JsonObject;
420
467
  }
421
468
 
422
469
  function splitRawRecord(
@@ -34,6 +34,7 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
34
34
  'provenance',
35
35
  'run_id',
36
36
  'task_wbs',
37
+ 'request_id',
37
38
  'imported_at',
38
39
  ],
39
40
  history_tool_call: [
@@ -279,6 +280,19 @@ function toolCallDurationUpdateOp(
279
280
  };
280
281
  }
281
282
 
283
+ /**
284
+ * Targeted UPDATE op attaching a result size to a previously-inserted
285
+ * `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
286
+ */
287
+ function toolCallResultBytesUpdateOp(recordHash: string, resultBytes: number): DbBatchOp {
288
+ return {
289
+ sql: `UPDATE history_tool_call
290
+ SET result_bytes = ?
291
+ WHERE record_hash = ?`,
292
+ params: [resultBytes, recordHash],
293
+ };
294
+ }
295
+
282
296
  /**
283
297
  * Query which of the given hashes already exist in the ledger, in chunks of at
284
298
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -713,5 +727,6 @@ export {
713
727
  targetTableFor,
714
728
  timestamp,
715
729
  toolCallDurationUpdateOp,
730
+ toolCallResultBytesUpdateOp,
716
731
  writeCheckpoint,
717
732
  };
package/src/mappers.ts CHANGED
@@ -27,6 +27,7 @@ const MESSAGE_MAPPER_KEYS: readonly string[] = [
27
27
  'provenance',
28
28
  'run_id',
29
29
  'task_wbs',
30
+ 'request_id',
30
31
  ];
31
32
 
32
33
  /** Columns the mapper may produce for history_tool_call. */
@@ -209,6 +210,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
209
210
  content_text: contentText ?? null,
210
211
  cwd: cwd ?? null,
211
212
  provenance: 'ambient',
213
+ request_id: s(raw.requestId) ?? null,
212
214
  },
213
215
  });
214
216
 
@@ -224,6 +226,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
224
226
  _messageSplitIndex: messageSplitIndex,
225
227
  session_id: sessionId,
226
228
  seq,
229
+ call_id: s(b.id),
227
230
  tool_name: String(b.name ?? ''),
228
231
  args_digest: argsDigest(b.input),
229
232
  args_raw: maybeArgsRaw('claude', String(b.name ?? ''), b.input),
@@ -554,6 +557,49 @@ export function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResult
554
557
  return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
555
558
  }
556
559
 
560
+ // ---------------------------------------------------------------------------
561
+ // Claude toolResult signals (task 0624 R2)
562
+ // ---------------------------------------------------------------------------
563
+
564
+ /** Signals carried by a Claude tool_result user line (task 0624 R2). */
565
+ export interface ClaudeToolResultTiming {
566
+ /** The `toolu_…` id from the answered tool_use block — joins `tool_use.id` exactly. */
567
+ toolCallId: string;
568
+ /** Message timestamp as epoch millis, when parseable. */
569
+ timestampMs: number | undefined;
570
+ /** Serialized size in bytes of the tool result payload. */
571
+ resultBytes: number;
572
+ }
573
+
574
+ /**
575
+ * Extract tool_result signals from a raw Claude Code record (task 0624 R2), or
576
+ * null when the record carries no tool_result block. Claude Code emits
577
+ * `type: "user"` lines whose `message.content` holds `{type: "tool_result",
578
+ * tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
579
+ * envelope. `resultBytes` prefers the model-visible block content and falls
580
+ * back to `toolUseResult`; both are null-safe. No wall time exists natively —
581
+ * claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
582
+ */
583
+ export function claudeToolResultTiming(raw: Record<string, unknown>): ClaudeToolResultTiming | null {
584
+ const content = o(raw.message).content;
585
+ if (!Array.isArray(content)) return null;
586
+ for (const block of content) {
587
+ if (typeof block !== 'object' || block === null) continue;
588
+ const b = block as Record<string, unknown>;
589
+ if (b.type !== 'tool_result') continue;
590
+ const toolCallId = s(b.tool_use_id);
591
+ if (toolCallId === undefined) continue;
592
+ const payload = b.content ?? raw.toolUseResult;
593
+ const resultBytes = JSON.stringify(payload ?? null)?.length ?? 2;
594
+ return {
595
+ toolCallId,
596
+ timestampMs: timestampToEpochMs(raw.timestamp),
597
+ resultBytes,
598
+ };
599
+ }
600
+ return null;
601
+ }
602
+
557
603
  // ---------------------------------------------------------------------------
558
604
  // Codex mapper
559
605
  // ---------------------------------------------------------------------------
@@ -172,6 +172,7 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
172
172
  processedLines,
173
173
  importedRecords,
174
174
  skippedDuplicates,
175
+ skippedCorruptLines: 0,
175
176
  unknownRecords: 0,
176
177
  parseErrors,
177
178
  validationErrors,
@@ -331,6 +332,7 @@ function emptyResult(mode: ImportMode): ImportResult {
331
332
  processedLines: 0,
332
333
  importedRecords: 0,
333
334
  skippedDuplicates: 0,
335
+ skippedCorruptLines: 0,
334
336
  unknownRecords: 0,
335
337
  parseErrors: [],
336
338
  validationErrors: [],
package/src/schema-sql.ts CHANGED
@@ -50,6 +50,7 @@ CREATE TABLE IF NOT EXISTS history_message (
50
50
  provenance TEXT NOT NULL,
51
51
  run_id TEXT,
52
52
  task_wbs TEXT,
53
+ request_id TEXT,
53
54
  imported_at TEXT NOT NULL
54
55
  );
55
56
 
package/src/sources.ts CHANGED
@@ -190,15 +190,21 @@ export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinitio
190
190
  GROK_FIELD_MAP,
191
191
  GROK_SCHEMA,
192
192
  ),
193
- agy: customSourceDefinition(
194
- 'agy',
195
- 'Antigravity CLI',
196
- ['.gemini/antigravity-cli/brain'],
197
- ['*.jsonl'],
198
- agySplit,
199
- AGY_FIELD_MAP,
200
- AGY_SCHEMA,
201
- ),
193
+ agy: {
194
+ ...customSourceDefinition(
195
+ 'agy',
196
+ 'Antigravity CLI',
197
+ ['.gemini/antigravity-cli/brain'],
198
+ ['*.jsonl'],
199
+ agySplit,
200
+ AGY_FIELD_MAP,
201
+ AGY_SCHEMA,
202
+ ),
203
+ // agy's producer interleaves non-JSON fragments and torn tail writes into its
204
+ // brain jsonl (task 0623: 789/89,818 lines, all unrecoverable). Skipping keeps
205
+ // the 99%+ good records importable without degrading the whole source.
206
+ corruptLinePolicy: 'skip',
207
+ },
202
208
  gemini: customSourceDefinition(
203
209
  'gemini',
204
210
  'Gemini CLI',
package/src/types.ts CHANGED
@@ -62,6 +62,13 @@ export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
62
62
  readonly displayName: string;
63
63
  readonly defaultRoots: readonly string[];
64
64
  readonly filePatterns: readonly string[];
65
+ /**
66
+ * Policy for lines that fail JSON parsing. `'error'` (default) records a parse error and
67
+ * degrades the import; `'skip'` counts the line and moves on — for producers whose logs
68
+ * are known to interleave non-JSON fragments and torn tail writes (agy task 0623:
69
+ * 789/89,818 lines unparseable, all unrecoverable garbage).
70
+ */
71
+ readonly corruptLinePolicy?: 'error' | 'skip';
65
72
  readonly targetTable: string;
66
73
  readonly splitConfig: SplitConfig;
67
74
  readonly fieldMap: Readonly<Record<string, string>>;
@@ -126,6 +133,8 @@ export interface ImportResult {
126
133
  readonly importedRecords: number;
127
134
  readonly skippedDuplicates: number;
128
135
  readonly unknownRecords: number;
136
+ /** Lines dropped by `corruptLinePolicy: 'skip'` instead of degrading the import. */
137
+ readonly skippedCorruptLines: number;
129
138
  readonly parseErrors: readonly ImportIssue[];
130
139
  readonly validationErrors: readonly ImportIssue[];
131
140
  readonly checkpointUpdates: number;