@gobing-ai/ts-llm-jsonl-importer 0.4.33 → 0.4.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAOjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAgMrH"}
1
+ {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAkRrH"}
package/dist/importer.js CHANGED
@@ -1,8 +1,44 @@
1
1
  import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
2
2
  import { sha256 } from './hash.js';
3
- import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, } from './jsonl-importer-dao.js';
3
+ import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, } from './jsonl-importer-dao.js';
4
+ import { ompToolResultTiming, timestampToEpochMs } from './mappers.js';
4
5
  import { redactRecord } from './redaction.js';
5
6
  import { resolveSourceDefinition } from './sources.js';
7
+ const TOOL_CALL_KEY_SEP = '\u0000';
8
+ function toolCallKey(source, sessionId, callId) {
9
+ return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
10
+ }
11
+ /** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
12
+ async function resolveToolCallRow(db, source, sessionId, callId) {
13
+ const row = await db.queryFirst(`SELECT tc.record_hash AS recordHash, m.ts AS messageTs
14
+ FROM history_tool_call tc
15
+ LEFT JOIN history_message m ON m.record_hash = tc.message_hash
16
+ WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`, source, sessionId, callId);
17
+ if (row === undefined || row === null)
18
+ return undefined;
19
+ return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
20
+ }
21
+ /**
22
+ * Compute the attachable duration for a toolResult, or null when nothing should
23
+ * be written (implausible fallback delta → row stays NULL → counts unmeasured).
24
+ */
25
+ function attachableDuration(timing, messageTsMs) {
26
+ if (timing.wallTimeMs !== undefined) {
27
+ // The tool's own measurement — no clamping, no sanity check.
28
+ return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
29
+ }
30
+ if (timing.timestampMs === undefined || messageTsMs === undefined)
31
+ return null;
32
+ const delta = timing.timestampMs - messageTsMs;
33
+ // Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
34
+ if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000)
35
+ return null;
36
+ return {
37
+ startedAt: new Date(messageTsMs).toISOString(),
38
+ completedAt: new Date(timing.timestampMs).toISOString(),
39
+ durationMs: Math.round(delta),
40
+ };
41
+ }
6
42
  /** Run the JSONL import pipeline for one source.
7
43
  *
8
44
  * @param source - Either a built-in source identifier resolved from the
@@ -32,6 +68,9 @@ export async function runJsonlImport(source, options) {
32
68
  // Full-mode desired set (R1, task 0504): every record hash the current source produces.
33
69
  // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
34
70
  const desiredHashes = new Set();
71
+ // Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
72
+ // (source, session_id, call_id) so a later toolResult line can attach its duration.
73
+ const toolCallRows = new Map();
35
74
  for (const file of files) {
36
75
  const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
37
76
  let lineNumber = 0;
@@ -92,6 +131,33 @@ export async function runJsonlImport(source, options) {
92
131
  // second pass, no checkpoint advance, no partial rows from this record.
93
132
  if (lineRejected)
94
133
  continue;
134
+ // Task 0564 R1 (omp): register this line's tool-call rows for later duration
135
+ // attach. Runs over ALL prepared entries — including duplicates — because a
136
+ // re-imported line's row already exists and its duration UPDATE still targets
137
+ // the same deterministic record_hash. The assistant message timestamp is the
138
+ // fallback's start bound; every call in one message shares it.
139
+ if (definition.source === 'omp') {
140
+ const assistantTs = new Map();
141
+ for (const entry of prepared) {
142
+ if (entry.split.targetTable === 'history_message' &&
143
+ entry.normalized.role === 'assistant' &&
144
+ typeof entry.normalized.session_id === 'string') {
145
+ assistantTs.set(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`, timestampToEpochMs(entry.normalized.ts));
146
+ }
147
+ }
148
+ for (const entry of prepared) {
149
+ const callId = entry.normalized.call_id;
150
+ if (entry.split.targetTable === 'history_tool_call' &&
151
+ typeof callId === 'string' &&
152
+ callId.length > 0 &&
153
+ typeof entry.normalized.session_id === 'string') {
154
+ toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
155
+ recordHash: entry.recordHash,
156
+ messageTsMs: assistantTs.get(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`),
157
+ });
158
+ }
159
+ }
160
+ }
95
161
  // Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
96
162
  // instead of a SELECT per record; duplicates are dropped before the write pass.
97
163
  const existing = await ledgerExistingHashes(options.db, prepared.map((e) => e.recordHash));
@@ -128,6 +194,32 @@ export async function runJsonlImport(source, options) {
128
194
  importedRecords += 1;
129
195
  }
130
196
  if (!options.dryRun && lineSucceeded) {
197
+ // Task 0564 R1 (omp): attach tool durations from this line's toolResult
198
+ // message. Unmatched toolCallIds attach nothing and never fail the import;
199
+ // an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
200
+ // this line's batch — idempotent, so re-imports and post-resume tail
201
+ // reprocessing write the same values.
202
+ if (definition.source === 'omp') {
203
+ const timing = ompToolResultTiming(raw);
204
+ if (timing !== null) {
205
+ const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult');
206
+ const sessionId = resultEntry?.normalized.session_id;
207
+ if (typeof sessionId === 'string') {
208
+ let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
209
+ if (pending === undefined) {
210
+ // The tool-call line sits behind this run's checkpoint — the
211
+ // row exists from an earlier import; resolve it from the DB.
212
+ pending = await resolveToolCallRow(options.db, resolvedSource, sessionId, timing.toolCallId);
213
+ }
214
+ if (pending !== undefined) {
215
+ const duration = attachableDuration(timing, pending.messageTsMs);
216
+ if (duration !== null) {
217
+ ops.push(toolCallDurationUpdateOp(pending.recordHash, duration.startedAt, duration.completedAt, duration.durationMs));
218
+ }
219
+ }
220
+ }
221
+ }
222
+ }
131
223
  // Checkpoint upsert joins the same batch so an accepted line can never be
132
224
  // re-imported after a mid-batch crash.
133
225
  ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
@@ -44,6 +44,16 @@ export declare function recordInsertOp(targetTable: string, recordHash: string,
44
44
  export declare function ledgerInsertOp(recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): DbBatchOp;
45
45
  /** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
46
46
  export declare function checkpointUpsertOp(source: string, sourceFile: string, line: number, now: ImportOptions['now']): DbBatchOp;
47
+ /**
48
+ * Targeted UPDATE op attaching a tool duration to a previously-inserted
49
+ * `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
50
+ * unindexed predicate. Idempotent: re-imports write the same values.
51
+ *
52
+ * Bounds are written alongside a FALLBACK duration so the figure is auditable;
53
+ * a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
54
+ * two measurement paths stay distinguishable.
55
+ */
56
+ declare function toolCallDurationUpdateOp(recordHash: string, startedAt: string | null, completedAt: string | null, durationMs: number | null): DbBatchOp;
47
57
  /**
48
58
  * Query which of the given hashes already exist in the ledger, in chunks of at
49
59
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -126,5 +136,5 @@ export declare function deleteCheckpointOperation(source: string, sourceFile: st
126
136
  /** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
127
137
  export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
128
138
  export type { CheckpointRow };
129
- export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
139
+ export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
130
140
  //# sourceMappingURL=jsonl-importer-dao.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAsDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
1
+ {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAuDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,eAAe,GAClB,CAAC"}
@@ -38,6 +38,7 @@ const TYPED_TABLE_COLUMNS = {
38
38
  'session_id',
39
39
  'seq',
40
40
  'tool_name',
41
+ 'call_id',
41
42
  'args_digest',
42
43
  'args_raw',
43
44
  'status',
@@ -194,6 +195,23 @@ export function checkpointUpsertOp(source, sourceFile, line, now) {
194
195
  params: [source, sourceFile, line, timestamp(now)],
195
196
  };
196
197
  }
198
+ /**
199
+ * Targeted UPDATE op attaching a tool duration to a previously-inserted
200
+ * `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
201
+ * unindexed predicate. Idempotent: re-imports write the same values.
202
+ *
203
+ * Bounds are written alongside a FALLBACK duration so the figure is auditable;
204
+ * a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
205
+ * two measurement paths stay distinguishable.
206
+ */
207
+ function toolCallDurationUpdateOp(recordHash, startedAt, completedAt, durationMs) {
208
+ return {
209
+ sql: `UPDATE history_tool_call
210
+ SET started_at = ?, completed_at = ?, duration_ms = ?
211
+ WHERE record_hash = ?`,
212
+ params: [startedAt, completedAt, durationMs, recordHash],
213
+ };
214
+ }
197
215
  /**
198
216
  * Query which of the given hashes already exist in the ledger, in chunks of at
199
217
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -468,4 +486,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
468
486
  }
469
487
  return operations;
470
488
  }
471
- export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
489
+ export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
package/dist/mappers.d.ts CHANGED
@@ -8,6 +8,26 @@ export declare function claudeSplit(raw: Record<string, unknown>): readonly Spli
8
8
  export declare function piSplit(raw: unknown): readonly SplitEntry[];
9
9
  /** Map an OMP JSONL record into one or more ETL split entries. */
10
10
  export declare function ompSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
11
+ /** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
12
+ export interface OmpToolResultTiming {
13
+ /** The toolCall id this result answers — joins `toolCall.id` exactly. */
14
+ toolCallId: string;
15
+ /** The tool's own measured wall time in ms, when present and finite. */
16
+ wallTimeMs: number | undefined;
17
+ /** Message timestamp as epoch millis, when parseable. */
18
+ timestampMs: number | undefined;
19
+ }
20
+ /** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
21
+ export declare function timestampToEpochMs(value: unknown): number | undefined;
22
+ /**
23
+ * Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
24
+ * when the record is not a toolResult message or carries no toolCallId. Live OMP
25
+ * emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
26
+ * content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
27
+ * measured wall time (48% of results in the sampled session) and `toolCallId`
28
+ * joins the originating `toolCall.id` exactly.
29
+ */
30
+ export declare function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null;
11
31
  /** Map a Codex JSONL record into one or more ETL split entries. */
12
32
  export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
13
33
  /** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
@@ -1 +1 @@
1
- {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAkExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAqDD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA6G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAuE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA2GxG;AAoBD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
1
+ {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAqDD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA6G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAuE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
package/dist/mappers.js CHANGED
@@ -31,6 +31,7 @@ const TOOL_CALL_MAPPER_KEYS = [
31
31
  'session_id',
32
32
  'seq',
33
33
  'tool_name',
34
+ 'call_id',
34
35
  'args_digest',
35
36
  'args_raw',
36
37
  'status',
@@ -383,6 +384,10 @@ export function ompSplit(raw, context) {
383
384
  session_id: sessionId,
384
385
  seq,
385
386
  tool_name: String(call.name ?? ''),
387
+ // Task 0564 R1: the tool's own call id is the exact join key a
388
+ // toolResult's `toolCallId` matches — it lets the streaming loop
389
+ // attach the tool's measured duration to the right row.
390
+ call_id: s(call.id),
386
391
  args_digest: argsDigest(call.input ?? call.arguments),
387
392
  args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
388
393
  status: 'ok',
@@ -413,6 +418,39 @@ function normalizeOmpToolCall(block) {
413
418
  return block;
414
419
  return null;
415
420
  }
421
+ /** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
422
+ export function timestampToEpochMs(value) {
423
+ if (typeof value === 'number' && Number.isFinite(value))
424
+ return value;
425
+ if (typeof value === 'string' && value.length > 0) {
426
+ const numeric = Number(value);
427
+ if (Number.isFinite(numeric))
428
+ return numeric;
429
+ const parsed = Date.parse(value);
430
+ if (!Number.isNaN(parsed))
431
+ return parsed;
432
+ }
433
+ return undefined;
434
+ }
435
+ /**
436
+ * Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
437
+ * when the record is not a toolResult message or carries no toolCallId. Live OMP
438
+ * emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
439
+ * content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
440
+ * measured wall time (48% of results in the sampled session) and `toolCallId`
441
+ * joins the originating `toolCall.id` exactly.
442
+ */
443
+ export function ompToolResultTiming(raw) {
444
+ const msg = o(raw.message);
445
+ if (String(msg.role ?? '').toLowerCase() !== 'toolresult')
446
+ return null;
447
+ const toolCallId = s(msg.toolCallId);
448
+ if (toolCallId === undefined)
449
+ return null;
450
+ const details = o(msg.details);
451
+ const wallTimeMs = typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
452
+ return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
453
+ }
416
454
  // ---------------------------------------------------------------------------
417
455
  // Codex mapper
418
456
  // ---------------------------------------------------------------------------
@@ -1 +1 @@
1
- {"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA4E9B,CAAC"}
1
+ {"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA6E9B,CAAC"}
@@ -62,6 +62,7 @@ CREATE TABLE IF NOT EXISTS history_tool_call (
62
62
  session_id TEXT NOT NULL,
63
63
  seq INTEGER NOT NULL,
64
64
  tool_name TEXT NOT NULL,
65
+ call_id TEXT,
65
66
  args_digest TEXT,
66
67
  args_raw TEXT,
67
68
  status TEXT NOT NULL,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gobing-ai/ts-llm-jsonl-importer",
3
- "version": "0.4.33",
3
+ "version": "0.4.35",
4
4
  "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
5
5
  "keywords": [
6
6
  "typescript",
@@ -47,9 +47,9 @@
47
47
  "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
48
48
  },
49
49
  "dependencies": {
50
- "@gobing-ai/ts-db": "^0.4.33",
51
- "@gobing-ai/ts-runtime": "^0.4.33",
52
- "@gobing-ai/ts-utils": "^0.4.33",
50
+ "@gobing-ai/ts-db": "^0.4.35",
51
+ "@gobing-ai/ts-runtime": "^0.4.35",
52
+ "@gobing-ai/ts-utils": "^0.4.35",
53
53
  "zod": "^4.1.0"
54
54
  },
55
55
  "devDependencies": {
package/src/importer.ts CHANGED
@@ -19,7 +19,9 @@ import {
19
19
  recordInsertOp,
20
20
  resetCheckpoints,
21
21
  targetTableFor,
22
+ toolCallDurationUpdateOp,
22
23
  } from './jsonl-importer-dao';
24
+ import { type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
23
25
  import { redactRecord } from './redaction';
24
26
  import { resolveSourceDefinition } from './sources';
25
27
  import type {
@@ -37,6 +39,83 @@ interface SplitRecord {
37
39
  readonly raw: JsonObject;
38
40
  }
39
41
 
42
+ // ---------------------------------------------------------------------------
43
+ // Task 0564 R1 — tool-call duration attach (omp)
44
+ //
45
+ // A toolCall row is emitted while handling its assistant message (SplitEntry is
46
+ // insert-only, so the duration cannot ride the mapper's split path); the
47
+ // duration arrives on a LATER toolResult line. The attach therefore lives in the
48
+ // streaming loop, keyed on (source, session_id, call_id) per the frozen contract:
49
+ // - `details.wallTimeMs` is the tool's own measured wall time — used as-is,
50
+ // rounded, NEVER clamped or second-guessed (guards are fallback-only);
51
+ // - otherwise the fallback `toolResult.timestamp − toolCall message timestamp`
52
+ // when both are parseable AND the delta is in [0, 3_600_000]; an implausible
53
+ // delta stays NULL so the unmeasured count stays honest;
54
+ // - started_at / completed_at are written alongside a FALLBACK figure so it is
55
+ // auditable (wallTimeMs rows keep both NULL — the two stay distinguishable);
56
+ // - an unmatched toolCallId attaches nothing and never fails the import.
57
+ //
58
+ // The in-memory map covers the common (fresh-import) case without extra queries;
59
+ // a toolResult whose tool-call line was imported by an earlier checkpointed run
60
+ // (resume) resolves through a DB lookup instead, so a mid-file checkpoint loses
61
+ // no earlier duration.
62
+ // ---------------------------------------------------------------------------
63
+
64
+ interface PendingToolCall {
65
+ readonly recordHash: string;
66
+ /** Assistant-message timestamp (epoch ms) — the fallback's start bound. */
67
+ readonly messageTsMs: number | undefined;
68
+ }
69
+
70
+ const TOOL_CALL_KEY_SEP = '\u0000';
71
+
72
+ function toolCallKey(source: string, sessionId: string, callId: string): string {
73
+ return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
74
+ }
75
+
76
+ /** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
77
+ async function resolveToolCallRow(
78
+ db: ImportOptions['db'],
79
+ source: string,
80
+ sessionId: string,
81
+ callId: string,
82
+ ): Promise<PendingToolCall | undefined> {
83
+ const row = await db.queryFirst<{ recordHash: string; messageTs: string | null }>(
84
+ `SELECT tc.record_hash AS recordHash, m.ts AS messageTs
85
+ FROM history_tool_call tc
86
+ LEFT JOIN history_message m ON m.record_hash = tc.message_hash
87
+ WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`,
88
+ source,
89
+ sessionId,
90
+ callId,
91
+ );
92
+ if (row === undefined || row === null) return undefined;
93
+ return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
94
+ }
95
+
96
+ /**
97
+ * Compute the attachable duration for a toolResult, or null when nothing should
98
+ * be written (implausible fallback delta → row stays NULL → counts unmeasured).
99
+ */
100
+ function attachableDuration(
101
+ timing: OmpToolResultTiming,
102
+ messageTsMs: number | undefined,
103
+ ): { startedAt: string | null; completedAt: string | null; durationMs: number } | null {
104
+ if (timing.wallTimeMs !== undefined) {
105
+ // The tool's own measurement — no clamping, no sanity check.
106
+ return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
107
+ }
108
+ if (timing.timestampMs === undefined || messageTsMs === undefined) return null;
109
+ const delta = timing.timestampMs - messageTsMs;
110
+ // Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
111
+ if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000) return null;
112
+ return {
113
+ startedAt: new Date(messageTsMs).toISOString(),
114
+ completedAt: new Date(timing.timestampMs).toISOString(),
115
+ durationMs: Math.round(delta),
116
+ };
117
+ }
118
+
40
119
  /** Run the JSONL import pipeline for one source.
41
120
  *
42
121
  * @param source - Either a built-in source identifier resolved from the
@@ -69,6 +148,10 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
69
148
  // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
70
149
  const desiredHashes = new Set<string>();
71
150
 
151
+ // Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
152
+ // (source, session_id, call_id) so a later toolResult line can attach its duration.
153
+ const toolCallRows = new Map<string, PendingToolCall>();
154
+
72
155
  for (const file of files) {
73
156
  const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
74
157
  let lineNumber = 0;
@@ -138,6 +221,43 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
138
221
  // second pass, no checkpoint advance, no partial rows from this record.
139
222
  if (lineRejected) continue;
140
223
 
224
+ // Task 0564 R1 (omp): register this line's tool-call rows for later duration
225
+ // attach. Runs over ALL prepared entries — including duplicates — because a
226
+ // re-imported line's row already exists and its duration UPDATE still targets
227
+ // the same deterministic record_hash. The assistant message timestamp is the
228
+ // fallback's start bound; every call in one message shares it.
229
+ if (definition.source === 'omp') {
230
+ const assistantTs = new Map<string, number | undefined>();
231
+ for (const entry of prepared) {
232
+ if (
233
+ entry.split.targetTable === 'history_message' &&
234
+ entry.normalized.role === 'assistant' &&
235
+ typeof entry.normalized.session_id === 'string'
236
+ ) {
237
+ assistantTs.set(
238
+ `${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
239
+ timestampToEpochMs(entry.normalized.ts),
240
+ );
241
+ }
242
+ }
243
+ for (const entry of prepared) {
244
+ const callId = entry.normalized.call_id;
245
+ if (
246
+ entry.split.targetTable === 'history_tool_call' &&
247
+ typeof callId === 'string' &&
248
+ callId.length > 0 &&
249
+ typeof entry.normalized.session_id === 'string'
250
+ ) {
251
+ toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
252
+ recordHash: entry.recordHash,
253
+ messageTsMs: assistantTs.get(
254
+ `${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
255
+ ),
256
+ });
257
+ }
258
+ }
259
+ }
260
+
141
261
  // Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
142
262
  // instead of a SELECT per record; duplicates are dropped before the write pass.
143
263
  const existing = await ledgerExistingHashes(
@@ -199,6 +319,47 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
199
319
  }
200
320
 
201
321
  if (!options.dryRun && lineSucceeded) {
322
+ // Task 0564 R1 (omp): attach tool durations from this line's toolResult
323
+ // message. Unmatched toolCallIds attach nothing and never fail the import;
324
+ // an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
325
+ // this line's batch — idempotent, so re-imports and post-resume tail
326
+ // reprocessing write the same values.
327
+ if (definition.source === 'omp') {
328
+ const timing = ompToolResultTiming(raw);
329
+ if (timing !== null) {
330
+ const resultEntry = prepared.find(
331
+ (entry) =>
332
+ entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult',
333
+ );
334
+ const sessionId = resultEntry?.normalized.session_id;
335
+ if (typeof sessionId === 'string') {
336
+ let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
337
+ if (pending === undefined) {
338
+ // The tool-call line sits behind this run's checkpoint — the
339
+ // row exists from an earlier import; resolve it from the DB.
340
+ pending = await resolveToolCallRow(
341
+ options.db,
342
+ resolvedSource,
343
+ sessionId,
344
+ timing.toolCallId,
345
+ );
346
+ }
347
+ if (pending !== undefined) {
348
+ const duration = attachableDuration(timing, pending.messageTsMs);
349
+ if (duration !== null) {
350
+ ops.push(
351
+ toolCallDurationUpdateOp(
352
+ pending.recordHash,
353
+ duration.startedAt,
354
+ duration.completedAt,
355
+ duration.durationMs,
356
+ ),
357
+ );
358
+ }
359
+ }
360
+ }
361
+ }
362
+ }
202
363
  // Checkpoint upsert joins the same batch so an accepted line can never be
203
364
  // re-imported after a mid-batch crash.
204
365
  ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
@@ -45,6 +45,7 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
45
45
  'session_id',
46
46
  'seq',
47
47
  'tool_name',
48
+ 'call_id',
48
49
  'args_digest',
49
50
  'args_raw',
50
51
  'status',
@@ -255,6 +256,29 @@ export function checkpointUpsertOp(
255
256
  };
256
257
  }
257
258
 
259
+ /**
260
+ * Targeted UPDATE op attaching a tool duration to a previously-inserted
261
+ * `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
262
+ * unindexed predicate. Idempotent: re-imports write the same values.
263
+ *
264
+ * Bounds are written alongside a FALLBACK duration so the figure is auditable;
265
+ * a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
266
+ * two measurement paths stay distinguishable.
267
+ */
268
+ function toolCallDurationUpdateOp(
269
+ recordHash: string,
270
+ startedAt: string | null,
271
+ completedAt: string | null,
272
+ durationMs: number | null,
273
+ ): DbBatchOp {
274
+ return {
275
+ sql: `UPDATE history_tool_call
276
+ SET started_at = ?, completed_at = ?, duration_ms = ?
277
+ WHERE record_hash = ?`,
278
+ params: [startedAt, completedAt, durationMs, recordHash],
279
+ };
280
+ }
281
+
258
282
  /**
259
283
  * Query which of the given hashes already exist in the ledger, in chunks of at
260
284
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -688,5 +712,6 @@ export {
688
712
  resetCheckpoints,
689
713
  targetTableFor,
690
714
  timestamp,
715
+ toolCallDurationUpdateOp,
691
716
  writeCheckpoint,
692
717
  };
package/src/mappers.ts CHANGED
@@ -35,6 +35,7 @@ const TOOL_CALL_MAPPER_KEYS: readonly string[] = [
35
35
  'session_id',
36
36
  'seq',
37
37
  'tool_name',
38
+ 'call_id',
38
39
  'args_digest',
39
40
  'args_raw',
40
41
  'status',
@@ -411,6 +412,10 @@ export function ompSplit(raw: Record<string, unknown>, context?: TransformContex
411
412
  session_id: sessionId,
412
413
  seq,
413
414
  tool_name: String(call.name ?? ''),
415
+ // Task 0564 R1: the tool's own call id is the exact join key a
416
+ // toolResult's `toolCallId` matches — it lets the streaming loop
417
+ // attach the tool's measured duration to the right row.
418
+ call_id: s(call.id),
414
419
  args_digest: argsDigest(call.input ?? call.arguments),
415
420
  args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
416
421
  status: 'ok',
@@ -441,6 +446,51 @@ function normalizeOmpToolCall(block: Record<string, unknown>): Record<string, un
441
446
  return null;
442
447
  }
443
448
 
449
+ // ---------------------------------------------------------------------------
450
+ // OMP toolResult timing signals (task 0564 R1)
451
+ // ---------------------------------------------------------------------------
452
+
453
+ /** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
454
+ export interface OmpToolResultTiming {
455
+ /** The toolCall id this result answers — joins `toolCall.id` exactly. */
456
+ toolCallId: string;
457
+ /** The tool's own measured wall time in ms, when present and finite. */
458
+ wallTimeMs: number | undefined;
459
+ /** Message timestamp as epoch millis, when parseable. */
460
+ timestampMs: number | undefined;
461
+ }
462
+
463
+ /** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
464
+ export function timestampToEpochMs(value: unknown): number | undefined {
465
+ if (typeof value === 'number' && Number.isFinite(value)) return value;
466
+ if (typeof value === 'string' && value.length > 0) {
467
+ const numeric = Number(value);
468
+ if (Number.isFinite(numeric)) return numeric;
469
+ const parsed = Date.parse(value);
470
+ if (!Number.isNaN(parsed)) return parsed;
471
+ }
472
+ return undefined;
473
+ }
474
+
475
+ /**
476
+ * Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
477
+ * when the record is not a toolResult message or carries no toolCallId. Live OMP
478
+ * emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
479
+ * content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
480
+ * measured wall time (48% of results in the sampled session) and `toolCallId`
481
+ * joins the originating `toolCall.id` exactly.
482
+ */
483
+ export function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null {
484
+ const msg = o(raw.message);
485
+ if (String(msg.role ?? '').toLowerCase() !== 'toolresult') return null;
486
+ const toolCallId = s(msg.toolCallId);
487
+ if (toolCallId === undefined) return null;
488
+ const details = o(msg.details);
489
+ const wallTimeMs =
490
+ typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
491
+ return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
492
+ }
493
+
444
494
  // ---------------------------------------------------------------------------
445
495
  // Codex mapper
446
496
  // ---------------------------------------------------------------------------
package/src/schema-sql.ts CHANGED
@@ -62,6 +62,7 @@ CREATE TABLE IF NOT EXISTS history_tool_call (
62
62
  session_id TEXT NOT NULL,
63
63
  seq INTEGER NOT NULL,
64
64
  tool_name TEXT NOT NULL,
65
+ call_id TEXT,
65
66
  args_digest TEXT,
66
67
  args_raw TEXT,
67
68
  status TEXT NOT NULL,