@gobing-ai/ts-llm-jsonl-importer 0.4.33 → 0.4.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAOjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAgMrH"}
1
+ {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAkRrH"}
package/dist/importer.js CHANGED
@@ -1,8 +1,44 @@
1
1
  import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
2
2
  import { sha256 } from './hash.js';
3
- import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, } from './jsonl-importer-dao.js';
3
+ import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, } from './jsonl-importer-dao.js';
4
+ import { ompToolResultTiming, timestampToEpochMs } from './mappers.js';
4
5
  import { redactRecord } from './redaction.js';
5
6
  import { resolveSourceDefinition } from './sources.js';
7
+ const TOOL_CALL_KEY_SEP = '\u0000';
8
+ function toolCallKey(source, sessionId, callId) {
9
+ return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
10
+ }
11
+ /** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
12
+ async function resolveToolCallRow(db, source, sessionId, callId) {
13
+ const row = await db.queryFirst(`SELECT tc.record_hash AS recordHash, m.ts AS messageTs
14
+ FROM history_tool_call tc
15
+ LEFT JOIN history_message m ON m.record_hash = tc.message_hash
16
+ WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`, source, sessionId, callId);
17
+ if (row === undefined || row === null)
18
+ return undefined;
19
+ return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
20
+ }
21
+ /**
22
+ * Compute the attachable duration for a toolResult, or null when nothing should
23
+ * be written (implausible fallback delta → row stays NULL → counts unmeasured).
24
+ */
25
+ function attachableDuration(timing, messageTsMs) {
26
+ if (timing.wallTimeMs !== undefined) {
27
+ // The tool's own measurement — no clamping, no sanity check.
28
+ return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
29
+ }
30
+ if (timing.timestampMs === undefined || messageTsMs === undefined)
31
+ return null;
32
+ const delta = timing.timestampMs - messageTsMs;
33
+ // Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
34
+ if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000)
35
+ return null;
36
+ return {
37
+ startedAt: new Date(messageTsMs).toISOString(),
38
+ completedAt: new Date(timing.timestampMs).toISOString(),
39
+ durationMs: Math.round(delta),
40
+ };
41
+ }
6
42
  /** Run the JSONL import pipeline for one source.
7
43
  *
8
44
  * @param source - Either a built-in source identifier resolved from the
@@ -32,6 +68,9 @@ export async function runJsonlImport(source, options) {
32
68
  // Full-mode desired set (R1, task 0504): every record hash the current source produces.
33
69
  // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
34
70
  const desiredHashes = new Set();
71
+ // Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
72
+ // (source, session_id, call_id) so a later toolResult line can attach its duration.
73
+ const toolCallRows = new Map();
35
74
  for (const file of files) {
36
75
  const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
37
76
  let lineNumber = 0;
@@ -92,6 +131,33 @@ export async function runJsonlImport(source, options) {
92
131
  // second pass, no checkpoint advance, no partial rows from this record.
93
132
  if (lineRejected)
94
133
  continue;
134
+ // Task 0564 R1 (omp): register this line's tool-call rows for later duration
135
+ // attach. Runs over ALL prepared entries — including duplicates — because a
136
+ // re-imported line's row already exists and its duration UPDATE still targets
137
+ // the same deterministic record_hash. The assistant message timestamp is the
138
+ // fallback's start bound; every call in one message shares it.
139
+ if (definition.source === 'omp') {
140
+ const assistantTs = new Map();
141
+ for (const entry of prepared) {
142
+ if (entry.split.targetTable === 'history_message' &&
143
+ entry.normalized.role === 'assistant' &&
144
+ typeof entry.normalized.session_id === 'string') {
145
+ assistantTs.set(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`, timestampToEpochMs(entry.normalized.ts));
146
+ }
147
+ }
148
+ for (const entry of prepared) {
149
+ const callId = entry.normalized.call_id;
150
+ if (entry.split.targetTable === 'history_tool_call' &&
151
+ typeof callId === 'string' &&
152
+ callId.length > 0 &&
153
+ typeof entry.normalized.session_id === 'string') {
154
+ toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
155
+ recordHash: entry.recordHash,
156
+ messageTsMs: assistantTs.get(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`),
157
+ });
158
+ }
159
+ }
160
+ }
95
161
  // Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
96
162
  // instead of a SELECT per record; duplicates are dropped before the write pass.
97
163
  const existing = await ledgerExistingHashes(options.db, prepared.map((e) => e.recordHash));
@@ -128,6 +194,32 @@ export async function runJsonlImport(source, options) {
128
194
  importedRecords += 1;
129
195
  }
130
196
  if (!options.dryRun && lineSucceeded) {
197
+ // Task 0564 R1 (omp): attach tool durations from this line's toolResult
198
+ // message. Unmatched toolCallIds attach nothing and never fail the import;
199
+ // an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
200
+ // this line's batch — idempotent, so re-imports and post-resume tail
201
+ // reprocessing write the same values.
202
+ if (definition.source === 'omp') {
203
+ const timing = ompToolResultTiming(raw);
204
+ if (timing !== null) {
205
+ const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult');
206
+ const sessionId = resultEntry?.normalized.session_id;
207
+ if (typeof sessionId === 'string') {
208
+ let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
209
+ if (pending === undefined) {
210
+ // The tool-call line sits behind this run's checkpoint — the
211
+ // row exists from an earlier import; resolve it from the DB.
212
+ pending = await resolveToolCallRow(options.db, resolvedSource, sessionId, timing.toolCallId);
213
+ }
214
+ if (pending !== undefined) {
215
+ const duration = attachableDuration(timing, pending.messageTsMs);
216
+ if (duration !== null) {
217
+ ops.push(toolCallDurationUpdateOp(pending.recordHash, duration.startedAt, duration.completedAt, duration.durationMs));
218
+ }
219
+ }
220
+ }
221
+ }
222
+ }
131
223
  // Checkpoint upsert joins the same batch so an accepted line can never be
132
224
  // re-imported after a mid-batch crash.
133
225
  ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
@@ -44,6 +44,16 @@ export declare function recordInsertOp(targetTable: string, recordHash: string,
44
44
  export declare function ledgerInsertOp(recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): DbBatchOp;
45
45
  /** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
46
46
  export declare function checkpointUpsertOp(source: string, sourceFile: string, line: number, now: ImportOptions['now']): DbBatchOp;
47
+ /**
48
+ * Targeted UPDATE op attaching a tool duration to a previously-inserted
49
+ * `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
50
+ * unindexed predicate. Idempotent: re-imports write the same values.
51
+ *
52
+ * Bounds are written alongside a FALLBACK duration so the figure is auditable;
53
+ * a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
54
+ * two measurement paths stay distinguishable.
55
+ */
56
+ declare function toolCallDurationUpdateOp(recordHash: string, startedAt: string | null, completedAt: string | null, durationMs: number | null): DbBatchOp;
47
57
  /**
48
58
  * Query which of the given hashes already exist in the ledger, in chunks of at
49
59
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -126,5 +136,5 @@ export declare function deleteCheckpointOperation(source: string, sourceFile: st
126
136
  /** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
127
137
  export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
128
138
  export type { CheckpointRow };
129
- export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
139
+ export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
130
140
  //# sourceMappingURL=jsonl-importer-dao.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAsDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
1
+ {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAuDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,eAAe,GAClB,CAAC"}
@@ -38,6 +38,7 @@ const TYPED_TABLE_COLUMNS = {
38
38
  'session_id',
39
39
  'seq',
40
40
  'tool_name',
41
+ 'call_id',
41
42
  'args_digest',
42
43
  'args_raw',
43
44
  'status',
@@ -194,6 +195,23 @@ export function checkpointUpsertOp(source, sourceFile, line, now) {
194
195
  params: [source, sourceFile, line, timestamp(now)],
195
196
  };
196
197
  }
198
+ /**
199
+ * Targeted UPDATE op attaching a tool duration to a previously-inserted
200
+ * `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
201
+ * unindexed predicate. Idempotent: re-imports write the same values.
202
+ *
203
+ * Bounds are written alongside a FALLBACK duration so the figure is auditable;
204
+ * a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
205
+ * two measurement paths stay distinguishable.
206
+ */
207
+ function toolCallDurationUpdateOp(recordHash, startedAt, completedAt, durationMs) {
208
+ return {
209
+ sql: `UPDATE history_tool_call
210
+ SET started_at = ?, completed_at = ?, duration_ms = ?
211
+ WHERE record_hash = ?`,
212
+ params: [startedAt, completedAt, durationMs, recordHash],
213
+ };
214
+ }
197
215
  /**
198
216
  * Query which of the given hashes already exist in the ledger, in chunks of at
199
217
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -468,4 +486,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
468
486
  }
469
487
  return operations;
470
488
  }
471
- export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
489
+ export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
package/dist/mappers.d.ts CHANGED
@@ -5,9 +5,29 @@ export declare function argsDigest(args: unknown): string;
5
5
  /** Map a Claude Code JSONL record into one or more ETL split entries. */
6
6
  export declare function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[];
7
7
  /** Map a Pi JSONL record into one or more ETL split entries. */
8
- export declare function piSplit(raw: unknown): readonly SplitEntry[];
8
+ export declare function piSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
9
9
  /** Map an OMP JSONL record into one or more ETL split entries. */
10
10
  export declare function ompSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
11
+ /** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
12
+ export interface OmpToolResultTiming {
13
+ /** The toolCall id this result answers — joins `toolCall.id` exactly. */
14
+ toolCallId: string;
15
+ /** The tool's own measured wall time in ms, when present and finite. */
16
+ wallTimeMs: number | undefined;
17
+ /** Message timestamp as epoch millis, when parseable. */
18
+ timestampMs: number | undefined;
19
+ }
20
+ /** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
21
+ export declare function timestampToEpochMs(value: unknown): number | undefined;
22
+ /**
23
+ * Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
24
+ * when the record is not a toolResult message or carries no toolCallId. Live OMP
25
+ * emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
26
+ * content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
27
+ * measured wall time (48% of results in the sampled session) and `toolCallId`
28
+ * joins the originating `toolCall.id` exactly.
29
+ */
30
+ export declare function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null;
11
31
  /** Map a Codex JSONL record into one or more ETL split entries. */
12
32
  export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
13
33
  /** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
@@ -1 +1 @@
1
- {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAkExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAqDD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA6G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAuE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA2GxG;AAoBD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
1
+ {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAqDD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA6G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAqHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
package/dist/mappers.js CHANGED
@@ -31,6 +31,7 @@ const TOOL_CALL_MAPPER_KEYS = [
31
31
  'session_id',
32
32
  'seq',
33
33
  'tool_name',
34
+ 'call_id',
34
35
  'args_digest',
35
36
  'args_raw',
36
37
  'status',
@@ -225,24 +226,62 @@ export function claudeSplit(raw) {
225
226
  // Pi mapper
226
227
  // ---------------------------------------------------------------------------
227
228
  /** Map a Pi JSONL record into one or more ETL split entries. */
228
- export function piSplit(raw) {
229
+ export function piSplit(raw, context) {
230
+ // Pi session files are top-level event envelopes: `{type, id, parentId, timestamp,
231
+ // message: {role, content, model, usage, provider, ...}}`. Newer records carry `recordType`
232
+ // instead of `type` with `role`/`text` at the top level. The top-level `id` is an event id,
233
+ // never a session id; the session key and sequence come from the source file (context).
234
+ const msg = raw.message;
235
+ const recordType = String(raw.type ?? raw.recordType ?? '');
236
+ const sessionId = sessionIdFromContext(context, raw);
237
+ const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
238
+ // Prefer the ISO `timestamp`; a numeric epoch-ms `ts` is converted, never stringified.
239
+ const ts = s(raw.timestamp, raw.createdAt) ??
240
+ (typeof raw.ts === 'number' && Number.isFinite(raw.ts) ? new Date(raw.ts).toISOString() : s(raw.ts)) ??
241
+ new Date(0).toISOString();
242
+ // Meta lifecycle/custom records collapse to one meta row keyed by the source session,
243
+ // never the unique event id, and never a guessed role.
244
+ if (recordType === 'title' ||
245
+ recordType === 'title_change' ||
246
+ recordType === 'service_tier_change' ||
247
+ recordType === 'session_info' ||
248
+ recordType === 'ttsr_injection' ||
249
+ recordType === 'session_init' ||
250
+ recordType === 'session' ||
251
+ recordType === 'model_change' ||
252
+ recordType === 'thinking_level_change' ||
253
+ recordType === 'compaction' ||
254
+ recordType === 'custom' ||
255
+ recordType === 'custom_message' ||
256
+ recordType.startsWith('custom.')) {
257
+ return [
258
+ {
259
+ targetTable: 'history_message',
260
+ record: {
261
+ session_id: sessionId,
262
+ seq,
263
+ role: 'meta',
264
+ record_type: recordType,
265
+ disposition: 'meta',
266
+ ts,
267
+ provenance: 'ambient',
268
+ },
269
+ },
270
+ ];
271
+ }
229
272
  const entries = [];
230
- const r = raw;
231
- const sessionId = s(r.id, o(r.session).id) ?? 'unknown';
232
- const seq = typeof r.seq === 'number' ? r.seq : 0;
233
- const ts = s(r.ts, r.timestamp, r.createdAt) ?? new Date(0).toISOString();
234
- const role = mapRole(r.type ?? r.role);
235
- const recordType = String(r.type ?? '');
236
- const model = s(r.model, o(r.message).model);
237
- const cwd = s(r.cwd, r.dir);
238
- const msg = r.message;
239
- const usage = msg?.usage;
273
+ const role = piRole(msg?.role ?? raw.role ?? recordType);
274
+ const model = s(raw.model, o(msg).model);
275
+ const cwd = s(raw.cwd, raw.dir);
276
+ const usage = (msg?.usage ?? raw.usage);
240
277
  const inputTokens = (usage?.input ?? usage?.input_tokens ?? undefined);
241
278
  const outputTokens = (usage?.output ?? usage?.output_tokens ?? undefined);
242
279
  const cacheRead = (usage?.cacheRead ?? usage?.cache_read_tokens ?? undefined);
243
280
  const cacheWrite = (usage?.cacheWrite ?? usage?.cache_write_tokens ?? undefined);
244
- const costObj = (msg?.cost ?? r.cost);
281
+ const costObj = (msg?.cost ?? raw.cost);
245
282
  const costUsd = typeof costObj?.total === 'number' ? costObj.total : computeCost(inputTokens, outputTokens, model);
283
+ const contentBlocks = msg?.content ?? raw.content;
284
+ const durationMs = typeof msg?.duration === 'number' && Number.isFinite(msg.duration) ? msg.duration : undefined;
246
285
  const messageSplitIndex = entries.length;
247
286
  entries.push({
248
287
  targetTable: 'history_message',
@@ -253,45 +292,64 @@ export function piSplit(raw) {
253
292
  record_type: recordType,
254
293
  disposition: 'keep',
255
294
  ts,
256
- duration_ms: undefined,
295
+ duration_ms: durationMs,
257
296
  model: model ?? null,
258
297
  input_tokens: inputTokens ?? null,
259
298
  output_tokens: outputTokens ?? null,
260
299
  cache_read_tokens: cacheRead ?? null,
261
300
  cache_write_tokens: cacheWrite ?? null,
262
301
  cost_usd: costUsd ?? null,
263
- content_text: s(r.content, r.text, msg?.content) ?? null,
302
+ content_text: extractContentText(contentBlocks) ?? s(raw.content, raw.text, msg?.content) ?? null,
264
303
  cwd: cwd ?? null,
265
304
  provenance: 'ambient',
266
305
  },
267
306
  });
268
- // Tool calls from toolCall blocks in assistant content
269
- if (Array.isArray(r.content) && role === 'assistant') {
270
- for (const block of r.content) {
271
- if (block?.toolCall) {
272
- const tc = block.toolCall;
273
- entries.push({
274
- targetTable: 'history_tool_call',
275
- record: {
276
- _messageSplitIndex: messageSplitIndex,
277
- session_id: sessionId,
278
- seq,
279
- tool_name: String(tc.name ?? ''),
280
- args_digest: argsDigest(tc.input ?? tc.arguments),
281
- args_raw: maybeArgsRaw('pi', String(tc.name ?? ''), tc.input ?? tc.arguments),
282
- status: 'ok',
283
- started_at: undefined,
284
- completed_at: undefined,
285
- duration_ms: undefined,
286
- result_bytes: undefined,
287
- error_text: undefined,
288
- },
289
- });
290
- }
307
+ // Tool calls from content blocks in assistant messages. normalizeOmpToolCall handles both
308
+ // the legacy nested `{toolCall: {...}}` block and pi's flat `{type: "toolCall", id, name,
309
+ // arguments}` block.
310
+ if (Array.isArray(contentBlocks) && role === 'assistant') {
311
+ for (const block of contentBlocks) {
312
+ const call = normalizeOmpToolCall(block);
313
+ if (call === null)
314
+ continue;
315
+ entries.push({
316
+ targetTable: 'history_tool_call',
317
+ record: {
318
+ _messageSplitIndex: messageSplitIndex,
319
+ session_id: sessionId,
320
+ seq,
321
+ tool_name: String(call.name ?? ''),
322
+ // The tool's own call id is the exact join key a toolResult's `toolCallId`
323
+ // matches (see ompSplit / task 0564 R1).
324
+ call_id: s(call.id),
325
+ args_digest: argsDigest(call.input ?? call.arguments),
326
+ args_raw: maybeArgsRaw('pi', String(call.name ?? ''), call.input ?? call.arguments),
327
+ status: 'ok',
328
+ started_at: undefined,
329
+ completed_at: undefined,
330
+ duration_ms: undefined,
331
+ result_bytes: undefined,
332
+ error_text: undefined,
333
+ },
334
+ });
291
335
  }
292
336
  }
293
337
  return entries;
294
338
  }
339
+ /**
340
+ * Canonicalize a pi message role. pi tool results and bash executions ride user turns (Claude
341
+ * protocol shape); everything unrecognized clamps to `unknown` so pi record types never leak
342
+ * into the role column (task 0577 AC2).
343
+ */
344
+ function piRole(type) {
345
+ const t = String(type ?? '').toLowerCase();
346
+ if (t === 'toolresult' || t === 'tool_result' || t === 'bashexecution' || t === 'tool')
347
+ return 'user';
348
+ const mapped = mapRole(t);
349
+ if (mapped === 'user' || mapped === 'assistant' || mapped === 'system')
350
+ return mapped;
351
+ return 'unknown';
352
+ }
295
353
  // ---------------------------------------------------------------------------
296
354
  // OMP mapper (near-identical to pi)
297
355
  // ---------------------------------------------------------------------------
@@ -383,6 +441,10 @@ export function ompSplit(raw, context) {
383
441
  session_id: sessionId,
384
442
  seq,
385
443
  tool_name: String(call.name ?? ''),
444
+ // Task 0564 R1: the tool's own call id is the exact join key a
445
+ // toolResult's `toolCallId` matches — it lets the streaming loop
446
+ // attach the tool's measured duration to the right row.
447
+ call_id: s(call.id),
386
448
  args_digest: argsDigest(call.input ?? call.arguments),
387
449
  args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
388
450
  status: 'ok',
@@ -413,6 +475,39 @@ function normalizeOmpToolCall(block) {
413
475
  return block;
414
476
  return null;
415
477
  }
478
+ /** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
479
+ export function timestampToEpochMs(value) {
480
+ if (typeof value === 'number' && Number.isFinite(value))
481
+ return value;
482
+ if (typeof value === 'string' && value.length > 0) {
483
+ const numeric = Number(value);
484
+ if (Number.isFinite(numeric))
485
+ return numeric;
486
+ const parsed = Date.parse(value);
487
+ if (!Number.isNaN(parsed))
488
+ return parsed;
489
+ }
490
+ return undefined;
491
+ }
492
+ /**
493
+ * Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
494
+ * when the record is not a toolResult message or carries no toolCallId. Live OMP
495
+ * emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
496
+ * content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
497
+ * measured wall time (48% of results in the sampled session) and `toolCallId`
498
+ * joins the originating `toolCall.id` exactly.
499
+ */
500
+ export function ompToolResultTiming(raw) {
501
+ const msg = o(raw.message);
502
+ if (String(msg.role ?? '').toLowerCase() !== 'toolresult')
503
+ return null;
504
+ const toolCallId = s(msg.toolCallId);
505
+ if (toolCallId === undefined)
506
+ return null;
507
+ const details = o(msg.details);
508
+ const wallTimeMs = typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
509
+ return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
510
+ }
416
511
  // ---------------------------------------------------------------------------
417
512
  // Codex mapper
418
513
  // ---------------------------------------------------------------------------
@@ -1 +1 @@
1
- {"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA4E9B,CAAC"}
1
+ {"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA6E9B,CAAC"}
@@ -62,6 +62,7 @@ CREATE TABLE IF NOT EXISTS history_tool_call (
62
62
  session_id TEXT NOT NULL,
63
63
  seq INTEGER NOT NULL,
64
64
  tool_name TEXT NOT NULL,
65
+ call_id TEXT,
65
66
  args_digest TEXT,
66
67
  args_raw TEXT,
67
68
  status TEXT NOT NULL,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gobing-ai/ts-llm-jsonl-importer",
3
- "version": "0.4.33",
3
+ "version": "0.4.36",
4
4
  "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
5
5
  "keywords": [
6
6
  "typescript",
@@ -47,9 +47,9 @@
47
47
  "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
48
48
  },
49
49
  "dependencies": {
50
- "@gobing-ai/ts-db": "^0.4.33",
51
- "@gobing-ai/ts-runtime": "^0.4.33",
52
- "@gobing-ai/ts-utils": "^0.4.33",
50
+ "@gobing-ai/ts-db": "^0.4.36",
51
+ "@gobing-ai/ts-runtime": "^0.4.36",
52
+ "@gobing-ai/ts-utils": "^0.4.36",
53
53
  "zod": "^4.1.0"
54
54
  },
55
55
  "devDependencies": {
package/src/importer.ts CHANGED
@@ -19,7 +19,9 @@ import {
19
19
  recordInsertOp,
20
20
  resetCheckpoints,
21
21
  targetTableFor,
22
+ toolCallDurationUpdateOp,
22
23
  } from './jsonl-importer-dao';
24
+ import { type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
23
25
  import { redactRecord } from './redaction';
24
26
  import { resolveSourceDefinition } from './sources';
25
27
  import type {
@@ -37,6 +39,83 @@ interface SplitRecord {
37
39
  readonly raw: JsonObject;
38
40
  }
39
41
 
42
+ // ---------------------------------------------------------------------------
43
+ // Task 0564 R1 — tool-call duration attach (omp)
44
+ //
45
+ // A toolCall row is emitted while handling its assistant message (SplitEntry is
46
+ // insert-only, so the duration cannot ride the mapper's split path); the
47
+ // duration arrives on a LATER toolResult line. The attach therefore lives in the
48
+ // streaming loop, keyed on (source, session_id, call_id) per the frozen contract:
49
+ // - `details.wallTimeMs` is the tool's own measured wall time — used as-is,
50
+ // rounded, NEVER clamped or second-guessed (guards are fallback-only);
51
+ // - otherwise the fallback `toolResult.timestamp − toolCall message timestamp`
52
+ // when both are parseable AND the delta is in [0, 3_600_000]; an implausible
53
+ // delta stays NULL so the unmeasured count stays honest;
54
+ // - started_at / completed_at are written alongside a FALLBACK figure so it is
55
+ // auditable (wallTimeMs rows keep both NULL — the two stay distinguishable);
56
+ // - an unmatched toolCallId attaches nothing and never fails the import.
57
+ //
58
+ // The in-memory map covers the common (fresh-import) case without extra queries;
59
+ // a toolResult whose tool-call line was imported by an earlier checkpointed run
60
+ // (resume) resolves through a DB lookup instead, so a mid-file checkpoint loses
61
+ // no earlier duration.
62
+ // ---------------------------------------------------------------------------
63
+
64
+ interface PendingToolCall {
65
+ readonly recordHash: string;
66
+ /** Assistant-message timestamp (epoch ms) — the fallback's start bound. */
67
+ readonly messageTsMs: number | undefined;
68
+ }
69
+
70
+ const TOOL_CALL_KEY_SEP = '\u0000';
71
+
72
+ function toolCallKey(source: string, sessionId: string, callId: string): string {
73
+ return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
74
+ }
75
+
76
+ /** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
77
+ async function resolveToolCallRow(
78
+ db: ImportOptions['db'],
79
+ source: string,
80
+ sessionId: string,
81
+ callId: string,
82
+ ): Promise<PendingToolCall | undefined> {
83
+ const row = await db.queryFirst<{ recordHash: string; messageTs: string | null }>(
84
+ `SELECT tc.record_hash AS recordHash, m.ts AS messageTs
85
+ FROM history_tool_call tc
86
+ LEFT JOIN history_message m ON m.record_hash = tc.message_hash
87
+ WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`,
88
+ source,
89
+ sessionId,
90
+ callId,
91
+ );
92
+ if (row === undefined || row === null) return undefined;
93
+ return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
94
+ }
95
+
96
+ /**
97
+ * Compute the attachable duration for a toolResult, or null when nothing should
98
+ * be written (implausible fallback delta → row stays NULL → counts unmeasured).
99
+ */
100
+ function attachableDuration(
101
+ timing: OmpToolResultTiming,
102
+ messageTsMs: number | undefined,
103
+ ): { startedAt: string | null; completedAt: string | null; durationMs: number } | null {
104
+ if (timing.wallTimeMs !== undefined) {
105
+ // The tool's own measurement — no clamping, no sanity check.
106
+ return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
107
+ }
108
+ if (timing.timestampMs === undefined || messageTsMs === undefined) return null;
109
+ const delta = timing.timestampMs - messageTsMs;
110
+ // Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
111
+ if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000) return null;
112
+ return {
113
+ startedAt: new Date(messageTsMs).toISOString(),
114
+ completedAt: new Date(timing.timestampMs).toISOString(),
115
+ durationMs: Math.round(delta),
116
+ };
117
+ }
118
+
40
119
  /** Run the JSONL import pipeline for one source.
41
120
  *
42
121
  * @param source - Either a built-in source identifier resolved from the
@@ -69,6 +148,10 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
69
148
  // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
70
149
  const desiredHashes = new Set<string>();
71
150
 
151
+ // Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
152
+ // (source, session_id, call_id) so a later toolResult line can attach its duration.
153
+ const toolCallRows = new Map<string, PendingToolCall>();
154
+
72
155
  for (const file of files) {
73
156
  const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
74
157
  let lineNumber = 0;
@@ -138,6 +221,43 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
138
221
  // second pass, no checkpoint advance, no partial rows from this record.
139
222
  if (lineRejected) continue;
140
223
 
224
+ // Task 0564 R1 (omp): register this line's tool-call rows for later duration
225
+ // attach. Runs over ALL prepared entries — including duplicates — because a
226
+ // re-imported line's row already exists and its duration UPDATE still targets
227
+ // the same deterministic record_hash. The assistant message timestamp is the
228
+ // fallback's start bound; every call in one message shares it.
229
+ if (definition.source === 'omp') {
230
+ const assistantTs = new Map<string, number | undefined>();
231
+ for (const entry of prepared) {
232
+ if (
233
+ entry.split.targetTable === 'history_message' &&
234
+ entry.normalized.role === 'assistant' &&
235
+ typeof entry.normalized.session_id === 'string'
236
+ ) {
237
+ assistantTs.set(
238
+ `${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
239
+ timestampToEpochMs(entry.normalized.ts),
240
+ );
241
+ }
242
+ }
243
+ for (const entry of prepared) {
244
+ const callId = entry.normalized.call_id;
245
+ if (
246
+ entry.split.targetTable === 'history_tool_call' &&
247
+ typeof callId === 'string' &&
248
+ callId.length > 0 &&
249
+ typeof entry.normalized.session_id === 'string'
250
+ ) {
251
+ toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
252
+ recordHash: entry.recordHash,
253
+ messageTsMs: assistantTs.get(
254
+ `${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
255
+ ),
256
+ });
257
+ }
258
+ }
259
+ }
260
+
141
261
  // Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
142
262
  // instead of a SELECT per record; duplicates are dropped before the write pass.
143
263
  const existing = await ledgerExistingHashes(
@@ -199,6 +319,47 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
199
319
  }
200
320
 
201
321
  if (!options.dryRun && lineSucceeded) {
322
+ // Task 0564 R1 (omp): attach tool durations from this line's toolResult
323
+ // message. Unmatched toolCallIds attach nothing and never fail the import;
324
+ // an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
325
+ // this line's batch — idempotent, so re-imports and post-resume tail
326
+ // reprocessing write the same values.
327
+ if (definition.source === 'omp') {
328
+ const timing = ompToolResultTiming(raw);
329
+ if (timing !== null) {
330
+ const resultEntry = prepared.find(
331
+ (entry) =>
332
+ entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult',
333
+ );
334
+ const sessionId = resultEntry?.normalized.session_id;
335
+ if (typeof sessionId === 'string') {
336
+ let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
337
+ if (pending === undefined) {
338
+ // The tool-call line sits behind this run's checkpoint — the
339
+ // row exists from an earlier import; resolve it from the DB.
340
+ pending = await resolveToolCallRow(
341
+ options.db,
342
+ resolvedSource,
343
+ sessionId,
344
+ timing.toolCallId,
345
+ );
346
+ }
347
+ if (pending !== undefined) {
348
+ const duration = attachableDuration(timing, pending.messageTsMs);
349
+ if (duration !== null) {
350
+ ops.push(
351
+ toolCallDurationUpdateOp(
352
+ pending.recordHash,
353
+ duration.startedAt,
354
+ duration.completedAt,
355
+ duration.durationMs,
356
+ ),
357
+ );
358
+ }
359
+ }
360
+ }
361
+ }
362
+ }
202
363
  // Checkpoint upsert joins the same batch so an accepted line can never be
203
364
  // re-imported after a mid-batch crash.
204
365
  ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
@@ -45,6 +45,7 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
45
45
  'session_id',
46
46
  'seq',
47
47
  'tool_name',
48
+ 'call_id',
48
49
  'args_digest',
49
50
  'args_raw',
50
51
  'status',
@@ -255,6 +256,29 @@ export function checkpointUpsertOp(
255
256
  };
256
257
  }
257
258
 
259
+ /**
260
+ * Targeted UPDATE op attaching a tool duration to a previously-inserted
261
+ * `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
262
+ * unindexed predicate. Idempotent: re-imports write the same values.
263
+ *
264
+ * Bounds are written alongside a FALLBACK duration so the figure is auditable;
265
+ * a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
266
+ * two measurement paths stay distinguishable.
267
+ */
268
+ function toolCallDurationUpdateOp(
269
+ recordHash: string,
270
+ startedAt: string | null,
271
+ completedAt: string | null,
272
+ durationMs: number | null,
273
+ ): DbBatchOp {
274
+ return {
275
+ sql: `UPDATE history_tool_call
276
+ SET started_at = ?, completed_at = ?, duration_ms = ?
277
+ WHERE record_hash = ?`,
278
+ params: [startedAt, completedAt, durationMs, recordHash],
279
+ };
280
+ }
281
+
258
282
  /**
259
283
  * Query which of the given hashes already exist in the ledger, in chunks of at
260
284
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -688,5 +712,6 @@ export {
688
712
  resetCheckpoints,
689
713
  targetTableFor,
690
714
  timestamp,
715
+ toolCallDurationUpdateOp,
691
716
  writeCheckpoint,
692
717
  };
package/src/mappers.ts CHANGED
@@ -35,6 +35,7 @@ const TOOL_CALL_MAPPER_KEYS: readonly string[] = [
35
35
  'session_id',
36
36
  'seq',
37
37
  'tool_name',
38
+ 'call_id',
38
39
  'args_digest',
39
40
  'args_raw',
40
41
  'status',
@@ -240,25 +241,67 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
240
241
  // ---------------------------------------------------------------------------
241
242
 
242
243
  /** Map a Pi JSONL record into one or more ETL split entries. */
243
- export function piSplit(raw: unknown): readonly SplitEntry[] {
244
+ export function piSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[] {
245
+ // Pi session files are top-level event envelopes: `{type, id, parentId, timestamp,
246
+ // message: {role, content, model, usage, provider, ...}}`. Newer records carry `recordType`
247
+ // instead of `type` with `role`/`text` at the top level. The top-level `id` is an event id,
248
+ // never a session id; the session key and sequence come from the source file (context).
249
+ const msg = raw.message as Record<string, unknown> | undefined;
250
+ const recordType = String(raw.type ?? raw.recordType ?? '');
251
+ const sessionId = sessionIdFromContext(context, raw);
252
+ const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
253
+ // Prefer the ISO `timestamp`; a numeric epoch-ms `ts` is converted, never stringified.
254
+ const ts =
255
+ s(raw.timestamp, raw.createdAt) ??
256
+ (typeof raw.ts === 'number' && Number.isFinite(raw.ts) ? new Date(raw.ts).toISOString() : s(raw.ts)) ??
257
+ new Date(0).toISOString();
258
+
259
+ // Meta lifecycle/custom records collapse to one meta row keyed by the source session,
260
+ // never the unique event id, and never a guessed role.
261
+ if (
262
+ recordType === 'title' ||
263
+ recordType === 'title_change' ||
264
+ recordType === 'service_tier_change' ||
265
+ recordType === 'session_info' ||
266
+ recordType === 'ttsr_injection' ||
267
+ recordType === 'session_init' ||
268
+ recordType === 'session' ||
269
+ recordType === 'model_change' ||
270
+ recordType === 'thinking_level_change' ||
271
+ recordType === 'compaction' ||
272
+ recordType === 'custom' ||
273
+ recordType === 'custom_message' ||
274
+ recordType.startsWith('custom.')
275
+ ) {
276
+ return [
277
+ {
278
+ targetTable: 'history_message',
279
+ record: {
280
+ session_id: sessionId,
281
+ seq,
282
+ role: 'meta',
283
+ record_type: recordType,
284
+ disposition: 'meta',
285
+ ts,
286
+ provenance: 'ambient',
287
+ },
288
+ },
289
+ ];
290
+ }
291
+
244
292
  const entries: SplitEntry[] = [];
245
- const r = raw as Record<string, unknown>;
246
- const sessionId = s(r.id, o(r.session).id) ?? 'unknown';
247
- const seq = typeof r.seq === 'number' ? r.seq : 0;
248
- const ts = s(r.ts, r.timestamp, r.createdAt) ?? new Date(0).toISOString();
249
- const role = mapRole(r.type ?? r.role);
250
- const recordType = String(r.type ?? '');
251
- const model = s(r.model, o(r.message).model);
252
- const cwd = s(r.cwd, r.dir);
253
-
254
- const msg = r.message as Record<string, unknown> | undefined;
255
- const usage = msg?.usage as Record<string, unknown> | undefined;
293
+ const role = piRole(msg?.role ?? raw.role ?? recordType);
294
+ const model = s(raw.model, o(msg).model);
295
+ const cwd = s(raw.cwd, raw.dir);
296
+ const usage = (msg?.usage ?? raw.usage) as Record<string, unknown> | undefined;
256
297
  const inputTokens = (usage?.input ?? usage?.input_tokens ?? undefined) as number | undefined;
257
298
  const outputTokens = (usage?.output ?? usage?.output_tokens ?? undefined) as number | undefined;
258
299
  const cacheRead = (usage?.cacheRead ?? usage?.cache_read_tokens ?? undefined) as number | undefined;
259
300
  const cacheWrite = (usage?.cacheWrite ?? usage?.cache_write_tokens ?? undefined) as number | undefined;
260
- const costObj = (msg?.cost ?? r.cost) as Record<string, unknown> | undefined;
301
+ const costObj = (msg?.cost ?? raw.cost) as Record<string, unknown> | undefined;
261
302
  const costUsd = typeof costObj?.total === 'number' ? costObj.total : computeCost(inputTokens, outputTokens, model);
303
+ const contentBlocks = msg?.content ?? raw.content;
304
+ const durationMs = typeof msg?.duration === 'number' && Number.isFinite(msg.duration) ? msg.duration : undefined;
262
305
 
263
306
  const messageSplitIndex = entries.length;
264
307
 
@@ -271,48 +314,65 @@ export function piSplit(raw: unknown): readonly SplitEntry[] {
271
314
  record_type: recordType,
272
315
  disposition: 'keep',
273
316
  ts,
274
- duration_ms: undefined,
317
+ duration_ms: durationMs,
275
318
  model: model ?? null,
276
319
  input_tokens: inputTokens ?? null,
277
320
  output_tokens: outputTokens ?? null,
278
321
  cache_read_tokens: cacheRead ?? null,
279
322
  cache_write_tokens: cacheWrite ?? null,
280
323
  cost_usd: costUsd ?? null,
281
- content_text: s(r.content, r.text, msg?.content) ?? null,
324
+ content_text: extractContentText(contentBlocks) ?? s(raw.content, raw.text, msg?.content) ?? null,
282
325
  cwd: cwd ?? null,
283
326
  provenance: 'ambient',
284
327
  },
285
328
  });
286
329
 
287
- // Tool calls from toolCall blocks in assistant content
288
- if (Array.isArray(r.content) && role === 'assistant') {
289
- for (const block of r.content as Record<string, unknown>[]) {
290
- if (block?.toolCall) {
291
- const tc = block.toolCall as Record<string, unknown>;
292
- entries.push({
293
- targetTable: 'history_tool_call',
294
- record: {
295
- _messageSplitIndex: messageSplitIndex,
296
- session_id: sessionId,
297
- seq,
298
- tool_name: String(tc.name ?? ''),
299
- args_digest: argsDigest(tc.input ?? tc.arguments),
300
- args_raw: maybeArgsRaw('pi', String(tc.name ?? ''), tc.input ?? tc.arguments),
301
- status: 'ok',
302
- started_at: undefined,
303
- completed_at: undefined,
304
- duration_ms: undefined,
305
- result_bytes: undefined,
306
- error_text: undefined,
307
- },
308
- });
309
- }
330
+ // Tool calls from content blocks in assistant messages. normalizeOmpToolCall handles both
331
+ // the legacy nested `{toolCall: {...}}` block and pi's flat `{type: "toolCall", id, name,
332
+ // arguments}` block.
333
+ if (Array.isArray(contentBlocks) && role === 'assistant') {
334
+ for (const block of contentBlocks as Record<string, unknown>[]) {
335
+ const call = normalizeOmpToolCall(block);
336
+ if (call === null) continue;
337
+ entries.push({
338
+ targetTable: 'history_tool_call',
339
+ record: {
340
+ _messageSplitIndex: messageSplitIndex,
341
+ session_id: sessionId,
342
+ seq,
343
+ tool_name: String(call.name ?? ''),
344
+ // The tool's own call id is the exact join key a toolResult's `toolCallId`
345
+ // matches (see ompSplit / task 0564 R1).
346
+ call_id: s(call.id),
347
+ args_digest: argsDigest(call.input ?? call.arguments),
348
+ args_raw: maybeArgsRaw('pi', String(call.name ?? ''), call.input ?? call.arguments),
349
+ status: 'ok',
350
+ started_at: undefined,
351
+ completed_at: undefined,
352
+ duration_ms: undefined,
353
+ result_bytes: undefined,
354
+ error_text: undefined,
355
+ },
356
+ });
310
357
  }
311
358
  }
312
359
 
313
360
  return entries;
314
361
  }
315
362
 
363
+ /**
364
+ * Canonicalize a pi message role. pi tool results and bash executions ride user turns (Claude
365
+ * protocol shape); everything unrecognized clamps to `unknown` so pi record types never leak
366
+ * into the role column (task 0577 AC2).
367
+ */
368
+ function piRole(type: unknown): string {
369
+ const t = String(type ?? '').toLowerCase();
370
+ if (t === 'toolresult' || t === 'tool_result' || t === 'bashexecution' || t === 'tool') return 'user';
371
+ const mapped = mapRole(t);
372
+ if (mapped === 'user' || mapped === 'assistant' || mapped === 'system') return mapped;
373
+ return 'unknown';
374
+ }
375
+
316
376
  // ---------------------------------------------------------------------------
317
377
  // OMP mapper (near-identical to pi)
318
378
  // ---------------------------------------------------------------------------
@@ -411,6 +471,10 @@ export function ompSplit(raw: Record<string, unknown>, context?: TransformContex
411
471
  session_id: sessionId,
412
472
  seq,
413
473
  tool_name: String(call.name ?? ''),
474
+ // Task 0564 R1: the tool's own call id is the exact join key a
475
+ // toolResult's `toolCallId` matches — it lets the streaming loop
476
+ // attach the tool's measured duration to the right row.
477
+ call_id: s(call.id),
414
478
  args_digest: argsDigest(call.input ?? call.arguments),
415
479
  args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
416
480
  status: 'ok',
@@ -441,6 +505,51 @@ function normalizeOmpToolCall(block: Record<string, unknown>): Record<string, un
441
505
  return null;
442
506
  }
443
507
 
508
+ // ---------------------------------------------------------------------------
509
+ // OMP toolResult timing signals (task 0564 R1)
510
+ // ---------------------------------------------------------------------------
511
+
512
+ /** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
513
+ export interface OmpToolResultTiming {
514
+ /** The toolCall id this result answers — joins `toolCall.id` exactly. */
515
+ toolCallId: string;
516
+ /** The tool's own measured wall time in ms, when present and finite. */
517
+ wallTimeMs: number | undefined;
518
+ /** Message timestamp as epoch millis, when parseable. */
519
+ timestampMs: number | undefined;
520
+ }
521
+
522
+ /** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
523
+ export function timestampToEpochMs(value: unknown): number | undefined {
524
+ if (typeof value === 'number' && Number.isFinite(value)) return value;
525
+ if (typeof value === 'string' && value.length > 0) {
526
+ const numeric = Number(value);
527
+ if (Number.isFinite(numeric)) return numeric;
528
+ const parsed = Date.parse(value);
529
+ if (!Number.isNaN(parsed)) return parsed;
530
+ }
531
+ return undefined;
532
+ }
533
+
534
+ /**
535
+ * Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
536
+ * when the record is not a toolResult message or carries no toolCallId. Live OMP
537
+ * emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
538
+ * content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
539
+ * measured wall time (48% of results in the sampled session) and `toolCallId`
540
+ * joins the originating `toolCall.id` exactly.
541
+ */
542
+ export function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null {
543
+ const msg = o(raw.message);
544
+ if (String(msg.role ?? '').toLowerCase() !== 'toolresult') return null;
545
+ const toolCallId = s(msg.toolCallId);
546
+ if (toolCallId === undefined) return null;
547
+ const details = o(msg.details);
548
+ const wallTimeMs =
549
+ typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
550
+ return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
551
+ }
552
+
444
553
  // ---------------------------------------------------------------------------
445
554
  // Codex mapper
446
555
  // ---------------------------------------------------------------------------
package/src/schema-sql.ts CHANGED
@@ -62,6 +62,7 @@ CREATE TABLE IF NOT EXISTS history_tool_call (
62
62
  session_id TEXT NOT NULL,
63
63
  seq INTEGER NOT NULL,
64
64
  tool_name TEXT NOT NULL,
65
+ call_id TEXT,
65
66
  args_digest TEXT,
66
67
  args_raw TEXT,
67
68
  status TEXT NOT NULL,