@gobing-ai/ts-llm-jsonl-importer 0.4.33 → 0.4.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +93 -1
- package/dist/jsonl-importer-dao.d.ts +11 -1
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +19 -1
- package/dist/mappers.d.ts +21 -1
- package/dist/mappers.d.ts.map +1 -1
- package/dist/mappers.js +132 -37
- package/dist/schema-sql.d.ts.map +1 -1
- package/dist/schema-sql.js +1 -0
- package/package.json +4 -4
- package/src/importer.ts +161 -0
- package/src/jsonl-importer-dao.ts +25 -0
- package/src/mappers.ts +147 -38
- package/src/schema-sql.ts +1 -0
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAkRrH"}
|
package/dist/importer.js
CHANGED
|
@@ -1,8 +1,44 @@
|
|
|
1
1
|
import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
|
|
2
2
|
import { sha256 } from './hash.js';
|
|
3
|
-
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, } from './jsonl-importer-dao.js';
|
|
3
|
+
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, } from './jsonl-importer-dao.js';
|
|
4
|
+
import { ompToolResultTiming, timestampToEpochMs } from './mappers.js';
|
|
4
5
|
import { redactRecord } from './redaction.js';
|
|
5
6
|
import { resolveSourceDefinition } from './sources.js';
|
|
7
|
+
const TOOL_CALL_KEY_SEP = '\u0000';
|
|
8
|
+
function toolCallKey(source, sessionId, callId) {
|
|
9
|
+
return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
|
|
10
|
+
}
|
|
11
|
+
/** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
|
|
12
|
+
async function resolveToolCallRow(db, source, sessionId, callId) {
|
|
13
|
+
const row = await db.queryFirst(`SELECT tc.record_hash AS recordHash, m.ts AS messageTs
|
|
14
|
+
FROM history_tool_call tc
|
|
15
|
+
LEFT JOIN history_message m ON m.record_hash = tc.message_hash
|
|
16
|
+
WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`, source, sessionId, callId);
|
|
17
|
+
if (row === undefined || row === null)
|
|
18
|
+
return undefined;
|
|
19
|
+
return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Compute the attachable duration for a toolResult, or null when nothing should
|
|
23
|
+
* be written (implausible fallback delta → row stays NULL → counts unmeasured).
|
|
24
|
+
*/
|
|
25
|
+
function attachableDuration(timing, messageTsMs) {
|
|
26
|
+
if (timing.wallTimeMs !== undefined) {
|
|
27
|
+
// The tool's own measurement — no clamping, no sanity check.
|
|
28
|
+
return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
|
|
29
|
+
}
|
|
30
|
+
if (timing.timestampMs === undefined || messageTsMs === undefined)
|
|
31
|
+
return null;
|
|
32
|
+
const delta = timing.timestampMs - messageTsMs;
|
|
33
|
+
// Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
|
|
34
|
+
if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000)
|
|
35
|
+
return null;
|
|
36
|
+
return {
|
|
37
|
+
startedAt: new Date(messageTsMs).toISOString(),
|
|
38
|
+
completedAt: new Date(timing.timestampMs).toISOString(),
|
|
39
|
+
durationMs: Math.round(delta),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
6
42
|
/** Run the JSONL import pipeline for one source.
|
|
7
43
|
*
|
|
8
44
|
* @param source - Either a built-in source identifier resolved from the
|
|
@@ -32,6 +68,9 @@ export async function runJsonlImport(source, options) {
|
|
|
32
68
|
// Full-mode desired set (R1, task 0504): every record hash the current source produces.
|
|
33
69
|
// ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
|
|
34
70
|
const desiredHashes = new Set();
|
|
71
|
+
// Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
|
|
72
|
+
// (source, session_id, call_id) so a later toolResult line can attach its duration.
|
|
73
|
+
const toolCallRows = new Map();
|
|
35
74
|
for (const file of files) {
|
|
36
75
|
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
|
|
37
76
|
let lineNumber = 0;
|
|
@@ -92,6 +131,33 @@ export async function runJsonlImport(source, options) {
|
|
|
92
131
|
// second pass, no checkpoint advance, no partial rows from this record.
|
|
93
132
|
if (lineRejected)
|
|
94
133
|
continue;
|
|
134
|
+
// Task 0564 R1 (omp): register this line's tool-call rows for later duration
|
|
135
|
+
// attach. Runs over ALL prepared entries — including duplicates — because a
|
|
136
|
+
// re-imported line's row already exists and its duration UPDATE still targets
|
|
137
|
+
// the same deterministic record_hash. The assistant message timestamp is the
|
|
138
|
+
// fallback's start bound; every call in one message shares it.
|
|
139
|
+
if (definition.source === 'omp') {
|
|
140
|
+
const assistantTs = new Map();
|
|
141
|
+
for (const entry of prepared) {
|
|
142
|
+
if (entry.split.targetTable === 'history_message' &&
|
|
143
|
+
entry.normalized.role === 'assistant' &&
|
|
144
|
+
typeof entry.normalized.session_id === 'string') {
|
|
145
|
+
assistantTs.set(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`, timestampToEpochMs(entry.normalized.ts));
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
for (const entry of prepared) {
|
|
149
|
+
const callId = entry.normalized.call_id;
|
|
150
|
+
if (entry.split.targetTable === 'history_tool_call' &&
|
|
151
|
+
typeof callId === 'string' &&
|
|
152
|
+
callId.length > 0 &&
|
|
153
|
+
typeof entry.normalized.session_id === 'string') {
|
|
154
|
+
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
155
|
+
recordHash: entry.recordHash,
|
|
156
|
+
messageTsMs: assistantTs.get(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`),
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
95
161
|
// Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
|
|
96
162
|
// instead of a SELECT per record; duplicates are dropped before the write pass.
|
|
97
163
|
const existing = await ledgerExistingHashes(options.db, prepared.map((e) => e.recordHash));
|
|
@@ -128,6 +194,32 @@ export async function runJsonlImport(source, options) {
|
|
|
128
194
|
importedRecords += 1;
|
|
129
195
|
}
|
|
130
196
|
if (!options.dryRun && lineSucceeded) {
|
|
197
|
+
// Task 0564 R1 (omp): attach tool durations from this line's toolResult
|
|
198
|
+
// message. Unmatched toolCallIds attach nothing and never fail the import;
|
|
199
|
+
// an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
|
|
200
|
+
// this line's batch — idempotent, so re-imports and post-resume tail
|
|
201
|
+
// reprocessing write the same values.
|
|
202
|
+
if (definition.source === 'omp') {
|
|
203
|
+
const timing = ompToolResultTiming(raw);
|
|
204
|
+
if (timing !== null) {
|
|
205
|
+
const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult');
|
|
206
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
207
|
+
if (typeof sessionId === 'string') {
|
|
208
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
209
|
+
if (pending === undefined) {
|
|
210
|
+
// The tool-call line sits behind this run's checkpoint — the
|
|
211
|
+
// row exists from an earlier import; resolve it from the DB.
|
|
212
|
+
pending = await resolveToolCallRow(options.db, resolvedSource, sessionId, timing.toolCallId);
|
|
213
|
+
}
|
|
214
|
+
if (pending !== undefined) {
|
|
215
|
+
const duration = attachableDuration(timing, pending.messageTsMs);
|
|
216
|
+
if (duration !== null) {
|
|
217
|
+
ops.push(toolCallDurationUpdateOp(pending.recordHash, duration.startedAt, duration.completedAt, duration.durationMs));
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
131
223
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
132
224
|
// re-imported after a mid-batch crash.
|
|
133
225
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
|
@@ -44,6 +44,16 @@ export declare function recordInsertOp(targetTable: string, recordHash: string,
|
|
|
44
44
|
export declare function ledgerInsertOp(recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): DbBatchOp;
|
|
45
45
|
/** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
|
|
46
46
|
export declare function checkpointUpsertOp(source: string, sourceFile: string, line: number, now: ImportOptions['now']): DbBatchOp;
|
|
47
|
+
/**
|
|
48
|
+
* Targeted UPDATE op attaching a tool duration to a previously-inserted
|
|
49
|
+
* `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
|
|
50
|
+
* unindexed predicate. Idempotent: re-imports write the same values.
|
|
51
|
+
*
|
|
52
|
+
* Bounds are written alongside a FALLBACK duration so the figure is auditable;
|
|
53
|
+
* a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
|
|
54
|
+
* two measurement paths stay distinguishable.
|
|
55
|
+
*/
|
|
56
|
+
declare function toolCallDurationUpdateOp(recordHash: string, startedAt: string | null, completedAt: string | null, durationMs: number | null): DbBatchOp;
|
|
47
57
|
/**
|
|
48
58
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
49
59
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -126,5 +136,5 @@ export declare function deleteCheckpointOperation(source: string, sourceFile: st
|
|
|
126
136
|
/** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
|
|
127
137
|
export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
|
|
128
138
|
export type { CheckpointRow };
|
|
129
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
|
139
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
|
130
140
|
//# sourceMappingURL=jsonl-importer-dao.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAuDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,eAAe,GAClB,CAAC"}
|
|
@@ -38,6 +38,7 @@ const TYPED_TABLE_COLUMNS = {
|
|
|
38
38
|
'session_id',
|
|
39
39
|
'seq',
|
|
40
40
|
'tool_name',
|
|
41
|
+
'call_id',
|
|
41
42
|
'args_digest',
|
|
42
43
|
'args_raw',
|
|
43
44
|
'status',
|
|
@@ -194,6 +195,23 @@ export function checkpointUpsertOp(source, sourceFile, line, now) {
|
|
|
194
195
|
params: [source, sourceFile, line, timestamp(now)],
|
|
195
196
|
};
|
|
196
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* Targeted UPDATE op attaching a tool duration to a previously-inserted
|
|
200
|
+
* `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
|
|
201
|
+
* unindexed predicate. Idempotent: re-imports write the same values.
|
|
202
|
+
*
|
|
203
|
+
* Bounds are written alongside a FALLBACK duration so the figure is auditable;
|
|
204
|
+
* a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
|
|
205
|
+
* two measurement paths stay distinguishable.
|
|
206
|
+
*/
|
|
207
|
+
function toolCallDurationUpdateOp(recordHash, startedAt, completedAt, durationMs) {
|
|
208
|
+
return {
|
|
209
|
+
sql: `UPDATE history_tool_call
|
|
210
|
+
SET started_at = ?, completed_at = ?, duration_ms = ?
|
|
211
|
+
WHERE record_hash = ?`,
|
|
212
|
+
params: [startedAt, completedAt, durationMs, recordHash],
|
|
213
|
+
};
|
|
214
|
+
}
|
|
197
215
|
/**
|
|
198
216
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
199
217
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -468,4 +486,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
|
|
|
468
486
|
}
|
|
469
487
|
return operations;
|
|
470
488
|
}
|
|
471
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
|
489
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
package/dist/mappers.d.ts
CHANGED
|
@@ -5,9 +5,29 @@ export declare function argsDigest(args: unknown): string;
|
|
|
5
5
|
/** Map a Claude Code JSONL record into one or more ETL split entries. */
|
|
6
6
|
export declare function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
7
7
|
/** Map a Pi JSONL record into one or more ETL split entries. */
|
|
8
|
-
export declare function piSplit(raw: unknown): readonly SplitEntry[];
|
|
8
|
+
export declare function piSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
9
9
|
/** Map an OMP JSONL record into one or more ETL split entries. */
|
|
10
10
|
export declare function ompSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
11
|
+
/** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
|
|
12
|
+
export interface OmpToolResultTiming {
|
|
13
|
+
/** The toolCall id this result answers — joins `toolCall.id` exactly. */
|
|
14
|
+
toolCallId: string;
|
|
15
|
+
/** The tool's own measured wall time in ms, when present and finite. */
|
|
16
|
+
wallTimeMs: number | undefined;
|
|
17
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
18
|
+
timestampMs: number | undefined;
|
|
19
|
+
}
|
|
20
|
+
/** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
|
|
21
|
+
export declare function timestampToEpochMs(value: unknown): number | undefined;
|
|
22
|
+
/**
|
|
23
|
+
* Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
|
|
24
|
+
* when the record is not a toolResult message or carries no toolCallId. Live OMP
|
|
25
|
+
* emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
|
|
26
|
+
* content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
|
|
27
|
+
* measured wall time (48% of results in the sampled session) and `toolCallId`
|
|
28
|
+
* joins the originating `toolCall.id` exactly.
|
|
29
|
+
*/
|
|
30
|
+
export declare function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null;
|
|
11
31
|
/** Map a Codex JSONL record into one or more ETL split entries. */
|
|
12
32
|
export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
13
33
|
/** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
|
package/dist/mappers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAqDD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA6G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAqHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
|
package/dist/mappers.js
CHANGED
|
@@ -31,6 +31,7 @@ const TOOL_CALL_MAPPER_KEYS = [
|
|
|
31
31
|
'session_id',
|
|
32
32
|
'seq',
|
|
33
33
|
'tool_name',
|
|
34
|
+
'call_id',
|
|
34
35
|
'args_digest',
|
|
35
36
|
'args_raw',
|
|
36
37
|
'status',
|
|
@@ -225,24 +226,62 @@ export function claudeSplit(raw) {
|
|
|
225
226
|
// Pi mapper
|
|
226
227
|
// ---------------------------------------------------------------------------
|
|
227
228
|
/** Map a Pi JSONL record into one or more ETL split entries. */
|
|
228
|
-
export function piSplit(raw) {
|
|
229
|
+
export function piSplit(raw, context) {
|
|
230
|
+
// Pi session files are top-level event envelopes: `{type, id, parentId, timestamp,
|
|
231
|
+
// message: {role, content, model, usage, provider, ...}}`. Newer records carry `recordType`
|
|
232
|
+
// instead of `type` with `role`/`text` at the top level. The top-level `id` is an event id,
|
|
233
|
+
// never a session id; the session key and sequence come from the source file (context).
|
|
234
|
+
const msg = raw.message;
|
|
235
|
+
const recordType = String(raw.type ?? raw.recordType ?? '');
|
|
236
|
+
const sessionId = sessionIdFromContext(context, raw);
|
|
237
|
+
const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
|
|
238
|
+
// Prefer the ISO `timestamp`; a numeric epoch-ms `ts` is converted, never stringified.
|
|
239
|
+
const ts = s(raw.timestamp, raw.createdAt) ??
|
|
240
|
+
(typeof raw.ts === 'number' && Number.isFinite(raw.ts) ? new Date(raw.ts).toISOString() : s(raw.ts)) ??
|
|
241
|
+
new Date(0).toISOString();
|
|
242
|
+
// Meta lifecycle/custom records collapse to one meta row keyed by the source session,
|
|
243
|
+
// never the unique event id, and never a guessed role.
|
|
244
|
+
if (recordType === 'title' ||
|
|
245
|
+
recordType === 'title_change' ||
|
|
246
|
+
recordType === 'service_tier_change' ||
|
|
247
|
+
recordType === 'session_info' ||
|
|
248
|
+
recordType === 'ttsr_injection' ||
|
|
249
|
+
recordType === 'session_init' ||
|
|
250
|
+
recordType === 'session' ||
|
|
251
|
+
recordType === 'model_change' ||
|
|
252
|
+
recordType === 'thinking_level_change' ||
|
|
253
|
+
recordType === 'compaction' ||
|
|
254
|
+
recordType === 'custom' ||
|
|
255
|
+
recordType === 'custom_message' ||
|
|
256
|
+
recordType.startsWith('custom.')) {
|
|
257
|
+
return [
|
|
258
|
+
{
|
|
259
|
+
targetTable: 'history_message',
|
|
260
|
+
record: {
|
|
261
|
+
session_id: sessionId,
|
|
262
|
+
seq,
|
|
263
|
+
role: 'meta',
|
|
264
|
+
record_type: recordType,
|
|
265
|
+
disposition: 'meta',
|
|
266
|
+
ts,
|
|
267
|
+
provenance: 'ambient',
|
|
268
|
+
},
|
|
269
|
+
},
|
|
270
|
+
];
|
|
271
|
+
}
|
|
229
272
|
const entries = [];
|
|
230
|
-
const
|
|
231
|
-
const
|
|
232
|
-
const
|
|
233
|
-
const
|
|
234
|
-
const role = mapRole(r.type ?? r.role);
|
|
235
|
-
const recordType = String(r.type ?? '');
|
|
236
|
-
const model = s(r.model, o(r.message).model);
|
|
237
|
-
const cwd = s(r.cwd, r.dir);
|
|
238
|
-
const msg = r.message;
|
|
239
|
-
const usage = msg?.usage;
|
|
273
|
+
const role = piRole(msg?.role ?? raw.role ?? recordType);
|
|
274
|
+
const model = s(raw.model, o(msg).model);
|
|
275
|
+
const cwd = s(raw.cwd, raw.dir);
|
|
276
|
+
const usage = (msg?.usage ?? raw.usage);
|
|
240
277
|
const inputTokens = (usage?.input ?? usage?.input_tokens ?? undefined);
|
|
241
278
|
const outputTokens = (usage?.output ?? usage?.output_tokens ?? undefined);
|
|
242
279
|
const cacheRead = (usage?.cacheRead ?? usage?.cache_read_tokens ?? undefined);
|
|
243
280
|
const cacheWrite = (usage?.cacheWrite ?? usage?.cache_write_tokens ?? undefined);
|
|
244
|
-
const costObj = (msg?.cost ??
|
|
281
|
+
const costObj = (msg?.cost ?? raw.cost);
|
|
245
282
|
const costUsd = typeof costObj?.total === 'number' ? costObj.total : computeCost(inputTokens, outputTokens, model);
|
|
283
|
+
const contentBlocks = msg?.content ?? raw.content;
|
|
284
|
+
const durationMs = typeof msg?.duration === 'number' && Number.isFinite(msg.duration) ? msg.duration : undefined;
|
|
246
285
|
const messageSplitIndex = entries.length;
|
|
247
286
|
entries.push({
|
|
248
287
|
targetTable: 'history_message',
|
|
@@ -253,45 +292,64 @@ export function piSplit(raw) {
|
|
|
253
292
|
record_type: recordType,
|
|
254
293
|
disposition: 'keep',
|
|
255
294
|
ts,
|
|
256
|
-
duration_ms:
|
|
295
|
+
duration_ms: durationMs,
|
|
257
296
|
model: model ?? null,
|
|
258
297
|
input_tokens: inputTokens ?? null,
|
|
259
298
|
output_tokens: outputTokens ?? null,
|
|
260
299
|
cache_read_tokens: cacheRead ?? null,
|
|
261
300
|
cache_write_tokens: cacheWrite ?? null,
|
|
262
301
|
cost_usd: costUsd ?? null,
|
|
263
|
-
content_text: s(
|
|
302
|
+
content_text: extractContentText(contentBlocks) ?? s(raw.content, raw.text, msg?.content) ?? null,
|
|
264
303
|
cwd: cwd ?? null,
|
|
265
304
|
provenance: 'ambient',
|
|
266
305
|
},
|
|
267
306
|
});
|
|
268
|
-
// Tool calls from
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
307
|
+
// Tool calls from content blocks in assistant messages. normalizeOmpToolCall handles both
|
|
308
|
+
// the legacy nested `{toolCall: {...}}` block and pi's flat `{type: "toolCall", id, name,
|
|
309
|
+
// arguments}` block.
|
|
310
|
+
if (Array.isArray(contentBlocks) && role === 'assistant') {
|
|
311
|
+
for (const block of contentBlocks) {
|
|
312
|
+
const call = normalizeOmpToolCall(block);
|
|
313
|
+
if (call === null)
|
|
314
|
+
continue;
|
|
315
|
+
entries.push({
|
|
316
|
+
targetTable: 'history_tool_call',
|
|
317
|
+
record: {
|
|
318
|
+
_messageSplitIndex: messageSplitIndex,
|
|
319
|
+
session_id: sessionId,
|
|
320
|
+
seq,
|
|
321
|
+
tool_name: String(call.name ?? ''),
|
|
322
|
+
// The tool's own call id is the exact join key a toolResult's `toolCallId`
|
|
323
|
+
// matches (see ompSplit / task 0564 R1).
|
|
324
|
+
call_id: s(call.id),
|
|
325
|
+
args_digest: argsDigest(call.input ?? call.arguments),
|
|
326
|
+
args_raw: maybeArgsRaw('pi', String(call.name ?? ''), call.input ?? call.arguments),
|
|
327
|
+
status: 'ok',
|
|
328
|
+
started_at: undefined,
|
|
329
|
+
completed_at: undefined,
|
|
330
|
+
duration_ms: undefined,
|
|
331
|
+
result_bytes: undefined,
|
|
332
|
+
error_text: undefined,
|
|
333
|
+
},
|
|
334
|
+
});
|
|
291
335
|
}
|
|
292
336
|
}
|
|
293
337
|
return entries;
|
|
294
338
|
}
|
|
339
|
+
/**
|
|
340
|
+
* Canonicalize a pi message role. pi tool results and bash executions ride user turns (Claude
|
|
341
|
+
* protocol shape); everything unrecognized clamps to `unknown` so pi record types never leak
|
|
342
|
+
* into the role column (task 0577 AC2).
|
|
343
|
+
*/
|
|
344
|
+
function piRole(type) {
|
|
345
|
+
const t = String(type ?? '').toLowerCase();
|
|
346
|
+
if (t === 'toolresult' || t === 'tool_result' || t === 'bashexecution' || t === 'tool')
|
|
347
|
+
return 'user';
|
|
348
|
+
const mapped = mapRole(t);
|
|
349
|
+
if (mapped === 'user' || mapped === 'assistant' || mapped === 'system')
|
|
350
|
+
return mapped;
|
|
351
|
+
return 'unknown';
|
|
352
|
+
}
|
|
295
353
|
// ---------------------------------------------------------------------------
|
|
296
354
|
// OMP mapper (near-identical to pi)
|
|
297
355
|
// ---------------------------------------------------------------------------
|
|
@@ -383,6 +441,10 @@ export function ompSplit(raw, context) {
|
|
|
383
441
|
session_id: sessionId,
|
|
384
442
|
seq,
|
|
385
443
|
tool_name: String(call.name ?? ''),
|
|
444
|
+
// Task 0564 R1: the tool's own call id is the exact join key a
|
|
445
|
+
// toolResult's `toolCallId` matches — it lets the streaming loop
|
|
446
|
+
// attach the tool's measured duration to the right row.
|
|
447
|
+
call_id: s(call.id),
|
|
386
448
|
args_digest: argsDigest(call.input ?? call.arguments),
|
|
387
449
|
args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
|
|
388
450
|
status: 'ok',
|
|
@@ -413,6 +475,39 @@ function normalizeOmpToolCall(block) {
|
|
|
413
475
|
return block;
|
|
414
476
|
return null;
|
|
415
477
|
}
|
|
478
|
+
/** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
|
|
479
|
+
export function timestampToEpochMs(value) {
|
|
480
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
481
|
+
return value;
|
|
482
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
483
|
+
const numeric = Number(value);
|
|
484
|
+
if (Number.isFinite(numeric))
|
|
485
|
+
return numeric;
|
|
486
|
+
const parsed = Date.parse(value);
|
|
487
|
+
if (!Number.isNaN(parsed))
|
|
488
|
+
return parsed;
|
|
489
|
+
}
|
|
490
|
+
return undefined;
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
|
|
494
|
+
* when the record is not a toolResult message or carries no toolCallId. Live OMP
|
|
495
|
+
* emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
|
|
496
|
+
* content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
|
|
497
|
+
* measured wall time (48% of results in the sampled session) and `toolCallId`
|
|
498
|
+
* joins the originating `toolCall.id` exactly.
|
|
499
|
+
*/
|
|
500
|
+
export function ompToolResultTiming(raw) {
|
|
501
|
+
const msg = o(raw.message);
|
|
502
|
+
if (String(msg.role ?? '').toLowerCase() !== 'toolresult')
|
|
503
|
+
return null;
|
|
504
|
+
const toolCallId = s(msg.toolCallId);
|
|
505
|
+
if (toolCallId === undefined)
|
|
506
|
+
return null;
|
|
507
|
+
const details = o(msg.details);
|
|
508
|
+
const wallTimeMs = typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
|
|
509
|
+
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
510
|
+
}
|
|
416
511
|
// ---------------------------------------------------------------------------
|
|
417
512
|
// Codex mapper
|
|
418
513
|
// ---------------------------------------------------------------------------
|
package/dist/schema-sql.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,
|
|
1
|
+
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA6E9B,CAAC"}
|
package/dist/schema-sql.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.36",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.36",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.36",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.36",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
package/src/importer.ts
CHANGED
|
@@ -19,7 +19,9 @@ import {
|
|
|
19
19
|
recordInsertOp,
|
|
20
20
|
resetCheckpoints,
|
|
21
21
|
targetTableFor,
|
|
22
|
+
toolCallDurationUpdateOp,
|
|
22
23
|
} from './jsonl-importer-dao';
|
|
24
|
+
import { type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
|
|
23
25
|
import { redactRecord } from './redaction';
|
|
24
26
|
import { resolveSourceDefinition } from './sources';
|
|
25
27
|
import type {
|
|
@@ -37,6 +39,83 @@ interface SplitRecord {
|
|
|
37
39
|
readonly raw: JsonObject;
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
// Task 0564 R1 — tool-call duration attach (omp)
|
|
44
|
+
//
|
|
45
|
+
// A toolCall row is emitted while handling its assistant message (SplitEntry is
|
|
46
|
+
// insert-only, so the duration cannot ride the mapper's split path); the
|
|
47
|
+
// duration arrives on a LATER toolResult line. The attach therefore lives in the
|
|
48
|
+
// streaming loop, keyed on (source, session_id, call_id) per the frozen contract:
|
|
49
|
+
// - `details.wallTimeMs` is the tool's own measured wall time — used as-is,
|
|
50
|
+
// rounded, NEVER clamped or second-guessed (guards are fallback-only);
|
|
51
|
+
// - otherwise the fallback `toolResult.timestamp − toolCall message timestamp`
|
|
52
|
+
// when both are parseable AND the delta is in [0, 3_600_000]; an implausible
|
|
53
|
+
// delta stays NULL so the unmeasured count stays honest;
|
|
54
|
+
// - started_at / completed_at are written alongside a FALLBACK figure so it is
|
|
55
|
+
// auditable (wallTimeMs rows keep both NULL — the two stay distinguishable);
|
|
56
|
+
// - an unmatched toolCallId attaches nothing and never fails the import.
|
|
57
|
+
//
|
|
58
|
+
// The in-memory map covers the common (fresh-import) case without extra queries;
|
|
59
|
+
// a toolResult whose tool-call line was imported by an earlier checkpointed run
|
|
60
|
+
// (resume) resolves through a DB lookup instead, so a mid-file checkpoint loses
|
|
61
|
+
// no earlier duration.
|
|
62
|
+
// ---------------------------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
interface PendingToolCall {
|
|
65
|
+
readonly recordHash: string;
|
|
66
|
+
/** Assistant-message timestamp (epoch ms) — the fallback's start bound. */
|
|
67
|
+
readonly messageTsMs: number | undefined;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const TOOL_CALL_KEY_SEP = '\u0000';
|
|
71
|
+
|
|
72
|
+
function toolCallKey(source: string, sessionId: string, callId: string): string {
|
|
73
|
+
return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
|
|
77
|
+
async function resolveToolCallRow(
|
|
78
|
+
db: ImportOptions['db'],
|
|
79
|
+
source: string,
|
|
80
|
+
sessionId: string,
|
|
81
|
+
callId: string,
|
|
82
|
+
): Promise<PendingToolCall | undefined> {
|
|
83
|
+
const row = await db.queryFirst<{ recordHash: string; messageTs: string | null }>(
|
|
84
|
+
`SELECT tc.record_hash AS recordHash, m.ts AS messageTs
|
|
85
|
+
FROM history_tool_call tc
|
|
86
|
+
LEFT JOIN history_message m ON m.record_hash = tc.message_hash
|
|
87
|
+
WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`,
|
|
88
|
+
source,
|
|
89
|
+
sessionId,
|
|
90
|
+
callId,
|
|
91
|
+
);
|
|
92
|
+
if (row === undefined || row === null) return undefined;
|
|
93
|
+
return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Compute the attachable duration for a toolResult, or null when nothing should
|
|
98
|
+
* be written (implausible fallback delta → row stays NULL → counts unmeasured).
|
|
99
|
+
*/
|
|
100
|
+
function attachableDuration(
|
|
101
|
+
timing: OmpToolResultTiming,
|
|
102
|
+
messageTsMs: number | undefined,
|
|
103
|
+
): { startedAt: string | null; completedAt: string | null; durationMs: number } | null {
|
|
104
|
+
if (timing.wallTimeMs !== undefined) {
|
|
105
|
+
// The tool's own measurement — no clamping, no sanity check.
|
|
106
|
+
return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
|
|
107
|
+
}
|
|
108
|
+
if (timing.timestampMs === undefined || messageTsMs === undefined) return null;
|
|
109
|
+
const delta = timing.timestampMs - messageTsMs;
|
|
110
|
+
// Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
|
|
111
|
+
if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000) return null;
|
|
112
|
+
return {
|
|
113
|
+
startedAt: new Date(messageTsMs).toISOString(),
|
|
114
|
+
completedAt: new Date(timing.timestampMs).toISOString(),
|
|
115
|
+
durationMs: Math.round(delta),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
40
119
|
/** Run the JSONL import pipeline for one source.
|
|
41
120
|
*
|
|
42
121
|
* @param source - Either a built-in source identifier resolved from the
|
|
@@ -69,6 +148,10 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
69
148
|
// ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
|
|
70
149
|
const desiredHashes = new Set<string>();
|
|
71
150
|
|
|
151
|
+
// Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
|
|
152
|
+
// (source, session_id, call_id) so a later toolResult line can attach its duration.
|
|
153
|
+
const toolCallRows = new Map<string, PendingToolCall>();
|
|
154
|
+
|
|
72
155
|
for (const file of files) {
|
|
73
156
|
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
|
|
74
157
|
let lineNumber = 0;
|
|
@@ -138,6 +221,43 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
138
221
|
// second pass, no checkpoint advance, no partial rows from this record.
|
|
139
222
|
if (lineRejected) continue;
|
|
140
223
|
|
|
224
|
+
// Task 0564 R1 (omp): register this line's tool-call rows for later duration
|
|
225
|
+
// attach. Runs over ALL prepared entries — including duplicates — because a
|
|
226
|
+
// re-imported line's row already exists and its duration UPDATE still targets
|
|
227
|
+
// the same deterministic record_hash. The assistant message timestamp is the
|
|
228
|
+
// fallback's start bound; every call in one message shares it.
|
|
229
|
+
if (definition.source === 'omp') {
|
|
230
|
+
const assistantTs = new Map<string, number | undefined>();
|
|
231
|
+
for (const entry of prepared) {
|
|
232
|
+
if (
|
|
233
|
+
entry.split.targetTable === 'history_message' &&
|
|
234
|
+
entry.normalized.role === 'assistant' &&
|
|
235
|
+
typeof entry.normalized.session_id === 'string'
|
|
236
|
+
) {
|
|
237
|
+
assistantTs.set(
|
|
238
|
+
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
239
|
+
timestampToEpochMs(entry.normalized.ts),
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
for (const entry of prepared) {
|
|
244
|
+
const callId = entry.normalized.call_id;
|
|
245
|
+
if (
|
|
246
|
+
entry.split.targetTable === 'history_tool_call' &&
|
|
247
|
+
typeof callId === 'string' &&
|
|
248
|
+
callId.length > 0 &&
|
|
249
|
+
typeof entry.normalized.session_id === 'string'
|
|
250
|
+
) {
|
|
251
|
+
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
252
|
+
recordHash: entry.recordHash,
|
|
253
|
+
messageTsMs: assistantTs.get(
|
|
254
|
+
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
255
|
+
),
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
141
261
|
// Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
|
|
142
262
|
// instead of a SELECT per record; duplicates are dropped before the write pass.
|
|
143
263
|
const existing = await ledgerExistingHashes(
|
|
@@ -199,6 +319,47 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
199
319
|
}
|
|
200
320
|
|
|
201
321
|
if (!options.dryRun && lineSucceeded) {
|
|
322
|
+
// Task 0564 R1 (omp): attach tool durations from this line's toolResult
|
|
323
|
+
// message. Unmatched toolCallIds attach nothing and never fail the import;
|
|
324
|
+
// an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
|
|
325
|
+
// this line's batch — idempotent, so re-imports and post-resume tail
|
|
326
|
+
// reprocessing write the same values.
|
|
327
|
+
if (definition.source === 'omp') {
|
|
328
|
+
const timing = ompToolResultTiming(raw);
|
|
329
|
+
if (timing !== null) {
|
|
330
|
+
const resultEntry = prepared.find(
|
|
331
|
+
(entry) =>
|
|
332
|
+
entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult',
|
|
333
|
+
);
|
|
334
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
335
|
+
if (typeof sessionId === 'string') {
|
|
336
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
337
|
+
if (pending === undefined) {
|
|
338
|
+
// The tool-call line sits behind this run's checkpoint — the
|
|
339
|
+
// row exists from an earlier import; resolve it from the DB.
|
|
340
|
+
pending = await resolveToolCallRow(
|
|
341
|
+
options.db,
|
|
342
|
+
resolvedSource,
|
|
343
|
+
sessionId,
|
|
344
|
+
timing.toolCallId,
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
if (pending !== undefined) {
|
|
348
|
+
const duration = attachableDuration(timing, pending.messageTsMs);
|
|
349
|
+
if (duration !== null) {
|
|
350
|
+
ops.push(
|
|
351
|
+
toolCallDurationUpdateOp(
|
|
352
|
+
pending.recordHash,
|
|
353
|
+
duration.startedAt,
|
|
354
|
+
duration.completedAt,
|
|
355
|
+
duration.durationMs,
|
|
356
|
+
),
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
202
363
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
203
364
|
// re-imported after a mid-batch crash.
|
|
204
365
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
|
@@ -45,6 +45,7 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
|
45
45
|
'session_id',
|
|
46
46
|
'seq',
|
|
47
47
|
'tool_name',
|
|
48
|
+
'call_id',
|
|
48
49
|
'args_digest',
|
|
49
50
|
'args_raw',
|
|
50
51
|
'status',
|
|
@@ -255,6 +256,29 @@ export function checkpointUpsertOp(
|
|
|
255
256
|
};
|
|
256
257
|
}
|
|
257
258
|
|
|
259
|
+
/**
|
|
260
|
+
* Targeted UPDATE op attaching a tool duration to a previously-inserted
|
|
261
|
+
* `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
|
|
262
|
+
* unindexed predicate. Idempotent: re-imports write the same values.
|
|
263
|
+
*
|
|
264
|
+
* Bounds are written alongside a FALLBACK duration so the figure is auditable;
|
|
265
|
+
* a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
|
|
266
|
+
* two measurement paths stay distinguishable.
|
|
267
|
+
*/
|
|
268
|
+
function toolCallDurationUpdateOp(
|
|
269
|
+
recordHash: string,
|
|
270
|
+
startedAt: string | null,
|
|
271
|
+
completedAt: string | null,
|
|
272
|
+
durationMs: number | null,
|
|
273
|
+
): DbBatchOp {
|
|
274
|
+
return {
|
|
275
|
+
sql: `UPDATE history_tool_call
|
|
276
|
+
SET started_at = ?, completed_at = ?, duration_ms = ?
|
|
277
|
+
WHERE record_hash = ?`,
|
|
278
|
+
params: [startedAt, completedAt, durationMs, recordHash],
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
258
282
|
/**
|
|
259
283
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
260
284
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -688,5 +712,6 @@ export {
|
|
|
688
712
|
resetCheckpoints,
|
|
689
713
|
targetTableFor,
|
|
690
714
|
timestamp,
|
|
715
|
+
toolCallDurationUpdateOp,
|
|
691
716
|
writeCheckpoint,
|
|
692
717
|
};
|
package/src/mappers.ts
CHANGED
|
@@ -35,6 +35,7 @@ const TOOL_CALL_MAPPER_KEYS: readonly string[] = [
|
|
|
35
35
|
'session_id',
|
|
36
36
|
'seq',
|
|
37
37
|
'tool_name',
|
|
38
|
+
'call_id',
|
|
38
39
|
'args_digest',
|
|
39
40
|
'args_raw',
|
|
40
41
|
'status',
|
|
@@ -240,25 +241,67 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
240
241
|
// ---------------------------------------------------------------------------
|
|
241
242
|
|
|
242
243
|
/** Map a Pi JSONL record into one or more ETL split entries. */
|
|
243
|
-
export function piSplit(raw: unknown): readonly SplitEntry[] {
|
|
244
|
+
export function piSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[] {
|
|
245
|
+
// Pi session files are top-level event envelopes: `{type, id, parentId, timestamp,
|
|
246
|
+
// message: {role, content, model, usage, provider, ...}}`. Newer records carry `recordType`
|
|
247
|
+
// instead of `type` with `role`/`text` at the top level. The top-level `id` is an event id,
|
|
248
|
+
// never a session id; the session key and sequence come from the source file (context).
|
|
249
|
+
const msg = raw.message as Record<string, unknown> | undefined;
|
|
250
|
+
const recordType = String(raw.type ?? raw.recordType ?? '');
|
|
251
|
+
const sessionId = sessionIdFromContext(context, raw);
|
|
252
|
+
const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
|
|
253
|
+
// Prefer the ISO `timestamp`; a numeric epoch-ms `ts` is converted, never stringified.
|
|
254
|
+
const ts =
|
|
255
|
+
s(raw.timestamp, raw.createdAt) ??
|
|
256
|
+
(typeof raw.ts === 'number' && Number.isFinite(raw.ts) ? new Date(raw.ts).toISOString() : s(raw.ts)) ??
|
|
257
|
+
new Date(0).toISOString();
|
|
258
|
+
|
|
259
|
+
// Meta lifecycle/custom records collapse to one meta row keyed by the source session,
|
|
260
|
+
// never the unique event id, and never a guessed role.
|
|
261
|
+
if (
|
|
262
|
+
recordType === 'title' ||
|
|
263
|
+
recordType === 'title_change' ||
|
|
264
|
+
recordType === 'service_tier_change' ||
|
|
265
|
+
recordType === 'session_info' ||
|
|
266
|
+
recordType === 'ttsr_injection' ||
|
|
267
|
+
recordType === 'session_init' ||
|
|
268
|
+
recordType === 'session' ||
|
|
269
|
+
recordType === 'model_change' ||
|
|
270
|
+
recordType === 'thinking_level_change' ||
|
|
271
|
+
recordType === 'compaction' ||
|
|
272
|
+
recordType === 'custom' ||
|
|
273
|
+
recordType === 'custom_message' ||
|
|
274
|
+
recordType.startsWith('custom.')
|
|
275
|
+
) {
|
|
276
|
+
return [
|
|
277
|
+
{
|
|
278
|
+
targetTable: 'history_message',
|
|
279
|
+
record: {
|
|
280
|
+
session_id: sessionId,
|
|
281
|
+
seq,
|
|
282
|
+
role: 'meta',
|
|
283
|
+
record_type: recordType,
|
|
284
|
+
disposition: 'meta',
|
|
285
|
+
ts,
|
|
286
|
+
provenance: 'ambient',
|
|
287
|
+
},
|
|
288
|
+
},
|
|
289
|
+
];
|
|
290
|
+
}
|
|
291
|
+
|
|
244
292
|
const entries: SplitEntry[] = [];
|
|
245
|
-
const
|
|
246
|
-
const
|
|
247
|
-
const
|
|
248
|
-
const
|
|
249
|
-
const role = mapRole(r.type ?? r.role);
|
|
250
|
-
const recordType = String(r.type ?? '');
|
|
251
|
-
const model = s(r.model, o(r.message).model);
|
|
252
|
-
const cwd = s(r.cwd, r.dir);
|
|
253
|
-
|
|
254
|
-
const msg = r.message as Record<string, unknown> | undefined;
|
|
255
|
-
const usage = msg?.usage as Record<string, unknown> | undefined;
|
|
293
|
+
const role = piRole(msg?.role ?? raw.role ?? recordType);
|
|
294
|
+
const model = s(raw.model, o(msg).model);
|
|
295
|
+
const cwd = s(raw.cwd, raw.dir);
|
|
296
|
+
const usage = (msg?.usage ?? raw.usage) as Record<string, unknown> | undefined;
|
|
256
297
|
const inputTokens = (usage?.input ?? usage?.input_tokens ?? undefined) as number | undefined;
|
|
257
298
|
const outputTokens = (usage?.output ?? usage?.output_tokens ?? undefined) as number | undefined;
|
|
258
299
|
const cacheRead = (usage?.cacheRead ?? usage?.cache_read_tokens ?? undefined) as number | undefined;
|
|
259
300
|
const cacheWrite = (usage?.cacheWrite ?? usage?.cache_write_tokens ?? undefined) as number | undefined;
|
|
260
|
-
const costObj = (msg?.cost ??
|
|
301
|
+
const costObj = (msg?.cost ?? raw.cost) as Record<string, unknown> | undefined;
|
|
261
302
|
const costUsd = typeof costObj?.total === 'number' ? costObj.total : computeCost(inputTokens, outputTokens, model);
|
|
303
|
+
const contentBlocks = msg?.content ?? raw.content;
|
|
304
|
+
const durationMs = typeof msg?.duration === 'number' && Number.isFinite(msg.duration) ? msg.duration : undefined;
|
|
262
305
|
|
|
263
306
|
const messageSplitIndex = entries.length;
|
|
264
307
|
|
|
@@ -271,48 +314,65 @@ export function piSplit(raw: unknown): readonly SplitEntry[] {
|
|
|
271
314
|
record_type: recordType,
|
|
272
315
|
disposition: 'keep',
|
|
273
316
|
ts,
|
|
274
|
-
duration_ms:
|
|
317
|
+
duration_ms: durationMs,
|
|
275
318
|
model: model ?? null,
|
|
276
319
|
input_tokens: inputTokens ?? null,
|
|
277
320
|
output_tokens: outputTokens ?? null,
|
|
278
321
|
cache_read_tokens: cacheRead ?? null,
|
|
279
322
|
cache_write_tokens: cacheWrite ?? null,
|
|
280
323
|
cost_usd: costUsd ?? null,
|
|
281
|
-
content_text: s(
|
|
324
|
+
content_text: extractContentText(contentBlocks) ?? s(raw.content, raw.text, msg?.content) ?? null,
|
|
282
325
|
cwd: cwd ?? null,
|
|
283
326
|
provenance: 'ambient',
|
|
284
327
|
},
|
|
285
328
|
});
|
|
286
329
|
|
|
287
|
-
// Tool calls from
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
330
|
+
// Tool calls from content blocks in assistant messages. normalizeOmpToolCall handles both
|
|
331
|
+
// the legacy nested `{toolCall: {...}}` block and pi's flat `{type: "toolCall", id, name,
|
|
332
|
+
// arguments}` block.
|
|
333
|
+
if (Array.isArray(contentBlocks) && role === 'assistant') {
|
|
334
|
+
for (const block of contentBlocks as Record<string, unknown>[]) {
|
|
335
|
+
const call = normalizeOmpToolCall(block);
|
|
336
|
+
if (call === null) continue;
|
|
337
|
+
entries.push({
|
|
338
|
+
targetTable: 'history_tool_call',
|
|
339
|
+
record: {
|
|
340
|
+
_messageSplitIndex: messageSplitIndex,
|
|
341
|
+
session_id: sessionId,
|
|
342
|
+
seq,
|
|
343
|
+
tool_name: String(call.name ?? ''),
|
|
344
|
+
// The tool's own call id is the exact join key a toolResult's `toolCallId`
|
|
345
|
+
// matches (see ompSplit / task 0564 R1).
|
|
346
|
+
call_id: s(call.id),
|
|
347
|
+
args_digest: argsDigest(call.input ?? call.arguments),
|
|
348
|
+
args_raw: maybeArgsRaw('pi', String(call.name ?? ''), call.input ?? call.arguments),
|
|
349
|
+
status: 'ok',
|
|
350
|
+
started_at: undefined,
|
|
351
|
+
completed_at: undefined,
|
|
352
|
+
duration_ms: undefined,
|
|
353
|
+
result_bytes: undefined,
|
|
354
|
+
error_text: undefined,
|
|
355
|
+
},
|
|
356
|
+
});
|
|
310
357
|
}
|
|
311
358
|
}
|
|
312
359
|
|
|
313
360
|
return entries;
|
|
314
361
|
}
|
|
315
362
|
|
|
363
|
+
/**
|
|
364
|
+
* Canonicalize a pi message role. pi tool results and bash executions ride user turns (Claude
|
|
365
|
+
* protocol shape); everything unrecognized clamps to `unknown` so pi record types never leak
|
|
366
|
+
* into the role column (task 0577 AC2).
|
|
367
|
+
*/
|
|
368
|
+
function piRole(type: unknown): string {
|
|
369
|
+
const t = String(type ?? '').toLowerCase();
|
|
370
|
+
if (t === 'toolresult' || t === 'tool_result' || t === 'bashexecution' || t === 'tool') return 'user';
|
|
371
|
+
const mapped = mapRole(t);
|
|
372
|
+
if (mapped === 'user' || mapped === 'assistant' || mapped === 'system') return mapped;
|
|
373
|
+
return 'unknown';
|
|
374
|
+
}
|
|
375
|
+
|
|
316
376
|
// ---------------------------------------------------------------------------
|
|
317
377
|
// OMP mapper (near-identical to pi)
|
|
318
378
|
// ---------------------------------------------------------------------------
|
|
@@ -411,6 +471,10 @@ export function ompSplit(raw: Record<string, unknown>, context?: TransformContex
|
|
|
411
471
|
session_id: sessionId,
|
|
412
472
|
seq,
|
|
413
473
|
tool_name: String(call.name ?? ''),
|
|
474
|
+
// Task 0564 R1: the tool's own call id is the exact join key a
|
|
475
|
+
// toolResult's `toolCallId` matches — it lets the streaming loop
|
|
476
|
+
// attach the tool's measured duration to the right row.
|
|
477
|
+
call_id: s(call.id),
|
|
414
478
|
args_digest: argsDigest(call.input ?? call.arguments),
|
|
415
479
|
args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
|
|
416
480
|
status: 'ok',
|
|
@@ -441,6 +505,51 @@ function normalizeOmpToolCall(block: Record<string, unknown>): Record<string, un
|
|
|
441
505
|
return null;
|
|
442
506
|
}
|
|
443
507
|
|
|
508
|
+
// ---------------------------------------------------------------------------
|
|
509
|
+
// OMP toolResult timing signals (task 0564 R1)
|
|
510
|
+
// ---------------------------------------------------------------------------
|
|
511
|
+
|
|
512
|
+
/** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
|
|
513
|
+
export interface OmpToolResultTiming {
|
|
514
|
+
/** The toolCall id this result answers — joins `toolCall.id` exactly. */
|
|
515
|
+
toolCallId: string;
|
|
516
|
+
/** The tool's own measured wall time in ms, when present and finite. */
|
|
517
|
+
wallTimeMs: number | undefined;
|
|
518
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
519
|
+
timestampMs: number | undefined;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
|
|
523
|
+
export function timestampToEpochMs(value: unknown): number | undefined {
|
|
524
|
+
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
|
525
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
526
|
+
const numeric = Number(value);
|
|
527
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
528
|
+
const parsed = Date.parse(value);
|
|
529
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
530
|
+
}
|
|
531
|
+
return undefined;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/**
|
|
535
|
+
* Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
|
|
536
|
+
* when the record is not a toolResult message or carries no toolCallId. Live OMP
|
|
537
|
+
* emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
|
|
538
|
+
* content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
|
|
539
|
+
* measured wall time (48% of results in the sampled session) and `toolCallId`
|
|
540
|
+
* joins the originating `toolCall.id` exactly.
|
|
541
|
+
*/
|
|
542
|
+
export function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null {
|
|
543
|
+
const msg = o(raw.message);
|
|
544
|
+
if (String(msg.role ?? '').toLowerCase() !== 'toolresult') return null;
|
|
545
|
+
const toolCallId = s(msg.toolCallId);
|
|
546
|
+
if (toolCallId === undefined) return null;
|
|
547
|
+
const details = o(msg.details);
|
|
548
|
+
const wallTimeMs =
|
|
549
|
+
typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
|
|
550
|
+
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
551
|
+
}
|
|
552
|
+
|
|
444
553
|
// ---------------------------------------------------------------------------
|
|
445
554
|
// Codex mapper
|
|
446
555
|
// ---------------------------------------------------------------------------
|