@gobing-ai/ts-llm-jsonl-importer 0.4.32 → 0.4.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +93 -1
- package/dist/jsonl-importer-dao.d.ts +11 -1
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +20 -1
- package/dist/mappers.d.ts +20 -0
- package/dist/mappers.d.ts.map +1 -1
- package/dist/mappers.js +93 -14
- package/dist/schema-sql.d.ts.map +1 -1
- package/dist/schema-sql.js +2 -0
- package/package.json +4 -4
- package/src/importer.ts +161 -0
- package/src/jsonl-importer-dao.ts +26 -0
- package/src/mappers.ts +104 -13
- package/src/schema-sql.ts +2 -0
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAkRrH"}
|
package/dist/importer.js
CHANGED
|
@@ -1,8 +1,44 @@
|
|
|
1
1
|
import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
|
|
2
2
|
import { sha256 } from './hash.js';
|
|
3
|
-
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, } from './jsonl-importer-dao.js';
|
|
3
|
+
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, } from './jsonl-importer-dao.js';
|
|
4
|
+
import { ompToolResultTiming, timestampToEpochMs } from './mappers.js';
|
|
4
5
|
import { redactRecord } from './redaction.js';
|
|
5
6
|
import { resolveSourceDefinition } from './sources.js';
|
|
7
|
+
const TOOL_CALL_KEY_SEP = '\u0000';
|
|
8
|
+
function toolCallKey(source, sessionId, callId) {
|
|
9
|
+
return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
|
|
10
|
+
}
|
|
11
|
+
/** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
|
|
12
|
+
async function resolveToolCallRow(db, source, sessionId, callId) {
|
|
13
|
+
const row = await db.queryFirst(`SELECT tc.record_hash AS recordHash, m.ts AS messageTs
|
|
14
|
+
FROM history_tool_call tc
|
|
15
|
+
LEFT JOIN history_message m ON m.record_hash = tc.message_hash
|
|
16
|
+
WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`, source, sessionId, callId);
|
|
17
|
+
if (row === undefined || row === null)
|
|
18
|
+
return undefined;
|
|
19
|
+
return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Compute the attachable duration for a toolResult, or null when nothing should
|
|
23
|
+
* be written (implausible fallback delta → row stays NULL → counts unmeasured).
|
|
24
|
+
*/
|
|
25
|
+
function attachableDuration(timing, messageTsMs) {
|
|
26
|
+
if (timing.wallTimeMs !== undefined) {
|
|
27
|
+
// The tool's own measurement — no clamping, no sanity check.
|
|
28
|
+
return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
|
|
29
|
+
}
|
|
30
|
+
if (timing.timestampMs === undefined || messageTsMs === undefined)
|
|
31
|
+
return null;
|
|
32
|
+
const delta = timing.timestampMs - messageTsMs;
|
|
33
|
+
// Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
|
|
34
|
+
if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000)
|
|
35
|
+
return null;
|
|
36
|
+
return {
|
|
37
|
+
startedAt: new Date(messageTsMs).toISOString(),
|
|
38
|
+
completedAt: new Date(timing.timestampMs).toISOString(),
|
|
39
|
+
durationMs: Math.round(delta),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
6
42
|
/** Run the JSONL import pipeline for one source.
|
|
7
43
|
*
|
|
8
44
|
* @param source - Either a built-in source identifier resolved from the
|
|
@@ -32,6 +68,9 @@ export async function runJsonlImport(source, options) {
|
|
|
32
68
|
// Full-mode desired set (R1, task 0504): every record hash the current source produces.
|
|
33
69
|
// ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
|
|
34
70
|
const desiredHashes = new Set();
|
|
71
|
+
// Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
|
|
72
|
+
// (source, session_id, call_id) so a later toolResult line can attach its duration.
|
|
73
|
+
const toolCallRows = new Map();
|
|
35
74
|
for (const file of files) {
|
|
36
75
|
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
|
|
37
76
|
let lineNumber = 0;
|
|
@@ -92,6 +131,33 @@ export async function runJsonlImport(source, options) {
|
|
|
92
131
|
// second pass, no checkpoint advance, no partial rows from this record.
|
|
93
132
|
if (lineRejected)
|
|
94
133
|
continue;
|
|
134
|
+
// Task 0564 R1 (omp): register this line's tool-call rows for later duration
|
|
135
|
+
// attach. Runs over ALL prepared entries — including duplicates — because a
|
|
136
|
+
// re-imported line's row already exists and its duration UPDATE still targets
|
|
137
|
+
// the same deterministic record_hash. The assistant message timestamp is the
|
|
138
|
+
// fallback's start bound; every call in one message shares it.
|
|
139
|
+
if (definition.source === 'omp') {
|
|
140
|
+
const assistantTs = new Map();
|
|
141
|
+
for (const entry of prepared) {
|
|
142
|
+
if (entry.split.targetTable === 'history_message' &&
|
|
143
|
+
entry.normalized.role === 'assistant' &&
|
|
144
|
+
typeof entry.normalized.session_id === 'string') {
|
|
145
|
+
assistantTs.set(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`, timestampToEpochMs(entry.normalized.ts));
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
for (const entry of prepared) {
|
|
149
|
+
const callId = entry.normalized.call_id;
|
|
150
|
+
if (entry.split.targetTable === 'history_tool_call' &&
|
|
151
|
+
typeof callId === 'string' &&
|
|
152
|
+
callId.length > 0 &&
|
|
153
|
+
typeof entry.normalized.session_id === 'string') {
|
|
154
|
+
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
155
|
+
recordHash: entry.recordHash,
|
|
156
|
+
messageTsMs: assistantTs.get(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`),
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
95
161
|
// Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
|
|
96
162
|
// instead of a SELECT per record; duplicates are dropped before the write pass.
|
|
97
163
|
const existing = await ledgerExistingHashes(options.db, prepared.map((e) => e.recordHash));
|
|
@@ -128,6 +194,32 @@ export async function runJsonlImport(source, options) {
|
|
|
128
194
|
importedRecords += 1;
|
|
129
195
|
}
|
|
130
196
|
if (!options.dryRun && lineSucceeded) {
|
|
197
|
+
// Task 0564 R1 (omp): attach tool durations from this line's toolResult
|
|
198
|
+
// message. Unmatched toolCallIds attach nothing and never fail the import;
|
|
199
|
+
// an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
|
|
200
|
+
// this line's batch — idempotent, so re-imports and post-resume tail
|
|
201
|
+
// reprocessing write the same values.
|
|
202
|
+
if (definition.source === 'omp') {
|
|
203
|
+
const timing = ompToolResultTiming(raw);
|
|
204
|
+
if (timing !== null) {
|
|
205
|
+
const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult');
|
|
206
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
207
|
+
if (typeof sessionId === 'string') {
|
|
208
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
209
|
+
if (pending === undefined) {
|
|
210
|
+
// The tool-call line sits behind this run's checkpoint — the
|
|
211
|
+
// row exists from an earlier import; resolve it from the DB.
|
|
212
|
+
pending = await resolveToolCallRow(options.db, resolvedSource, sessionId, timing.toolCallId);
|
|
213
|
+
}
|
|
214
|
+
if (pending !== undefined) {
|
|
215
|
+
const duration = attachableDuration(timing, pending.messageTsMs);
|
|
216
|
+
if (duration !== null) {
|
|
217
|
+
ops.push(toolCallDurationUpdateOp(pending.recordHash, duration.startedAt, duration.completedAt, duration.durationMs));
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
131
223
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
132
224
|
// re-imported after a mid-batch crash.
|
|
133
225
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
|
@@ -44,6 +44,16 @@ export declare function recordInsertOp(targetTable: string, recordHash: string,
|
|
|
44
44
|
export declare function ledgerInsertOp(recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): DbBatchOp;
|
|
45
45
|
/** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
|
|
46
46
|
export declare function checkpointUpsertOp(source: string, sourceFile: string, line: number, now: ImportOptions['now']): DbBatchOp;
|
|
47
|
+
/**
|
|
48
|
+
* Targeted UPDATE op attaching a tool duration to a previously-inserted
|
|
49
|
+
* `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
|
|
50
|
+
* unindexed predicate. Idempotent: re-imports write the same values.
|
|
51
|
+
*
|
|
52
|
+
* Bounds are written alongside a FALLBACK duration so the figure is auditable;
|
|
53
|
+
* a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
|
|
54
|
+
* two measurement paths stay distinguishable.
|
|
55
|
+
*/
|
|
56
|
+
declare function toolCallDurationUpdateOp(recordHash: string, startedAt: string | null, completedAt: string | null, durationMs: number | null): DbBatchOp;
|
|
47
57
|
/**
|
|
48
58
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
49
59
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -126,5 +136,5 @@ export declare function deleteCheckpointOperation(source: string, sourceFile: st
|
|
|
126
136
|
/** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
|
|
127
137
|
export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
|
|
128
138
|
export type { CheckpointRow };
|
|
129
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
|
139
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
|
130
140
|
//# sourceMappingURL=jsonl-importer-dao.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAuDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,eAAe,GAClB,CAAC"}
|
|
@@ -38,7 +38,9 @@ const TYPED_TABLE_COLUMNS = {
|
|
|
38
38
|
'session_id',
|
|
39
39
|
'seq',
|
|
40
40
|
'tool_name',
|
|
41
|
+
'call_id',
|
|
41
42
|
'args_digest',
|
|
43
|
+
'args_raw',
|
|
42
44
|
'status',
|
|
43
45
|
'started_at',
|
|
44
46
|
'completed_at',
|
|
@@ -193,6 +195,23 @@ export function checkpointUpsertOp(source, sourceFile, line, now) {
|
|
|
193
195
|
params: [source, sourceFile, line, timestamp(now)],
|
|
194
196
|
};
|
|
195
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* Targeted UPDATE op attaching a tool duration to a previously-inserted
|
|
200
|
+
* `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
|
|
201
|
+
* unindexed predicate. Idempotent: re-imports write the same values.
|
|
202
|
+
*
|
|
203
|
+
* Bounds are written alongside a FALLBACK duration so the figure is auditable;
|
|
204
|
+
* a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
|
|
205
|
+
* two measurement paths stay distinguishable.
|
|
206
|
+
*/
|
|
207
|
+
function toolCallDurationUpdateOp(recordHash, startedAt, completedAt, durationMs) {
|
|
208
|
+
return {
|
|
209
|
+
sql: `UPDATE history_tool_call
|
|
210
|
+
SET started_at = ?, completed_at = ?, duration_ms = ?
|
|
211
|
+
WHERE record_hash = ?`,
|
|
212
|
+
params: [startedAt, completedAt, durationMs, recordHash],
|
|
213
|
+
};
|
|
214
|
+
}
|
|
196
215
|
/**
|
|
197
216
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
198
217
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -467,4 +486,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
|
|
|
467
486
|
}
|
|
468
487
|
return operations;
|
|
469
488
|
}
|
|
470
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
|
489
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
package/dist/mappers.d.ts
CHANGED
|
@@ -8,6 +8,26 @@ export declare function claudeSplit(raw: Record<string, unknown>): readonly Spli
|
|
|
8
8
|
export declare function piSplit(raw: unknown): readonly SplitEntry[];
|
|
9
9
|
/** Map an OMP JSONL record into one or more ETL split entries. */
|
|
10
10
|
export declare function ompSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
11
|
+
/** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
|
|
12
|
+
export interface OmpToolResultTiming {
|
|
13
|
+
/** The toolCall id this result answers — joins `toolCall.id` exactly. */
|
|
14
|
+
toolCallId: string;
|
|
15
|
+
/** The tool's own measured wall time in ms, when present and finite. */
|
|
16
|
+
wallTimeMs: number | undefined;
|
|
17
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
18
|
+
timestampMs: number | undefined;
|
|
19
|
+
}
|
|
20
|
+
/** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
|
|
21
|
+
export declare function timestampToEpochMs(value: unknown): number | undefined;
|
|
22
|
+
/**
|
|
23
|
+
* Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
|
|
24
|
+
* when the record is not a toolResult message or carries no toolCallId. Live OMP
|
|
25
|
+
* emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
|
|
26
|
+
* content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
|
|
27
|
+
* measured wall time (48% of results in the sampled session) and `toolCallId`
|
|
28
|
+
* joins the originating `toolCall.id` exactly.
|
|
29
|
+
*/
|
|
30
|
+
export declare function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null;
|
|
11
31
|
/** Map a Codex JSONL record into one or more ETL split entries. */
|
|
12
32
|
export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
13
33
|
/** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
|
package/dist/mappers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAqDD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA6G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAuE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
|
package/dist/mappers.js
CHANGED
|
@@ -31,7 +31,9 @@ const TOOL_CALL_MAPPER_KEYS = [
|
|
|
31
31
|
'session_id',
|
|
32
32
|
'seq',
|
|
33
33
|
'tool_name',
|
|
34
|
+
'call_id',
|
|
34
35
|
'args_digest',
|
|
36
|
+
'args_raw',
|
|
35
37
|
'status',
|
|
36
38
|
'started_at',
|
|
37
39
|
'completed_at',
|
|
@@ -50,12 +52,13 @@ function identityFieldMap(keys) {
|
|
|
50
52
|
// ---------------------------------------------------------------------------
|
|
51
53
|
// Source-specific helpers
|
|
52
54
|
// ---------------------------------------------------------------------------
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
55
|
+
// Launch provenance (`spur-run` vs `ambient`) is a fact only the import host knows:
|
|
56
|
+
// a session is spur-launched iff its (source, session_id) appears in the host's
|
|
57
|
+
// run→session mapping (spur: `history_run_session`, tasks 0557/0558). The mapper
|
|
58
|
+
// cannot see that, so every row imports as `ambient` and the host corrects mapped
|
|
59
|
+
// sessions after import. The old cwd-substring heuristic (path contains `/spur`)
|
|
60
|
+
// was deleted — it mislabelled ambient sessions that merely ran inside a spur
|
|
61
|
+
// directory (task 0559 R5).
|
|
59
62
|
/** Compute args_digest: sha256 of stable-jsonified, key-sorted, redacted args. */
|
|
60
63
|
export function argsDigest(args) {
|
|
61
64
|
const redacted = redactArgs(args);
|
|
@@ -81,6 +84,36 @@ function redactArgs(args) {
|
|
|
81
84
|
}
|
|
82
85
|
return args;
|
|
83
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* Per-source todo-writing tool names whose raw arguments are retained for phase detection.
|
|
89
|
+
*
|
|
90
|
+
* Evidence: 0489 R4 confirmed omp/pi/claude; codex/grok/agy probed task 0553 R3 from real
|
|
91
|
+
* session JSONL. agy has no on-disk session format (VS Code fork); gemini deferred by the
|
|
92
|
+
* 2026-08-06 source-support ruling.
|
|
93
|
+
*/
|
|
94
|
+
const TODO_TOOL_ALLOWLIST = {
|
|
95
|
+
claude: ['TodoWrite'],
|
|
96
|
+
pi: ['todo'],
|
|
97
|
+
omp: ['TodoWrite', 'todo'],
|
|
98
|
+
codex: ['update_plan'],
|
|
99
|
+
grok: ['todo_write'],
|
|
100
|
+
agy: [],
|
|
101
|
+
gemini: [],
|
|
102
|
+
};
|
|
103
|
+
/**
|
|
104
|
+
* Return JSON-stringified raw args when the tool is a todo-writing tool for its source,
|
|
105
|
+
* otherwise undefined. Codex `arguments` arrives as a JSON string already — store as-is.
|
|
106
|
+
*/
|
|
107
|
+
function maybeArgsRaw(source, toolName, args) {
|
|
108
|
+
const allow = TODO_TOOL_ALLOWLIST[source];
|
|
109
|
+
if (allow === undefined || !allow.includes(toolName))
|
|
110
|
+
return undefined;
|
|
111
|
+
if (typeof args === 'string')
|
|
112
|
+
return args;
|
|
113
|
+
if (args === undefined || args === null)
|
|
114
|
+
return undefined;
|
|
115
|
+
return JSON.stringify(args);
|
|
116
|
+
}
|
|
84
117
|
// ---------------------------------------------------------------------------
|
|
85
118
|
// Claude mapper
|
|
86
119
|
// ---------------------------------------------------------------------------
|
|
@@ -107,7 +140,7 @@ export function claudeSplit(raw) {
|
|
|
107
140
|
ts,
|
|
108
141
|
duration_ms: raw.durationMs,
|
|
109
142
|
content_text: null,
|
|
110
|
-
provenance:
|
|
143
|
+
provenance: 'ambient',
|
|
111
144
|
},
|
|
112
145
|
});
|
|
113
146
|
return entries;
|
|
@@ -122,7 +155,7 @@ export function claudeSplit(raw) {
|
|
|
122
155
|
record_type: recordType,
|
|
123
156
|
disposition: 'meta',
|
|
124
157
|
ts,
|
|
125
|
-
provenance:
|
|
158
|
+
provenance: 'ambient',
|
|
126
159
|
},
|
|
127
160
|
});
|
|
128
161
|
return entries;
|
|
@@ -157,7 +190,7 @@ export function claudeSplit(raw) {
|
|
|
157
190
|
cost_usd: costUsd ?? null,
|
|
158
191
|
content_text: contentText ?? null,
|
|
159
192
|
cwd: cwd ?? null,
|
|
160
|
-
provenance:
|
|
193
|
+
provenance: 'ambient',
|
|
161
194
|
},
|
|
162
195
|
});
|
|
163
196
|
// Tool calls from content blocks
|
|
@@ -175,6 +208,7 @@ export function claudeSplit(raw) {
|
|
|
175
208
|
seq,
|
|
176
209
|
tool_name: String(b.name ?? ''),
|
|
177
210
|
args_digest: argsDigest(b.input),
|
|
211
|
+
args_raw: maybeArgsRaw('claude', String(b.name ?? ''), b.input),
|
|
178
212
|
status: 'ok',
|
|
179
213
|
started_at: undefined,
|
|
180
214
|
completed_at: undefined,
|
|
@@ -229,7 +263,7 @@ export function piSplit(raw) {
|
|
|
229
263
|
cost_usd: costUsd ?? null,
|
|
230
264
|
content_text: s(r.content, r.text, msg?.content) ?? null,
|
|
231
265
|
cwd: cwd ?? null,
|
|
232
|
-
provenance:
|
|
266
|
+
provenance: 'ambient',
|
|
233
267
|
},
|
|
234
268
|
});
|
|
235
269
|
// Tool calls from toolCall blocks in assistant content
|
|
@@ -245,6 +279,7 @@ export function piSplit(raw) {
|
|
|
245
279
|
seq,
|
|
246
280
|
tool_name: String(tc.name ?? ''),
|
|
247
281
|
args_digest: argsDigest(tc.input ?? tc.arguments),
|
|
282
|
+
args_raw: maybeArgsRaw('pi', String(tc.name ?? ''), tc.input ?? tc.arguments),
|
|
248
283
|
status: 'ok',
|
|
249
284
|
started_at: undefined,
|
|
250
285
|
completed_at: undefined,
|
|
@@ -334,7 +369,7 @@ export function ompSplit(raw, context) {
|
|
|
334
369
|
cost_usd: costUsd ?? null,
|
|
335
370
|
content_text: extractContentText(contentBlocks) ?? s(raw.content, raw.text, msg?.content) ?? null,
|
|
336
371
|
cwd: cwd ?? null,
|
|
337
|
-
provenance:
|
|
372
|
+
provenance: 'ambient',
|
|
338
373
|
},
|
|
339
374
|
});
|
|
340
375
|
if (Array.isArray(contentBlocks) && role === 'assistant') {
|
|
@@ -349,7 +384,12 @@ export function ompSplit(raw, context) {
|
|
|
349
384
|
session_id: sessionId,
|
|
350
385
|
seq,
|
|
351
386
|
tool_name: String(call.name ?? ''),
|
|
387
|
+
// Task 0564 R1: the tool's own call id is the exact join key a
|
|
388
|
+
// toolResult's `toolCallId` matches — it lets the streaming loop
|
|
389
|
+
// attach the tool's measured duration to the right row.
|
|
390
|
+
call_id: s(call.id),
|
|
352
391
|
args_digest: argsDigest(call.input ?? call.arguments),
|
|
392
|
+
args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
|
|
353
393
|
status: 'ok',
|
|
354
394
|
started_at: undefined,
|
|
355
395
|
completed_at: undefined,
|
|
@@ -378,6 +418,39 @@ function normalizeOmpToolCall(block) {
|
|
|
378
418
|
return block;
|
|
379
419
|
return null;
|
|
380
420
|
}
|
|
421
|
+
/** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
|
|
422
|
+
export function timestampToEpochMs(value) {
|
|
423
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
424
|
+
return value;
|
|
425
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
426
|
+
const numeric = Number(value);
|
|
427
|
+
if (Number.isFinite(numeric))
|
|
428
|
+
return numeric;
|
|
429
|
+
const parsed = Date.parse(value);
|
|
430
|
+
if (!Number.isNaN(parsed))
|
|
431
|
+
return parsed;
|
|
432
|
+
}
|
|
433
|
+
return undefined;
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
|
|
437
|
+
* when the record is not a toolResult message or carries no toolCallId. Live OMP
|
|
438
|
+
* emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
|
|
439
|
+
* content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
|
|
440
|
+
* measured wall time (48% of results in the sampled session) and `toolCallId`
|
|
441
|
+
* joins the originating `toolCall.id` exactly.
|
|
442
|
+
*/
|
|
443
|
+
export function ompToolResultTiming(raw) {
|
|
444
|
+
const msg = o(raw.message);
|
|
445
|
+
if (String(msg.role ?? '').toLowerCase() !== 'toolresult')
|
|
446
|
+
return null;
|
|
447
|
+
const toolCallId = s(msg.toolCallId);
|
|
448
|
+
if (toolCallId === undefined)
|
|
449
|
+
return null;
|
|
450
|
+
const details = o(msg.details);
|
|
451
|
+
const wallTimeMs = typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
|
|
452
|
+
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
453
|
+
}
|
|
381
454
|
// ---------------------------------------------------------------------------
|
|
382
455
|
// Codex mapper
|
|
383
456
|
// ---------------------------------------------------------------------------
|
|
@@ -438,7 +511,7 @@ export function codexSplit(raw) {
|
|
|
438
511
|
cost_usd: costUsd ?? null,
|
|
439
512
|
content_text: s(payload.content, payload.text, raw.content, raw.text) ?? null,
|
|
440
513
|
cwd: cwd ?? null,
|
|
441
|
-
provenance:
|
|
514
|
+
provenance: 'ambient',
|
|
442
515
|
},
|
|
443
516
|
});
|
|
444
517
|
// Tool calls: function_call in response_item
|
|
@@ -458,6 +531,7 @@ export function codexSplit(raw) {
|
|
|
458
531
|
seq,
|
|
459
532
|
tool_name: String(fc.name ?? ''),
|
|
460
533
|
args_digest: argsDigest(fc.arguments),
|
|
534
|
+
args_raw: maybeArgsRaw('codex', String(fc.name ?? ''), fc.arguments),
|
|
461
535
|
status: 'ok',
|
|
462
536
|
started_at: undefined,
|
|
463
537
|
completed_at: undefined,
|
|
@@ -570,6 +644,7 @@ export function agySplit(raw) {
|
|
|
570
644
|
seq: typeof tc.step_index === 'number' ? tc.step_index : seq,
|
|
571
645
|
tool_name: String(tc.name ?? tc.tool_name ?? ''),
|
|
572
646
|
args_digest: argsDigest(tc.input ?? tc.arguments),
|
|
647
|
+
args_raw: maybeArgsRaw('agy', String(tc.name ?? tc.tool_name ?? ''), tc.input ?? tc.arguments),
|
|
573
648
|
status: 'ok',
|
|
574
649
|
started_at: undefined,
|
|
575
650
|
completed_at: undefined,
|
|
@@ -604,7 +679,7 @@ export function geminiSplit(raw, context) {
|
|
|
604
679
|
disposition: 'meta',
|
|
605
680
|
ts,
|
|
606
681
|
content_text: s(state.summary) ?? null,
|
|
607
|
-
provenance:
|
|
682
|
+
provenance: 'ambient',
|
|
608
683
|
},
|
|
609
684
|
},
|
|
610
685
|
];
|
|
@@ -631,7 +706,7 @@ export function geminiSplit(raw, context) {
|
|
|
631
706
|
cost_usd: null,
|
|
632
707
|
content_text: content,
|
|
633
708
|
cwd: null,
|
|
634
|
-
provenance:
|
|
709
|
+
provenance: 'ambient',
|
|
635
710
|
},
|
|
636
711
|
},
|
|
637
712
|
];
|
|
@@ -651,6 +726,7 @@ export function geminiSplit(raw, context) {
|
|
|
651
726
|
seq,
|
|
652
727
|
tool_name: s(tool.name, tool.displayName) ?? 'unknown',
|
|
653
728
|
args_digest: argsDigest(tool.args),
|
|
729
|
+
args_raw: maybeArgsRaw('gemini', s(tool.name, tool.displayName) ?? '', tool.args),
|
|
654
730
|
status,
|
|
655
731
|
started_at: timestamp,
|
|
656
732
|
completed_at: timestamp,
|
|
@@ -895,6 +971,7 @@ export function grokSplit(raw) {
|
|
|
895
971
|
seq,
|
|
896
972
|
tool_name: n.toolName,
|
|
897
973
|
args_digest: argsDigest(n.toolArgs),
|
|
974
|
+
args_raw: maybeArgsRaw('grok', n.toolName, n.toolArgs),
|
|
898
975
|
status: n.toolStatus || 'ok',
|
|
899
976
|
started_at: ts,
|
|
900
977
|
completed_at: undefined,
|
|
@@ -935,6 +1012,7 @@ export function grokSplit(raw) {
|
|
|
935
1012
|
seq,
|
|
936
1013
|
tool_name: n.toolName,
|
|
937
1014
|
args_digest: null,
|
|
1015
|
+
args_raw: undefined,
|
|
938
1016
|
status: n.toolStatus || 'ok',
|
|
939
1017
|
started_at: undefined,
|
|
940
1018
|
completed_at: ts,
|
|
@@ -977,6 +1055,7 @@ export function grokSplit(raw) {
|
|
|
977
1055
|
seq,
|
|
978
1056
|
tool_name: n.toolName,
|
|
979
1057
|
args_digest: argsDigest(n.toolArgs),
|
|
1058
|
+
args_raw: maybeArgsRaw('grok', n.toolName, n.toolArgs),
|
|
980
1059
|
status: n.toolStatus || 'ok',
|
|
981
1060
|
started_at: recordType === 'tool_call' ? ts : undefined,
|
|
982
1061
|
completed_at: recordType === 'tool_call_update' ? ts : undefined,
|
package/dist/schema-sql.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,
|
|
1
|
+
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA6E9B,CAAC"}
|
package/dist/schema-sql.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.35",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.35",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.35",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.35",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
package/src/importer.ts
CHANGED
|
@@ -19,7 +19,9 @@ import {
|
|
|
19
19
|
recordInsertOp,
|
|
20
20
|
resetCheckpoints,
|
|
21
21
|
targetTableFor,
|
|
22
|
+
toolCallDurationUpdateOp,
|
|
22
23
|
} from './jsonl-importer-dao';
|
|
24
|
+
import { type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
|
|
23
25
|
import { redactRecord } from './redaction';
|
|
24
26
|
import { resolveSourceDefinition } from './sources';
|
|
25
27
|
import type {
|
|
@@ -37,6 +39,83 @@ interface SplitRecord {
|
|
|
37
39
|
readonly raw: JsonObject;
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
// Task 0564 R1 — tool-call duration attach (omp)
|
|
44
|
+
//
|
|
45
|
+
// A toolCall row is emitted while handling its assistant message (SplitEntry is
|
|
46
|
+
// insert-only, so the duration cannot ride the mapper's split path); the
|
|
47
|
+
// duration arrives on a LATER toolResult line. The attach therefore lives in the
|
|
48
|
+
// streaming loop, keyed on (source, session_id, call_id) per the frozen contract:
|
|
49
|
+
// - `details.wallTimeMs` is the tool's own measured wall time — used as-is,
|
|
50
|
+
// rounded, NEVER clamped or second-guessed (guards are fallback-only);
|
|
51
|
+
// - otherwise the fallback `toolResult.timestamp − toolCall message timestamp`
|
|
52
|
+
// when both are parseable AND the delta is in [0, 3_600_000]; an implausible
|
|
53
|
+
// delta stays NULL so the unmeasured count stays honest;
|
|
54
|
+
// - started_at / completed_at are written alongside a FALLBACK figure so it is
|
|
55
|
+
// auditable (wallTimeMs rows keep both NULL — the two stay distinguishable);
|
|
56
|
+
// - an unmatched toolCallId attaches nothing and never fails the import.
|
|
57
|
+
//
|
|
58
|
+
// The in-memory map covers the common (fresh-import) case without extra queries;
|
|
59
|
+
// a toolResult whose tool-call line was imported by an earlier checkpointed run
|
|
60
|
+
// (resume) resolves through a DB lookup instead, so a mid-file checkpoint loses
|
|
61
|
+
// no earlier duration.
|
|
62
|
+
// ---------------------------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
interface PendingToolCall {
|
|
65
|
+
readonly recordHash: string;
|
|
66
|
+
/** Assistant-message timestamp (epoch ms) — the fallback's start bound. */
|
|
67
|
+
readonly messageTsMs: number | undefined;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const TOOL_CALL_KEY_SEP = '\u0000';
|
|
71
|
+
|
|
72
|
+
function toolCallKey(source: string, sessionId: string, callId: string): string {
|
|
73
|
+
return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
|
|
77
|
+
async function resolveToolCallRow(
|
|
78
|
+
db: ImportOptions['db'],
|
|
79
|
+
source: string,
|
|
80
|
+
sessionId: string,
|
|
81
|
+
callId: string,
|
|
82
|
+
): Promise<PendingToolCall | undefined> {
|
|
83
|
+
const row = await db.queryFirst<{ recordHash: string; messageTs: string | null }>(
|
|
84
|
+
`SELECT tc.record_hash AS recordHash, m.ts AS messageTs
|
|
85
|
+
FROM history_tool_call tc
|
|
86
|
+
LEFT JOIN history_message m ON m.record_hash = tc.message_hash
|
|
87
|
+
WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`,
|
|
88
|
+
source,
|
|
89
|
+
sessionId,
|
|
90
|
+
callId,
|
|
91
|
+
);
|
|
92
|
+
if (row === undefined || row === null) return undefined;
|
|
93
|
+
return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Compute the attachable duration for a toolResult, or null when nothing should
|
|
98
|
+
* be written (implausible fallback delta → row stays NULL → counts unmeasured).
|
|
99
|
+
*/
|
|
100
|
+
function attachableDuration(
|
|
101
|
+
timing: OmpToolResultTiming,
|
|
102
|
+
messageTsMs: number | undefined,
|
|
103
|
+
): { startedAt: string | null; completedAt: string | null; durationMs: number } | null {
|
|
104
|
+
if (timing.wallTimeMs !== undefined) {
|
|
105
|
+
// The tool's own measurement — no clamping, no sanity check.
|
|
106
|
+
return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
|
|
107
|
+
}
|
|
108
|
+
if (timing.timestampMs === undefined || messageTsMs === undefined) return null;
|
|
109
|
+
const delta = timing.timestampMs - messageTsMs;
|
|
110
|
+
// Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
|
|
111
|
+
if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000) return null;
|
|
112
|
+
return {
|
|
113
|
+
startedAt: new Date(messageTsMs).toISOString(),
|
|
114
|
+
completedAt: new Date(timing.timestampMs).toISOString(),
|
|
115
|
+
durationMs: Math.round(delta),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
40
119
|
/** Run the JSONL import pipeline for one source.
|
|
41
120
|
*
|
|
42
121
|
* @param source - Either a built-in source identifier resolved from the
|
|
@@ -69,6 +148,10 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
69
148
|
// ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
|
|
70
149
|
const desiredHashes = new Set<string>();
|
|
71
150
|
|
|
151
|
+
// Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
|
|
152
|
+
// (source, session_id, call_id) so a later toolResult line can attach its duration.
|
|
153
|
+
const toolCallRows = new Map<string, PendingToolCall>();
|
|
154
|
+
|
|
72
155
|
for (const file of files) {
|
|
73
156
|
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
|
|
74
157
|
let lineNumber = 0;
|
|
@@ -138,6 +221,43 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
138
221
|
// second pass, no checkpoint advance, no partial rows from this record.
|
|
139
222
|
if (lineRejected) continue;
|
|
140
223
|
|
|
224
|
+
// Task 0564 R1 (omp): register this line's tool-call rows for later duration
|
|
225
|
+
// attach. Runs over ALL prepared entries — including duplicates — because a
|
|
226
|
+
// re-imported line's row already exists and its duration UPDATE still targets
|
|
227
|
+
// the same deterministic record_hash. The assistant message timestamp is the
|
|
228
|
+
// fallback's start bound; every call in one message shares it.
|
|
229
|
+
if (definition.source === 'omp') {
|
|
230
|
+
const assistantTs = new Map<string, number | undefined>();
|
|
231
|
+
for (const entry of prepared) {
|
|
232
|
+
if (
|
|
233
|
+
entry.split.targetTable === 'history_message' &&
|
|
234
|
+
entry.normalized.role === 'assistant' &&
|
|
235
|
+
typeof entry.normalized.session_id === 'string'
|
|
236
|
+
) {
|
|
237
|
+
assistantTs.set(
|
|
238
|
+
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
239
|
+
timestampToEpochMs(entry.normalized.ts),
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
for (const entry of prepared) {
|
|
244
|
+
const callId = entry.normalized.call_id;
|
|
245
|
+
if (
|
|
246
|
+
entry.split.targetTable === 'history_tool_call' &&
|
|
247
|
+
typeof callId === 'string' &&
|
|
248
|
+
callId.length > 0 &&
|
|
249
|
+
typeof entry.normalized.session_id === 'string'
|
|
250
|
+
) {
|
|
251
|
+
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
252
|
+
recordHash: entry.recordHash,
|
|
253
|
+
messageTsMs: assistantTs.get(
|
|
254
|
+
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
255
|
+
),
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
141
261
|
// Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
|
|
142
262
|
// instead of a SELECT per record; duplicates are dropped before the write pass.
|
|
143
263
|
const existing = await ledgerExistingHashes(
|
|
@@ -199,6 +319,47 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
199
319
|
}
|
|
200
320
|
|
|
201
321
|
if (!options.dryRun && lineSucceeded) {
|
|
322
|
+
// Task 0564 R1 (omp): attach tool durations from this line's toolResult
|
|
323
|
+
// message. Unmatched toolCallIds attach nothing and never fail the import;
|
|
324
|
+
// an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
|
|
325
|
+
// this line's batch — idempotent, so re-imports and post-resume tail
|
|
326
|
+
// reprocessing write the same values.
|
|
327
|
+
if (definition.source === 'omp') {
|
|
328
|
+
const timing = ompToolResultTiming(raw);
|
|
329
|
+
if (timing !== null) {
|
|
330
|
+
const resultEntry = prepared.find(
|
|
331
|
+
(entry) =>
|
|
332
|
+
entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult',
|
|
333
|
+
);
|
|
334
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
335
|
+
if (typeof sessionId === 'string') {
|
|
336
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
337
|
+
if (pending === undefined) {
|
|
338
|
+
// The tool-call line sits behind this run's checkpoint — the
|
|
339
|
+
// row exists from an earlier import; resolve it from the DB.
|
|
340
|
+
pending = await resolveToolCallRow(
|
|
341
|
+
options.db,
|
|
342
|
+
resolvedSource,
|
|
343
|
+
sessionId,
|
|
344
|
+
timing.toolCallId,
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
if (pending !== undefined) {
|
|
348
|
+
const duration = attachableDuration(timing, pending.messageTsMs);
|
|
349
|
+
if (duration !== null) {
|
|
350
|
+
ops.push(
|
|
351
|
+
toolCallDurationUpdateOp(
|
|
352
|
+
pending.recordHash,
|
|
353
|
+
duration.startedAt,
|
|
354
|
+
duration.completedAt,
|
|
355
|
+
duration.durationMs,
|
|
356
|
+
),
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
202
363
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
203
364
|
// re-imported after a mid-batch crash.
|
|
204
365
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
|
@@ -45,7 +45,9 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
|
45
45
|
'session_id',
|
|
46
46
|
'seq',
|
|
47
47
|
'tool_name',
|
|
48
|
+
'call_id',
|
|
48
49
|
'args_digest',
|
|
50
|
+
'args_raw',
|
|
49
51
|
'status',
|
|
50
52
|
'started_at',
|
|
51
53
|
'completed_at',
|
|
@@ -254,6 +256,29 @@ export function checkpointUpsertOp(
|
|
|
254
256
|
};
|
|
255
257
|
}
|
|
256
258
|
|
|
259
|
+
/**
|
|
260
|
+
* Targeted UPDATE op attaching a tool duration to a previously-inserted
|
|
261
|
+
* `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
|
|
262
|
+
* unindexed predicate. Idempotent: re-imports write the same values.
|
|
263
|
+
*
|
|
264
|
+
* Bounds are written alongside a FALLBACK duration so the figure is auditable;
|
|
265
|
+
* a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
|
|
266
|
+
* two measurement paths stay distinguishable.
|
|
267
|
+
*/
|
|
268
|
+
function toolCallDurationUpdateOp(
|
|
269
|
+
recordHash: string,
|
|
270
|
+
startedAt: string | null,
|
|
271
|
+
completedAt: string | null,
|
|
272
|
+
durationMs: number | null,
|
|
273
|
+
): DbBatchOp {
|
|
274
|
+
return {
|
|
275
|
+
sql: `UPDATE history_tool_call
|
|
276
|
+
SET started_at = ?, completed_at = ?, duration_ms = ?
|
|
277
|
+
WHERE record_hash = ?`,
|
|
278
|
+
params: [startedAt, completedAt, durationMs, recordHash],
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
257
282
|
/**
|
|
258
283
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
259
284
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -687,5 +712,6 @@ export {
|
|
|
687
712
|
resetCheckpoints,
|
|
688
713
|
targetTableFor,
|
|
689
714
|
timestamp,
|
|
715
|
+
toolCallDurationUpdateOp,
|
|
690
716
|
writeCheckpoint,
|
|
691
717
|
};
|
package/src/mappers.ts
CHANGED
|
@@ -35,7 +35,9 @@ const TOOL_CALL_MAPPER_KEYS: readonly string[] = [
|
|
|
35
35
|
'session_id',
|
|
36
36
|
'seq',
|
|
37
37
|
'tool_name',
|
|
38
|
+
'call_id',
|
|
38
39
|
'args_digest',
|
|
40
|
+
'args_raw',
|
|
39
41
|
'status',
|
|
40
42
|
'started_at',
|
|
41
43
|
'completed_at',
|
|
@@ -57,11 +59,13 @@ function identityFieldMap(keys: readonly string[]): Record<string, string> {
|
|
|
57
59
|
// Source-specific helpers
|
|
58
60
|
// ---------------------------------------------------------------------------
|
|
59
61
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
// Launch provenance (`spur-run` vs `ambient`) is a fact only the import host knows:
|
|
63
|
+
// a session is spur-launched iff its (source, session_id) appears in the host's
|
|
64
|
+
// run→session mapping (spur: `history_run_session`, tasks 0557/0558). The mapper
|
|
65
|
+
// cannot see that, so every row imports as `ambient` and the host corrects mapped
|
|
66
|
+
// sessions after import. The old cwd-substring heuristic (path contains `/spur`)
|
|
67
|
+
// was deleted — it mislabelled ambient sessions that merely ran inside a spur
|
|
68
|
+
// directory (task 0559 R5).
|
|
65
69
|
|
|
66
70
|
/** Compute args_digest: sha256 of stable-jsonified, key-sorted, redacted args. */
|
|
67
71
|
export function argsDigest(args: unknown): string {
|
|
@@ -87,6 +91,35 @@ function redactArgs(args: unknown): unknown {
|
|
|
87
91
|
return args;
|
|
88
92
|
}
|
|
89
93
|
|
|
94
|
+
/**
|
|
95
|
+
* Per-source todo-writing tool names whose raw arguments are retained for phase detection.
|
|
96
|
+
*
|
|
97
|
+
* Evidence: 0489 R4 confirmed omp/pi/claude; codex/grok/agy probed task 0553 R3 from real
|
|
98
|
+
* session JSONL. agy has no on-disk session format (VS Code fork); gemini deferred by the
|
|
99
|
+
* 2026-08-06 source-support ruling.
|
|
100
|
+
*/
|
|
101
|
+
const TODO_TOOL_ALLOWLIST: Readonly<Record<string, readonly string[]>> = {
|
|
102
|
+
claude: ['TodoWrite'],
|
|
103
|
+
pi: ['todo'],
|
|
104
|
+
omp: ['TodoWrite', 'todo'],
|
|
105
|
+
codex: ['update_plan'],
|
|
106
|
+
grok: ['todo_write'],
|
|
107
|
+
agy: [],
|
|
108
|
+
gemini: [],
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Return JSON-stringified raw args when the tool is a todo-writing tool for its source,
|
|
113
|
+
* otherwise undefined. Codex `arguments` arrives as a JSON string already — store as-is.
|
|
114
|
+
*/
|
|
115
|
+
function maybeArgsRaw(source: string, toolName: string, args: unknown): string | undefined {
|
|
116
|
+
const allow = TODO_TOOL_ALLOWLIST[source];
|
|
117
|
+
if (allow === undefined || !allow.includes(toolName)) return undefined;
|
|
118
|
+
if (typeof args === 'string') return args;
|
|
119
|
+
if (args === undefined || args === null) return undefined;
|
|
120
|
+
return JSON.stringify(args);
|
|
121
|
+
}
|
|
122
|
+
|
|
90
123
|
// ---------------------------------------------------------------------------
|
|
91
124
|
// Claude mapper
|
|
92
125
|
// ---------------------------------------------------------------------------
|
|
@@ -115,7 +148,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
115
148
|
ts,
|
|
116
149
|
duration_ms: raw.durationMs,
|
|
117
150
|
content_text: null,
|
|
118
|
-
provenance:
|
|
151
|
+
provenance: 'ambient',
|
|
119
152
|
},
|
|
120
153
|
});
|
|
121
154
|
return entries;
|
|
@@ -130,7 +163,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
130
163
|
record_type: recordType,
|
|
131
164
|
disposition: 'meta',
|
|
132
165
|
ts,
|
|
133
|
-
provenance:
|
|
166
|
+
provenance: 'ambient',
|
|
134
167
|
},
|
|
135
168
|
});
|
|
136
169
|
return entries;
|
|
@@ -169,7 +202,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
169
202
|
cost_usd: costUsd ?? null,
|
|
170
203
|
content_text: contentText ?? null,
|
|
171
204
|
cwd: cwd ?? null,
|
|
172
|
-
provenance:
|
|
205
|
+
provenance: 'ambient',
|
|
173
206
|
},
|
|
174
207
|
});
|
|
175
208
|
|
|
@@ -187,6 +220,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
187
220
|
seq,
|
|
188
221
|
tool_name: String(b.name ?? ''),
|
|
189
222
|
args_digest: argsDigest(b.input),
|
|
223
|
+
args_raw: maybeArgsRaw('claude', String(b.name ?? ''), b.input),
|
|
190
224
|
status: 'ok',
|
|
191
225
|
started_at: undefined,
|
|
192
226
|
completed_at: undefined,
|
|
@@ -247,7 +281,7 @@ export function piSplit(raw: unknown): readonly SplitEntry[] {
|
|
|
247
281
|
cost_usd: costUsd ?? null,
|
|
248
282
|
content_text: s(r.content, r.text, msg?.content) ?? null,
|
|
249
283
|
cwd: cwd ?? null,
|
|
250
|
-
provenance:
|
|
284
|
+
provenance: 'ambient',
|
|
251
285
|
},
|
|
252
286
|
});
|
|
253
287
|
|
|
@@ -264,6 +298,7 @@ export function piSplit(raw: unknown): readonly SplitEntry[] {
|
|
|
264
298
|
seq,
|
|
265
299
|
tool_name: String(tc.name ?? ''),
|
|
266
300
|
args_digest: argsDigest(tc.input ?? tc.arguments),
|
|
301
|
+
args_raw: maybeArgsRaw('pi', String(tc.name ?? ''), tc.input ?? tc.arguments),
|
|
267
302
|
status: 'ok',
|
|
268
303
|
started_at: undefined,
|
|
269
304
|
completed_at: undefined,
|
|
@@ -362,7 +397,7 @@ export function ompSplit(raw: Record<string, unknown>, context?: TransformContex
|
|
|
362
397
|
cost_usd: costUsd ?? null,
|
|
363
398
|
content_text: extractContentText(contentBlocks) ?? s(raw.content, raw.text, msg?.content) ?? null,
|
|
364
399
|
cwd: cwd ?? null,
|
|
365
|
-
provenance:
|
|
400
|
+
provenance: 'ambient',
|
|
366
401
|
},
|
|
367
402
|
});
|
|
368
403
|
|
|
@@ -377,7 +412,12 @@ export function ompSplit(raw: Record<string, unknown>, context?: TransformContex
|
|
|
377
412
|
session_id: sessionId,
|
|
378
413
|
seq,
|
|
379
414
|
tool_name: String(call.name ?? ''),
|
|
415
|
+
// Task 0564 R1: the tool's own call id is the exact join key a
|
|
416
|
+
// toolResult's `toolCallId` matches — it lets the streaming loop
|
|
417
|
+
// attach the tool's measured duration to the right row.
|
|
418
|
+
call_id: s(call.id),
|
|
380
419
|
args_digest: argsDigest(call.input ?? call.arguments),
|
|
420
|
+
args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
|
|
381
421
|
status: 'ok',
|
|
382
422
|
started_at: undefined,
|
|
383
423
|
completed_at: undefined,
|
|
@@ -406,6 +446,51 @@ function normalizeOmpToolCall(block: Record<string, unknown>): Record<string, un
|
|
|
406
446
|
return null;
|
|
407
447
|
}
|
|
408
448
|
|
|
449
|
+
// ---------------------------------------------------------------------------
|
|
450
|
+
// OMP toolResult timing signals (task 0564 R1)
|
|
451
|
+
// ---------------------------------------------------------------------------
|
|
452
|
+
|
|
453
|
+
/** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
|
|
454
|
+
export interface OmpToolResultTiming {
|
|
455
|
+
/** The toolCall id this result answers — joins `toolCall.id` exactly. */
|
|
456
|
+
toolCallId: string;
|
|
457
|
+
/** The tool's own measured wall time in ms, when present and finite. */
|
|
458
|
+
wallTimeMs: number | undefined;
|
|
459
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
460
|
+
timestampMs: number | undefined;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
|
|
464
|
+
export function timestampToEpochMs(value: unknown): number | undefined {
|
|
465
|
+
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
|
466
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
467
|
+
const numeric = Number(value);
|
|
468
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
469
|
+
const parsed = Date.parse(value);
|
|
470
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
471
|
+
}
|
|
472
|
+
return undefined;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
|
|
477
|
+
* when the record is not a toolResult message or carries no toolCallId. Live OMP
|
|
478
|
+
* emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
|
|
479
|
+
* content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
|
|
480
|
+
* measured wall time (48% of results in the sampled session) and `toolCallId`
|
|
481
|
+
* joins the originating `toolCall.id` exactly.
|
|
482
|
+
*/
|
|
483
|
+
export function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null {
|
|
484
|
+
const msg = o(raw.message);
|
|
485
|
+
if (String(msg.role ?? '').toLowerCase() !== 'toolresult') return null;
|
|
486
|
+
const toolCallId = s(msg.toolCallId);
|
|
487
|
+
if (toolCallId === undefined) return null;
|
|
488
|
+
const details = o(msg.details);
|
|
489
|
+
const wallTimeMs =
|
|
490
|
+
typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
|
|
491
|
+
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
492
|
+
}
|
|
493
|
+
|
|
409
494
|
// ---------------------------------------------------------------------------
|
|
410
495
|
// Codex mapper
|
|
411
496
|
// ---------------------------------------------------------------------------
|
|
@@ -475,7 +560,7 @@ export function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
475
560
|
cost_usd: costUsd ?? null,
|
|
476
561
|
content_text: s(payload.content, payload.text, raw.content, raw.text) ?? null,
|
|
477
562
|
cwd: cwd ?? null,
|
|
478
|
-
provenance:
|
|
563
|
+
provenance: 'ambient',
|
|
479
564
|
},
|
|
480
565
|
});
|
|
481
566
|
|
|
@@ -496,6 +581,7 @@ export function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
496
581
|
seq,
|
|
497
582
|
tool_name: String(fc.name ?? ''),
|
|
498
583
|
args_digest: argsDigest(fc.arguments),
|
|
584
|
+
args_raw: maybeArgsRaw('codex', String(fc.name ?? ''), fc.arguments),
|
|
499
585
|
status: 'ok',
|
|
500
586
|
started_at: undefined,
|
|
501
587
|
completed_at: undefined,
|
|
@@ -615,6 +701,7 @@ export function agySplit(raw: Record<string, unknown>): readonly SplitEntry[] {
|
|
|
615
701
|
seq: typeof tc.step_index === 'number' ? tc.step_index : seq,
|
|
616
702
|
tool_name: String(tc.name ?? tc.tool_name ?? ''),
|
|
617
703
|
args_digest: argsDigest(tc.input ?? tc.arguments),
|
|
704
|
+
args_raw: maybeArgsRaw('agy', String(tc.name ?? tc.tool_name ?? ''), tc.input ?? tc.arguments),
|
|
618
705
|
status: 'ok',
|
|
619
706
|
started_at: undefined,
|
|
620
707
|
completed_at: undefined,
|
|
@@ -653,7 +740,7 @@ export function geminiSplit(raw: Record<string, unknown>, context?: TransformCon
|
|
|
653
740
|
disposition: 'meta',
|
|
654
741
|
ts,
|
|
655
742
|
content_text: s(state.summary) ?? null,
|
|
656
|
-
provenance:
|
|
743
|
+
provenance: 'ambient',
|
|
657
744
|
},
|
|
658
745
|
},
|
|
659
746
|
];
|
|
@@ -681,7 +768,7 @@ export function geminiSplit(raw: Record<string, unknown>, context?: TransformCon
|
|
|
681
768
|
cost_usd: null,
|
|
682
769
|
content_text: content,
|
|
683
770
|
cwd: null,
|
|
684
|
-
provenance:
|
|
771
|
+
provenance: 'ambient',
|
|
685
772
|
},
|
|
686
773
|
},
|
|
687
774
|
];
|
|
@@ -701,6 +788,7 @@ export function geminiSplit(raw: Record<string, unknown>, context?: TransformCon
|
|
|
701
788
|
seq,
|
|
702
789
|
tool_name: s(tool.name, tool.displayName) ?? 'unknown',
|
|
703
790
|
args_digest: argsDigest(tool.args),
|
|
791
|
+
args_raw: maybeArgsRaw('gemini', s(tool.name, tool.displayName) ?? '', tool.args),
|
|
704
792
|
status,
|
|
705
793
|
started_at: timestamp,
|
|
706
794
|
completed_at: timestamp,
|
|
@@ -988,6 +1076,7 @@ export function grokSplit(raw: Record<string, unknown>): readonly SplitEntry[] {
|
|
|
988
1076
|
seq,
|
|
989
1077
|
tool_name: n.toolName,
|
|
990
1078
|
args_digest: argsDigest(n.toolArgs),
|
|
1079
|
+
args_raw: maybeArgsRaw('grok', n.toolName, n.toolArgs),
|
|
991
1080
|
status: n.toolStatus || 'ok',
|
|
992
1081
|
started_at: ts,
|
|
993
1082
|
completed_at: undefined,
|
|
@@ -1029,6 +1118,7 @@ export function grokSplit(raw: Record<string, unknown>): readonly SplitEntry[] {
|
|
|
1029
1118
|
seq,
|
|
1030
1119
|
tool_name: n.toolName,
|
|
1031
1120
|
args_digest: null,
|
|
1121
|
+
args_raw: undefined,
|
|
1032
1122
|
status: n.toolStatus || 'ok',
|
|
1033
1123
|
started_at: undefined,
|
|
1034
1124
|
completed_at: ts,
|
|
@@ -1072,6 +1162,7 @@ export function grokSplit(raw: Record<string, unknown>): readonly SplitEntry[] {
|
|
|
1072
1162
|
seq,
|
|
1073
1163
|
tool_name: n.toolName,
|
|
1074
1164
|
args_digest: argsDigest(n.toolArgs),
|
|
1165
|
+
args_raw: maybeArgsRaw('grok', n.toolName, n.toolArgs),
|
|
1075
1166
|
status: n.toolStatus || 'ok',
|
|
1076
1167
|
started_at: recordType === 'tool_call' ? ts : undefined,
|
|
1077
1168
|
completed_at: recordType === 'tool_call_update' ? ts : undefined,
|
package/src/schema-sql.ts
CHANGED