@gobing-ai/ts-llm-jsonl-importer 0.4.33 → 0.4.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +93 -1
- package/dist/jsonl-importer-dao.d.ts +11 -1
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +19 -1
- package/dist/mappers.d.ts +20 -0
- package/dist/mappers.d.ts.map +1 -1
- package/dist/mappers.js +38 -0
- package/dist/schema-sql.d.ts.map +1 -1
- package/dist/schema-sql.js +1 -0
- package/package.json +4 -4
- package/src/importer.ts +161 -0
- package/src/jsonl-importer-dao.ts +25 -0
- package/src/mappers.ts +50 -0
- package/src/schema-sql.ts +1 -0
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAkRrH"}
|
package/dist/importer.js
CHANGED
|
@@ -1,8 +1,44 @@
|
|
|
1
1
|
import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
|
|
2
2
|
import { sha256 } from './hash.js';
|
|
3
|
-
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, } from './jsonl-importer-dao.js';
|
|
3
|
+
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, } from './jsonl-importer-dao.js';
|
|
4
|
+
import { ompToolResultTiming, timestampToEpochMs } from './mappers.js';
|
|
4
5
|
import { redactRecord } from './redaction.js';
|
|
5
6
|
import { resolveSourceDefinition } from './sources.js';
|
|
7
|
+
const TOOL_CALL_KEY_SEP = '\u0000';
|
|
8
|
+
function toolCallKey(source, sessionId, callId) {
|
|
9
|
+
return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
|
|
10
|
+
}
|
|
11
|
+
/** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
|
|
12
|
+
async function resolveToolCallRow(db, source, sessionId, callId) {
|
|
13
|
+
const row = await db.queryFirst(`SELECT tc.record_hash AS recordHash, m.ts AS messageTs
|
|
14
|
+
FROM history_tool_call tc
|
|
15
|
+
LEFT JOIN history_message m ON m.record_hash = tc.message_hash
|
|
16
|
+
WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`, source, sessionId, callId);
|
|
17
|
+
if (row === undefined || row === null)
|
|
18
|
+
return undefined;
|
|
19
|
+
return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Compute the attachable duration for a toolResult, or null when nothing should
|
|
23
|
+
* be written (implausible fallback delta → row stays NULL → counts unmeasured).
|
|
24
|
+
*/
|
|
25
|
+
function attachableDuration(timing, messageTsMs) {
|
|
26
|
+
if (timing.wallTimeMs !== undefined) {
|
|
27
|
+
// The tool's own measurement — no clamping, no sanity check.
|
|
28
|
+
return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
|
|
29
|
+
}
|
|
30
|
+
if (timing.timestampMs === undefined || messageTsMs === undefined)
|
|
31
|
+
return null;
|
|
32
|
+
const delta = timing.timestampMs - messageTsMs;
|
|
33
|
+
// Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
|
|
34
|
+
if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000)
|
|
35
|
+
return null;
|
|
36
|
+
return {
|
|
37
|
+
startedAt: new Date(messageTsMs).toISOString(),
|
|
38
|
+
completedAt: new Date(timing.timestampMs).toISOString(),
|
|
39
|
+
durationMs: Math.round(delta),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
6
42
|
/** Run the JSONL import pipeline for one source.
|
|
7
43
|
*
|
|
8
44
|
* @param source - Either a built-in source identifier resolved from the
|
|
@@ -32,6 +68,9 @@ export async function runJsonlImport(source, options) {
|
|
|
32
68
|
// Full-mode desired set (R1, task 0504): every record hash the current source produces.
|
|
33
69
|
// ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
|
|
34
70
|
const desiredHashes = new Set();
|
|
71
|
+
// Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
|
|
72
|
+
// (source, session_id, call_id) so a later toolResult line can attach its duration.
|
|
73
|
+
const toolCallRows = new Map();
|
|
35
74
|
for (const file of files) {
|
|
36
75
|
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
|
|
37
76
|
let lineNumber = 0;
|
|
@@ -92,6 +131,33 @@ export async function runJsonlImport(source, options) {
|
|
|
92
131
|
// second pass, no checkpoint advance, no partial rows from this record.
|
|
93
132
|
if (lineRejected)
|
|
94
133
|
continue;
|
|
134
|
+
// Task 0564 R1 (omp): register this line's tool-call rows for later duration
|
|
135
|
+
// attach. Runs over ALL prepared entries — including duplicates — because a
|
|
136
|
+
// re-imported line's row already exists and its duration UPDATE still targets
|
|
137
|
+
// the same deterministic record_hash. The assistant message timestamp is the
|
|
138
|
+
// fallback's start bound; every call in one message shares it.
|
|
139
|
+
if (definition.source === 'omp') {
|
|
140
|
+
const assistantTs = new Map();
|
|
141
|
+
for (const entry of prepared) {
|
|
142
|
+
if (entry.split.targetTable === 'history_message' &&
|
|
143
|
+
entry.normalized.role === 'assistant' &&
|
|
144
|
+
typeof entry.normalized.session_id === 'string') {
|
|
145
|
+
assistantTs.set(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`, timestampToEpochMs(entry.normalized.ts));
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
for (const entry of prepared) {
|
|
149
|
+
const callId = entry.normalized.call_id;
|
|
150
|
+
if (entry.split.targetTable === 'history_tool_call' &&
|
|
151
|
+
typeof callId === 'string' &&
|
|
152
|
+
callId.length > 0 &&
|
|
153
|
+
typeof entry.normalized.session_id === 'string') {
|
|
154
|
+
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
155
|
+
recordHash: entry.recordHash,
|
|
156
|
+
messageTsMs: assistantTs.get(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`),
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
95
161
|
// Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
|
|
96
162
|
// instead of a SELECT per record; duplicates are dropped before the write pass.
|
|
97
163
|
const existing = await ledgerExistingHashes(options.db, prepared.map((e) => e.recordHash));
|
|
@@ -128,6 +194,32 @@ export async function runJsonlImport(source, options) {
|
|
|
128
194
|
importedRecords += 1;
|
|
129
195
|
}
|
|
130
196
|
if (!options.dryRun && lineSucceeded) {
|
|
197
|
+
// Task 0564 R1 (omp): attach tool durations from this line's toolResult
|
|
198
|
+
// message. Unmatched toolCallIds attach nothing and never fail the import;
|
|
199
|
+
// an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
|
|
200
|
+
// this line's batch — idempotent, so re-imports and post-resume tail
|
|
201
|
+
// reprocessing write the same values.
|
|
202
|
+
if (definition.source === 'omp') {
|
|
203
|
+
const timing = ompToolResultTiming(raw);
|
|
204
|
+
if (timing !== null) {
|
|
205
|
+
const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult');
|
|
206
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
207
|
+
if (typeof sessionId === 'string') {
|
|
208
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
209
|
+
if (pending === undefined) {
|
|
210
|
+
// The tool-call line sits behind this run's checkpoint — the
|
|
211
|
+
// row exists from an earlier import; resolve it from the DB.
|
|
212
|
+
pending = await resolveToolCallRow(options.db, resolvedSource, sessionId, timing.toolCallId);
|
|
213
|
+
}
|
|
214
|
+
if (pending !== undefined) {
|
|
215
|
+
const duration = attachableDuration(timing, pending.messageTsMs);
|
|
216
|
+
if (duration !== null) {
|
|
217
|
+
ops.push(toolCallDurationUpdateOp(pending.recordHash, duration.startedAt, duration.completedAt, duration.durationMs));
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
131
223
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
132
224
|
// re-imported after a mid-batch crash.
|
|
133
225
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
|
@@ -44,6 +44,16 @@ export declare function recordInsertOp(targetTable: string, recordHash: string,
|
|
|
44
44
|
export declare function ledgerInsertOp(recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): DbBatchOp;
|
|
45
45
|
/** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
|
|
46
46
|
export declare function checkpointUpsertOp(source: string, sourceFile: string, line: number, now: ImportOptions['now']): DbBatchOp;
|
|
47
|
+
/**
|
|
48
|
+
* Targeted UPDATE op attaching a tool duration to a previously-inserted
|
|
49
|
+
* `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
|
|
50
|
+
* unindexed predicate. Idempotent: re-imports write the same values.
|
|
51
|
+
*
|
|
52
|
+
* Bounds are written alongside a FALLBACK duration so the figure is auditable;
|
|
53
|
+
* a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
|
|
54
|
+
* two measurement paths stay distinguishable.
|
|
55
|
+
*/
|
|
56
|
+
declare function toolCallDurationUpdateOp(recordHash: string, startedAt: string | null, completedAt: string | null, durationMs: number | null): DbBatchOp;
|
|
47
57
|
/**
|
|
48
58
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
49
59
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -126,5 +136,5 @@ export declare function deleteCheckpointOperation(source: string, sourceFile: st
|
|
|
126
136
|
/** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
|
|
127
137
|
export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
|
|
128
138
|
export type { CheckpointRow };
|
|
129
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
|
139
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
|
130
140
|
//# sourceMappingURL=jsonl-importer-dao.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAuDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,eAAe,GAClB,CAAC"}
|
|
@@ -38,6 +38,7 @@ const TYPED_TABLE_COLUMNS = {
|
|
|
38
38
|
'session_id',
|
|
39
39
|
'seq',
|
|
40
40
|
'tool_name',
|
|
41
|
+
'call_id',
|
|
41
42
|
'args_digest',
|
|
42
43
|
'args_raw',
|
|
43
44
|
'status',
|
|
@@ -194,6 +195,23 @@ export function checkpointUpsertOp(source, sourceFile, line, now) {
|
|
|
194
195
|
params: [source, sourceFile, line, timestamp(now)],
|
|
195
196
|
};
|
|
196
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* Targeted UPDATE op attaching a tool duration to a previously-inserted
|
|
200
|
+
* `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
|
|
201
|
+
* unindexed predicate. Idempotent: re-imports write the same values.
|
|
202
|
+
*
|
|
203
|
+
* Bounds are written alongside a FALLBACK duration so the figure is auditable;
|
|
204
|
+
* a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
|
|
205
|
+
* two measurement paths stay distinguishable.
|
|
206
|
+
*/
|
|
207
|
+
function toolCallDurationUpdateOp(recordHash, startedAt, completedAt, durationMs) {
|
|
208
|
+
return {
|
|
209
|
+
sql: `UPDATE history_tool_call
|
|
210
|
+
SET started_at = ?, completed_at = ?, duration_ms = ?
|
|
211
|
+
WHERE record_hash = ?`,
|
|
212
|
+
params: [startedAt, completedAt, durationMs, recordHash],
|
|
213
|
+
};
|
|
214
|
+
}
|
|
197
215
|
/**
|
|
198
216
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
199
217
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -468,4 +486,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
|
|
|
468
486
|
}
|
|
469
487
|
return operations;
|
|
470
488
|
}
|
|
471
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
|
489
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
package/dist/mappers.d.ts
CHANGED
|
@@ -8,6 +8,26 @@ export declare function claudeSplit(raw: Record<string, unknown>): readonly Spli
|
|
|
8
8
|
export declare function piSplit(raw: unknown): readonly SplitEntry[];
|
|
9
9
|
/** Map an OMP JSONL record into one or more ETL split entries. */
|
|
10
10
|
export declare function ompSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
11
|
+
/** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
|
|
12
|
+
export interface OmpToolResultTiming {
|
|
13
|
+
/** The toolCall id this result answers — joins `toolCall.id` exactly. */
|
|
14
|
+
toolCallId: string;
|
|
15
|
+
/** The tool's own measured wall time in ms, when present and finite. */
|
|
16
|
+
wallTimeMs: number | undefined;
|
|
17
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
18
|
+
timestampMs: number | undefined;
|
|
19
|
+
}
|
|
20
|
+
/** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
|
|
21
|
+
export declare function timestampToEpochMs(value: unknown): number | undefined;
|
|
22
|
+
/**
|
|
23
|
+
* Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
|
|
24
|
+
* when the record is not a toolResult message or carries no toolCallId. Live OMP
|
|
25
|
+
* emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
|
|
26
|
+
* content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
|
|
27
|
+
* measured wall time (48% of results in the sampled session) and `toolCallId`
|
|
28
|
+
* joins the originating `toolCall.id` exactly.
|
|
29
|
+
*/
|
|
30
|
+
export declare function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null;
|
|
11
31
|
/** Map a Codex JSONL record into one or more ETL split entries. */
|
|
12
32
|
export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
13
33
|
/** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
|
package/dist/mappers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAqDD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA6G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAuE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
|
package/dist/mappers.js
CHANGED
|
@@ -31,6 +31,7 @@ const TOOL_CALL_MAPPER_KEYS = [
|
|
|
31
31
|
'session_id',
|
|
32
32
|
'seq',
|
|
33
33
|
'tool_name',
|
|
34
|
+
'call_id',
|
|
34
35
|
'args_digest',
|
|
35
36
|
'args_raw',
|
|
36
37
|
'status',
|
|
@@ -383,6 +384,10 @@ export function ompSplit(raw, context) {
|
|
|
383
384
|
session_id: sessionId,
|
|
384
385
|
seq,
|
|
385
386
|
tool_name: String(call.name ?? ''),
|
|
387
|
+
// Task 0564 R1: the tool's own call id is the exact join key a
|
|
388
|
+
// toolResult's `toolCallId` matches — it lets the streaming loop
|
|
389
|
+
// attach the tool's measured duration to the right row.
|
|
390
|
+
call_id: s(call.id),
|
|
386
391
|
args_digest: argsDigest(call.input ?? call.arguments),
|
|
387
392
|
args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
|
|
388
393
|
status: 'ok',
|
|
@@ -413,6 +418,39 @@ function normalizeOmpToolCall(block) {
|
|
|
413
418
|
return block;
|
|
414
419
|
return null;
|
|
415
420
|
}
|
|
421
|
+
/** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
|
|
422
|
+
export function timestampToEpochMs(value) {
|
|
423
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
424
|
+
return value;
|
|
425
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
426
|
+
const numeric = Number(value);
|
|
427
|
+
if (Number.isFinite(numeric))
|
|
428
|
+
return numeric;
|
|
429
|
+
const parsed = Date.parse(value);
|
|
430
|
+
if (!Number.isNaN(parsed))
|
|
431
|
+
return parsed;
|
|
432
|
+
}
|
|
433
|
+
return undefined;
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
|
|
437
|
+
* when the record is not a toolResult message or carries no toolCallId. Live OMP
|
|
438
|
+
* emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
|
|
439
|
+
* content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
|
|
440
|
+
* measured wall time (48% of results in the sampled session) and `toolCallId`
|
|
441
|
+
* joins the originating `toolCall.id` exactly.
|
|
442
|
+
*/
|
|
443
|
+
export function ompToolResultTiming(raw) {
|
|
444
|
+
const msg = o(raw.message);
|
|
445
|
+
if (String(msg.role ?? '').toLowerCase() !== 'toolresult')
|
|
446
|
+
return null;
|
|
447
|
+
const toolCallId = s(msg.toolCallId);
|
|
448
|
+
if (toolCallId === undefined)
|
|
449
|
+
return null;
|
|
450
|
+
const details = o(msg.details);
|
|
451
|
+
const wallTimeMs = typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
|
|
452
|
+
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
453
|
+
}
|
|
416
454
|
// ---------------------------------------------------------------------------
|
|
417
455
|
// Codex mapper
|
|
418
456
|
// ---------------------------------------------------------------------------
|
package/dist/schema-sql.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,
|
|
1
|
+
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA6E9B,CAAC"}
|
package/dist/schema-sql.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.35",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.35",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.35",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.35",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
package/src/importer.ts
CHANGED
|
@@ -19,7 +19,9 @@ import {
|
|
|
19
19
|
recordInsertOp,
|
|
20
20
|
resetCheckpoints,
|
|
21
21
|
targetTableFor,
|
|
22
|
+
toolCallDurationUpdateOp,
|
|
22
23
|
} from './jsonl-importer-dao';
|
|
24
|
+
import { type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
|
|
23
25
|
import { redactRecord } from './redaction';
|
|
24
26
|
import { resolveSourceDefinition } from './sources';
|
|
25
27
|
import type {
|
|
@@ -37,6 +39,83 @@ interface SplitRecord {
|
|
|
37
39
|
readonly raw: JsonObject;
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
// Task 0564 R1 — tool-call duration attach (omp)
|
|
44
|
+
//
|
|
45
|
+
// A toolCall row is emitted while handling its assistant message (SplitEntry is
|
|
46
|
+
// insert-only, so the duration cannot ride the mapper's split path); the
|
|
47
|
+
// duration arrives on a LATER toolResult line. The attach therefore lives in the
|
|
48
|
+
// streaming loop, keyed on (source, session_id, call_id) per the frozen contract:
|
|
49
|
+
// - `details.wallTimeMs` is the tool's own measured wall time — used as-is,
|
|
50
|
+
// rounded, NEVER clamped or second-guessed (guards are fallback-only);
|
|
51
|
+
// - otherwise the fallback `toolResult.timestamp − toolCall message timestamp`
|
|
52
|
+
// when both are parseable AND the delta is in [0, 3_600_000]; an implausible
|
|
53
|
+
// delta stays NULL so the unmeasured count stays honest;
|
|
54
|
+
// - started_at / completed_at are written alongside a FALLBACK figure so it is
|
|
55
|
+
// auditable (wallTimeMs rows keep both NULL — the two stay distinguishable);
|
|
56
|
+
// - an unmatched toolCallId attaches nothing and never fails the import.
|
|
57
|
+
//
|
|
58
|
+
// The in-memory map covers the common (fresh-import) case without extra queries;
|
|
59
|
+
// a toolResult whose tool-call line was imported by an earlier checkpointed run
|
|
60
|
+
// (resume) resolves through a DB lookup instead, so a mid-file checkpoint loses
|
|
61
|
+
// no earlier duration.
|
|
62
|
+
// ---------------------------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
interface PendingToolCall {
|
|
65
|
+
readonly recordHash: string;
|
|
66
|
+
/** Assistant-message timestamp (epoch ms) — the fallback's start bound. */
|
|
67
|
+
readonly messageTsMs: number | undefined;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const TOOL_CALL_KEY_SEP = '\u0000';
|
|
71
|
+
|
|
72
|
+
function toolCallKey(source: string, sessionId: string, callId: string): string {
|
|
73
|
+
return `${source}${TOOL_CALL_KEY_SEP}${sessionId}${TOOL_CALL_KEY_SEP}${callId}`;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Resolve a tool-call row this run did not see (its line is behind the checkpoint). */
|
|
77
|
+
async function resolveToolCallRow(
|
|
78
|
+
db: ImportOptions['db'],
|
|
79
|
+
source: string,
|
|
80
|
+
sessionId: string,
|
|
81
|
+
callId: string,
|
|
82
|
+
): Promise<PendingToolCall | undefined> {
|
|
83
|
+
const row = await db.queryFirst<{ recordHash: string; messageTs: string | null }>(
|
|
84
|
+
`SELECT tc.record_hash AS recordHash, m.ts AS messageTs
|
|
85
|
+
FROM history_tool_call tc
|
|
86
|
+
LEFT JOIN history_message m ON m.record_hash = tc.message_hash
|
|
87
|
+
WHERE tc.source = ? AND tc.session_id = ? AND tc.call_id = ?`,
|
|
88
|
+
source,
|
|
89
|
+
sessionId,
|
|
90
|
+
callId,
|
|
91
|
+
);
|
|
92
|
+
if (row === undefined || row === null) return undefined;
|
|
93
|
+
return { recordHash: row.recordHash, messageTsMs: timestampToEpochMs(row.messageTs) };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Compute the attachable duration for a toolResult, or null when nothing should
|
|
98
|
+
* be written (implausible fallback delta → row stays NULL → counts unmeasured).
|
|
99
|
+
*/
|
|
100
|
+
function attachableDuration(
|
|
101
|
+
timing: OmpToolResultTiming,
|
|
102
|
+
messageTsMs: number | undefined,
|
|
103
|
+
): { startedAt: string | null; completedAt: string | null; durationMs: number } | null {
|
|
104
|
+
if (timing.wallTimeMs !== undefined) {
|
|
105
|
+
// The tool's own measurement — no clamping, no sanity check.
|
|
106
|
+
return { startedAt: null, completedAt: null, durationMs: Math.round(timing.wallTimeMs) };
|
|
107
|
+
}
|
|
108
|
+
if (timing.timestampMs === undefined || messageTsMs === undefined) return null;
|
|
109
|
+
const delta = timing.timestampMs - messageTsMs;
|
|
110
|
+
// Guard rails apply to the fallback ONLY: negative or > 1h is implausible.
|
|
111
|
+
if (!Number.isFinite(delta) || delta < 0 || delta > 3_600_000) return null;
|
|
112
|
+
return {
|
|
113
|
+
startedAt: new Date(messageTsMs).toISOString(),
|
|
114
|
+
completedAt: new Date(timing.timestampMs).toISOString(),
|
|
115
|
+
durationMs: Math.round(delta),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
40
119
|
/** Run the JSONL import pipeline for one source.
|
|
41
120
|
*
|
|
42
121
|
* @param source - Either a built-in source identifier resolved from the
|
|
@@ -69,6 +148,10 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
69
148
|
// ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
|
|
70
149
|
const desiredHashes = new Set<string>();
|
|
71
150
|
|
|
151
|
+
// Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
|
|
152
|
+
// (source, session_id, call_id) so a later toolResult line can attach its duration.
|
|
153
|
+
const toolCallRows = new Map<string, PendingToolCall>();
|
|
154
|
+
|
|
72
155
|
for (const file of files) {
|
|
73
156
|
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
|
|
74
157
|
let lineNumber = 0;
|
|
@@ -138,6 +221,43 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
138
221
|
// second pass, no checkpoint advance, no partial rows from this record.
|
|
139
222
|
if (lineRejected) continue;
|
|
140
223
|
|
|
224
|
+
// Task 0564 R1 (omp): register this line's tool-call rows for later duration
|
|
225
|
+
// attach. Runs over ALL prepared entries — including duplicates — because a
|
|
226
|
+
// re-imported line's row already exists and its duration UPDATE still targets
|
|
227
|
+
// the same deterministic record_hash. The assistant message timestamp is the
|
|
228
|
+
// fallback's start bound; every call in one message shares it.
|
|
229
|
+
if (definition.source === 'omp') {
|
|
230
|
+
const assistantTs = new Map<string, number | undefined>();
|
|
231
|
+
for (const entry of prepared) {
|
|
232
|
+
if (
|
|
233
|
+
entry.split.targetTable === 'history_message' &&
|
|
234
|
+
entry.normalized.role === 'assistant' &&
|
|
235
|
+
typeof entry.normalized.session_id === 'string'
|
|
236
|
+
) {
|
|
237
|
+
assistantTs.set(
|
|
238
|
+
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
239
|
+
timestampToEpochMs(entry.normalized.ts),
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
for (const entry of prepared) {
|
|
244
|
+
const callId = entry.normalized.call_id;
|
|
245
|
+
if (
|
|
246
|
+
entry.split.targetTable === 'history_tool_call' &&
|
|
247
|
+
typeof callId === 'string' &&
|
|
248
|
+
callId.length > 0 &&
|
|
249
|
+
typeof entry.normalized.session_id === 'string'
|
|
250
|
+
) {
|
|
251
|
+
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
252
|
+
recordHash: entry.recordHash,
|
|
253
|
+
messageTsMs: assistantTs.get(
|
|
254
|
+
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
255
|
+
),
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
141
261
|
// Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
|
|
142
262
|
// instead of a SELECT per record; duplicates are dropped before the write pass.
|
|
143
263
|
const existing = await ledgerExistingHashes(
|
|
@@ -199,6 +319,47 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
199
319
|
}
|
|
200
320
|
|
|
201
321
|
if (!options.dryRun && lineSucceeded) {
|
|
322
|
+
// Task 0564 R1 (omp): attach tool durations from this line's toolResult
|
|
323
|
+
// message. Unmatched toolCallIds attach nothing and never fail the import;
|
|
324
|
+
// an implausible fallback delta stays NULL (unmeasured). The UPDATEs ride
|
|
325
|
+
// this line's batch — idempotent, so re-imports and post-resume tail
|
|
326
|
+
// reprocessing write the same values.
|
|
327
|
+
if (definition.source === 'omp') {
|
|
328
|
+
const timing = ompToolResultTiming(raw);
|
|
329
|
+
if (timing !== null) {
|
|
330
|
+
const resultEntry = prepared.find(
|
|
331
|
+
(entry) =>
|
|
332
|
+
entry.split.targetTable === 'history_message' && entry.normalized.role === 'toolresult',
|
|
333
|
+
);
|
|
334
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
335
|
+
if (typeof sessionId === 'string') {
|
|
336
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
337
|
+
if (pending === undefined) {
|
|
338
|
+
// The tool-call line sits behind this run's checkpoint — the
|
|
339
|
+
// row exists from an earlier import; resolve it from the DB.
|
|
340
|
+
pending = await resolveToolCallRow(
|
|
341
|
+
options.db,
|
|
342
|
+
resolvedSource,
|
|
343
|
+
sessionId,
|
|
344
|
+
timing.toolCallId,
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
if (pending !== undefined) {
|
|
348
|
+
const duration = attachableDuration(timing, pending.messageTsMs);
|
|
349
|
+
if (duration !== null) {
|
|
350
|
+
ops.push(
|
|
351
|
+
toolCallDurationUpdateOp(
|
|
352
|
+
pending.recordHash,
|
|
353
|
+
duration.startedAt,
|
|
354
|
+
duration.completedAt,
|
|
355
|
+
duration.durationMs,
|
|
356
|
+
),
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
202
363
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
203
364
|
// re-imported after a mid-batch crash.
|
|
204
365
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
|
@@ -45,6 +45,7 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
|
45
45
|
'session_id',
|
|
46
46
|
'seq',
|
|
47
47
|
'tool_name',
|
|
48
|
+
'call_id',
|
|
48
49
|
'args_digest',
|
|
49
50
|
'args_raw',
|
|
50
51
|
'status',
|
|
@@ -255,6 +256,29 @@ export function checkpointUpsertOp(
|
|
|
255
256
|
};
|
|
256
257
|
}
|
|
257
258
|
|
|
259
|
+
/**
|
|
260
|
+
* Targeted UPDATE op attaching a tool duration to a previously-inserted
|
|
261
|
+
* `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
|
|
262
|
+
* unindexed predicate. Idempotent: re-imports write the same values.
|
|
263
|
+
*
|
|
264
|
+
* Bounds are written alongside a FALLBACK duration so the figure is auditable;
|
|
265
|
+
* a wallTimeMs-derived duration keeps `started_at`/`completed_at` NULL so the
|
|
266
|
+
* two measurement paths stay distinguishable.
|
|
267
|
+
*/
|
|
268
|
+
function toolCallDurationUpdateOp(
|
|
269
|
+
recordHash: string,
|
|
270
|
+
startedAt: string | null,
|
|
271
|
+
completedAt: string | null,
|
|
272
|
+
durationMs: number | null,
|
|
273
|
+
): DbBatchOp {
|
|
274
|
+
return {
|
|
275
|
+
sql: `UPDATE history_tool_call
|
|
276
|
+
SET started_at = ?, completed_at = ?, duration_ms = ?
|
|
277
|
+
WHERE record_hash = ?`,
|
|
278
|
+
params: [startedAt, completedAt, durationMs, recordHash],
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
258
282
|
/**
|
|
259
283
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
260
284
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -688,5 +712,6 @@ export {
|
|
|
688
712
|
resetCheckpoints,
|
|
689
713
|
targetTableFor,
|
|
690
714
|
timestamp,
|
|
715
|
+
toolCallDurationUpdateOp,
|
|
691
716
|
writeCheckpoint,
|
|
692
717
|
};
|
package/src/mappers.ts
CHANGED
|
@@ -35,6 +35,7 @@ const TOOL_CALL_MAPPER_KEYS: readonly string[] = [
|
|
|
35
35
|
'session_id',
|
|
36
36
|
'seq',
|
|
37
37
|
'tool_name',
|
|
38
|
+
'call_id',
|
|
38
39
|
'args_digest',
|
|
39
40
|
'args_raw',
|
|
40
41
|
'status',
|
|
@@ -411,6 +412,10 @@ export function ompSplit(raw: Record<string, unknown>, context?: TransformContex
|
|
|
411
412
|
session_id: sessionId,
|
|
412
413
|
seq,
|
|
413
414
|
tool_name: String(call.name ?? ''),
|
|
415
|
+
// Task 0564 R1: the tool's own call id is the exact join key a
|
|
416
|
+
// toolResult's `toolCallId` matches — it lets the streaming loop
|
|
417
|
+
// attach the tool's measured duration to the right row.
|
|
418
|
+
call_id: s(call.id),
|
|
414
419
|
args_digest: argsDigest(call.input ?? call.arguments),
|
|
415
420
|
args_raw: maybeArgsRaw('omp', String(call.name ?? ''), call.input ?? call.arguments),
|
|
416
421
|
status: 'ok',
|
|
@@ -441,6 +446,51 @@ function normalizeOmpToolCall(block: Record<string, unknown>): Record<string, un
|
|
|
441
446
|
return null;
|
|
442
447
|
}
|
|
443
448
|
|
|
449
|
+
// ---------------------------------------------------------------------------
|
|
450
|
+
// OMP toolResult timing signals (task 0564 R1)
|
|
451
|
+
// ---------------------------------------------------------------------------
|
|
452
|
+
|
|
453
|
+
/** Timing signals carried by an OMP toolResult message envelope (task 0564 R1). */
|
|
454
|
+
export interface OmpToolResultTiming {
|
|
455
|
+
/** The toolCall id this result answers — joins `toolCall.id` exactly. */
|
|
456
|
+
toolCallId: string;
|
|
457
|
+
/** The tool's own measured wall time in ms, when present and finite. */
|
|
458
|
+
wallTimeMs: number | undefined;
|
|
459
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
460
|
+
timestampMs: number | undefined;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/** Parse a message timestamp (epoch-millis number, numeric string, or ISO) to epoch millis. */
|
|
464
|
+
export function timestampToEpochMs(value: unknown): number | undefined {
|
|
465
|
+
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
|
466
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
467
|
+
const numeric = Number(value);
|
|
468
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
469
|
+
const parsed = Date.parse(value);
|
|
470
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
471
|
+
}
|
|
472
|
+
return undefined;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Extract toolResult timing signals from a raw OMP record (task 0564 R1), or null
|
|
477
|
+
* when the record is not a toolResult message or carries no toolCallId. Live OMP
|
|
478
|
+
* emits `role: "toolResult"` message envelopes with `{toolCallId, toolName,
|
|
479
|
+
* content, details, isError, timestamp}`; `details.wallTimeMs` is the tool's own
|
|
480
|
+
* measured wall time (48% of results in the sampled session) and `toolCallId`
|
|
481
|
+
* joins the originating `toolCall.id` exactly.
|
|
482
|
+
*/
|
|
483
|
+
export function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null {
|
|
484
|
+
const msg = o(raw.message);
|
|
485
|
+
if (String(msg.role ?? '').toLowerCase() !== 'toolresult') return null;
|
|
486
|
+
const toolCallId = s(msg.toolCallId);
|
|
487
|
+
if (toolCallId === undefined) return null;
|
|
488
|
+
const details = o(msg.details);
|
|
489
|
+
const wallTimeMs =
|
|
490
|
+
typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
|
|
491
|
+
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
492
|
+
}
|
|
493
|
+
|
|
444
494
|
// ---------------------------------------------------------------------------
|
|
445
495
|
// Codex mapper
|
|
446
496
|
// ---------------------------------------------------------------------------
|