@gobing-ai/ts-llm-jsonl-importer 0.4.40 → 0.4.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +36 -10
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/jsonl-importer-dao.d.ts +6 -1
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +14 -1
- package/dist/mappers.d.ts +29 -2
- package/dist/mappers.d.ts.map +1 -1
- package/dist/mappers.js +78 -5
- package/dist/schema-sql.d.ts.map +1 -1
- package/dist/schema-sql.js +1 -0
- package/package.json +4 -4
- package/src/importer.ts +46 -14
- package/src/index.ts +1 -0
- package/src/jsonl-importer-dao.ts +15 -0
- package/src/mappers.ts +94 -5
- package/src/schema-sql.ts +1 -0
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAsTrH"}
|
package/dist/importer.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
|
|
2
2
|
import { sha256 } from './hash.js';
|
|
3
|
-
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, } from './jsonl-importer-dao.js';
|
|
4
|
-
import { ompToolResultTiming, timestampToEpochMs } from './mappers.js';
|
|
3
|
+
import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, } from './jsonl-importer-dao.js';
|
|
4
|
+
import { claudeToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers.js';
|
|
5
5
|
import { redactRecord } from './redaction.js';
|
|
6
6
|
import { resolveSourceDefinition } from './sources.js';
|
|
7
7
|
const TOOL_CALL_KEY_SEP = '\u0000';
|
|
@@ -140,13 +140,18 @@ export async function runJsonlImport(source, options) {
|
|
|
140
140
|
// re-imported line's row already exists and its duration UPDATE still targets
|
|
141
141
|
// the same deterministic record_hash. The assistant message timestamp is the
|
|
142
142
|
// fallback's start bound; every call in one message shares it.
|
|
143
|
-
if (definition.source === 'omp') {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
143
|
+
if (definition.source === 'omp' || definition.source === 'claude') {
|
|
144
|
+
// omp only: assistant-message timestamps are the fallback duration's
|
|
145
|
+
// start bound. claude attaches result_bytes only — no fallback
|
|
146
|
+
// duration exists natively (never-fabricate, 0624 R2).
|
|
147
|
+
const assistantTs = definition.source === 'omp' ? new Map() : null;
|
|
148
|
+
if (assistantTs !== null) {
|
|
149
|
+
for (const entry of prepared) {
|
|
150
|
+
if (entry.split.targetTable === 'history_message' &&
|
|
151
|
+
entry.normalized.role === 'assistant' &&
|
|
152
|
+
typeof entry.normalized.session_id === 'string') {
|
|
153
|
+
assistantTs.set(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`, timestampToEpochMs(entry.normalized.ts));
|
|
154
|
+
}
|
|
150
155
|
}
|
|
151
156
|
}
|
|
152
157
|
for (const entry of prepared) {
|
|
@@ -157,7 +162,7 @@ export async function runJsonlImport(source, options) {
|
|
|
157
162
|
typeof entry.normalized.session_id === 'string') {
|
|
158
163
|
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
159
164
|
recordHash: entry.recordHash,
|
|
160
|
-
messageTsMs: assistantTs
|
|
165
|
+
messageTsMs: assistantTs?.get(`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`),
|
|
161
166
|
});
|
|
162
167
|
}
|
|
163
168
|
}
|
|
@@ -224,6 +229,27 @@ export async function runJsonlImport(source, options) {
|
|
|
224
229
|
}
|
|
225
230
|
}
|
|
226
231
|
}
|
|
232
|
+
// Task 0624 R2 (claude): attach result sizes from this line's
|
|
233
|
+
// tool_result blocks. Pairing is by `tool_use_id` → `call_id`
|
|
234
|
+
// (source ∘ session ∘ call), the same key the omp branch uses.
|
|
235
|
+
// No duration is written — claude has no native wall time and the
|
|
236
|
+
// guarded timestamp fallback does not extend to it (0624 Design).
|
|
237
|
+
if (definition.source === 'claude') {
|
|
238
|
+
const timing = claudeToolResultTiming(raw);
|
|
239
|
+
if (timing !== null) {
|
|
240
|
+
const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message');
|
|
241
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
242
|
+
if (typeof sessionId === 'string') {
|
|
243
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
244
|
+
if (pending === undefined) {
|
|
245
|
+
pending = await resolveToolCallRow(options.db, resolvedSource, sessionId, timing.toolCallId);
|
|
246
|
+
}
|
|
247
|
+
if (pending !== undefined) {
|
|
248
|
+
ops.push(toolCallResultBytesUpdateOp(pending.recordHash, timing.resultBytes));
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
227
253
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
228
254
|
// re-imported after a mid-batch crash.
|
|
229
255
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
package/dist/index.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ export { HistoryImportError } from './errors';
|
|
|
2
2
|
export { sha256, stableJson } from './hash';
|
|
3
3
|
export { runJsonlImport } from './importer';
|
|
4
4
|
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
|
|
5
|
+
export { sessionIdFromSourcePath } from './mappers';
|
|
5
6
|
export { type OpenCodeImportOptions, runOpenCodeImport } from './opencode-importer';
|
|
6
7
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
7
8
|
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAC1F,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAC1F,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@ export { HistoryImportError } from './errors.js';
|
|
|
2
2
|
export { sha256, stableJson } from './hash.js';
|
|
3
3
|
export { runJsonlImport } from './importer.js';
|
|
4
4
|
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao.js';
|
|
5
|
+
export { sessionIdFromSourcePath } from './mappers.js';
|
|
5
6
|
export { runOpenCodeImport } from './opencode-importer.js';
|
|
6
7
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
|
|
7
8
|
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
|
|
@@ -54,6 +54,11 @@ export declare function checkpointUpsertOp(source: string, sourceFile: string, l
|
|
|
54
54
|
* two measurement paths stay distinguishable.
|
|
55
55
|
*/
|
|
56
56
|
declare function toolCallDurationUpdateOp(recordHash: string, startedAt: string | null, completedAt: string | null, durationMs: number | null): DbBatchOp;
|
|
57
|
+
/**
|
|
58
|
+
* Targeted UPDATE op attaching a result size to a previously-inserted
|
|
59
|
+
* `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
|
|
60
|
+
*/
|
|
61
|
+
declare function toolCallResultBytesUpdateOp(recordHash: string, resultBytes: number): DbBatchOp;
|
|
57
62
|
/**
|
|
58
63
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
59
64
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -136,5 +141,5 @@ export declare function deleteCheckpointOperation(source: string, sourceFile: st
|
|
|
136
141
|
/** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
|
|
137
142
|
export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
|
|
138
143
|
export type { CheckpointRow };
|
|
139
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
|
144
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };
|
|
140
145
|
//# sourceMappingURL=jsonl-importer-dao.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAwDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,CAOvF;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,2BAA2B,EAC3B,eAAe,GAClB,CAAC"}
|
|
@@ -27,6 +27,7 @@ const TYPED_TABLE_COLUMNS = {
|
|
|
27
27
|
'provenance',
|
|
28
28
|
'run_id',
|
|
29
29
|
'task_wbs',
|
|
30
|
+
'request_id',
|
|
30
31
|
'imported_at',
|
|
31
32
|
],
|
|
32
33
|
history_tool_call: [
|
|
@@ -212,6 +213,18 @@ function toolCallDurationUpdateOp(recordHash, startedAt, completedAt, durationMs
|
|
|
212
213
|
params: [startedAt, completedAt, durationMs, recordHash],
|
|
213
214
|
};
|
|
214
215
|
}
|
|
216
|
+
/**
|
|
217
|
+
* Targeted UPDATE op attaching a result size to a previously-inserted
|
|
218
|
+
* `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
|
|
219
|
+
*/
|
|
220
|
+
function toolCallResultBytesUpdateOp(recordHash, resultBytes) {
|
|
221
|
+
return {
|
|
222
|
+
sql: `UPDATE history_tool_call
|
|
223
|
+
SET result_bytes = ?
|
|
224
|
+
WHERE record_hash = ?`,
|
|
225
|
+
params: [resultBytes, recordHash],
|
|
226
|
+
};
|
|
227
|
+
}
|
|
215
228
|
/**
|
|
216
229
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
217
230
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -486,4 +499,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
|
|
|
486
499
|
}
|
|
487
500
|
return operations;
|
|
488
501
|
}
|
|
489
|
-
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, writeCheckpoint, };
|
|
502
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };
|
package/dist/mappers.d.ts
CHANGED
|
@@ -33,12 +33,39 @@ export declare function timestampToEpochMs(value: unknown): number | undefined;
|
|
|
33
33
|
* joins the originating `toolCall.id` exactly.
|
|
34
34
|
*/
|
|
35
35
|
export declare function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResultTiming | null;
|
|
36
|
+
/** Signals carried by a Claude tool_result user line (task 0624 R2). */
|
|
37
|
+
export interface ClaudeToolResultTiming {
|
|
38
|
+
/** The `toolu_…` id from the answered tool_use block — joins `tool_use.id` exactly. */
|
|
39
|
+
toolCallId: string;
|
|
40
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
41
|
+
timestampMs: number | undefined;
|
|
42
|
+
/** Serialized size in bytes of the tool result payload. */
|
|
43
|
+
resultBytes: number;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Extract tool_result signals from a raw Claude Code record (task 0624 R2), or
|
|
47
|
+
* null when the record carries no tool_result block. Claude Code emits
|
|
48
|
+
* `type: "user"` lines whose `message.content` holds `{type: "tool_result",
|
|
49
|
+
* tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
|
|
50
|
+
* envelope. `resultBytes` prefers the model-visible block content and falls
|
|
51
|
+
* back to `toolUseResult`; both are null-safe. No wall time exists natively —
|
|
52
|
+
* claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
|
|
53
|
+
*/
|
|
54
|
+
export declare function claudeToolResultTiming(raw: Record<string, unknown>): ClaudeToolResultTiming | null;
|
|
36
55
|
/** Map a Codex JSONL record into one or more ETL split entries. */
|
|
37
|
-
export declare function codexSplit(raw: Record<string, unknown
|
|
56
|
+
export declare function codexSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
38
57
|
/** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
|
|
39
|
-
export declare function agySplit(raw: Record<string, unknown
|
|
58
|
+
export declare function agySplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
40
59
|
/** Map Gemini CLI message, tool, and session-state events into the forensic contract tables. */
|
|
41
60
|
export declare function geminiSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
61
|
+
/**
|
|
62
|
+
* Pure helper extracting canonical session identity from source transcript paths (task 0638 R4).
|
|
63
|
+
*
|
|
64
|
+
* Normalizes POSIX/Windows separators and returns only valid UUID-shaped identifiers:
|
|
65
|
+
* - agy: segment immediately following `/brain/`
|
|
66
|
+
* - codex: trailing UUID before `.jsonl` in a filename beginning with `rollout-`
|
|
67
|
+
*/
|
|
68
|
+
export declare function sessionIdFromSourcePath(source: string, sourceFile: string): string | undefined;
|
|
42
69
|
/** Map a Grok JSONL record into one or more ETL split entries. */
|
|
43
70
|
export declare function grokSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
44
71
|
/** Compute a stable field-shape key from a JSON object: sorted, lowercased, +-joined top-level keys. */
|
package/dist/mappers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAoExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAwCD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAMhG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkH/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,wEAAwE;AACxE,MAAM,WAAW,sBAAsB;IACnC,uFAAuF;IACvF,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,sBAAsB,GAAG,IAAI,CAkBlG;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAgL1G;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHxG;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAMD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAgB9F;AAiMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAgFD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
|
package/dist/mappers.js
CHANGED
|
@@ -24,6 +24,7 @@ const MESSAGE_MAPPER_KEYS = [
|
|
|
24
24
|
'provenance',
|
|
25
25
|
'run_id',
|
|
26
26
|
'task_wbs',
|
|
27
|
+
'request_id',
|
|
27
28
|
];
|
|
28
29
|
/** Columns the mapper may produce for history_tool_call. */
|
|
29
30
|
const TOOL_CALL_MAPPER_KEYS = [
|
|
@@ -194,6 +195,7 @@ export function claudeSplit(raw) {
|
|
|
194
195
|
content_text: contentText ?? null,
|
|
195
196
|
cwd: cwd ?? null,
|
|
196
197
|
provenance: 'ambient',
|
|
198
|
+
request_id: s(raw.requestId) ?? null,
|
|
197
199
|
},
|
|
198
200
|
});
|
|
199
201
|
// Tool calls from content blocks
|
|
@@ -209,6 +211,7 @@ export function claudeSplit(raw) {
|
|
|
209
211
|
_messageSplitIndex: messageSplitIndex,
|
|
210
212
|
session_id: sessionId,
|
|
211
213
|
seq,
|
|
214
|
+
call_id: s(b.id),
|
|
212
215
|
tool_name: String(b.name ?? ''),
|
|
213
216
|
args_digest: argsDigest(b.input),
|
|
214
217
|
args_raw: maybeArgsRaw('claude', String(b.name ?? ''), b.input),
|
|
@@ -510,20 +513,61 @@ export function ompToolResultTiming(raw) {
|
|
|
510
513
|
const wallTimeMs = typeof details.wallTimeMs === 'number' && Number.isFinite(details.wallTimeMs) ? details.wallTimeMs : undefined;
|
|
511
514
|
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
512
515
|
}
|
|
516
|
+
/**
|
|
517
|
+
* Extract tool_result signals from a raw Claude Code record (task 0624 R2), or
|
|
518
|
+
* null when the record carries no tool_result block. Claude Code emits
|
|
519
|
+
* `type: "user"` lines whose `message.content` holds `{type: "tool_result",
|
|
520
|
+
* tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
|
|
521
|
+
* envelope. `resultBytes` prefers the model-visible block content and falls
|
|
522
|
+
* back to `toolUseResult`; both are null-safe. No wall time exists natively —
|
|
523
|
+
* claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
|
|
524
|
+
*/
|
|
525
|
+
export function claudeToolResultTiming(raw) {
|
|
526
|
+
const content = o(raw.message).content;
|
|
527
|
+
if (!Array.isArray(content))
|
|
528
|
+
return null;
|
|
529
|
+
for (const block of content) {
|
|
530
|
+
if (typeof block !== 'object' || block === null)
|
|
531
|
+
continue;
|
|
532
|
+
const b = block;
|
|
533
|
+
if (b.type !== 'tool_result')
|
|
534
|
+
continue;
|
|
535
|
+
const toolCallId = s(b.tool_use_id);
|
|
536
|
+
if (toolCallId === undefined)
|
|
537
|
+
continue;
|
|
538
|
+
const payload = b.content ?? raw.toolUseResult;
|
|
539
|
+
const resultBytes = JSON.stringify(payload ?? null)?.length ?? 2;
|
|
540
|
+
return {
|
|
541
|
+
toolCallId,
|
|
542
|
+
timestampMs: timestampToEpochMs(raw.timestamp),
|
|
543
|
+
resultBytes,
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
return null;
|
|
547
|
+
}
|
|
513
548
|
// ---------------------------------------------------------------------------
|
|
514
549
|
// Codex mapper
|
|
515
550
|
// ---------------------------------------------------------------------------
|
|
516
551
|
/** Map a Codex JSONL record into one or more ETL split entries. */
|
|
517
|
-
export function codexSplit(raw) {
|
|
552
|
+
export function codexSplit(raw, context) {
|
|
518
553
|
const entries = [];
|
|
519
554
|
const payload = (raw.payload ?? raw);
|
|
520
|
-
const sessionId = s(raw.session_id, o(raw.session_meta).id, raw.id) ?? 'unknown';
|
|
521
555
|
const seq = typeof raw.seq === 'number' ? raw.seq : 0;
|
|
522
556
|
const ts = timestampOf(raw.timestamp, raw.ts, payload.ts);
|
|
523
557
|
const recordType = String(raw.type ?? '');
|
|
524
558
|
const payloadType = String(payload.type ?? '');
|
|
559
|
+
const isShortFormat = recordType === '' && raw.id !== undefined && raw.timestamp !== undefined && raw.instructions !== undefined;
|
|
560
|
+
let explicitSessionId = s(raw.session_id, o(raw.session_meta).id);
|
|
561
|
+
if (explicitSessionId === undefined && recordType === 'session_meta') {
|
|
562
|
+
explicitSessionId = s(payload.id, o(raw.payload).id);
|
|
563
|
+
}
|
|
564
|
+
if (explicitSessionId === undefined && isShortFormat) {
|
|
565
|
+
explicitSessionId = s(raw.id);
|
|
566
|
+
}
|
|
567
|
+
const pathSessionId = context?.sourceFile ? sessionIdFromSourcePath('codex', context.sourceFile) : undefined;
|
|
568
|
+
const sessionId = explicitSessionId ?? pathSessionId ?? 'unknown';
|
|
525
569
|
// Check for older short format
|
|
526
|
-
if (
|
|
570
|
+
if (isShortFormat) {
|
|
527
571
|
return [
|
|
528
572
|
{
|
|
529
573
|
targetTable: 'history_message',
|
|
@@ -676,10 +720,12 @@ export function codexSplit(raw) {
|
|
|
676
720
|
// AGY mapper (Antigravity CLI)
|
|
677
721
|
// ---------------------------------------------------------------------------
|
|
678
722
|
/** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
|
|
679
|
-
export function agySplit(raw) {
|
|
723
|
+
export function agySplit(raw, context) {
|
|
680
724
|
const entries = [];
|
|
681
725
|
const recordType = String(raw.type ?? '');
|
|
682
|
-
const
|
|
726
|
+
const explicitSessionId = s(raw.session_id, raw.conversation_id);
|
|
727
|
+
const pathSessionId = context?.sourceFile ? sessionIdFromSourcePath('agy', context.sourceFile) : undefined;
|
|
728
|
+
const sessionId = explicitSessionId ?? pathSessionId ?? 'unknown';
|
|
683
729
|
const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.step_index === 'number' ? raw.step_index : 0;
|
|
684
730
|
const ts = timestampOf(raw.created_at, raw.timestamp, raw.ts);
|
|
685
731
|
let role;
|
|
@@ -867,6 +913,33 @@ export function geminiSplit(raw, context) {
|
|
|
867
913
|
}
|
|
868
914
|
return entries;
|
|
869
915
|
}
|
|
916
|
+
const UUID_PATTERN = '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}';
|
|
917
|
+
const AGY_BRAIN_REGEX = new RegExp(`(?:^|\\/)brain\\/(${UUID_PATTERN})(?:\\/|$)`, 'i');
|
|
918
|
+
const CODEX_ROLLOUT_REGEX = new RegExp(`(?:^|-|_)(${UUID_PATTERN})$`, 'i');
|
|
919
|
+
/**
|
|
920
|
+
* Pure helper extracting canonical session identity from source transcript paths (task 0638 R4).
|
|
921
|
+
*
|
|
922
|
+
* Normalizes POSIX/Windows separators and returns only valid UUID-shaped identifiers:
|
|
923
|
+
* - agy: segment immediately following `/brain/`
|
|
924
|
+
* - codex: trailing UUID before `.jsonl` in a filename beginning with `rollout-`
|
|
925
|
+
*/
|
|
926
|
+
export function sessionIdFromSourcePath(source, sourceFile) {
|
|
927
|
+
const normalized = sourceFile.replace(/\\/g, '/');
|
|
928
|
+
const sLower = source.toLowerCase();
|
|
929
|
+
if (sLower === 'agy' || sLower === 'antigravity') {
|
|
930
|
+
const match = normalized.match(AGY_BRAIN_REGEX);
|
|
931
|
+
return match?.[1];
|
|
932
|
+
}
|
|
933
|
+
if (sLower === 'codex') {
|
|
934
|
+
const basename = normalized.split('/').at(-1) ?? '';
|
|
935
|
+
if (basename.startsWith('rollout-') && basename.endsWith('.jsonl')) {
|
|
936
|
+
const stem = basename.slice(0, -6);
|
|
937
|
+
const match = stem.match(CODEX_ROLLOUT_REGEX);
|
|
938
|
+
return match?.[1];
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
return undefined;
|
|
942
|
+
}
|
|
870
943
|
function sessionIdFromContext(context, raw) {
|
|
871
944
|
if (context !== undefined) {
|
|
872
945
|
const file = context.sourceFile.split('/').at(-1);
|
package/dist/schema-sql.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,
|
|
1
|
+
{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA8E9B,CAAC"}
|
package/dist/schema-sql.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.42",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.42",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.42",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.42",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
package/src/importer.ts
CHANGED
|
@@ -20,8 +20,9 @@ import {
|
|
|
20
20
|
resetCheckpoints,
|
|
21
21
|
targetTableFor,
|
|
22
22
|
toolCallDurationUpdateOp,
|
|
23
|
+
toolCallResultBytesUpdateOp,
|
|
23
24
|
} from './jsonl-importer-dao';
|
|
24
|
-
import { type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
|
|
25
|
+
import { claudeToolResultTiming, type OmpToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers';
|
|
25
26
|
import { redactRecord } from './redaction';
|
|
26
27
|
import { resolveSourceDefinition } from './sources';
|
|
27
28
|
import type {
|
|
@@ -230,18 +231,23 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
230
231
|
// re-imported line's row already exists and its duration UPDATE still targets
|
|
231
232
|
// the same deterministic record_hash. The assistant message timestamp is the
|
|
232
233
|
// fallback's start bound; every call in one message shares it.
|
|
233
|
-
if (definition.source === 'omp') {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
)
|
|
234
|
+
if (definition.source === 'omp' || definition.source === 'claude') {
|
|
235
|
+
// omp only: assistant-message timestamps are the fallback duration's
|
|
236
|
+
// start bound. claude attaches result_bytes only — no fallback
|
|
237
|
+
// duration exists natively (never-fabricate, 0624 R2).
|
|
238
|
+
const assistantTs = definition.source === 'omp' ? new Map<string, number | undefined>() : null;
|
|
239
|
+
if (assistantTs !== null) {
|
|
240
|
+
for (const entry of prepared) {
|
|
241
|
+
if (
|
|
242
|
+
entry.split.targetTable === 'history_message' &&
|
|
243
|
+
entry.normalized.role === 'assistant' &&
|
|
244
|
+
typeof entry.normalized.session_id === 'string'
|
|
245
|
+
) {
|
|
246
|
+
assistantTs.set(
|
|
247
|
+
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
248
|
+
timestampToEpochMs(entry.normalized.ts),
|
|
249
|
+
);
|
|
250
|
+
}
|
|
245
251
|
}
|
|
246
252
|
}
|
|
247
253
|
for (const entry of prepared) {
|
|
@@ -254,7 +260,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
254
260
|
) {
|
|
255
261
|
toolCallRows.set(toolCallKey(resolvedSource, entry.normalized.session_id, callId), {
|
|
256
262
|
recordHash: entry.recordHash,
|
|
257
|
-
messageTsMs: assistantTs
|
|
263
|
+
messageTsMs: assistantTs?.get(
|
|
258
264
|
`${entry.normalized.session_id}${TOOL_CALL_KEY_SEP}${entry.normalized.seq}`,
|
|
259
265
|
),
|
|
260
266
|
});
|
|
@@ -364,6 +370,32 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
|
|
|
364
370
|
}
|
|
365
371
|
}
|
|
366
372
|
}
|
|
373
|
+
// Task 0624 R2 (claude): attach result sizes from this line's
|
|
374
|
+
// tool_result blocks. Pairing is by `tool_use_id` → `call_id`
|
|
375
|
+
// (source ∘ session ∘ call), the same key the omp branch uses.
|
|
376
|
+
// No duration is written — claude has no native wall time and the
|
|
377
|
+
// guarded timestamp fallback does not extend to it (0624 Design).
|
|
378
|
+
if (definition.source === 'claude') {
|
|
379
|
+
const timing = claudeToolResultTiming(raw);
|
|
380
|
+
if (timing !== null) {
|
|
381
|
+
const resultEntry = prepared.find((entry) => entry.split.targetTable === 'history_message');
|
|
382
|
+
const sessionId = resultEntry?.normalized.session_id;
|
|
383
|
+
if (typeof sessionId === 'string') {
|
|
384
|
+
let pending = toolCallRows.get(toolCallKey(resolvedSource, sessionId, timing.toolCallId));
|
|
385
|
+
if (pending === undefined) {
|
|
386
|
+
pending = await resolveToolCallRow(
|
|
387
|
+
options.db,
|
|
388
|
+
resolvedSource,
|
|
389
|
+
sessionId,
|
|
390
|
+
timing.toolCallId,
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
if (pending !== undefined) {
|
|
394
|
+
ops.push(toolCallResultBytesUpdateOp(pending.recordHash, timing.resultBytes));
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
367
399
|
// Checkpoint upsert joins the same batch so an accepted line can never be
|
|
368
400
|
// re-imported after a mid-batch crash.
|
|
369
401
|
ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
|
package/src/index.ts
CHANGED
|
@@ -2,6 +2,7 @@ export { HistoryImportError } from './errors';
|
|
|
2
2
|
export { sha256, stableJson } from './hash';
|
|
3
3
|
export { runJsonlImport } from './importer';
|
|
4
4
|
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
|
|
5
|
+
export { sessionIdFromSourcePath } from './mappers';
|
|
5
6
|
export { type OpenCodeImportOptions, runOpenCodeImport } from './opencode-importer';
|
|
6
7
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
7
8
|
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
@@ -34,6 +34,7 @@ const TYPED_TABLE_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
|
34
34
|
'provenance',
|
|
35
35
|
'run_id',
|
|
36
36
|
'task_wbs',
|
|
37
|
+
'request_id',
|
|
37
38
|
'imported_at',
|
|
38
39
|
],
|
|
39
40
|
history_tool_call: [
|
|
@@ -279,6 +280,19 @@ function toolCallDurationUpdateOp(
|
|
|
279
280
|
};
|
|
280
281
|
}
|
|
281
282
|
|
|
283
|
+
/**
|
|
284
|
+
* Targeted UPDATE op attaching a result size to a previously-inserted
|
|
285
|
+
* `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
|
|
286
|
+
*/
|
|
287
|
+
function toolCallResultBytesUpdateOp(recordHash: string, resultBytes: number): DbBatchOp {
|
|
288
|
+
return {
|
|
289
|
+
sql: `UPDATE history_tool_call
|
|
290
|
+
SET result_bytes = ?
|
|
291
|
+
WHERE record_hash = ?`,
|
|
292
|
+
params: [resultBytes, recordHash],
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
|
|
282
296
|
/**
|
|
283
297
|
* Query which of the given hashes already exist in the ledger, in chunks of at
|
|
284
298
|
* most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
|
|
@@ -713,5 +727,6 @@ export {
|
|
|
713
727
|
targetTableFor,
|
|
714
728
|
timestamp,
|
|
715
729
|
toolCallDurationUpdateOp,
|
|
730
|
+
toolCallResultBytesUpdateOp,
|
|
716
731
|
writeCheckpoint,
|
|
717
732
|
};
|
package/src/mappers.ts
CHANGED
|
@@ -27,6 +27,7 @@ const MESSAGE_MAPPER_KEYS: readonly string[] = [
|
|
|
27
27
|
'provenance',
|
|
28
28
|
'run_id',
|
|
29
29
|
'task_wbs',
|
|
30
|
+
'request_id',
|
|
30
31
|
];
|
|
31
32
|
|
|
32
33
|
/** Columns the mapper may produce for history_tool_call. */
|
|
@@ -209,6 +210,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
209
210
|
content_text: contentText ?? null,
|
|
210
211
|
cwd: cwd ?? null,
|
|
211
212
|
provenance: 'ambient',
|
|
213
|
+
request_id: s(raw.requestId) ?? null,
|
|
212
214
|
},
|
|
213
215
|
});
|
|
214
216
|
|
|
@@ -224,6 +226,7 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
224
226
|
_messageSplitIndex: messageSplitIndex,
|
|
225
227
|
session_id: sessionId,
|
|
226
228
|
seq,
|
|
229
|
+
call_id: s(b.id),
|
|
227
230
|
tool_name: String(b.name ?? ''),
|
|
228
231
|
args_digest: argsDigest(b.input),
|
|
229
232
|
args_raw: maybeArgsRaw('claude', String(b.name ?? ''), b.input),
|
|
@@ -554,23 +557,78 @@ export function ompToolResultTiming(raw: Record<string, unknown>): OmpToolResult
|
|
|
554
557
|
return { toolCallId, wallTimeMs, timestampMs: timestampToEpochMs(raw.timestamp ?? msg.timestamp) };
|
|
555
558
|
}
|
|
556
559
|
|
|
560
|
+
// ---------------------------------------------------------------------------
|
|
561
|
+
// Claude toolResult signals (task 0624 R2)
|
|
562
|
+
// ---------------------------------------------------------------------------
|
|
563
|
+
|
|
564
|
+
/** Signals carried by a Claude tool_result user line (task 0624 R2). */
|
|
565
|
+
export interface ClaudeToolResultTiming {
|
|
566
|
+
/** The `toolu_…` id from the answered tool_use block — joins `tool_use.id` exactly. */
|
|
567
|
+
toolCallId: string;
|
|
568
|
+
/** Message timestamp as epoch millis, when parseable. */
|
|
569
|
+
timestampMs: number | undefined;
|
|
570
|
+
/** Serialized size in bytes of the tool result payload. */
|
|
571
|
+
resultBytes: number;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Extract tool_result signals from a raw Claude Code record (task 0624 R2), or
|
|
576
|
+
* null when the record carries no tool_result block. Claude Code emits
|
|
577
|
+
* `type: "user"` lines whose `message.content` holds `{type: "tool_result",
|
|
578
|
+
* tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
|
|
579
|
+
* envelope. `resultBytes` prefers the model-visible block content and falls
|
|
580
|
+
* back to `toolUseResult`; both are null-safe. No wall time exists natively —
|
|
581
|
+
* claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
|
|
582
|
+
*/
|
|
583
|
+
export function claudeToolResultTiming(raw: Record<string, unknown>): ClaudeToolResultTiming | null {
|
|
584
|
+
const content = o(raw.message).content;
|
|
585
|
+
if (!Array.isArray(content)) return null;
|
|
586
|
+
for (const block of content) {
|
|
587
|
+
if (typeof block !== 'object' || block === null) continue;
|
|
588
|
+
const b = block as Record<string, unknown>;
|
|
589
|
+
if (b.type !== 'tool_result') continue;
|
|
590
|
+
const toolCallId = s(b.tool_use_id);
|
|
591
|
+
if (toolCallId === undefined) continue;
|
|
592
|
+
const payload = b.content ?? raw.toolUseResult;
|
|
593
|
+
const resultBytes = JSON.stringify(payload ?? null)?.length ?? 2;
|
|
594
|
+
return {
|
|
595
|
+
toolCallId,
|
|
596
|
+
timestampMs: timestampToEpochMs(raw.timestamp),
|
|
597
|
+
resultBytes,
|
|
598
|
+
};
|
|
599
|
+
}
|
|
600
|
+
return null;
|
|
601
|
+
}
|
|
602
|
+
|
|
557
603
|
// ---------------------------------------------------------------------------
|
|
558
604
|
// Codex mapper
|
|
559
605
|
// ---------------------------------------------------------------------------
|
|
560
606
|
|
|
561
607
|
/** Map a Codex JSONL record into one or more ETL split entries. */
|
|
562
|
-
export function codexSplit(raw: Record<string, unknown
|
|
608
|
+
export function codexSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[] {
|
|
563
609
|
const entries: SplitEntry[] = [];
|
|
564
610
|
|
|
565
611
|
const payload = (raw.payload ?? raw) as Record<string, unknown>;
|
|
566
|
-
const sessionId = s(raw.session_id, o(raw.session_meta).id, raw.id) ?? 'unknown';
|
|
567
612
|
const seq = typeof raw.seq === 'number' ? raw.seq : 0;
|
|
568
613
|
const ts = timestampOf(raw.timestamp, raw.ts, payload.ts);
|
|
569
614
|
const recordType = String(raw.type ?? '');
|
|
570
615
|
const payloadType = String(payload.type ?? '');
|
|
571
616
|
|
|
617
|
+
const isShortFormat =
|
|
618
|
+
recordType === '' && raw.id !== undefined && raw.timestamp !== undefined && raw.instructions !== undefined;
|
|
619
|
+
|
|
620
|
+
let explicitSessionId = s(raw.session_id, o(raw.session_meta).id);
|
|
621
|
+
if (explicitSessionId === undefined && recordType === 'session_meta') {
|
|
622
|
+
explicitSessionId = s(payload.id, o(raw.payload).id);
|
|
623
|
+
}
|
|
624
|
+
if (explicitSessionId === undefined && isShortFormat) {
|
|
625
|
+
explicitSessionId = s(raw.id);
|
|
626
|
+
}
|
|
627
|
+
const pathSessionId = context?.sourceFile ? sessionIdFromSourcePath('codex', context.sourceFile) : undefined;
|
|
628
|
+
const sessionId = explicitSessionId ?? pathSessionId ?? 'unknown';
|
|
629
|
+
|
|
572
630
|
// Check for older short format
|
|
573
|
-
if (
|
|
631
|
+
if (isShortFormat) {
|
|
574
632
|
return [
|
|
575
633
|
{
|
|
576
634
|
targetTable: 'history_message',
|
|
@@ -730,10 +788,12 @@ export function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
730
788
|
// ---------------------------------------------------------------------------
|
|
731
789
|
|
|
732
790
|
/** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
|
|
733
|
-
export function agySplit(raw: Record<string, unknown
|
|
791
|
+
export function agySplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[] {
|
|
734
792
|
const entries: SplitEntry[] = [];
|
|
735
793
|
const recordType = String(raw.type ?? '');
|
|
736
|
-
const
|
|
794
|
+
const explicitSessionId = s(raw.session_id, raw.conversation_id);
|
|
795
|
+
const pathSessionId = context?.sourceFile ? sessionIdFromSourcePath('agy', context.sourceFile) : undefined;
|
|
796
|
+
const sessionId = explicitSessionId ?? pathSessionId ?? 'unknown';
|
|
737
797
|
const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.step_index === 'number' ? raw.step_index : 0;
|
|
738
798
|
const ts = timestampOf(raw.created_at, raw.timestamp, raw.ts);
|
|
739
799
|
|
|
@@ -933,6 +993,35 @@ export function geminiSplit(raw: Record<string, unknown>, context?: TransformCon
|
|
|
933
993
|
return entries;
|
|
934
994
|
}
|
|
935
995
|
|
|
996
|
+
const UUID_PATTERN = '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}';
|
|
997
|
+
const AGY_BRAIN_REGEX = new RegExp(`(?:^|\\/)brain\\/(${UUID_PATTERN})(?:\\/|$)`, 'i');
|
|
998
|
+
const CODEX_ROLLOUT_REGEX = new RegExp(`(?:^|-|_)(${UUID_PATTERN})$`, 'i');
|
|
999
|
+
|
|
1000
|
+
/**
|
|
1001
|
+
* Pure helper extracting canonical session identity from source transcript paths (task 0638 R4).
|
|
1002
|
+
*
|
|
1003
|
+
* Normalizes POSIX/Windows separators and returns only valid UUID-shaped identifiers:
|
|
1004
|
+
* - agy: segment immediately following `/brain/`
|
|
1005
|
+
* - codex: trailing UUID before `.jsonl` in a filename beginning with `rollout-`
|
|
1006
|
+
*/
|
|
1007
|
+
export function sessionIdFromSourcePath(source: string, sourceFile: string): string | undefined {
|
|
1008
|
+
const normalized = sourceFile.replace(/\\/g, '/');
|
|
1009
|
+
const sLower = source.toLowerCase();
|
|
1010
|
+
if (sLower === 'agy' || sLower === 'antigravity') {
|
|
1011
|
+
const match = normalized.match(AGY_BRAIN_REGEX);
|
|
1012
|
+
return match?.[1];
|
|
1013
|
+
}
|
|
1014
|
+
if (sLower === 'codex') {
|
|
1015
|
+
const basename = normalized.split('/').at(-1) ?? '';
|
|
1016
|
+
if (basename.startsWith('rollout-') && basename.endsWith('.jsonl')) {
|
|
1017
|
+
const stem = basename.slice(0, -6);
|
|
1018
|
+
const match = stem.match(CODEX_ROLLOUT_REGEX);
|
|
1019
|
+
return match?.[1];
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
return undefined;
|
|
1023
|
+
}
|
|
1024
|
+
|
|
936
1025
|
function sessionIdFromContext(context: TransformContext | undefined, raw: Record<string, unknown>): string {
|
|
937
1026
|
if (context !== undefined) {
|
|
938
1027
|
const file = context.sourceFile.split('/').at(-1);
|