@gobing-ai/ts-llm-jsonl-importer 0.4.36 → 0.4.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/mappers.d.ts +5 -0
- package/dist/mappers.d.ts.map +1 -1
- package/dist/mappers.js +159 -72
- package/dist/opencode-importer.d.ts.map +1 -1
- package/dist/opencode-importer.js +2 -0
- package/dist/schema-sql.js +1 -1
- package/package.json +4 -4
- package/src/mappers.ts +162 -76
- package/src/opencode-importer.ts +2 -0
- package/src/schema-sql.ts +1 -1
package/dist/mappers.d.ts
CHANGED
|
@@ -2,6 +2,11 @@ import { z } from 'zod';
|
|
|
2
2
|
import type { JsonObject, SplitEntry, TransformContext } from './types';
|
|
3
3
|
/** Compute args_digest: sha256 of stable-jsonified, key-sorted, redacted args. */
|
|
4
4
|
export declare function argsDigest(args: unknown): string;
|
|
5
|
+
/**
|
|
6
|
+
* Return JSON-stringified raw args when the tool is a todo-writing tool for its source,
|
|
7
|
+
* otherwise undefined. Codex `arguments` arrives as a JSON string already — store as-is.
|
|
8
|
+
*/
|
|
9
|
+
export declare function maybeArgsRaw(source: string, toolName: string, args: unknown): string | undefined;
|
|
5
10
|
/** Map a Claude Code JSONL record into one or more ETL split entries. */
|
|
6
11
|
export declare function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
7
12
|
/** Map a Pi JSONL record into one or more ETL split entries. */
|
package/dist/mappers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAwCD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAMhG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAgH/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAoK9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAiMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAgFD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
|
package/dist/mappers.js
CHANGED
|
@@ -89,14 +89,17 @@ function redactArgs(args) {
|
|
|
89
89
|
*
|
|
90
90
|
* Evidence: 0489 R4 confirmed omp/pi/claude; codex/grok/agy probed task 0553 R3 from real
|
|
91
91
|
* session JSONL. agy has no on-disk session format (VS Code fork); gemini deferred by the
|
|
92
|
-
* 2026-08-06 source-support ruling.
|
|
92
|
+
* 2026-08-06 source-support ruling. Task 0578 R3: omp `todo_write`, pi `manage_todo_list`,
|
|
93
|
+
* opencode `todowrite`/`todoread` measured live in .spur/spur.db (3,237/21, 1,043, 379/2
|
|
94
|
+
* rows with NULL args_raw).
|
|
93
95
|
*/
|
|
94
96
|
const TODO_TOOL_ALLOWLIST = {
|
|
95
97
|
claude: ['TodoWrite'],
|
|
96
|
-
|
|
97
|
-
omp: ['TodoWrite', 'todo'],
|
|
98
|
+
opencode: ['todowrite', 'todoread'],
|
|
98
99
|
codex: ['update_plan'],
|
|
100
|
+
omp: ['TodoWrite', 'todo', 'todo_write'],
|
|
99
101
|
grok: ['todo_write'],
|
|
102
|
+
pi: ['todo', 'manage_todo_list'],
|
|
100
103
|
agy: [],
|
|
101
104
|
gemini: [],
|
|
102
105
|
};
|
|
@@ -104,7 +107,7 @@ const TODO_TOOL_ALLOWLIST = {
|
|
|
104
107
|
* Return JSON-stringified raw args when the tool is a todo-writing tool for its source,
|
|
105
108
|
* otherwise undefined. Codex `arguments` arrives as a JSON string already — store as-is.
|
|
106
109
|
*/
|
|
107
|
-
function maybeArgsRaw(source, toolName, args) {
|
|
110
|
+
export function maybeArgsRaw(source, toolName, args) {
|
|
108
111
|
const allow = TODO_TOOL_ALLOWLIST[source];
|
|
109
112
|
if (allow === undefined || !allow.includes(toolName))
|
|
110
113
|
return undefined;
|
|
@@ -122,10 +125,10 @@ export function claudeSplit(raw) {
|
|
|
122
125
|
const entries = [];
|
|
123
126
|
const sessionId = s(raw.sessionId, raw.conversation_uuid) ?? 'unknown';
|
|
124
127
|
const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.messageIndex === 'number' ? raw.messageIndex : 0;
|
|
125
|
-
const ts =
|
|
128
|
+
const ts = timestampOf(raw.ts, raw.timestamp, raw.createdAt);
|
|
126
129
|
const role = mapRole(raw.type ?? raw.role);
|
|
127
130
|
const recordType = String(raw.type ?? '');
|
|
128
|
-
const model = s(raw.model);
|
|
131
|
+
const model = s(raw.model, o(raw.message).model);
|
|
129
132
|
const cwd = s(raw.cwd, raw.dir);
|
|
130
133
|
// Check for system/turn-duration records
|
|
131
134
|
if (raw.subtype === 'turn_duration' && typeof raw.durationMs === 'number') {
|
|
@@ -162,13 +165,13 @@ export function claudeSplit(raw) {
|
|
|
162
165
|
}
|
|
163
166
|
// Main message
|
|
164
167
|
const hasUsage = typeof raw.usage === 'object' && raw.usage !== null;
|
|
165
|
-
const usage = hasUsage ? raw.usage :
|
|
166
|
-
const inputTokens =
|
|
167
|
-
const outputTokens =
|
|
168
|
-
const cacheRead = (usage?.cacheReadTokens ?? usage?.cache_read_tokens ??
|
|
169
|
-
const cacheWrite = (usage?.cacheWriteTokens ?? usage?.cache_write_tokens ??
|
|
168
|
+
const usage = (hasUsage ? raw.usage : o(raw.message).usage);
|
|
169
|
+
const inputTokens = (raw.inputTokens ?? usage?.input_tokens);
|
|
170
|
+
const outputTokens = (raw.outputTokens ?? usage?.output_tokens);
|
|
171
|
+
const cacheRead = (usage?.cacheReadTokens ?? usage?.cache_read_tokens ?? usage?.cache_read_input_tokens);
|
|
172
|
+
const cacheWrite = (usage?.cacheWriteTokens ?? usage?.cache_write_tokens ?? usage?.cache_creation_input_tokens);
|
|
170
173
|
const costUsd = computeCost(inputTokens, outputTokens, model);
|
|
171
|
-
const contentBlocks = raw.content;
|
|
174
|
+
const contentBlocks = raw.content ?? o(raw.message).content;
|
|
172
175
|
const contentText = extractContentText(contentBlocks);
|
|
173
176
|
const messageSplitIndex = entries.length;
|
|
174
177
|
entries.push({
|
|
@@ -235,10 +238,9 @@ export function piSplit(raw, context) {
|
|
|
235
238
|
const recordType = String(raw.type ?? raw.recordType ?? '');
|
|
236
239
|
const sessionId = sessionIdFromContext(context, raw);
|
|
237
240
|
const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
|
|
238
|
-
// Prefer the ISO `timestamp`;
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
new Date(0).toISOString();
|
|
241
|
+
// Prefer the ISO `timestamp`; numeric epoch values are converted (task 0580 R6), never
|
|
242
|
+
// stringified, and a missing ts persists as null rather than a 1970 sentinel (D4).
|
|
243
|
+
const ts = timestampOf(raw.timestamp, raw.createdAt, raw.ts);
|
|
242
244
|
// Meta lifecycle/custom records collapse to one meta row keyed by the source session,
|
|
243
245
|
// never the unique event id, and never a guessed role.
|
|
244
246
|
if (recordType === 'title' ||
|
|
@@ -363,7 +365,7 @@ export function ompSplit(raw, context) {
|
|
|
363
365
|
const recordType = String(raw.type ?? '');
|
|
364
366
|
const sessionId = sessionIdFromContext(context, raw);
|
|
365
367
|
const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
|
|
366
|
-
const ts =
|
|
368
|
+
const ts = timestampOf(raw.ts, raw.timestamp, raw.createdAt);
|
|
367
369
|
// Meta lifecycle/custom records — including the current `custom.*` events and non-message
|
|
368
370
|
// lifecycle types — collapse to one meta row keyed by the source session, never the unique
|
|
369
371
|
// event id, and never a guessed role.
|
|
@@ -517,8 +519,9 @@ export function codexSplit(raw) {
|
|
|
517
519
|
const payload = (raw.payload ?? raw);
|
|
518
520
|
const sessionId = s(raw.session_id, o(raw.session_meta).id, raw.id) ?? 'unknown';
|
|
519
521
|
const seq = typeof raw.seq === 'number' ? raw.seq : 0;
|
|
520
|
-
const ts =
|
|
522
|
+
const ts = timestampOf(raw.timestamp, raw.ts, payload.ts);
|
|
521
523
|
const recordType = String(raw.type ?? '');
|
|
524
|
+
const payloadType = String(payload.type ?? '');
|
|
522
525
|
// Check for older short format
|
|
523
526
|
if (recordType === '' && raw.id && raw.timestamp && raw.instructions) {
|
|
524
527
|
return [
|
|
@@ -536,18 +539,77 @@ export function codexSplit(raw) {
|
|
|
536
539
|
},
|
|
537
540
|
];
|
|
538
541
|
}
|
|
539
|
-
|
|
542
|
+
// Codex JSONL is a transport envelope: the top-level `type` (response_item / event_msg /
|
|
543
|
+
// turn_context / ...) is bookkeeping, never a role. Real roles live in
|
|
544
|
+
// `payload.role` for response_item message items (task 0580 D1: passing recordType to
|
|
545
|
+
// mapRole leaked 153k response_item / 91k event_msg rows into the role column).
|
|
546
|
+
let role;
|
|
547
|
+
let disposition;
|
|
548
|
+
let contentText;
|
|
549
|
+
let usage;
|
|
550
|
+
if (recordType === 'response_item') {
|
|
551
|
+
if (payloadType === 'message') {
|
|
552
|
+
role = mapRole(payload.role);
|
|
553
|
+
disposition = 'keep';
|
|
554
|
+
contentText = extractContentText(payload.content) ?? s(payload.text) ?? null;
|
|
555
|
+
}
|
|
556
|
+
else if (payloadType === 'agent_message') {
|
|
557
|
+
role = 'assistant';
|
|
558
|
+
disposition = 'keep';
|
|
559
|
+
contentText = extractContentText(payload.content) ?? s(payload.text, payload.message) ?? null;
|
|
560
|
+
}
|
|
561
|
+
else if (payloadType === 'reasoning') {
|
|
562
|
+
role = 'assistant';
|
|
563
|
+
disposition = 'keep';
|
|
564
|
+
contentText = extractContentText(payload.summary) ?? s(payload.text) ?? null;
|
|
565
|
+
}
|
|
566
|
+
else if (payloadType === 'function_call' || payloadType === 'custom_tool_call') {
|
|
567
|
+
role = 'tool';
|
|
568
|
+
disposition = 'keep';
|
|
569
|
+
contentText = null;
|
|
570
|
+
}
|
|
571
|
+
else {
|
|
572
|
+
role = 'meta';
|
|
573
|
+
disposition = 'meta';
|
|
574
|
+
contentText = s(payload.text) ?? null;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
else if (recordType === 'user' || recordType === 'assistant') {
|
|
578
|
+
// Legacy (pre-rollout) envelope: top-level type carried the role directly.
|
|
579
|
+
role = mapRole(recordType);
|
|
580
|
+
disposition = 'keep';
|
|
581
|
+
contentText = extractContentText(payload.content) ?? s(raw.content, payload.text) ?? null;
|
|
582
|
+
}
|
|
583
|
+
else if (recordType === 'message') {
|
|
584
|
+
role = mapRole(raw.role ?? payload.role);
|
|
585
|
+
disposition = 'keep';
|
|
586
|
+
contentText = extractContentText(payload.content ?? raw.content) ?? s(payload.text, raw.text) ?? null;
|
|
587
|
+
}
|
|
588
|
+
else if (recordType === 'event_msg') {
|
|
589
|
+
role = 'meta';
|
|
590
|
+
disposition = 'meta';
|
|
591
|
+
contentText = s(payload.message, payload.text) ?? null;
|
|
592
|
+
// Per-turn usage lives on token_count transport events (task 0580 R2): real files
|
|
593
|
+
// carry `payload.info.last_token_usage`; the legacy `payload.token_count` path is kept.
|
|
594
|
+
usage = o(o(payload.info).last_token_usage);
|
|
595
|
+
}
|
|
596
|
+
else {
|
|
597
|
+
role = 'meta';
|
|
598
|
+
disposition = 'meta';
|
|
599
|
+
contentText = s(payload.text, raw.instructions) ?? null;
|
|
600
|
+
}
|
|
540
601
|
const model = s(raw.model, payload.model, o(o(raw.turn_context).payload).model);
|
|
541
602
|
const cwd = s(raw.cwd, raw.dir);
|
|
542
603
|
const tokenCount = payload.token_count;
|
|
543
|
-
const
|
|
544
|
-
const
|
|
604
|
+
const usageInput = (usage?.input_tokens ?? tokenCount?.input ?? tokenCount?.input_tokens ?? undefined);
|
|
605
|
+
const usageOutput = (usage?.output_tokens ?? tokenCount?.output ?? tokenCount?.output_tokens ?? undefined);
|
|
606
|
+
const cacheRead = (usage?.cached_input_tokens ?? undefined);
|
|
545
607
|
const turnCtx = raw.turn_context;
|
|
546
608
|
const turnPayload = turnCtx?.payload;
|
|
547
609
|
const turnInputTokens = turnPayload?.input_tokens;
|
|
548
610
|
const turnOutputTokens = turnPayload?.output_tokens;
|
|
549
|
-
const finalInput =
|
|
550
|
-
const finalOutput =
|
|
611
|
+
const finalInput = usageInput !== undefined ? usageInput : turnInputTokens;
|
|
612
|
+
const finalOutput = usageOutput !== undefined ? usageOutput : turnOutputTokens;
|
|
551
613
|
const costUsd = computeCost(finalInput, finalOutput, model);
|
|
552
614
|
const messageSplitIndex = entries.length;
|
|
553
615
|
entries.push({
|
|
@@ -557,47 +619,56 @@ export function codexSplit(raw) {
|
|
|
557
619
|
seq,
|
|
558
620
|
role,
|
|
559
621
|
record_type: recordType,
|
|
560
|
-
disposition
|
|
622
|
+
disposition,
|
|
561
623
|
ts,
|
|
562
624
|
duration_ms: undefined,
|
|
563
625
|
model: model ?? null,
|
|
564
626
|
input_tokens: finalInput ?? null,
|
|
565
627
|
output_tokens: finalOutput ?? null,
|
|
566
|
-
cache_read_tokens: null,
|
|
628
|
+
cache_read_tokens: cacheRead ?? null,
|
|
567
629
|
cache_write_tokens: null,
|
|
568
630
|
cost_usd: costUsd ?? null,
|
|
569
|
-
content_text:
|
|
631
|
+
content_text: contentText,
|
|
570
632
|
cwd: cwd ?? null,
|
|
571
633
|
provenance: 'ambient',
|
|
572
634
|
},
|
|
573
635
|
});
|
|
574
|
-
// Tool calls
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
636
|
+
// Tool calls. Live rollouts emit each call as its own response_item whose payload IS the
|
|
637
|
+
// item (`payload.type === "function_call" | "custom_tool_call"`, name/call_id/arguments or
|
|
638
|
+
// input at payload level); the legacy nested `payload.response_item[_s]` shape is kept.
|
|
639
|
+
const toolItems = [];
|
|
640
|
+
if ((recordType === 'response_item' || recordType === 'function_call' || recordType === 'custom_tool_call') &&
|
|
641
|
+
(payloadType === 'function_call' || payloadType === 'custom_tool_call')) {
|
|
642
|
+
toolItems.push(payload);
|
|
643
|
+
}
|
|
644
|
+
else if (payload.response_item) {
|
|
645
|
+
toolItems.push(payload.response_item);
|
|
646
|
+
}
|
|
647
|
+
else if (Array.isArray(payload.response_items)) {
|
|
648
|
+
toolItems.push(...payload.response_items);
|
|
649
|
+
}
|
|
650
|
+
for (const item of toolItems) {
|
|
651
|
+
const fc = item.function_call ?? item;
|
|
652
|
+
const toolName = s(fc.name) ?? '';
|
|
653
|
+
const args = fc.arguments ?? fc.input;
|
|
654
|
+
entries.push({
|
|
655
|
+
targetTable: 'history_tool_call',
|
|
656
|
+
record: {
|
|
657
|
+
_messageSplitIndex: messageSplitIndex,
|
|
658
|
+
session_id: sessionId,
|
|
659
|
+
seq,
|
|
660
|
+
tool_name: toolName,
|
|
661
|
+
call_id: s(fc.call_id),
|
|
662
|
+
args_digest: argsDigest(args),
|
|
663
|
+
args_raw: maybeArgsRaw('codex', toolName, args),
|
|
664
|
+
status: 'ok',
|
|
665
|
+
started_at: undefined,
|
|
666
|
+
completed_at: undefined,
|
|
667
|
+
duration_ms: undefined,
|
|
668
|
+
result_bytes: undefined,
|
|
669
|
+
error_text: undefined,
|
|
670
|
+
},
|
|
671
|
+
});
|
|
601
672
|
}
|
|
602
673
|
return entries;
|
|
603
674
|
}
|
|
@@ -610,7 +681,7 @@ export function agySplit(raw) {
|
|
|
610
681
|
const recordType = String(raw.type ?? '');
|
|
611
682
|
const sessionId = s(raw.session_id, raw.conversation_id) ?? 'unknown';
|
|
612
683
|
const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.step_index === 'number' ? raw.step_index : 0;
|
|
613
|
-
const ts =
|
|
684
|
+
const ts = timestampOf(raw.created_at, raw.timestamp, raw.ts);
|
|
614
685
|
let role;
|
|
615
686
|
let disposition;
|
|
616
687
|
let contentText = null;
|
|
@@ -723,7 +794,7 @@ export function geminiSplit(raw, context) {
|
|
|
723
794
|
const state = o(raw.$set);
|
|
724
795
|
const sessionId = sessionIdFromContext(context, raw);
|
|
725
796
|
const seq = context?.sourceLine ?? 0;
|
|
726
|
-
const ts =
|
|
797
|
+
const ts = timestampOf(raw.timestamp, raw.startTime, raw.lastUpdated, state.lastUpdated);
|
|
727
798
|
if (recordType === 'session' || recordType === 'state') {
|
|
728
799
|
return [
|
|
729
800
|
{
|
|
@@ -858,7 +929,12 @@ function normalizeGrokRecord(raw) {
|
|
|
858
929
|
: undefined));
|
|
859
930
|
const usage = (body.usage ?? raw.usage);
|
|
860
931
|
const toolMeta = meta?.['x.ai/tool'];
|
|
861
|
-
|
|
932
|
+
// The tool's real name is `_meta["x.ai/tool"].name` (or `kind`); `title` is a human label
|
|
933
|
+
// like "Read `/path`" — used only as a last resort, stripped at the first backtick so the
|
|
934
|
+
// argument payload never pollutes tool_name (task 0580 D3).
|
|
935
|
+
const titleFallback = s(body.title)?.split('`')[0]?.trim();
|
|
936
|
+
const toolName = s(toolMeta?.name, body.kind, body.tool_name, body.name, raw.tool_name, raw.name) ??
|
|
937
|
+
(titleFallback !== undefined && titleFallback.length > 0 && titleFallback.length <= 40 ? titleFallback : '');
|
|
862
938
|
const toolArgs = body.rawInput ?? body.input ?? body.arguments ?? toolMeta?.input ?? raw.input ?? raw.arguments;
|
|
863
939
|
const toolStatus = s(body.status, raw.outcome, raw.status) ?? 'ok';
|
|
864
940
|
const durationMs = typeof body.duration_ms === 'number'
|
|
@@ -888,20 +964,7 @@ function normalizeGrokRecord(raw) {
|
|
|
888
964
|
};
|
|
889
965
|
}
|
|
890
966
|
function normalizeTs(value) {
|
|
891
|
-
|
|
892
|
-
// Grok timestamps are unix seconds (sometimes with fractional).
|
|
893
|
-
const ms = value > 1e12 ? value : value * 1000;
|
|
894
|
-
return new Date(ms).toISOString();
|
|
895
|
-
}
|
|
896
|
-
if (typeof value === 'string' && value.length > 0) {
|
|
897
|
-
const asNum = Number(value);
|
|
898
|
-
if (Number.isFinite(asNum) && /^\d+(\.\d+)?$/.test(value.trim())) {
|
|
899
|
-
const ms = asNum > 1e12 ? asNum : asNum * 1000;
|
|
900
|
-
return new Date(ms).toISOString();
|
|
901
|
-
}
|
|
902
|
-
return value;
|
|
903
|
-
}
|
|
904
|
-
return new Date(0).toISOString();
|
|
967
|
+
return timestampOf(value);
|
|
905
968
|
}
|
|
906
969
|
function extractGrokContent(content) {
|
|
907
970
|
if (typeof content === 'string')
|
|
@@ -1265,9 +1328,31 @@ function s(...values) {
|
|
|
1265
1328
|
}
|
|
1266
1329
|
return undefined;
|
|
1267
1330
|
}
|
|
1331
|
+
/**
|
|
1332
|
+
* First usable timestamp: an ISO-ish string passes through, a finite number (or digit-string)
|
|
1333
|
+
* is converted from epoch seconds/millis (>1e12 treated as ms). Returns undefined when nothing
|
|
1334
|
+
* usable exists — mappers must persist null ts rather than a 1970 sentinel (task 0580 D4).
|
|
1335
|
+
*/
|
|
1336
|
+
function timestampOf(...values) {
|
|
1337
|
+
for (const value of values) {
|
|
1338
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
1339
|
+
const ms = value > 1e12 ? value : value * 1000;
|
|
1340
|
+
return new Date(ms).toISOString();
|
|
1341
|
+
}
|
|
1342
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
1343
|
+
const asNum = Number(value);
|
|
1344
|
+
if (Number.isFinite(asNum) && /^\d+(\.\d+)?$/.test(value.trim())) {
|
|
1345
|
+
const ms = asNum > 1e12 ? asNum : asNum * 1000;
|
|
1346
|
+
return new Date(ms).toISOString();
|
|
1347
|
+
}
|
|
1348
|
+
return value;
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
return undefined;
|
|
1352
|
+
}
|
|
1268
1353
|
function mapRole(type) {
|
|
1269
1354
|
const t = String(type ?? '').toLowerCase();
|
|
1270
|
-
if (t === 'user' || t === 'human')
|
|
1355
|
+
if (t === 'user' || t === 'human' || t === 'developer')
|
|
1271
1356
|
return 'user';
|
|
1272
1357
|
if (t === 'assistant' || t === 'ai' || t === 'model')
|
|
1273
1358
|
return 'assistant';
|
|
@@ -1296,7 +1381,9 @@ function extractContentText(content) {
|
|
|
1296
1381
|
if (typeof block !== 'object' || block === null)
|
|
1297
1382
|
continue;
|
|
1298
1383
|
const b = block;
|
|
1299
|
-
|
|
1384
|
+
// Claude emits {type:"text",text}; codex emits {type:"input_text"|"output_text"|
|
|
1385
|
+
// "summary_text",text} — any block carrying a string `text` is text content.
|
|
1386
|
+
if (typeof b.text === 'string') {
|
|
1300
1387
|
parts.push(b.text);
|
|
1301
1388
|
}
|
|
1302
1389
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAgB/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAoI7F"}
|
|
@@ -2,6 +2,7 @@ import { createDbAdapter } from '@gobing-ai/ts-db';
|
|
|
2
2
|
import { ambientRuntimePaths, createNodeFileSystem, joinPath, } from '@gobing-ai/ts-runtime';
|
|
3
3
|
import { sha256 } from './hash.js';
|
|
4
4
|
import { applyHistoryImportSchema, deleteCheckpointOperation, openCodeBulkWriteOperations, openCodeDeleteOperations, readOpenCodeExistingEntries, readOpenCodeMessages, readOpenCodeParts, } from './jsonl-importer-dao.js';
|
|
5
|
+
import { maybeArgsRaw } from './mappers.js';
|
|
5
6
|
import { redactRecord } from './redaction.js';
|
|
6
7
|
const SOURCE = 'opencode';
|
|
7
8
|
const PAGE_SIZE = 250;
|
|
@@ -202,6 +203,7 @@ function prepareEntries(row, data, parts, sourceFile, redactionRules) {
|
|
|
202
203
|
session_id: row.session_id,
|
|
203
204
|
seq: started ?? created,
|
|
204
205
|
tool_name: stringValue(part.tool) ?? 'unknown',
|
|
206
|
+
args_raw: maybeArgsRaw(SOURCE, stringValue(part.tool) ?? 'unknown', state.input),
|
|
205
207
|
args_digest: sha256(state.input ?? null),
|
|
206
208
|
status,
|
|
207
209
|
started_at: started === undefined ? null : isoTime(started),
|
package/dist/schema-sql.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.38",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.38",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.38",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.38",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
package/src/mappers.ts
CHANGED
|
@@ -96,14 +96,17 @@ function redactArgs(args: unknown): unknown {
|
|
|
96
96
|
*
|
|
97
97
|
* Evidence: 0489 R4 confirmed omp/pi/claude; codex/grok/agy probed task 0553 R3 from real
|
|
98
98
|
* session JSONL. agy has no on-disk session format (VS Code fork); gemini deferred by the
|
|
99
|
-
* 2026-08-06 source-support ruling.
|
|
99
|
+
* 2026-08-06 source-support ruling. Task 0578 R3: omp `todo_write`, pi `manage_todo_list`,
|
|
100
|
+
* opencode `todowrite`/`todoread` measured live in .spur/spur.db (3,237/21, 1,043, 379/2
|
|
101
|
+
* rows with NULL args_raw).
|
|
100
102
|
*/
|
|
101
103
|
const TODO_TOOL_ALLOWLIST: Readonly<Record<string, readonly string[]>> = {
|
|
102
104
|
claude: ['TodoWrite'],
|
|
103
|
-
|
|
104
|
-
omp: ['TodoWrite', 'todo'],
|
|
105
|
+
opencode: ['todowrite', 'todoread'],
|
|
105
106
|
codex: ['update_plan'],
|
|
107
|
+
omp: ['TodoWrite', 'todo', 'todo_write'],
|
|
106
108
|
grok: ['todo_write'],
|
|
109
|
+
pi: ['todo', 'manage_todo_list'],
|
|
107
110
|
agy: [],
|
|
108
111
|
gemini: [],
|
|
109
112
|
};
|
|
@@ -112,7 +115,7 @@ const TODO_TOOL_ALLOWLIST: Readonly<Record<string, readonly string[]>> = {
|
|
|
112
115
|
* Return JSON-stringified raw args when the tool is a todo-writing tool for its source,
|
|
113
116
|
* otherwise undefined. Codex `arguments` arrives as a JSON string already — store as-is.
|
|
114
117
|
*/
|
|
115
|
-
function maybeArgsRaw(source: string, toolName: string, args: unknown): string | undefined {
|
|
118
|
+
export function maybeArgsRaw(source: string, toolName: string, args: unknown): string | undefined {
|
|
116
119
|
const allow = TODO_TOOL_ALLOWLIST[source];
|
|
117
120
|
if (allow === undefined || !allow.includes(toolName)) return undefined;
|
|
118
121
|
if (typeof args === 'string') return args;
|
|
@@ -129,10 +132,10 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
129
132
|
const entries: SplitEntry[] = [];
|
|
130
133
|
const sessionId = s(raw.sessionId, raw.conversation_uuid) ?? 'unknown';
|
|
131
134
|
const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.messageIndex === 'number' ? raw.messageIndex : 0;
|
|
132
|
-
const ts =
|
|
135
|
+
const ts = timestampOf(raw.ts, raw.timestamp, raw.createdAt);
|
|
133
136
|
const role = mapRole(raw.type ?? raw.role);
|
|
134
137
|
const recordType = String(raw.type ?? '');
|
|
135
|
-
const model = s(raw.model);
|
|
138
|
+
const model = s(raw.model, o(raw.message).model);
|
|
136
139
|
const cwd = s(raw.cwd, raw.dir);
|
|
137
140
|
|
|
138
141
|
// Check for system/turn-duration records
|
|
@@ -171,14 +174,17 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
171
174
|
|
|
172
175
|
// Main message
|
|
173
176
|
const hasUsage = typeof raw.usage === 'object' && raw.usage !== null;
|
|
174
|
-
const usage = hasUsage ? (raw.usage as Record<string, unknown>
|
|
175
|
-
const inputTokens =
|
|
176
|
-
const outputTokens =
|
|
177
|
-
const cacheRead = (usage?.cacheReadTokens ?? usage?.cache_read_tokens ??
|
|
178
|
-
|
|
177
|
+
const usage = (hasUsage ? raw.usage : o(raw.message).usage) as Record<string, unknown> | undefined;
|
|
178
|
+
const inputTokens = ((raw.inputTokens as number | undefined) ?? usage?.input_tokens) as number | undefined;
|
|
179
|
+
const outputTokens = ((raw.outputTokens as number | undefined) ?? usage?.output_tokens) as number | undefined;
|
|
180
|
+
const cacheRead = (usage?.cacheReadTokens ?? usage?.cache_read_tokens ?? usage?.cache_read_input_tokens) as
|
|
181
|
+
| number
|
|
182
|
+
| undefined;
|
|
183
|
+
const cacheWrite = (usage?.cacheWriteTokens ?? usage?.cache_write_tokens ?? usage?.cache_creation_input_tokens) as
|
|
184
|
+
| number
|
|
185
|
+
| undefined;
|
|
179
186
|
const costUsd = computeCost(inputTokens, outputTokens, model);
|
|
180
|
-
|
|
181
|
-
const contentBlocks = raw.content;
|
|
187
|
+
const contentBlocks = raw.content ?? o(raw.message).content;
|
|
182
188
|
const contentText = extractContentText(contentBlocks);
|
|
183
189
|
|
|
184
190
|
const messageSplitIndex = entries.length;
|
|
@@ -250,11 +256,9 @@ export function piSplit(raw: Record<string, unknown>, context?: TransformContext
|
|
|
250
256
|
const recordType = String(raw.type ?? raw.recordType ?? '');
|
|
251
257
|
const sessionId = sessionIdFromContext(context, raw);
|
|
252
258
|
const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
|
|
253
|
-
// Prefer the ISO `timestamp`;
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
(typeof raw.ts === 'number' && Number.isFinite(raw.ts) ? new Date(raw.ts).toISOString() : s(raw.ts)) ??
|
|
257
|
-
new Date(0).toISOString();
|
|
259
|
+
// Prefer the ISO `timestamp`; numeric epoch values are converted (task 0580 R6), never
|
|
260
|
+
// stringified, and a missing ts persists as null rather than a 1970 sentinel (D4).
|
|
261
|
+
const ts = timestampOf(raw.timestamp, raw.createdAt, raw.ts);
|
|
258
262
|
|
|
259
263
|
// Meta lifecycle/custom records collapse to one meta row keyed by the source session,
|
|
260
264
|
// never the unique event id, and never a guessed role.
|
|
@@ -387,7 +391,7 @@ export function ompSplit(raw: Record<string, unknown>, context?: TransformContex
|
|
|
387
391
|
const recordType = String(raw.type ?? '');
|
|
388
392
|
const sessionId = sessionIdFromContext(context, raw);
|
|
389
393
|
const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
|
|
390
|
-
const ts =
|
|
394
|
+
const ts = timestampOf(raw.ts, raw.timestamp, raw.createdAt);
|
|
391
395
|
|
|
392
396
|
// Meta lifecycle/custom records — including the current `custom.*` events and non-message
|
|
393
397
|
// lifecycle types — collapse to one meta row keyed by the source session, never the unique
|
|
@@ -561,8 +565,9 @@ export function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
561
565
|
const payload = (raw.payload ?? raw) as Record<string, unknown>;
|
|
562
566
|
const sessionId = s(raw.session_id, o(raw.session_meta).id, raw.id) ?? 'unknown';
|
|
563
567
|
const seq = typeof raw.seq === 'number' ? raw.seq : 0;
|
|
564
|
-
const ts =
|
|
568
|
+
const ts = timestampOf(raw.timestamp, raw.ts, payload.ts);
|
|
565
569
|
const recordType = String(raw.type ?? '');
|
|
570
|
+
const payloadType = String(payload.type ?? '');
|
|
566
571
|
|
|
567
572
|
// Check for older short format
|
|
568
573
|
if (recordType === '' && raw.id && raw.timestamp && raw.instructions) {
|
|
@@ -582,21 +587,77 @@ export function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
582
587
|
];
|
|
583
588
|
}
|
|
584
589
|
|
|
585
|
-
|
|
590
|
+
// Codex JSONL is a transport envelope: the top-level `type` (response_item / event_msg /
|
|
591
|
+
// turn_context / ...) is bookkeeping, never a role. Real roles live in
|
|
592
|
+
// `payload.role` for response_item message items (task 0580 D1: passing recordType to
|
|
593
|
+
// mapRole leaked 153k response_item / 91k event_msg rows into the role column).
|
|
594
|
+
let role: string;
|
|
595
|
+
let disposition: string;
|
|
596
|
+
let contentText: string | null;
|
|
597
|
+
let usage: Record<string, unknown> | undefined;
|
|
598
|
+
if (recordType === 'response_item') {
|
|
599
|
+
if (payloadType === 'message') {
|
|
600
|
+
role = mapRole(payload.role);
|
|
601
|
+
disposition = 'keep';
|
|
602
|
+
contentText = extractContentText(payload.content) ?? s(payload.text) ?? null;
|
|
603
|
+
} else if (payloadType === 'agent_message') {
|
|
604
|
+
role = 'assistant';
|
|
605
|
+
disposition = 'keep';
|
|
606
|
+
contentText = extractContentText(payload.content) ?? s(payload.text, payload.message) ?? null;
|
|
607
|
+
} else if (payloadType === 'reasoning') {
|
|
608
|
+
role = 'assistant';
|
|
609
|
+
disposition = 'keep';
|
|
610
|
+
contentText = extractContentText(payload.summary) ?? s(payload.text) ?? null;
|
|
611
|
+
} else if (payloadType === 'function_call' || payloadType === 'custom_tool_call') {
|
|
612
|
+
role = 'tool';
|
|
613
|
+
disposition = 'keep';
|
|
614
|
+
contentText = null;
|
|
615
|
+
} else {
|
|
616
|
+
role = 'meta';
|
|
617
|
+
disposition = 'meta';
|
|
618
|
+
contentText = s(payload.text) ?? null;
|
|
619
|
+
}
|
|
620
|
+
} else if (recordType === 'user' || recordType === 'assistant') {
|
|
621
|
+
// Legacy (pre-rollout) envelope: top-level type carried the role directly.
|
|
622
|
+
role = mapRole(recordType);
|
|
623
|
+
disposition = 'keep';
|
|
624
|
+
contentText = extractContentText(payload.content) ?? s(raw.content, payload.text) ?? null;
|
|
625
|
+
} else if (recordType === 'message') {
|
|
626
|
+
role = mapRole(raw.role ?? payload.role);
|
|
627
|
+
disposition = 'keep';
|
|
628
|
+
contentText = extractContentText(payload.content ?? raw.content) ?? s(payload.text, raw.text) ?? null;
|
|
629
|
+
} else if (recordType === 'event_msg') {
|
|
630
|
+
role = 'meta';
|
|
631
|
+
disposition = 'meta';
|
|
632
|
+
contentText = s(payload.message, payload.text) ?? null;
|
|
633
|
+
// Per-turn usage lives on token_count transport events (task 0580 R2): real files
|
|
634
|
+
// carry `payload.info.last_token_usage`; the legacy `payload.token_count` path is kept.
|
|
635
|
+
usage = o(o(payload.info).last_token_usage);
|
|
636
|
+
} else {
|
|
637
|
+
role = 'meta';
|
|
638
|
+
disposition = 'meta';
|
|
639
|
+
contentText = s(payload.text, raw.instructions) ?? null;
|
|
640
|
+
}
|
|
641
|
+
|
|
586
642
|
const model = s(raw.model, payload.model, o(o(raw.turn_context).payload).model);
|
|
587
643
|
const cwd = s(raw.cwd, raw.dir);
|
|
588
644
|
|
|
589
645
|
const tokenCount = payload.token_count as Record<string, unknown> | undefined;
|
|
590
|
-
const
|
|
591
|
-
|
|
646
|
+
const usageInput = (usage?.input_tokens ?? tokenCount?.input ?? tokenCount?.input_tokens ?? undefined) as
|
|
647
|
+
| number
|
|
648
|
+
| undefined;
|
|
649
|
+
const usageOutput = (usage?.output_tokens ?? tokenCount?.output ?? tokenCount?.output_tokens ?? undefined) as
|
|
650
|
+
| number
|
|
651
|
+
| undefined;
|
|
652
|
+
const cacheRead = (usage?.cached_input_tokens ?? undefined) as number | undefined;
|
|
592
653
|
|
|
593
654
|
const turnCtx = raw.turn_context as Record<string, unknown> | undefined;
|
|
594
655
|
const turnPayload = turnCtx?.payload as Record<string, unknown> | undefined;
|
|
595
656
|
const turnInputTokens = turnPayload?.input_tokens as number | undefined;
|
|
596
657
|
const turnOutputTokens = turnPayload?.output_tokens as number | undefined;
|
|
597
658
|
|
|
598
|
-
const finalInput =
|
|
599
|
-
const finalOutput =
|
|
659
|
+
const finalInput = usageInput !== undefined ? usageInput : turnInputTokens;
|
|
660
|
+
const finalOutput = usageOutput !== undefined ? usageOutput : turnOutputTokens;
|
|
600
661
|
const costUsd = computeCost(finalInput, finalOutput, model);
|
|
601
662
|
|
|
602
663
|
const messageSplitIndex = entries.length;
|
|
@@ -608,48 +669,57 @@ export function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[]
|
|
|
608
669
|
seq,
|
|
609
670
|
role,
|
|
610
671
|
record_type: recordType,
|
|
611
|
-
disposition
|
|
672
|
+
disposition,
|
|
612
673
|
ts,
|
|
613
674
|
duration_ms: undefined,
|
|
614
675
|
model: model ?? null,
|
|
615
676
|
input_tokens: finalInput ?? null,
|
|
616
677
|
output_tokens: finalOutput ?? null,
|
|
617
|
-
cache_read_tokens: null,
|
|
678
|
+
cache_read_tokens: cacheRead ?? null,
|
|
618
679
|
cache_write_tokens: null,
|
|
619
680
|
cost_usd: costUsd ?? null,
|
|
620
|
-
content_text:
|
|
681
|
+
content_text: contentText,
|
|
621
682
|
cwd: cwd ?? null,
|
|
622
683
|
provenance: 'ambient',
|
|
623
684
|
},
|
|
624
685
|
});
|
|
625
686
|
|
|
626
|
-
// Tool calls
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
687
|
+
// Tool calls. Live rollouts emit each call as its own response_item whose payload IS the
|
|
688
|
+
// item (`payload.type === "function_call" | "custom_tool_call"`, name/call_id/arguments or
|
|
689
|
+
// input at payload level); the legacy nested `payload.response_item[_s]` shape is kept.
|
|
690
|
+
const toolItems: Record<string, unknown>[] = [];
|
|
691
|
+
if (
|
|
692
|
+
(recordType === 'response_item' || recordType === 'function_call' || recordType === 'custom_tool_call') &&
|
|
693
|
+
(payloadType === 'function_call' || payloadType === 'custom_tool_call')
|
|
694
|
+
) {
|
|
695
|
+
toolItems.push(payload);
|
|
696
|
+
} else if (payload.response_item) {
|
|
697
|
+
toolItems.push(payload.response_item as Record<string, unknown>);
|
|
698
|
+
} else if (Array.isArray(payload.response_items)) {
|
|
699
|
+
toolItems.push(...(payload.response_items as Record<string, unknown>[]));
|
|
700
|
+
}
|
|
701
|
+
for (const item of toolItems) {
|
|
702
|
+
const fc = (item.function_call as Record<string, unknown> | undefined) ?? item;
|
|
703
|
+
const toolName = s(fc.name) ?? '';
|
|
704
|
+
const args = fc.arguments ?? fc.input;
|
|
705
|
+
entries.push({
|
|
706
|
+
targetTable: 'history_tool_call',
|
|
707
|
+
record: {
|
|
708
|
+
_messageSplitIndex: messageSplitIndex,
|
|
709
|
+
session_id: sessionId,
|
|
710
|
+
seq,
|
|
711
|
+
tool_name: toolName,
|
|
712
|
+
call_id: s(fc.call_id),
|
|
713
|
+
args_digest: argsDigest(args),
|
|
714
|
+
args_raw: maybeArgsRaw('codex', toolName, args),
|
|
715
|
+
status: 'ok',
|
|
716
|
+
started_at: undefined,
|
|
717
|
+
completed_at: undefined,
|
|
718
|
+
duration_ms: undefined,
|
|
719
|
+
result_bytes: undefined,
|
|
720
|
+
error_text: undefined,
|
|
721
|
+
},
|
|
722
|
+
});
|
|
653
723
|
}
|
|
654
724
|
|
|
655
725
|
return entries;
|
|
@@ -665,7 +735,7 @@ export function agySplit(raw: Record<string, unknown>): readonly SplitEntry[] {
|
|
|
665
735
|
const recordType = String(raw.type ?? '');
|
|
666
736
|
const sessionId = s(raw.session_id, raw.conversation_id) ?? 'unknown';
|
|
667
737
|
const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.step_index === 'number' ? raw.step_index : 0;
|
|
668
|
-
const ts =
|
|
738
|
+
const ts = timestampOf(raw.created_at, raw.timestamp, raw.ts);
|
|
669
739
|
|
|
670
740
|
let role: string;
|
|
671
741
|
let disposition: string;
|
|
@@ -785,7 +855,7 @@ export function geminiSplit(raw: Record<string, unknown>, context?: TransformCon
|
|
|
785
855
|
const state = o(raw.$set);
|
|
786
856
|
const sessionId = sessionIdFromContext(context, raw);
|
|
787
857
|
const seq = context?.sourceLine ?? 0;
|
|
788
|
-
const ts =
|
|
858
|
+
const ts = timestampOf(raw.timestamp, raw.startTime, raw.lastUpdated, state.lastUpdated);
|
|
789
859
|
|
|
790
860
|
if (recordType === 'session' || recordType === 'state') {
|
|
791
861
|
return [
|
|
@@ -903,7 +973,7 @@ function numberOrNull(value: unknown): number | null {
|
|
|
903
973
|
function normalizeGrokRecord(raw: Record<string, unknown>): {
|
|
904
974
|
recordType: string;
|
|
905
975
|
sessionId: string;
|
|
906
|
-
ts: string;
|
|
976
|
+
ts: string | undefined;
|
|
907
977
|
seq: number;
|
|
908
978
|
body: Record<string, unknown>;
|
|
909
979
|
model: string | undefined;
|
|
@@ -958,8 +1028,13 @@ function normalizeGrokRecord(raw: Record<string, unknown>): {
|
|
|
958
1028
|
);
|
|
959
1029
|
const usage = (body.usage ?? raw.usage) as Record<string, unknown> | undefined;
|
|
960
1030
|
const toolMeta = meta?.['x.ai/tool'] as Record<string, unknown> | undefined;
|
|
1031
|
+
// The tool's real name is `_meta["x.ai/tool"].name` (or `kind`); `title` is a human label
|
|
1032
|
+
// like "Read `/path`" — used only as a last resort, stripped at the first backtick so the
|
|
1033
|
+
// argument payload never pollutes tool_name (task 0580 D3).
|
|
1034
|
+
const titleFallback = s(body.title)?.split('`')[0]?.trim();
|
|
961
1035
|
const toolName =
|
|
962
|
-
s(body.
|
|
1036
|
+
s(toolMeta?.name, body.kind, body.tool_name, body.name, raw.tool_name, raw.name) ??
|
|
1037
|
+
(titleFallback !== undefined && titleFallback.length > 0 && titleFallback.length <= 40 ? titleFallback : '');
|
|
963
1038
|
const toolArgs = body.rawInput ?? body.input ?? body.arguments ?? toolMeta?.input ?? raw.input ?? raw.arguments;
|
|
964
1039
|
const toolStatus = s(body.status, raw.outcome, raw.status) ?? 'ok';
|
|
965
1040
|
const durationMs =
|
|
@@ -992,21 +1067,8 @@ function normalizeGrokRecord(raw: Record<string, unknown>): {
|
|
|
992
1067
|
};
|
|
993
1068
|
}
|
|
994
1069
|
|
|
995
|
-
function normalizeTs(value: unknown): string {
|
|
996
|
-
|
|
997
|
-
// Grok timestamps are unix seconds (sometimes with fractional).
|
|
998
|
-
const ms = value > 1e12 ? value : value * 1000;
|
|
999
|
-
return new Date(ms).toISOString();
|
|
1000
|
-
}
|
|
1001
|
-
if (typeof value === 'string' && value.length > 0) {
|
|
1002
|
-
const asNum = Number(value);
|
|
1003
|
-
if (Number.isFinite(asNum) && /^\d+(\.\d+)?$/.test(value.trim())) {
|
|
1004
|
-
const ms = asNum > 1e12 ? asNum : asNum * 1000;
|
|
1005
|
-
return new Date(ms).toISOString();
|
|
1006
|
-
}
|
|
1007
|
-
return value;
|
|
1008
|
-
}
|
|
1009
|
-
return new Date(0).toISOString();
|
|
1070
|
+
function normalizeTs(value: unknown): string | undefined {
|
|
1071
|
+
return timestampOf(value);
|
|
1010
1072
|
}
|
|
1011
1073
|
|
|
1012
1074
|
function extractGrokContent(content: unknown): string | null {
|
|
@@ -1382,10 +1444,32 @@ function s(...values: readonly unknown[]): string | undefined {
|
|
|
1382
1444
|
}
|
|
1383
1445
|
return undefined;
|
|
1384
1446
|
}
|
|
1447
|
+
/**
|
|
1448
|
+
* First usable timestamp: an ISO-ish string passes through, a finite number (or digit-string)
|
|
1449
|
+
* is converted from epoch seconds/millis (>1e12 treated as ms). Returns undefined when nothing
|
|
1450
|
+
* usable exists — mappers must persist null ts rather than a 1970 sentinel (task 0580 D4).
|
|
1451
|
+
*/
|
|
1452
|
+
function timestampOf(...values: readonly unknown[]): string | undefined {
|
|
1453
|
+
for (const value of values) {
|
|
1454
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
1455
|
+
const ms = value > 1e12 ? value : value * 1000;
|
|
1456
|
+
return new Date(ms).toISOString();
|
|
1457
|
+
}
|
|
1458
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
1459
|
+
const asNum = Number(value);
|
|
1460
|
+
if (Number.isFinite(asNum) && /^\d+(\.\d+)?$/.test(value.trim())) {
|
|
1461
|
+
const ms = asNum > 1e12 ? asNum : asNum * 1000;
|
|
1462
|
+
return new Date(ms).toISOString();
|
|
1463
|
+
}
|
|
1464
|
+
return value;
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
return undefined;
|
|
1468
|
+
}
|
|
1385
1469
|
|
|
1386
1470
|
function mapRole(type: unknown): string {
|
|
1387
1471
|
const t = String(type ?? '').toLowerCase();
|
|
1388
|
-
if (t === 'user' || t === 'human') return 'user';
|
|
1472
|
+
if (t === 'user' || t === 'human' || t === 'developer') return 'user';
|
|
1389
1473
|
if (t === 'assistant' || t === 'ai' || t === 'model') return 'assistant';
|
|
1390
1474
|
if (t === 'system' || t === 'error') return 'system';
|
|
1391
1475
|
return t || 'unknown';
|
|
@@ -1412,7 +1496,9 @@ function extractContentText(content: unknown): string | undefined {
|
|
|
1412
1496
|
for (const block of content) {
|
|
1413
1497
|
if (typeof block !== 'object' || block === null) continue;
|
|
1414
1498
|
const b = block as Record<string, unknown>;
|
|
1415
|
-
|
|
1499
|
+
// Claude emits {type:"text",text}; codex emits {type:"input_text"|"output_text"|
|
|
1500
|
+
// "summary_text",text} — any block carrying a string `text` is text content.
|
|
1501
|
+
if (typeof b.text === 'string') {
|
|
1416
1502
|
parts.push(b.text);
|
|
1417
1503
|
}
|
|
1418
1504
|
}
|
package/src/opencode-importer.ts
CHANGED
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
readOpenCodeMessages,
|
|
20
20
|
readOpenCodeParts,
|
|
21
21
|
} from './jsonl-importer-dao';
|
|
22
|
+
import { maybeArgsRaw } from './mappers';
|
|
22
23
|
import { redactRecord } from './redaction';
|
|
23
24
|
import type { ImportIssue, ImportMode, ImportResult, JsonObject, ReconcileSummary, RedactionRule } from './types';
|
|
24
25
|
|
|
@@ -262,6 +263,7 @@ function prepareEntries(
|
|
|
262
263
|
session_id: row.session_id,
|
|
263
264
|
seq: started ?? created,
|
|
264
265
|
tool_name: stringValue(part.tool) ?? 'unknown',
|
|
266
|
+
args_raw: maybeArgsRaw(SOURCE, stringValue(part.tool) ?? 'unknown', state.input),
|
|
265
267
|
args_digest: sha256(state.input ?? null),
|
|
266
268
|
status,
|
|
267
269
|
started_at: started === undefined ? null : isoTime(started),
|