@gobing-ai/ts-llm-jsonl-importer 0.4.36 → 0.4.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/mappers.d.ts CHANGED
@@ -2,6 +2,11 @@ import { z } from 'zod';
2
2
  import type { JsonObject, SplitEntry, TransformContext } from './types';
3
3
  /** Compute args_digest: sha256 of stable-jsonified, key-sorted, redacted args. */
4
4
  export declare function argsDigest(args: unknown): string;
5
+ /**
6
+ * Return JSON-stringified raw args when the tool is a todo-writing tool for its source,
7
+ * otherwise undefined. Codex `arguments` arrives as a JSON string already — store as-is.
8
+ */
9
+ export declare function maybeArgsRaw(source: string, toolName: string, args: unknown): string | undefined;
5
10
  /** Map a Claude Code JSONL record into one or more ETL split entries. */
6
11
  export declare function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[];
7
12
  /** Map a Pi JSONL record into one or more ETL split entries. */
@@ -1 +1 @@
1
- {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAqDD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA6G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAqHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
1
+ {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAmExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAwCD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAMhG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAgH/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAoK9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAiMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAgFD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
package/dist/mappers.js CHANGED
@@ -89,14 +89,17 @@ function redactArgs(args) {
89
89
  *
90
90
  * Evidence: 0489 R4 confirmed omp/pi/claude; codex/grok/agy probed task 0553 R3 from real
91
91
  * session JSONL. agy has no on-disk session format (VS Code fork); gemini deferred by the
92
- * 2026-08-06 source-support ruling.
92
+ * 2026-08-06 source-support ruling. Task 0578 R3: omp `todo_write`, pi `manage_todo_list`,
93
+ * opencode `todowrite`/`todoread` measured live in .spur/spur.db (3,237/21, 1,043, 379/2
94
+ * rows with NULL args_raw).
93
95
  */
94
96
  const TODO_TOOL_ALLOWLIST = {
95
97
  claude: ['TodoWrite'],
96
- pi: ['todo'],
97
- omp: ['TodoWrite', 'todo'],
98
+ opencode: ['todowrite', 'todoread'],
98
99
  codex: ['update_plan'],
100
+ omp: ['TodoWrite', 'todo', 'todo_write'],
99
101
  grok: ['todo_write'],
102
+ pi: ['todo', 'manage_todo_list'],
100
103
  agy: [],
101
104
  gemini: [],
102
105
  };
@@ -104,7 +107,7 @@ const TODO_TOOL_ALLOWLIST = {
104
107
  * Return JSON-stringified raw args when the tool is a todo-writing tool for its source,
105
108
  * otherwise undefined. Codex `arguments` arrives as a JSON string already — store as-is.
106
109
  */
107
- function maybeArgsRaw(source, toolName, args) {
110
+ export function maybeArgsRaw(source, toolName, args) {
108
111
  const allow = TODO_TOOL_ALLOWLIST[source];
109
112
  if (allow === undefined || !allow.includes(toolName))
110
113
  return undefined;
@@ -122,10 +125,10 @@ export function claudeSplit(raw) {
122
125
  const entries = [];
123
126
  const sessionId = s(raw.sessionId, raw.conversation_uuid) ?? 'unknown';
124
127
  const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.messageIndex === 'number' ? raw.messageIndex : 0;
125
- const ts = s(raw.ts, raw.timestamp, raw.createdAt) ?? new Date(0).toISOString();
128
+ const ts = timestampOf(raw.ts, raw.timestamp, raw.createdAt);
126
129
  const role = mapRole(raw.type ?? raw.role);
127
130
  const recordType = String(raw.type ?? '');
128
- const model = s(raw.model);
131
+ const model = s(raw.model, o(raw.message).model);
129
132
  const cwd = s(raw.cwd, raw.dir);
130
133
  // Check for system/turn-duration records
131
134
  if (raw.subtype === 'turn_duration' && typeof raw.durationMs === 'number') {
@@ -162,13 +165,13 @@ export function claudeSplit(raw) {
162
165
  }
163
166
  // Main message
164
167
  const hasUsage = typeof raw.usage === 'object' && raw.usage !== null;
165
- const usage = hasUsage ? raw.usage : undefined;
166
- const inputTokens = typeof raw.inputTokens === 'number' ? raw.inputTokens : undefined;
167
- const outputTokens = typeof raw.outputTokens === 'number' ? raw.outputTokens : undefined;
168
- const cacheRead = (usage?.cacheReadTokens ?? usage?.cache_read_tokens ?? undefined);
169
- const cacheWrite = (usage?.cacheWriteTokens ?? usage?.cache_write_tokens ?? undefined);
168
+ const usage = (hasUsage ? raw.usage : o(raw.message).usage);
169
+ const inputTokens = (raw.inputTokens ?? usage?.input_tokens);
170
+ const outputTokens = (raw.outputTokens ?? usage?.output_tokens);
171
+ const cacheRead = (usage?.cacheReadTokens ?? usage?.cache_read_tokens ?? usage?.cache_read_input_tokens);
172
+ const cacheWrite = (usage?.cacheWriteTokens ?? usage?.cache_write_tokens ?? usage?.cache_creation_input_tokens);
170
173
  const costUsd = computeCost(inputTokens, outputTokens, model);
171
- const contentBlocks = raw.content;
174
+ const contentBlocks = raw.content ?? o(raw.message).content;
172
175
  const contentText = extractContentText(contentBlocks);
173
176
  const messageSplitIndex = entries.length;
174
177
  entries.push({
@@ -235,10 +238,9 @@ export function piSplit(raw, context) {
235
238
  const recordType = String(raw.type ?? raw.recordType ?? '');
236
239
  const sessionId = sessionIdFromContext(context, raw);
237
240
  const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
238
- // Prefer the ISO `timestamp`; a numeric epoch-ms `ts` is converted, never stringified.
239
- const ts = s(raw.timestamp, raw.createdAt) ??
240
- (typeof raw.ts === 'number' && Number.isFinite(raw.ts) ? new Date(raw.ts).toISOString() : s(raw.ts)) ??
241
- new Date(0).toISOString();
241
+ // Prefer the ISO `timestamp`; numeric epoch values are converted (task 0580 R6), never
242
+ // stringified, and a missing ts persists as null rather than a 1970 sentinel (D4).
243
+ const ts = timestampOf(raw.timestamp, raw.createdAt, raw.ts);
242
244
  // Meta lifecycle/custom records collapse to one meta row keyed by the source session,
243
245
  // never the unique event id, and never a guessed role.
244
246
  if (recordType === 'title' ||
@@ -363,7 +365,7 @@ export function ompSplit(raw, context) {
363
365
  const recordType = String(raw.type ?? '');
364
366
  const sessionId = sessionIdFromContext(context, raw);
365
367
  const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
366
- const ts = s(raw.ts, raw.timestamp, raw.createdAt) ?? new Date(0).toISOString();
368
+ const ts = timestampOf(raw.ts, raw.timestamp, raw.createdAt);
367
369
  // Meta lifecycle/custom records — including the current `custom.*` events and non-message
368
370
  // lifecycle types — collapse to one meta row keyed by the source session, never the unique
369
371
  // event id, and never a guessed role.
@@ -517,8 +519,9 @@ export function codexSplit(raw) {
517
519
  const payload = (raw.payload ?? raw);
518
520
  const sessionId = s(raw.session_id, o(raw.session_meta).id, raw.id) ?? 'unknown';
519
521
  const seq = typeof raw.seq === 'number' ? raw.seq : 0;
520
- const ts = s(raw.timestamp, raw.ts, payload.ts) ?? new Date(0).toISOString();
522
+ const ts = timestampOf(raw.timestamp, raw.ts, payload.ts);
521
523
  const recordType = String(raw.type ?? '');
524
+ const payloadType = String(payload.type ?? '');
522
525
  // Check for older short format
523
526
  if (recordType === '' && raw.id && raw.timestamp && raw.instructions) {
524
527
  return [
@@ -536,18 +539,77 @@ export function codexSplit(raw) {
536
539
  },
537
540
  ];
538
541
  }
539
- const role = mapRole(recordType);
542
+ // Codex JSONL is a transport envelope: the top-level `type` (response_item / event_msg /
543
+ // turn_context / ...) is bookkeeping, never a role. Real roles live in
544
+ // `payload.role` for response_item message items (task 0580 D1: passing recordType to
545
+ // mapRole leaked 153k response_item / 91k event_msg rows into the role column).
546
+ let role;
547
+ let disposition;
548
+ let contentText;
549
+ let usage;
550
+ if (recordType === 'response_item') {
551
+ if (payloadType === 'message') {
552
+ role = mapRole(payload.role);
553
+ disposition = 'keep';
554
+ contentText = extractContentText(payload.content) ?? s(payload.text) ?? null;
555
+ }
556
+ else if (payloadType === 'agent_message') {
557
+ role = 'assistant';
558
+ disposition = 'keep';
559
+ contentText = extractContentText(payload.content) ?? s(payload.text, payload.message) ?? null;
560
+ }
561
+ else if (payloadType === 'reasoning') {
562
+ role = 'assistant';
563
+ disposition = 'keep';
564
+ contentText = extractContentText(payload.summary) ?? s(payload.text) ?? null;
565
+ }
566
+ else if (payloadType === 'function_call' || payloadType === 'custom_tool_call') {
567
+ role = 'tool';
568
+ disposition = 'keep';
569
+ contentText = null;
570
+ }
571
+ else {
572
+ role = 'meta';
573
+ disposition = 'meta';
574
+ contentText = s(payload.text) ?? null;
575
+ }
576
+ }
577
+ else if (recordType === 'user' || recordType === 'assistant') {
578
+ // Legacy (pre-rollout) envelope: top-level type carried the role directly.
579
+ role = mapRole(recordType);
580
+ disposition = 'keep';
581
+ contentText = extractContentText(payload.content) ?? s(raw.content, payload.text) ?? null;
582
+ }
583
+ else if (recordType === 'message') {
584
+ role = mapRole(raw.role ?? payload.role);
585
+ disposition = 'keep';
586
+ contentText = extractContentText(payload.content ?? raw.content) ?? s(payload.text, raw.text) ?? null;
587
+ }
588
+ else if (recordType === 'event_msg') {
589
+ role = 'meta';
590
+ disposition = 'meta';
591
+ contentText = s(payload.message, payload.text) ?? null;
592
+ // Per-turn usage lives on token_count transport events (task 0580 R2): real files
593
+ // carry `payload.info.last_token_usage`; the legacy `payload.token_count` path is kept.
594
+ usage = o(o(payload.info).last_token_usage);
595
+ }
596
+ else {
597
+ role = 'meta';
598
+ disposition = 'meta';
599
+ contentText = s(payload.text, raw.instructions) ?? null;
600
+ }
540
601
  const model = s(raw.model, payload.model, o(o(raw.turn_context).payload).model);
541
602
  const cwd = s(raw.cwd, raw.dir);
542
603
  const tokenCount = payload.token_count;
543
- const inputTokens = (tokenCount?.input ?? tokenCount?.input_tokens ?? undefined);
544
- const outputTokens = (tokenCount?.output ?? tokenCount?.output_tokens ?? undefined);
604
+ const usageInput = (usage?.input_tokens ?? tokenCount?.input ?? tokenCount?.input_tokens ?? undefined);
605
+ const usageOutput = (usage?.output_tokens ?? tokenCount?.output ?? tokenCount?.output_tokens ?? undefined);
606
+ const cacheRead = (usage?.cached_input_tokens ?? undefined);
545
607
  const turnCtx = raw.turn_context;
546
608
  const turnPayload = turnCtx?.payload;
547
609
  const turnInputTokens = turnPayload?.input_tokens;
548
610
  const turnOutputTokens = turnPayload?.output_tokens;
549
- const finalInput = inputTokens !== undefined ? inputTokens : turnInputTokens;
550
- const finalOutput = outputTokens !== undefined ? outputTokens : turnOutputTokens;
611
+ const finalInput = usageInput !== undefined ? usageInput : turnInputTokens;
612
+ const finalOutput = usageOutput !== undefined ? usageOutput : turnOutputTokens;
551
613
  const costUsd = computeCost(finalInput, finalOutput, model);
552
614
  const messageSplitIndex = entries.length;
553
615
  entries.push({
@@ -557,47 +619,56 @@ export function codexSplit(raw) {
557
619
  seq,
558
620
  role,
559
621
  record_type: recordType,
560
- disposition: 'keep',
622
+ disposition,
561
623
  ts,
562
624
  duration_ms: undefined,
563
625
  model: model ?? null,
564
626
  input_tokens: finalInput ?? null,
565
627
  output_tokens: finalOutput ?? null,
566
- cache_read_tokens: null,
628
+ cache_read_tokens: cacheRead ?? null,
567
629
  cache_write_tokens: null,
568
630
  cost_usd: costUsd ?? null,
569
- content_text: s(payload.content, payload.text, raw.content, raw.text) ?? null,
631
+ content_text: contentText,
570
632
  cwd: cwd ?? null,
571
633
  provenance: 'ambient',
572
634
  },
573
635
  });
574
- // Tool calls: function_call in response_item
575
- const responseItems = payload.response_item
576
- ? [payload.response_item]
577
- : Array.isArray(payload.response_items)
578
- ? payload.response_items
579
- : [];
580
- for (const item of responseItems) {
581
- if (item.function_call) {
582
- const fc = item.function_call;
583
- entries.push({
584
- targetTable: 'history_tool_call',
585
- record: {
586
- _messageSplitIndex: messageSplitIndex,
587
- session_id: sessionId,
588
- seq,
589
- tool_name: String(fc.name ?? ''),
590
- args_digest: argsDigest(fc.arguments),
591
- args_raw: maybeArgsRaw('codex', String(fc.name ?? ''), fc.arguments),
592
- status: 'ok',
593
- started_at: undefined,
594
- completed_at: undefined,
595
- duration_ms: undefined,
596
- result_bytes: undefined,
597
- error_text: undefined,
598
- },
599
- });
600
- }
636
+ // Tool calls. Live rollouts emit each call as its own response_item whose payload IS the
637
+ // item (`payload.type === "function_call" | "custom_tool_call"`, name/call_id/arguments or
638
+ // input at payload level); the legacy nested `payload.response_item[_s]` shape is kept.
639
+ const toolItems = [];
640
+ if ((recordType === 'response_item' || recordType === 'function_call' || recordType === 'custom_tool_call') &&
641
+ (payloadType === 'function_call' || payloadType === 'custom_tool_call')) {
642
+ toolItems.push(payload);
643
+ }
644
+ else if (payload.response_item) {
645
+ toolItems.push(payload.response_item);
646
+ }
647
+ else if (Array.isArray(payload.response_items)) {
648
+ toolItems.push(...payload.response_items);
649
+ }
650
+ for (const item of toolItems) {
651
+ const fc = item.function_call ?? item;
652
+ const toolName = s(fc.name) ?? '';
653
+ const args = fc.arguments ?? fc.input;
654
+ entries.push({
655
+ targetTable: 'history_tool_call',
656
+ record: {
657
+ _messageSplitIndex: messageSplitIndex,
658
+ session_id: sessionId,
659
+ seq,
660
+ tool_name: toolName,
661
+ call_id: s(fc.call_id),
662
+ args_digest: argsDigest(args),
663
+ args_raw: maybeArgsRaw('codex', toolName, args),
664
+ status: 'ok',
665
+ started_at: undefined,
666
+ completed_at: undefined,
667
+ duration_ms: undefined,
668
+ result_bytes: undefined,
669
+ error_text: undefined,
670
+ },
671
+ });
601
672
  }
602
673
  return entries;
603
674
  }
@@ -610,7 +681,7 @@ export function agySplit(raw) {
610
681
  const recordType = String(raw.type ?? '');
611
682
  const sessionId = s(raw.session_id, raw.conversation_id) ?? 'unknown';
612
683
  const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.step_index === 'number' ? raw.step_index : 0;
613
- const ts = s(raw.created_at, raw.timestamp, raw.ts) ?? new Date(0).toISOString();
684
+ const ts = timestampOf(raw.created_at, raw.timestamp, raw.ts);
614
685
  let role;
615
686
  let disposition;
616
687
  let contentText = null;
@@ -723,7 +794,7 @@ export function geminiSplit(raw, context) {
723
794
  const state = o(raw.$set);
724
795
  const sessionId = sessionIdFromContext(context, raw);
725
796
  const seq = context?.sourceLine ?? 0;
726
- const ts = s(raw.timestamp, raw.startTime, raw.lastUpdated, state.lastUpdated) ?? new Date(0).toISOString();
797
+ const ts = timestampOf(raw.timestamp, raw.startTime, raw.lastUpdated, state.lastUpdated);
727
798
  if (recordType === 'session' || recordType === 'state') {
728
799
  return [
729
800
  {
@@ -858,7 +929,12 @@ function normalizeGrokRecord(raw) {
858
929
  : undefined));
859
930
  const usage = (body.usage ?? raw.usage);
860
931
  const toolMeta = meta?.['x.ai/tool'];
861
- const toolName = s(body.title, body.tool_name, body.name, toolMeta?.name, toolMeta?.label, raw.tool_name, raw.name) ?? '';
932
+ // The tool's real name is `_meta["x.ai/tool"].name` (or `kind`); `title` is a human label
933
+ // like "Read `/path`" — used only as a last resort, stripped at the first backtick so the
934
+ // argument payload never pollutes tool_name (task 0580 D3).
935
+ const titleFallback = s(body.title)?.split('`')[0]?.trim();
936
+ const toolName = s(toolMeta?.name, body.kind, body.tool_name, body.name, raw.tool_name, raw.name) ??
937
+ (titleFallback !== undefined && titleFallback.length > 0 && titleFallback.length <= 40 ? titleFallback : '');
862
938
  const toolArgs = body.rawInput ?? body.input ?? body.arguments ?? toolMeta?.input ?? raw.input ?? raw.arguments;
863
939
  const toolStatus = s(body.status, raw.outcome, raw.status) ?? 'ok';
864
940
  const durationMs = typeof body.duration_ms === 'number'
@@ -888,20 +964,7 @@ function normalizeGrokRecord(raw) {
888
964
  };
889
965
  }
890
966
  function normalizeTs(value) {
891
- if (typeof value === 'number' && Number.isFinite(value)) {
892
- // Grok timestamps are unix seconds (sometimes with fractional).
893
- const ms = value > 1e12 ? value : value * 1000;
894
- return new Date(ms).toISOString();
895
- }
896
- if (typeof value === 'string' && value.length > 0) {
897
- const asNum = Number(value);
898
- if (Number.isFinite(asNum) && /^\d+(\.\d+)?$/.test(value.trim())) {
899
- const ms = asNum > 1e12 ? asNum : asNum * 1000;
900
- return new Date(ms).toISOString();
901
- }
902
- return value;
903
- }
904
- return new Date(0).toISOString();
967
+ return timestampOf(value);
905
968
  }
906
969
  function extractGrokContent(content) {
907
970
  if (typeof content === 'string')
@@ -1265,9 +1328,31 @@ function s(...values) {
1265
1328
  }
1266
1329
  return undefined;
1267
1330
  }
1331
+ /**
1332
+ * First usable timestamp: an ISO-ish string passes through, a finite number (or digit-string)
1333
+ * is converted from epoch seconds/millis (>1e12 treated as ms). Returns undefined when nothing
1334
+ * usable exists — mappers must persist null ts rather than a 1970 sentinel (task 0580 D4).
1335
+ */
1336
+ function timestampOf(...values) {
1337
+ for (const value of values) {
1338
+ if (typeof value === 'number' && Number.isFinite(value)) {
1339
+ const ms = value > 1e12 ? value : value * 1000;
1340
+ return new Date(ms).toISOString();
1341
+ }
1342
+ if (typeof value === 'string' && value.length > 0) {
1343
+ const asNum = Number(value);
1344
+ if (Number.isFinite(asNum) && /^\d+(\.\d+)?$/.test(value.trim())) {
1345
+ const ms = asNum > 1e12 ? asNum : asNum * 1000;
1346
+ return new Date(ms).toISOString();
1347
+ }
1348
+ return value;
1349
+ }
1350
+ }
1351
+ return undefined;
1352
+ }
1268
1353
  function mapRole(type) {
1269
1354
  const t = String(type ?? '').toLowerCase();
1270
- if (t === 'user' || t === 'human')
1355
+ if (t === 'user' || t === 'human' || t === 'developer')
1271
1356
  return 'user';
1272
1357
  if (t === 'assistant' || t === 'ai' || t === 'model')
1273
1358
  return 'assistant';
@@ -1296,7 +1381,9 @@ function extractContentText(content) {
1296
1381
  if (typeof block !== 'object' || block === null)
1297
1382
  continue;
1298
1383
  const b = block;
1299
- if (b.type === 'text' && typeof b.text === 'string') {
1384
+ // Claude emits {type:"text",text}; codex emits {type:"input_text"|"output_text"|
1385
+ // "summary_text",text} — any block carrying a string `text` is text content.
1386
+ if (typeof b.text === 'string') {
1300
1387
  parts.push(b.text);
1301
1388
  }
1302
1389
  }
@@ -1 +1 @@
1
- {"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAe/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAoI7F"}
1
+ {"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAgB/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAoI7F"}
@@ -2,6 +2,7 @@ import { createDbAdapter } from '@gobing-ai/ts-db';
2
2
  import { ambientRuntimePaths, createNodeFileSystem, joinPath, } from '@gobing-ai/ts-runtime';
3
3
  import { sha256 } from './hash.js';
4
4
  import { applyHistoryImportSchema, deleteCheckpointOperation, openCodeBulkWriteOperations, openCodeDeleteOperations, readOpenCodeExistingEntries, readOpenCodeMessages, readOpenCodeParts, } from './jsonl-importer-dao.js';
5
+ import { maybeArgsRaw } from './mappers.js';
5
6
  import { redactRecord } from './redaction.js';
6
7
  const SOURCE = 'opencode';
7
8
  const PAGE_SIZE = 250;
@@ -202,6 +203,7 @@ function prepareEntries(row, data, parts, sourceFile, redactionRules) {
202
203
  session_id: row.session_id,
203
204
  seq: started ?? created,
204
205
  tool_name: stringValue(part.tool) ?? 'unknown',
206
+ args_raw: maybeArgsRaw(SOURCE, stringValue(part.tool) ?? 'unknown', state.input),
205
207
  args_digest: sha256(state.input ?? null),
206
208
  status,
207
209
  started_at: started === undefined ? null : isoTime(started),
@@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS history_message (
37
37
  role TEXT NOT NULL,
38
38
  record_type TEXT NOT NULL,
39
39
  disposition TEXT NOT NULL,
40
- ts TEXT NOT NULL,
40
+ ts TEXT,
41
41
  duration_ms INTEGER,
42
42
  model TEXT,
43
43
  input_tokens INTEGER,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gobing-ai/ts-llm-jsonl-importer",
3
- "version": "0.4.36",
3
+ "version": "0.4.38",
4
4
  "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
5
5
  "keywords": [
6
6
  "typescript",
@@ -47,9 +47,9 @@
47
47
  "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
48
48
  },
49
49
  "dependencies": {
50
- "@gobing-ai/ts-db": "^0.4.36",
51
- "@gobing-ai/ts-runtime": "^0.4.36",
52
- "@gobing-ai/ts-utils": "^0.4.36",
50
+ "@gobing-ai/ts-db": "^0.4.38",
51
+ "@gobing-ai/ts-runtime": "^0.4.38",
52
+ "@gobing-ai/ts-utils": "^0.4.38",
53
53
  "zod": "^4.1.0"
54
54
  },
55
55
  "devDependencies": {
package/src/mappers.ts CHANGED
@@ -96,14 +96,17 @@ function redactArgs(args: unknown): unknown {
96
96
  *
97
97
  * Evidence: 0489 R4 confirmed omp/pi/claude; codex/grok/agy probed task 0553 R3 from real
98
98
  * session JSONL. agy has no on-disk session format (VS Code fork); gemini deferred by the
99
- * 2026-08-06 source-support ruling.
99
+ * 2026-08-06 source-support ruling. Task 0578 R3: omp `todo_write`, pi `manage_todo_list`,
100
+ * opencode `todowrite`/`todoread` measured live in .spur/spur.db (3,237/21, 1,043, 379/2
101
+ * rows with NULL args_raw).
100
102
  */
101
103
  const TODO_TOOL_ALLOWLIST: Readonly<Record<string, readonly string[]>> = {
102
104
  claude: ['TodoWrite'],
103
- pi: ['todo'],
104
- omp: ['TodoWrite', 'todo'],
105
+ opencode: ['todowrite', 'todoread'],
105
106
  codex: ['update_plan'],
107
+ omp: ['TodoWrite', 'todo', 'todo_write'],
106
108
  grok: ['todo_write'],
109
+ pi: ['todo', 'manage_todo_list'],
107
110
  agy: [],
108
111
  gemini: [],
109
112
  };
@@ -112,7 +115,7 @@ const TODO_TOOL_ALLOWLIST: Readonly<Record<string, readonly string[]>> = {
112
115
  * Return JSON-stringified raw args when the tool is a todo-writing tool for its source,
113
116
  * otherwise undefined. Codex `arguments` arrives as a JSON string already — store as-is.
114
117
  */
115
- function maybeArgsRaw(source: string, toolName: string, args: unknown): string | undefined {
118
+ export function maybeArgsRaw(source: string, toolName: string, args: unknown): string | undefined {
116
119
  const allow = TODO_TOOL_ALLOWLIST[source];
117
120
  if (allow === undefined || !allow.includes(toolName)) return undefined;
118
121
  if (typeof args === 'string') return args;
@@ -129,10 +132,10 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
129
132
  const entries: SplitEntry[] = [];
130
133
  const sessionId = s(raw.sessionId, raw.conversation_uuid) ?? 'unknown';
131
134
  const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.messageIndex === 'number' ? raw.messageIndex : 0;
132
- const ts = s(raw.ts, raw.timestamp, raw.createdAt) ?? new Date(0).toISOString();
135
+ const ts = timestampOf(raw.ts, raw.timestamp, raw.createdAt);
133
136
  const role = mapRole(raw.type ?? raw.role);
134
137
  const recordType = String(raw.type ?? '');
135
- const model = s(raw.model);
138
+ const model = s(raw.model, o(raw.message).model);
136
139
  const cwd = s(raw.cwd, raw.dir);
137
140
 
138
141
  // Check for system/turn-duration records
@@ -171,14 +174,17 @@ export function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[]
171
174
 
172
175
  // Main message
173
176
  const hasUsage = typeof raw.usage === 'object' && raw.usage !== null;
174
- const usage = hasUsage ? (raw.usage as Record<string, unknown>) : undefined;
175
- const inputTokens = typeof raw.inputTokens === 'number' ? raw.inputTokens : undefined;
176
- const outputTokens = typeof raw.outputTokens === 'number' ? raw.outputTokens : undefined;
177
- const cacheRead = (usage?.cacheReadTokens ?? usage?.cache_read_tokens ?? undefined) as number | undefined;
178
- const cacheWrite = (usage?.cacheWriteTokens ?? usage?.cache_write_tokens ?? undefined) as number | undefined;
177
+ const usage = (hasUsage ? raw.usage : o(raw.message).usage) as Record<string, unknown> | undefined;
178
+ const inputTokens = ((raw.inputTokens as number | undefined) ?? usage?.input_tokens) as number | undefined;
179
+ const outputTokens = ((raw.outputTokens as number | undefined) ?? usage?.output_tokens) as number | undefined;
180
+ const cacheRead = (usage?.cacheReadTokens ?? usage?.cache_read_tokens ?? usage?.cache_read_input_tokens) as
181
+ | number
182
+ | undefined;
183
+ const cacheWrite = (usage?.cacheWriteTokens ?? usage?.cache_write_tokens ?? usage?.cache_creation_input_tokens) as
184
+ | number
185
+ | undefined;
179
186
  const costUsd = computeCost(inputTokens, outputTokens, model);
180
-
181
- const contentBlocks = raw.content;
187
+ const contentBlocks = raw.content ?? o(raw.message).content;
182
188
  const contentText = extractContentText(contentBlocks);
183
189
 
184
190
  const messageSplitIndex = entries.length;
@@ -250,11 +256,9 @@ export function piSplit(raw: Record<string, unknown>, context?: TransformContext
250
256
  const recordType = String(raw.type ?? raw.recordType ?? '');
251
257
  const sessionId = sessionIdFromContext(context, raw);
252
258
  const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
253
- // Prefer the ISO `timestamp`; a numeric epoch-ms `ts` is converted, never stringified.
254
- const ts =
255
- s(raw.timestamp, raw.createdAt) ??
256
- (typeof raw.ts === 'number' && Number.isFinite(raw.ts) ? new Date(raw.ts).toISOString() : s(raw.ts)) ??
257
- new Date(0).toISOString();
259
+ // Prefer the ISO `timestamp`; numeric epoch values are converted (task 0580 R6), never
260
+ // stringified, and a missing ts persists as null rather than a 1970 sentinel (D4).
261
+ const ts = timestampOf(raw.timestamp, raw.createdAt, raw.ts);
258
262
 
259
263
  // Meta lifecycle/custom records collapse to one meta row keyed by the source session,
260
264
  // never the unique event id, and never a guessed role.
@@ -387,7 +391,7 @@ export function ompSplit(raw: Record<string, unknown>, context?: TransformContex
387
391
  const recordType = String(raw.type ?? '');
388
392
  const sessionId = sessionIdFromContext(context, raw);
389
393
  const seq = context?.sourceLine ?? (typeof raw.seq === 'number' ? raw.seq : 0);
390
- const ts = s(raw.ts, raw.timestamp, raw.createdAt) ?? new Date(0).toISOString();
394
+ const ts = timestampOf(raw.ts, raw.timestamp, raw.createdAt);
391
395
 
392
396
  // Meta lifecycle/custom records — including the current `custom.*` events and non-message
393
397
  // lifecycle types — collapse to one meta row keyed by the source session, never the unique
@@ -561,8 +565,9 @@ export function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[]
561
565
  const payload = (raw.payload ?? raw) as Record<string, unknown>;
562
566
  const sessionId = s(raw.session_id, o(raw.session_meta).id, raw.id) ?? 'unknown';
563
567
  const seq = typeof raw.seq === 'number' ? raw.seq : 0;
564
- const ts = s(raw.timestamp, raw.ts, payload.ts) ?? new Date(0).toISOString();
568
+ const ts = timestampOf(raw.timestamp, raw.ts, payload.ts);
565
569
  const recordType = String(raw.type ?? '');
570
+ const payloadType = String(payload.type ?? '');
566
571
 
567
572
  // Check for older short format
568
573
  if (recordType === '' && raw.id && raw.timestamp && raw.instructions) {
@@ -582,21 +587,77 @@ export function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[]
582
587
  ];
583
588
  }
584
589
 
585
- const role = mapRole(recordType);
590
+ // Codex JSONL is a transport envelope: the top-level `type` (response_item / event_msg /
591
+ // turn_context / ...) is bookkeeping, never a role. Real roles live in
592
+ // `payload.role` for response_item message items (task 0580 D1: passing recordType to
593
+ // mapRole leaked 153k response_item / 91k event_msg rows into the role column).
594
+ let role: string;
595
+ let disposition: string;
596
+ let contentText: string | null;
597
+ let usage: Record<string, unknown> | undefined;
598
+ if (recordType === 'response_item') {
599
+ if (payloadType === 'message') {
600
+ role = mapRole(payload.role);
601
+ disposition = 'keep';
602
+ contentText = extractContentText(payload.content) ?? s(payload.text) ?? null;
603
+ } else if (payloadType === 'agent_message') {
604
+ role = 'assistant';
605
+ disposition = 'keep';
606
+ contentText = extractContentText(payload.content) ?? s(payload.text, payload.message) ?? null;
607
+ } else if (payloadType === 'reasoning') {
608
+ role = 'assistant';
609
+ disposition = 'keep';
610
+ contentText = extractContentText(payload.summary) ?? s(payload.text) ?? null;
611
+ } else if (payloadType === 'function_call' || payloadType === 'custom_tool_call') {
612
+ role = 'tool';
613
+ disposition = 'keep';
614
+ contentText = null;
615
+ } else {
616
+ role = 'meta';
617
+ disposition = 'meta';
618
+ contentText = s(payload.text) ?? null;
619
+ }
620
+ } else if (recordType === 'user' || recordType === 'assistant') {
621
+ // Legacy (pre-rollout) envelope: top-level type carried the role directly.
622
+ role = mapRole(recordType);
623
+ disposition = 'keep';
624
+ contentText = extractContentText(payload.content) ?? s(raw.content, payload.text) ?? null;
625
+ } else if (recordType === 'message') {
626
+ role = mapRole(raw.role ?? payload.role);
627
+ disposition = 'keep';
628
+ contentText = extractContentText(payload.content ?? raw.content) ?? s(payload.text, raw.text) ?? null;
629
+ } else if (recordType === 'event_msg') {
630
+ role = 'meta';
631
+ disposition = 'meta';
632
+ contentText = s(payload.message, payload.text) ?? null;
633
+ // Per-turn usage lives on token_count transport events (task 0580 R2): real files
634
+ // carry `payload.info.last_token_usage`; the legacy `payload.token_count` path is kept.
635
+ usage = o(o(payload.info).last_token_usage);
636
+ } else {
637
+ role = 'meta';
638
+ disposition = 'meta';
639
+ contentText = s(payload.text, raw.instructions) ?? null;
640
+ }
641
+
586
642
  const model = s(raw.model, payload.model, o(o(raw.turn_context).payload).model);
587
643
  const cwd = s(raw.cwd, raw.dir);
588
644
 
589
645
  const tokenCount = payload.token_count as Record<string, unknown> | undefined;
590
- const inputTokens = (tokenCount?.input ?? tokenCount?.input_tokens ?? undefined) as number | undefined;
591
- const outputTokens = (tokenCount?.output ?? tokenCount?.output_tokens ?? undefined) as number | undefined;
646
+ const usageInput = (usage?.input_tokens ?? tokenCount?.input ?? tokenCount?.input_tokens ?? undefined) as
647
+ | number
648
+ | undefined;
649
+ const usageOutput = (usage?.output_tokens ?? tokenCount?.output ?? tokenCount?.output_tokens ?? undefined) as
650
+ | number
651
+ | undefined;
652
+ const cacheRead = (usage?.cached_input_tokens ?? undefined) as number | undefined;
592
653
 
593
654
  const turnCtx = raw.turn_context as Record<string, unknown> | undefined;
594
655
  const turnPayload = turnCtx?.payload as Record<string, unknown> | undefined;
595
656
  const turnInputTokens = turnPayload?.input_tokens as number | undefined;
596
657
  const turnOutputTokens = turnPayload?.output_tokens as number | undefined;
597
658
 
598
- const finalInput = inputTokens !== undefined ? inputTokens : turnInputTokens;
599
- const finalOutput = outputTokens !== undefined ? outputTokens : turnOutputTokens;
659
+ const finalInput = usageInput !== undefined ? usageInput : turnInputTokens;
660
+ const finalOutput = usageOutput !== undefined ? usageOutput : turnOutputTokens;
600
661
  const costUsd = computeCost(finalInput, finalOutput, model);
601
662
 
602
663
  const messageSplitIndex = entries.length;
@@ -608,48 +669,57 @@ export function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[]
608
669
  seq,
609
670
  role,
610
671
  record_type: recordType,
611
- disposition: 'keep',
672
+ disposition,
612
673
  ts,
613
674
  duration_ms: undefined,
614
675
  model: model ?? null,
615
676
  input_tokens: finalInput ?? null,
616
677
  output_tokens: finalOutput ?? null,
617
- cache_read_tokens: null,
678
+ cache_read_tokens: cacheRead ?? null,
618
679
  cache_write_tokens: null,
619
680
  cost_usd: costUsd ?? null,
620
- content_text: s(payload.content, payload.text, raw.content, raw.text) ?? null,
681
+ content_text: contentText,
621
682
  cwd: cwd ?? null,
622
683
  provenance: 'ambient',
623
684
  },
624
685
  });
625
686
 
626
- // Tool calls: function_call in response_item
627
- const responseItems = payload.response_item
628
- ? [payload.response_item as Record<string, unknown>]
629
- : Array.isArray(payload.response_items)
630
- ? (payload.response_items as Record<string, unknown>[])
631
- : [];
632
- for (const item of responseItems) {
633
- if (item.function_call) {
634
- const fc = item.function_call as Record<string, unknown>;
635
- entries.push({
636
- targetTable: 'history_tool_call',
637
- record: {
638
- _messageSplitIndex: messageSplitIndex,
639
- session_id: sessionId,
640
- seq,
641
- tool_name: String(fc.name ?? ''),
642
- args_digest: argsDigest(fc.arguments),
643
- args_raw: maybeArgsRaw('codex', String(fc.name ?? ''), fc.arguments),
644
- status: 'ok',
645
- started_at: undefined,
646
- completed_at: undefined,
647
- duration_ms: undefined,
648
- result_bytes: undefined,
649
- error_text: undefined,
650
- },
651
- });
652
- }
687
+ // Tool calls. Live rollouts emit each call as its own response_item whose payload IS the
688
+ // item (`payload.type === "function_call" | "custom_tool_call"`, name/call_id/arguments or
689
+ // input at payload level); the legacy nested `payload.response_item[_s]` shape is kept.
690
+ const toolItems: Record<string, unknown>[] = [];
691
+ if (
692
+ (recordType === 'response_item' || recordType === 'function_call' || recordType === 'custom_tool_call') &&
693
+ (payloadType === 'function_call' || payloadType === 'custom_tool_call')
694
+ ) {
695
+ toolItems.push(payload);
696
+ } else if (payload.response_item) {
697
+ toolItems.push(payload.response_item as Record<string, unknown>);
698
+ } else if (Array.isArray(payload.response_items)) {
699
+ toolItems.push(...(payload.response_items as Record<string, unknown>[]));
700
+ }
701
+ for (const item of toolItems) {
702
+ const fc = (item.function_call as Record<string, unknown> | undefined) ?? item;
703
+ const toolName = s(fc.name) ?? '';
704
+ const args = fc.arguments ?? fc.input;
705
+ entries.push({
706
+ targetTable: 'history_tool_call',
707
+ record: {
708
+ _messageSplitIndex: messageSplitIndex,
709
+ session_id: sessionId,
710
+ seq,
711
+ tool_name: toolName,
712
+ call_id: s(fc.call_id),
713
+ args_digest: argsDigest(args),
714
+ args_raw: maybeArgsRaw('codex', toolName, args),
715
+ status: 'ok',
716
+ started_at: undefined,
717
+ completed_at: undefined,
718
+ duration_ms: undefined,
719
+ result_bytes: undefined,
720
+ error_text: undefined,
721
+ },
722
+ });
653
723
  }
654
724
 
655
725
  return entries;
@@ -665,7 +735,7 @@ export function agySplit(raw: Record<string, unknown>): readonly SplitEntry[] {
665
735
  const recordType = String(raw.type ?? '');
666
736
  const sessionId = s(raw.session_id, raw.conversation_id) ?? 'unknown';
667
737
  const seq = typeof raw.seq === 'number' ? raw.seq : typeof raw.step_index === 'number' ? raw.step_index : 0;
668
- const ts = s(raw.created_at, raw.timestamp, raw.ts) ?? new Date(0).toISOString();
738
+ const ts = timestampOf(raw.created_at, raw.timestamp, raw.ts);
669
739
 
670
740
  let role: string;
671
741
  let disposition: string;
@@ -785,7 +855,7 @@ export function geminiSplit(raw: Record<string, unknown>, context?: TransformCon
785
855
  const state = o(raw.$set);
786
856
  const sessionId = sessionIdFromContext(context, raw);
787
857
  const seq = context?.sourceLine ?? 0;
788
- const ts = s(raw.timestamp, raw.startTime, raw.lastUpdated, state.lastUpdated) ?? new Date(0).toISOString();
858
+ const ts = timestampOf(raw.timestamp, raw.startTime, raw.lastUpdated, state.lastUpdated);
789
859
 
790
860
  if (recordType === 'session' || recordType === 'state') {
791
861
  return [
@@ -903,7 +973,7 @@ function numberOrNull(value: unknown): number | null {
903
973
  function normalizeGrokRecord(raw: Record<string, unknown>): {
904
974
  recordType: string;
905
975
  sessionId: string;
906
- ts: string;
976
+ ts: string | undefined;
907
977
  seq: number;
908
978
  body: Record<string, unknown>;
909
979
  model: string | undefined;
@@ -958,8 +1028,13 @@ function normalizeGrokRecord(raw: Record<string, unknown>): {
958
1028
  );
959
1029
  const usage = (body.usage ?? raw.usage) as Record<string, unknown> | undefined;
960
1030
  const toolMeta = meta?.['x.ai/tool'] as Record<string, unknown> | undefined;
1031
+ // The tool's real name is `_meta["x.ai/tool"].name` (or `kind`); `title` is a human label
1032
+ // like "Read `/path`" — used only as a last resort, stripped at the first backtick so the
1033
+ // argument payload never pollutes tool_name (task 0580 D3).
1034
+ const titleFallback = s(body.title)?.split('`')[0]?.trim();
961
1035
  const toolName =
962
- s(body.title, body.tool_name, body.name, toolMeta?.name, toolMeta?.label, raw.tool_name, raw.name) ?? '';
1036
+ s(toolMeta?.name, body.kind, body.tool_name, body.name, raw.tool_name, raw.name) ??
1037
+ (titleFallback !== undefined && titleFallback.length > 0 && titleFallback.length <= 40 ? titleFallback : '');
963
1038
  const toolArgs = body.rawInput ?? body.input ?? body.arguments ?? toolMeta?.input ?? raw.input ?? raw.arguments;
964
1039
  const toolStatus = s(body.status, raw.outcome, raw.status) ?? 'ok';
965
1040
  const durationMs =
@@ -992,21 +1067,8 @@ function normalizeGrokRecord(raw: Record<string, unknown>): {
992
1067
  };
993
1068
  }
994
1069
 
995
- function normalizeTs(value: unknown): string {
996
- if (typeof value === 'number' && Number.isFinite(value)) {
997
- // Grok timestamps are unix seconds (sometimes with fractional).
998
- const ms = value > 1e12 ? value : value * 1000;
999
- return new Date(ms).toISOString();
1000
- }
1001
- if (typeof value === 'string' && value.length > 0) {
1002
- const asNum = Number(value);
1003
- if (Number.isFinite(asNum) && /^\d+(\.\d+)?$/.test(value.trim())) {
1004
- const ms = asNum > 1e12 ? asNum : asNum * 1000;
1005
- return new Date(ms).toISOString();
1006
- }
1007
- return value;
1008
- }
1009
- return new Date(0).toISOString();
1070
+ function normalizeTs(value: unknown): string | undefined {
1071
+ return timestampOf(value);
1010
1072
  }
1011
1073
 
1012
1074
  function extractGrokContent(content: unknown): string | null {
@@ -1382,10 +1444,32 @@ function s(...values: readonly unknown[]): string | undefined {
1382
1444
  }
1383
1445
  return undefined;
1384
1446
  }
1447
+ /**
1448
+ * First usable timestamp: an ISO-ish string passes through, a finite number (or digit-string)
1449
+ * is converted from epoch seconds/millis (>1e12 treated as ms). Returns undefined when nothing
1450
+ * usable exists — mappers must persist null ts rather than a 1970 sentinel (task 0580 D4).
1451
+ */
1452
+ function timestampOf(...values: readonly unknown[]): string | undefined {
1453
+ for (const value of values) {
1454
+ if (typeof value === 'number' && Number.isFinite(value)) {
1455
+ const ms = value > 1e12 ? value : value * 1000;
1456
+ return new Date(ms).toISOString();
1457
+ }
1458
+ if (typeof value === 'string' && value.length > 0) {
1459
+ const asNum = Number(value);
1460
+ if (Number.isFinite(asNum) && /^\d+(\.\d+)?$/.test(value.trim())) {
1461
+ const ms = asNum > 1e12 ? asNum : asNum * 1000;
1462
+ return new Date(ms).toISOString();
1463
+ }
1464
+ return value;
1465
+ }
1466
+ }
1467
+ return undefined;
1468
+ }
1385
1469
 
1386
1470
  function mapRole(type: unknown): string {
1387
1471
  const t = String(type ?? '').toLowerCase();
1388
- if (t === 'user' || t === 'human') return 'user';
1472
+ if (t === 'user' || t === 'human' || t === 'developer') return 'user';
1389
1473
  if (t === 'assistant' || t === 'ai' || t === 'model') return 'assistant';
1390
1474
  if (t === 'system' || t === 'error') return 'system';
1391
1475
  return t || 'unknown';
@@ -1412,7 +1496,9 @@ function extractContentText(content: unknown): string | undefined {
1412
1496
  for (const block of content) {
1413
1497
  if (typeof block !== 'object' || block === null) continue;
1414
1498
  const b = block as Record<string, unknown>;
1415
- if (b.type === 'text' && typeof b.text === 'string') {
1499
+ // Claude emits {type:"text",text}; codex emits {type:"input_text"|"output_text"|
1500
+ // "summary_text",text} — any block carrying a string `text` is text content.
1501
+ if (typeof b.text === 'string') {
1416
1502
  parts.push(b.text);
1417
1503
  }
1418
1504
  }
@@ -19,6 +19,7 @@ import {
19
19
  readOpenCodeMessages,
20
20
  readOpenCodeParts,
21
21
  } from './jsonl-importer-dao';
22
+ import { maybeArgsRaw } from './mappers';
22
23
  import { redactRecord } from './redaction';
23
24
  import type { ImportIssue, ImportMode, ImportResult, JsonObject, ReconcileSummary, RedactionRule } from './types';
24
25
 
@@ -262,6 +263,7 @@ function prepareEntries(
262
263
  session_id: row.session_id,
263
264
  seq: started ?? created,
264
265
  tool_name: stringValue(part.tool) ?? 'unknown',
266
+ args_raw: maybeArgsRaw(SOURCE, stringValue(part.tool) ?? 'unknown', state.input),
265
267
  args_digest: sha256(state.input ?? null),
266
268
  status,
267
269
  started_at: started === undefined ? null : isoTime(started),
package/src/schema-sql.ts CHANGED
@@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS history_message (
37
37
  role TEXT NOT NULL,
38
38
  record_type TEXT NOT NULL,
39
39
  disposition TEXT NOT NULL,
40
- ts TEXT NOT NULL,
40
+ ts TEXT,
41
41
  duration_ms INTEGER,
42
42
  model TEXT,
43
43
  input_tokens INTEGER,