@gobing-ai/ts-llm-jsonl-importer 0.4.19 → 0.4.21

package/src/schema-sql.ts

@@ -80,4 +80,61 @@ CREATE TABLE IF NOT EXISTS history_etl_openclaw (
     payload_json TEXT NOT NULL,
     imported_at TEXT NOT NULL
 );
+
+CREATE TABLE IF NOT EXISTS history_message (
+    record_hash        TEXT PRIMARY KEY,
+    source             TEXT NOT NULL,
+    source_file        TEXT NOT NULL,
+    source_line        INTEGER NOT NULL,
+    session_id         TEXT NOT NULL,
+    seq                INTEGER NOT NULL,
+    turn_index         INTEGER,
+    role               TEXT NOT NULL,
+    record_type        TEXT NOT NULL,
+    disposition        TEXT NOT NULL,
+    ts                 TEXT NOT NULL,
+    duration_ms        INTEGER,
+    model              TEXT,
+    input_tokens       INTEGER,
+    output_tokens      INTEGER,
+    cache_read_tokens  INTEGER,
+    cache_write_tokens INTEGER,
+    cost_usd           REAL,
+    content_text       TEXT,
+    cwd                TEXT,
+    provenance         TEXT NOT NULL,
+    run_id             TEXT,
+    task_wbs           TEXT,
+    imported_at        TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_tool_call (
+    record_hash   TEXT PRIMARY KEY,
+    message_hash  TEXT NOT NULL,
+    source        TEXT NOT NULL,
+    source_file   TEXT NOT NULL,
+    source_line   INTEGER NOT NULL,
+    session_id    TEXT NOT NULL,
+    seq           INTEGER NOT NULL,
+    tool_name     TEXT NOT NULL,
+    args_digest   TEXT,
+    status        TEXT NOT NULL,
+    started_at    TEXT,
+    completed_at  TEXT,
+    duration_ms   INTEGER,
+    result_bytes  INTEGER,
+    error_text    TEXT,
+    imported_at   TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_history_message_session
+    ON history_message(source, session_id, seq);
+CREATE INDEX IF NOT EXISTS idx_history_message_ts
+    ON history_message(ts);
+CREATE INDEX IF NOT EXISTS idx_history_tool_call_session
+    ON history_tool_call(source, session_id, seq);
+CREATE INDEX IF NOT EXISTS idx_history_tool_call_tool_name
+    ON history_tool_call(tool_name);
+CREATE INDEX IF NOT EXISTS idx_history_tool_call_message_hash
+    ON history_tool_call(message_hash);
 `.trim();

package/src/sources.ts

@@ -1,5 +1,25 @@
 import { z } from 'zod';
 import { HistoryImportError } from './errors';
+import {
+    AGY_FIELD_MAP,
+    AGY_SCHEMA,
+    agySplit,
+    CLAUDE_FIELD_MAP,
+    CLAUDE_SCHEMA,
+    CODEX_FIELD_MAP,
+    CODEX_SCHEMA,
+    claudeSplit,
+    codexSplit,
+    GROK_FIELD_MAP,
+    GROK_SCHEMA,
+    grokSplit,
+    OMP_FIELD_MAP,
+    OMP_SCHEMA,
+    ompSplit,
+    PI_FIELD_MAP,
+    PI_SCHEMA,
+    piSplit,
+} from './mappers';
 import type { FieldTransform, JsonObject, LlmJsonlSource, SourceDefinition, TransformContext } from './types';
 
 const sourceRecordSchema = z
@@ -94,6 +114,30 @@ function sourceDefinition(
         schema: sourceRecordSchema,
     };
 }
+
+/** Build a source definition with a typed custom split for the forensic contract tables. */
+function customSourceDefinition(
+    source: LlmJsonlSource,
+    displayName: string,
+    defaultRoots: readonly string[],
+    filePatterns: readonly string[],
+    split: (raw: JsonObject) => readonly import('./types').SplitEntry[],
+    fieldMap: Readonly<Record<string, string>>,
+    schema: z.ZodType<JsonObject>,
+): SourceDefinition {
+    return {
+        source,
+        displayName,
+        defaultRoots,
+        filePatterns,
+        targetTable: `history_etl_${source}`,
+        splitConfig: { mode: 'custom', split },
+        fieldMap,
+        fieldTransforms: {},
+        schema,
+    };
+}
+
 /**
  * Pattern every importer-owned table must match.
  *
@@ -102,13 +146,56 @@ function sourceDefinition(
  * namespaces importer tables away from consumer schemas. Both built-in and
  * custom source definitions are held to this rule.
  */
-export const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
+export const VALID_TABLE_NAME = /^history_[a-z_]+$/;
 
 /** Built-in source definitions keyed by source identifier. */
 export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinition>> = {
-    pi: sourceDefinition('pi', 'Pi', ['.pi/history', '.pi'], ['*.jsonl', '*.json']),
-    claude: sourceDefinition('claude', 'Claude Code', ['.claude/projects', '.claude'], ['*.jsonl']),
-    codex: sourceDefinition('codex', 'Codex', ['.codex/sessions', '.codex'], ['*.jsonl']),
+    pi: customSourceDefinition('pi', 'Pi', ['.pi/agent/sessions'], ['*.jsonl'], piSplit, PI_FIELD_MAP, PI_SCHEMA),
+    claude: customSourceDefinition(
+        'claude',
+        'Claude Code',
+        ['.claude/projects'],
+        ['*.jsonl'],
+        claudeSplit,
+        CLAUDE_FIELD_MAP,
+        CLAUDE_SCHEMA,
+    ),
+    codex: customSourceDefinition(
+        'codex',
+        'Codex',
+        ['.codex/sessions'],
+        ['*.jsonl'],
+        codexSplit,
+        CODEX_FIELD_MAP,
+        CODEX_SCHEMA,
+    ),
+    omp: customSourceDefinition(
+        'omp',
+        'OMP',
+        ['.omp/agent/sessions'],
+        ['*.jsonl'],
+        ompSplit,
+        OMP_FIELD_MAP,
+        OMP_SCHEMA,
+    ),
+    grok: customSourceDefinition(
+        'grok',
+        'Grok',
+        ['.grok/sessions'],
+        ['*.jsonl'],
+        grokSplit,
+        GROK_FIELD_MAP,
+        GROK_SCHEMA,
+    ),
+    agy: customSourceDefinition(
+        'agy',
+        'Antigravity CLI',
+        ['.gemini/antigravity-cli/brain'],
+        ['*.jsonl'],
+        agySplit,
+        AGY_FIELD_MAP,
+        AGY_SCHEMA,
+    ),
     gemini: sourceDefinition('gemini', 'Gemini CLI', ['.gemini', '.config/gemini'], ['*.jsonl']),
     opencode: sourceDefinition('opencode', 'OpenCode', ['.opencode', '.local/share/opencode'], ['*.jsonl']),
     antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
@@ -186,7 +273,7 @@ export function resolveSourceDefinition(input: string | SourceDefinition): Sourc
 function assertValidTable(table: string, source: string): void {
     if (!VALID_TABLE_NAME.test(table)) {
         throw new HistoryImportError(
-            `Invalid history ETL target table for source "${source}": ${table}. Must match ${VALID_TABLE_NAME.source}.`,
+            `Invalid history target table for source "${source}": ${table}. Must match ${VALID_TABLE_NAME.source}.`,
             { source, table },
         );
     }

package/src/types.ts

@@ -3,11 +3,27 @@ import type { FileSystem, RuntimePaths } from '@gobing-ai/ts-runtime';
 import type { z } from 'zod';
 
 /** Built-in source identifiers supported by the importer. */
-export type LlmJsonlSource = 'pi' | 'claude' | 'codex' | 'gemini' | 'opencode' | 'antigravity' | 'openclaw';
+export type LlmJsonlSource =
+    | 'pi'
+    | 'claude'
+    | 'codex'
+    | 'gemini'
+    | 'opencode'
+    | 'antigravity'
+    | 'openclaw'
+    | 'omp'
+    | 'grok'
+    | 'agy';
 
 /** Import mode controlling checkpoint behavior. */
 export type ImportMode = 'full' | 'incremental' | 'force-file';
 
+/** One record emitted by a custom split, optionally routed to its own table. */
+export interface SplitEntry {
+    readonly targetTable?: string;
+    readonly record: JsonObject;
+}
+
 /** Split strategy for turning a raw JSONL object into one or more ETL records. */
 export type SplitConfig =
     | {
@@ -20,7 +36,7 @@ export type SplitConfig =
       }
     | {
           readonly mode: 'custom';
-          readonly split: (raw: JsonObject) => readonly JsonObject[];
+          readonly split: (raw: JsonObject) => readonly (JsonObject | SplitEntry)[];
           readonly targetTable?: string;
       };
 
@@ -94,6 +110,7 @@ export interface ImportResult {
     readonly processedLines: number;
     readonly importedRecords: number;
     readonly skippedDuplicates: number;
+    readonly unknownRecords: number;
     readonly parseErrors: readonly ImportIssue[];
     readonly validationErrors: readonly ImportIssue[];
     readonly checkpointUpdates: number;