@gobing-ai/ts-llm-jsonl-importer 0.2.1

package/README.md

@@ -0,0 +1,4 @@
+# @gobing-ai/ts-llm-jsonl-importer
+
+Generic JSONL importer for LLM agent history files. The package owns the import schema, source definitions, validation, redaction, deduplication, and checkpointing.
+

package/dist/hash.d.ts

@@ -0,0 +1,5 @@
+/** Serialize JSON with sorted object keys so equivalent records hash identically. */
+export declare function stableJson(value: unknown): string;
+/** Compute a SHA-256 hash for an already-normalized record. */
+export declare function sha256(value: unknown): string;
+//# sourceMappingURL=hash.d.ts.map

package/dist/hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAEA,qFAAqF;AACrF,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAYjD;AAED,+DAA+D;AAC/D,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAE7C"}

package/dist/hash.js

@@ -0,0 +1,19 @@
+import { createHash } from 'node:crypto';
+/** Serialize JSON with sorted object keys so equivalent records hash identically. */
+export function stableJson(value) {
+    if (Array.isArray(value)) {
+        return `[${value.map((entry) => stableJson(entry)).join(',')}]`;
+    }
+    if (value !== null && typeof value === 'object') {
+        const record = value;
+        return `{${Object.keys(record)
+            .sort()
+            .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
+            .join(',')}}`;
+    }
+    return JSON.stringify(value);
+}
+/** Compute a SHA-256 hash for an already-normalized record. */
+export function sha256(value) {
+    return createHash('sha256').update(stableJson(value)).digest('hex');
+}

package/dist/importer.d.ts

@@ -0,0 +1,6 @@
+import type { ImportOptions, ImportResult, LlmJsonlSource } from './types';
+/** Apply importer-owned schema to the target database. */
+export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void>;
+/** Run the JSONL import pipeline for one source definition. */
+export declare function runJsonlImport(source: LlmJsonlSource, options: ImportOptions): Promise<ImportResult>;
+//# sourceMappingURL=importer.d.ts.map

package/dist/importer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAEZ,cAAc,EAGjB,MAAM,SAAS,CAAC;AAajB,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,+DAA+D;AAC/D,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA4G1G"}

package/dist/importer.js

@@ -0,0 +1,226 @@
+import { resolve } from 'node:path';
+import { getFs, walkDir } from '@gobing-ai/ts-runtime';
+import { sha256 } from './hash.js';
+import { redactRecord } from './redaction.js';
+import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
+import { getSourceDefinition } from './sources.js';
+const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
+/** Apply importer-owned schema to the target database. */
+export async function applyHistoryImportSchema(db) {
+    for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
+        const sql = statement.trim();
+        if (sql.length > 0) {
+            await db.exec(sql);
+        }
+    }
+}
+/** Run the JSONL import pipeline for one source definition. */
+export async function runJsonlImport(source, options) {
+    const definition = getSourceDefinition(source);
+    await applyHistoryImportSchema(options.db);
+    const mode = options.mode ?? 'incremental';
+    const files = await discoverFiles(definition, options.roots, options.files);
+    if (mode === 'full' && !options.dryRun) {
+        await resetCheckpoints(options.db, source, files);
+    }
+    const parseErrors = [];
+    const validationErrors = [];
+    let processedLines = 0;
+    let importedRecords = 0;
+    let skippedDuplicates = 0;
+    let checkpointUpdates = 0;
+    for (const file of files) {
+        const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, source, file) : 0;
+        const lines = (await getFs().readFile(file)).split(/\r?\n/);
+        for (let index = 0; index < lines.length; index += 1) {
+            const lineNumber = index + 1;
+            const line = lines[index]?.trim();
+            if (line === undefined || line.length === 0 || lineNumber <= checkpoint)
+                continue;
+            processedLines += 1;
+            const raw = parseJsonLine(line, file, lineNumber, parseErrors);
+            if (raw === undefined)
+                continue;
+            const splitRecords = splitRawRecord(definition, raw);
+            let lineSucceeded = false;
+            for (let splitIndex = 0; splitIndex < splitRecords.length; splitIndex += 1) {
+                const split = splitRecords[splitIndex];
+                if (split === undefined)
+                    continue;
+                const normalized = normalizeRecord(definition, split.raw, {
+                    source,
+                    sourceFile: file,
+                    sourceLine: lineNumber,
+                    splitIndex,
+                });
+                const parsed = definition.schema.safeParse(normalized);
+                if (!parsed.success) {
+                    validationErrors.push({
+                        sourceFile: file,
+                        sourceLine: lineNumber,
+                        reason: parsed.error.issues.map((issue) => issue.message).join('; '),
+                    });
+                    continue;
+                }
+                const redacted = redactRecord(parsed.data, options.redactionRules);
+                lineSucceeded = true;
+                const recordHash = sha256({
+                    source,
+                    sourceFile: file,
+                    sourceLine: lineNumber,
+                    splitIndex,
+                    record: redacted,
+                });
+                if (await ledgerExists(options.db, recordHash)) {
+                    skippedDuplicates += 1;
+                    continue;
+                }
+                if (!options.dryRun) {
+                    await insertRecord(options.db, split.targetTable, recordHash, file, lineNumber, splitIndex, redacted, options.now);
+                    await insertLedger(options.db, recordHash, source, file, lineNumber, splitIndex, split.targetTable, options.now);
+                }
+                importedRecords += 1;
+            }
+            if (lineSucceeded && !options.dryRun) {
+                await writeCheckpoint(options.db, source, file, lineNumber, options.now);
+                checkpointUpdates += 1;
+            }
+        }
+    }
+    return {
+        source,
+        mode,
+        scannedFiles: files.length,
+        processedLines,
+        importedRecords,
+        skippedDuplicates,
+        parseErrors,
+        validationErrors,
+        checkpointUpdates,
+    };
+}
+function parseJsonLine(line, sourceFile, sourceLine, parseErrors) {
+    try {
+        const value = JSON.parse(line);
+        if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+            parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
+            return undefined;
+        }
+        return value;
+    }
+    catch (error) {
+        parseErrors.push({ sourceFile, sourceLine, reason: error instanceof Error ? error.message : 'Invalid JSON' });
+        return undefined;
+    }
+}
+function splitRawRecord(definition, raw) {
+    const targetTable = targetTableFor(definition.targetTable);
+    if (definition.splitConfig.mode === 'one-to-one') {
+        return [{ targetTable, raw }];
+    }
+    if (definition.splitConfig.mode === 'custom') {
+        const config = definition.splitConfig;
+        return config.split(raw).map((entry) => ({
+            targetTable: targetTableFor(config.targetTable ?? definition.targetTable),
+            raw: entry,
+        }));
+    }
+    const config = definition.splitConfig;
+    const nested = raw[config.field];
+    if (!Array.isArray(nested)) {
+        return [{ targetTable, raw }];
+    }
+    return nested
+        .filter((entry) => entry !== null && typeof entry === 'object' && !Array.isArray(entry))
+        .map((entry) => ({
+        targetTable: targetTableFor(config.targetTable ?? definition.targetTable),
+        raw: { ...raw, ...entry },
+    }));
+}
+function normalizeRecord(definition, raw, context) {
+    const normalized = {};
+    for (const [rawKey, targetKey] of Object.entries(definition.fieldMap)) {
+        if (rawKey in raw)
+            normalized[targetKey] = raw[rawKey];
+    }
+    for (const [targetKey, transform] of Object.entries(definition.fieldTransforms)) {
+        normalized[targetKey] = transform(normalized[targetKey], raw, context);
+    }
+    normalized.source = context.source;
+    normalized.source_file = context.sourceFile;
+    normalized.source_line = context.sourceLine;
+    normalized.split_index = context.splitIndex;
+    return normalized;
+}
+async function discoverFiles(definition, roots, files) {
+    if (files !== undefined && files.length > 0) {
+        return files.map((file) => resolve(file)).sort();
+    }
+    const fs = getFs();
+    const resolvedRoots = (roots ?? definition.defaultRoots).map((root) => resolve(root));
+    const found = new Set();
+    for (const root of resolvedRoots) {
+        if (!(await fs.exists(root)))
+            continue;
+        const stat = await fs.stat(root);
+        if (stat === null)
+            continue;
+        if (stat.isFile()) {
+            if (matchesPattern(root, definition.filePatterns))
+                found.add(root);
+            continue;
+        }
+        for (const file of await walkDir(root)) {
+            if (matchesPattern(file, definition.filePatterns))
+                found.add(file);
+        }
+    }
+    return [...found].sort();
+}
+function matchesPattern(path, patterns) {
+    return patterns.some((pattern) => {
+        if (pattern === '*.jsonl')
+            return path.endsWith('.jsonl');
+        if (pattern === '*.json')
+            return path.endsWith('.json');
+        return path.endsWith(pattern.replace(/^\*/, ''));
+    });
+}
+async function readCheckpoint(db, source, sourceFile) {
+    const row = await db.queryFirst('SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, sourceFile);
+    return row?.last_imported_line ?? 0;
+}
+async function resetCheckpoints(db, source, files) {
+    for (const file of files) {
+        await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
+    }
+}
+async function writeCheckpoint(db, source, sourceFile, line, now) {
+    await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
+         VALUES (?, ?, ?, ?)
+         ON CONFLICT(source, source_file) DO UPDATE SET
+            last_imported_line = excluded.last_imported_line,
+            updated_at = excluded.updated_at`, source, sourceFile, line, timestamp(now));
+}
+async function ledgerExists(db, recordHash) {
+    const row = await db.queryFirst('SELECT record_hash FROM history_import_ledger WHERE record_hash = ?', recordHash);
+    return row !== undefined && row !== null;
+}
+async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
+    const table = targetTableFor(targetTable);
+    await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
+         VALUES (?, ?, ?, ?, ?, ?)`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
+}
+async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
+    await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?)`, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now));
+}
+function targetTableFor(table) {
+    if (!VALID_TABLE_NAME.test(table)) {
+        throw new Error(`Invalid history ETL target table: ${table}`);
+    }
+    return table;
+}
+function timestamp(now) {
+    return (now?.() ?? new Date()).toISOString();
+}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export { sha256, stableJson } from './hash';
+export { applyHistoryImportSchema, runJsonlImport } from './importer';
+export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
+export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
+export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources';
+export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, RedactionRule, SourceDefinition, SplitConfig, TransformContext, } from './types';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACpE,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -0,0 +1,5 @@
+export { sha256, stableJson } from './hash.js';
+export { applyHistoryImportSchema, runJsonlImport } from './importer.js';
+export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
+export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
+export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources.js';

package/dist/redaction.d.ts

@@ -0,0 +1,8 @@
+import type { JsonObject, RedactionRule } from './types';
+/** Default redaction rules for common token, key, and email shapes in agent logs. */
+export declare const DEFAULT_REDACTION_RULES: readonly RedactionRule[];
+/** Redact supported scalar and composite JSON values recursively. */
+export declare function redactValue(value: unknown, rules?: readonly RedactionRule[]): unknown;
+/** Redact a normalized ETL record before hashing or writing it to the database. */
+export declare function redactRecord<TRecord extends JsonObject>(record: TRecord, rules?: readonly RedactionRule[]): TRecord;
+//# sourceMappingURL=redaction.d.ts.map

package/dist/redaction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redaction.d.ts","sourceRoot":"","sources":["../src/redaction.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEzD,qFAAqF;AACrF,eAAO,MAAM,uBAAuB,EAAE,SAAS,aAAa,EAgB3D,CAAC;AAEF,qEAAqE;AACrE,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,GAAE,SAAS,aAAa,EAA4B,GAAG,OAAO,CAa9G;AAED,mFAAmF;AACnF,wBAAgB,YAAY,CAAC,OAAO,SAAS,UAAU,EACnD,MAAM,EAAE,OAAO,EACf,KAAK,GAAE,SAAS,aAAa,EAA4B,GAC1D,OAAO,CAET"}

package/dist/redaction.js

@@ -0,0 +1,35 @@
+/** Default redaction rules for common token, key, and email shapes in agent logs. */
+export const DEFAULT_REDACTION_RULES = [
+    {
+        name: 'api-key',
+        pattern: /\b(?:sk|pk|ghp|github_pat|xox[baprs])-[-_a-zA-Z0-9]{12,}\b/g,
+        replacement: '[REDACTED:token]',
+    },
+    {
+        name: 'assignment-secret',
+        pattern: /\b(?:api[_-]?key|token|secret|password)\s*[:=]\s*["']?[^"',\s}]+/gi,
+        replacement: '[REDACTED:secret]',
+    },
+    {
+        name: 'email',
+        pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
+        replacement: '[REDACTED:email]',
+    },
+];
+/** Redact supported scalar and composite JSON values recursively. */
+export function redactValue(value, rules = DEFAULT_REDACTION_RULES) {
+    if (typeof value === 'string') {
+        return rules.reduce((current, rule) => current.replace(rule.pattern, rule.replacement), value);
+    }
+    if (Array.isArray(value)) {
+        return value.map((entry) => redactValue(entry, rules));
+    }
+    if (value !== null && typeof value === 'object') {
+        return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redactValue(entry, rules)]));
+    }
+    return value;
+}
+/** Redact a normalized ETL record before hashing or writing it to the database. */
+export function redactRecord(record, rules = DEFAULT_REDACTION_RULES) {
+    return redactValue(record, rules);
+}

package/dist/schema-sql.d.ts

@@ -0,0 +1,2 @@
+export declare const HISTORY_IMPORT_SCHEMA_SQL: string;
+//# sourceMappingURL=schema-sql.d.ts.map

package/dist/schema-sql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,yBAAyB,QAiF9B,CAAC"}

package/dist/schema-sql.js

@@ -0,0 +1,82 @@
+export const HISTORY_IMPORT_SCHEMA_SQL = `
+CREATE TABLE IF NOT EXISTS history_import_checkpoint (
+    source TEXT NOT NULL,
+    source_file TEXT NOT NULL,
+    last_imported_line INTEGER NOT NULL DEFAULT 0,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (source, source_file)
+);
+
+CREATE TABLE IF NOT EXISTS history_import_ledger (
+    record_hash TEXT PRIMARY KEY,
+    source TEXT NOT NULL,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    target_table TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_pi (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_claude (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_codex (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_gemini (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_opencode (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_antigravity (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_openclaw (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+`.trim();

package/dist/sources.d.ts

@@ -0,0 +1,6 @@
+import type { LlmJsonlSource, SourceDefinition } from './types';
+/** Built-in source definitions keyed by source identifier. */
+export declare const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinition>>;
+/** Resolve a built-in source definition by identifier. */
+export declare function getSourceDefinition(source: LlmJsonlSource): SourceDefinition;
+//# sourceMappingURL=sources.d.ts.map

package/dist/sources.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AA+F9G,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAQjF,CAAC;AAEF,0DAA0D;AAC1D,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAE5E"}

package/dist/sources.js

@@ -0,0 +1,87 @@
+import { z } from 'zod';
+const sourceRecordSchema = z
+    .object({
+    source_record_id: z.string().min(1),
+    created_at: z.string().min(1),
+    content: z.string().min(1),
+    role: z.string().optional(),
+    model: z.string().optional(),
+})
+    .passthrough();
+function firstString(...values) {
+    for (const value of values) {
+        if (typeof value === 'string' && value.length > 0)
+            return value;
+        if (typeof value === 'number' && Number.isFinite(value))
+            return String(value);
+    }
+    return undefined;
+}
+function defaultCreatedAt(value) {
+    return firstString(value) ?? new Date(0).toISOString();
+}
+function defaultRecordId(value, raw, context) {
+    return (firstString(value, raw.id, raw.uuid, raw.message_id, raw.messageId, raw.session_id, raw.sessionId, raw.conversation_id) ?? `${context.source}:${context.sourceFile}:${context.sourceLine}:${context.splitIndex}`);
+}
+function defaultContent(value, raw) {
+    return firstString(value, raw.content, raw.text, raw.message, raw.prompt, raw.response, raw.output);
+}
+function transforms() {
+    return {
+        source_record_id: defaultRecordId,
+        created_at: defaultCreatedAt,
+        content: defaultContent,
+        role: (value, raw) => firstString(value, raw.role, raw.type),
+        model: (value, raw) => firstString(value, raw.model, raw.model_name, raw.modelName),
+    };
+}
+function sourceDefinition(source, displayName, defaultRoots, filePatterns) {
+    return {
+        source,
+        displayName,
+        defaultRoots,
+        filePatterns,
+        targetTable: `history_etl_${source}`,
+        splitConfig: source === 'pi' ? { mode: 'one-to-many', field: 'messages' } : { mode: 'one-to-one' },
+        fieldMap: {
+            id: 'source_record_id',
+            uuid: 'source_record_id',
+            message_id: 'source_record_id',
+            messageId: 'source_record_id',
+            session_id: 'source_record_id',
+            sessionId: 'source_record_id',
+            conversation_id: 'source_record_id',
+            timestamp: 'created_at',
+            created_at: 'created_at',
+            createdAt: 'created_at',
+            time: 'created_at',
+            content: 'content',
+            text: 'content',
+            message: 'content',
+            prompt: 'content',
+            response: 'content',
+            output: 'content',
+            role: 'role',
+            type: 'role',
+            model: 'model',
+            model_name: 'model',
+            modelName: 'model',
+        },
+        fieldTransforms: transforms(),
+        schema: sourceRecordSchema,
+    };
+}
+/** Built-in source definitions keyed by source identifier. */
+export const SOURCE_DEFINITIONS = {
+    pi: sourceDefinition('pi', 'Pi', ['.pi/history', '.pi'], ['*.jsonl', '*.json']),
+    claude: sourceDefinition('claude', 'Claude Code', ['.claude/projects', '.claude'], ['*.jsonl']),
+    codex: sourceDefinition('codex', 'Codex', ['.codex/sessions', '.codex'], ['*.jsonl']),
+    gemini: sourceDefinition('gemini', 'Gemini CLI', ['.gemini', '.config/gemini'], ['*.jsonl']),
+    opencode: sourceDefinition('opencode', 'OpenCode', ['.opencode', '.local/share/opencode'], ['*.jsonl']),
+    antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
+    openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
+};
+/** Resolve a built-in source definition by identifier. */
+export function getSourceDefinition(source) {
+    return SOURCE_DEFINITIONS[source];
+}

package/dist/types.d.ts

@@ -0,0 +1,76 @@
+import type { DbAdapter } from '@gobing-ai/ts-db';
+import type { z } from 'zod';
+/** Built-in source identifiers supported by the importer. */
+export type LlmJsonlSource = 'pi' | 'claude' | 'codex' | 'gemini' | 'opencode' | 'antigravity' | 'openclaw';
+/** Import mode controlling checkpoint behavior. */
+export type ImportMode = 'full' | 'incremental' | 'force-file';
+/** Split strategy for turning a raw JSONL object into one or more ETL records. */
+export type SplitConfig = {
+    readonly mode: 'one-to-one';
+} | {
+    readonly mode: 'one-to-many';
+    readonly field: string;
+    readonly targetTable?: string;
+} | {
+    readonly mode: 'custom';
+    readonly split: (raw: JsonObject) => readonly JsonObject[];
+    readonly targetTable?: string;
+};
+/** JSON-compatible record used at the importer boundary. */
+export type JsonObject = Record<string, unknown>;
+/** Transform function for mapped canonical fields. */
+export type FieldTransform = (value: unknown, raw: JsonObject, context: TransformContext) => unknown;
+/** Context available while normalizing a split record. */
+export interface TransformContext {
+    readonly source: LlmJsonlSource;
+    readonly sourceFile: string;
+    readonly sourceLine: number;
+    readonly splitIndex: number;
+}
+/** Declarative importer configuration for one LLM history source. */
+export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
+    readonly source: LlmJsonlSource;
+    readonly displayName: string;
+    readonly defaultRoots: readonly string[];
+    readonly filePatterns: readonly string[];
+    readonly targetTable: string;
+    readonly splitConfig: SplitConfig;
+    readonly fieldMap: Readonly<Record<string, string>>;
+    readonly fieldTransforms: Readonly<Record<string, FieldTransform>>;
+    readonly schema: z.ZodType<TRecord>;
+}
+/** Redaction rule applied before hashing and persistence. */
+export interface RedactionRule {
+    readonly name: string;
+    readonly pattern: RegExp;
+    readonly replacement: string;
+}
+/** Options for one importer run. */
+export interface ImportOptions {
+    readonly db: DbAdapter;
+    readonly mode?: ImportMode;
+    readonly roots?: readonly string[];
+    readonly files?: readonly string[];
+    readonly dryRun?: boolean;
+    readonly redactionRules?: readonly RedactionRule[];
+    readonly now?: () => Date;
+}
+/** Structured importer error that can be reported without persisting raw input. */
+export interface ImportIssue {
+    readonly sourceFile: string;
+    readonly sourceLine: number;
+    readonly reason: string;
+}
+/** Summary produced by the importer control function. */
+export interface ImportResult {
+    readonly source: LlmJsonlSource;
+    readonly mode: ImportMode;
+    readonly scannedFiles: number;
+    readonly processedLines: number;
+    readonly importedRecords: number;
+    readonly skippedDuplicates: number;
+    readonly parseErrors: readonly ImportIssue[];
+    readonly validationErrors: readonly ImportIssue[];
+    readonly checkpointUpdates: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC;AAE5G,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,SAAS,UAAU,EAAE,CAAC;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAC7B;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACtC"}