@gobing-ai/ts-llm-jsonl-importer 0.2.1

package/package.json

@@ -0,0 +1,60 @@
+{
+    "name": "@gobing-ai/ts-llm-jsonl-importer",
+    "version": "0.2.1",
+    "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
+    "keywords": [
+        "typescript",
+        "jsonl",
+        "llm",
+        "history",
+        "etl"
+    ],
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/gobing-ai/ts-libs.git",
+        "directory": "packages/llm-jsonl-importer"
+    },
+    "author": "Robin Min <minlongbing@gmail.com>",
+    "contributors": [
+        "Robin Min <minlongbing@gmail.com>"
+    ],
+    "type": "module",
+    "private": false,
+    "sideEffects": false,
+    "license": "Apache-2.0",
+    "main": "./dist/index.js",
+    "types": "./dist/index.d.ts",
+    "exports": {
+        ".": {
+            "types": "./dist/index.d.ts",
+            "import": "./dist/index.js"
+        }
+    },
+    "files": [
+        "dist",
+        "src",
+        "README.md"
+    ],
+    "scripts": {
+        "build": "tsc -p tsconfig.build.json && bun ../../scripts/builder.ts fix-dist-esm-extensions dist",
+        "test": "NODE_ENV=test bun test --coverage --coverage-dir=.coverage --reporter=dots",
+        "test:full": "NODE_ENV=test bun test --update-snapshots --coverage --coverage-dir=.coverage",
+        "typecheck": "tsc --noEmit",
+        "lint": "biome check . && bun run typecheck",
+        "format": "biome check . --write",
+        "check": "bun run lint && bun run test",
+        "prepublishOnly": "bun run build",
+        "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
+    },
+    "dependencies": {
+        "@gobing-ai/ts-db": "workspace:*",
+        "@gobing-ai/ts-runtime": "workspace:*",
+        "zod": "^4.1.0"
+    },
+    "devDependencies": {
+        "@types/bun": "1.3.14"
+    },
+    "publishConfig": {
+        "access": "public"
+    }
+}

package/src/hash.ts

@@ -0,0 +1,21 @@
+import { createHash } from 'node:crypto';
+
+/** Serialize JSON with sorted object keys so equivalent records hash identically. */
+export function stableJson(value: unknown): string {
+    if (Array.isArray(value)) {
+        return `[${value.map((entry) => stableJson(entry)).join(',')}]`;
+    }
+    if (value !== null && typeof value === 'object') {
+        const record = value as Record<string, unknown>;
+        return `{${Object.keys(record)
+            .sort()
+            .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
+            .join(',')}}`;
+    }
+    return JSON.stringify(value);
+}
+
+/** Compute a SHA-256 hash for an already-normalized record. */
+export function sha256(value: unknown): string {
+    return createHash('sha256').update(stableJson(value)).digest('hex');
+}

package/src/importer.ts

@@ -0,0 +1,342 @@
+import { resolve } from 'node:path';
+import { getFs, walkDir } from '@gobing-ai/ts-runtime';
+import { sha256 } from './hash';
+import { redactRecord } from './redaction';
+import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
+import { getSourceDefinition } from './sources';
+import type {
+    ImportIssue,
+    ImportOptions,
+    ImportResult,
+    JsonObject,
+    LlmJsonlSource,
+    SourceDefinition,
+    TransformContext,
+} from './types';
+
+interface SplitRecord {
+    readonly targetTable: string;
+    readonly raw: JsonObject;
+}
+
+interface CheckpointRow {
+    readonly last_imported_line: number;
+}
+
+const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
+
+/** Apply importer-owned schema to the target database. */
+export async function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void> {
+    for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
+        const sql = statement.trim();
+        if (sql.length > 0) {
+            await db.exec(sql);
+        }
+    }
+}
+
+/** Run the JSONL import pipeline for one source definition. */
+export async function runJsonlImport(source: LlmJsonlSource, options: ImportOptions): Promise<ImportResult> {
+    const definition = getSourceDefinition(source);
+    await applyHistoryImportSchema(options.db);
+
+    const mode = options.mode ?? 'incremental';
+    const files = await discoverFiles(definition, options.roots, options.files);
+    if (mode === 'full' && !options.dryRun) {
+        await resetCheckpoints(options.db, source, files);
+    }
+
+    const parseErrors: ImportIssue[] = [];
+    const validationErrors: ImportIssue[] = [];
+    let processedLines = 0;
+    let importedRecords = 0;
+    let skippedDuplicates = 0;
+    let checkpointUpdates = 0;
+
+    for (const file of files) {
+        const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, source, file) : 0;
+        const lines = (await getFs().readFile(file)).split(/\r?\n/);
+
+        for (let index = 0; index < lines.length; index += 1) {
+            const lineNumber = index + 1;
+            const line = lines[index]?.trim();
+            if (line === undefined || line.length === 0 || lineNumber <= checkpoint) continue;
+            processedLines += 1;
+
+            const raw = parseJsonLine(line, file, lineNumber, parseErrors);
+            if (raw === undefined) continue;
+
+            const splitRecords = splitRawRecord(definition, raw);
+            let lineSucceeded = false;
+
+            for (let splitIndex = 0; splitIndex < splitRecords.length; splitIndex += 1) {
+                const split = splitRecords[splitIndex];
+                if (split === undefined) continue;
+                const normalized = normalizeRecord(definition, split.raw, {
+                    source,
+                    sourceFile: file,
+                    sourceLine: lineNumber,
+                    splitIndex,
+                });
+                const parsed = definition.schema.safeParse(normalized);
+                if (!parsed.success) {
+                    validationErrors.push({
+                        sourceFile: file,
+                        sourceLine: lineNumber,
+                        reason: parsed.error.issues.map((issue) => issue.message).join('; '),
+                    });
+                    continue;
+                }
+
+                const redacted = redactRecord(parsed.data, options.redactionRules);
+                lineSucceeded = true;
+                const recordHash = sha256({
+                    source,
+                    sourceFile: file,
+                    sourceLine: lineNumber,
+                    splitIndex,
+                    record: redacted,
+                });
+                if (await ledgerExists(options.db, recordHash)) {
+                    skippedDuplicates += 1;
+                    continue;
+                }
+                if (!options.dryRun) {
+                    await insertRecord(
+                        options.db,
+                        split.targetTable,
+                        recordHash,
+                        file,
+                        lineNumber,
+                        splitIndex,
+                        redacted,
+                        options.now,
+                    );
+                    await insertLedger(
+                        options.db,
+                        recordHash,
+                        source,
+                        file,
+                        lineNumber,
+                        splitIndex,
+                        split.targetTable,
+                        options.now,
+                    );
+                }
+                importedRecords += 1;
+            }
+
+            if (lineSucceeded && !options.dryRun) {
+                await writeCheckpoint(options.db, source, file, lineNumber, options.now);
+                checkpointUpdates += 1;
+            }
+        }
+    }
+
+    return {
+        source,
+        mode,
+        scannedFiles: files.length,
+        processedLines,
+        importedRecords,
+        skippedDuplicates,
+        parseErrors,
+        validationErrors,
+        checkpointUpdates,
+    };
+}
+
+function parseJsonLine(
+    line: string,
+    sourceFile: string,
+    sourceLine: number,
+    parseErrors: ImportIssue[],
+): JsonObject | undefined {
+    try {
+        const value = JSON.parse(line);
+        if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+            parseErrors.push({ sourceFile, sourceLine, reason: 'JSONL row must be an object' });
+            return undefined;
+        }
+        return value as JsonObject;
+    } catch (error) {
+        parseErrors.push({ sourceFile, sourceLine, reason: error instanceof Error ? error.message : 'Invalid JSON' });
+        return undefined;
+    }
+}
+
+function splitRawRecord(definition: SourceDefinition, raw: JsonObject): readonly SplitRecord[] {
+    const targetTable = targetTableFor(definition.targetTable);
+    if (definition.splitConfig.mode === 'one-to-one') {
+        return [{ targetTable, raw }];
+    }
+    if (definition.splitConfig.mode === 'custom') {
+        const config = definition.splitConfig;
+        return config.split(raw).map((entry) => ({
+            targetTable: targetTableFor(config.targetTable ?? definition.targetTable),
+            raw: entry,
+        }));
+    }
+
+    const config = definition.splitConfig;
+    const nested = raw[config.field];
+    if (!Array.isArray(nested)) {
+        return [{ targetTable, raw }];
+    }
+    return nested
+        .filter((entry): entry is JsonObject => entry !== null && typeof entry === 'object' && !Array.isArray(entry))
+        .map((entry) => ({
+            targetTable: targetTableFor(config.targetTable ?? definition.targetTable),
+            raw: { ...raw, ...entry },
+        }));
+}
+
+function normalizeRecord(definition: SourceDefinition, raw: JsonObject, context: TransformContext): JsonObject {
+    const normalized: JsonObject = {};
+    for (const [rawKey, targetKey] of Object.entries(definition.fieldMap)) {
+        if (rawKey in raw) normalized[targetKey] = raw[rawKey];
+    }
+    for (const [targetKey, transform] of Object.entries(definition.fieldTransforms)) {
+        normalized[targetKey] = transform(normalized[targetKey], raw, context);
+    }
+    normalized.source = context.source;
+    normalized.source_file = context.sourceFile;
+    normalized.source_line = context.sourceLine;
+    normalized.split_index = context.splitIndex;
+    return normalized;
+}
+
+async function discoverFiles(
+    definition: SourceDefinition,
+    roots: readonly string[] | undefined,
+    files: readonly string[] | undefined,
+): Promise<readonly string[]> {
+    if (files !== undefined && files.length > 0) {
+        return files.map((file) => resolve(file)).sort();
+    }
+
+    const fs = getFs();
+    const resolvedRoots = (roots ?? definition.defaultRoots).map((root) => resolve(root));
+    const found = new Set<string>();
+    for (const root of resolvedRoots) {
+        if (!(await fs.exists(root))) continue;
+        const stat = await fs.stat(root);
+        if (stat === null) continue;
+        if (stat.isFile()) {
+            if (matchesPattern(root, definition.filePatterns)) found.add(root);
+            continue;
+        }
+        for (const file of await walkDir(root)) {
+            if (matchesPattern(file, definition.filePatterns)) found.add(file);
+        }
+    }
+    return [...found].sort();
+}
+
+function matchesPattern(path: string, patterns: readonly string[]): boolean {
+    return patterns.some((pattern) => {
+        if (pattern === '*.jsonl') return path.endsWith('.jsonl');
+        if (pattern === '*.json') return path.endsWith('.json');
+        return path.endsWith(pattern.replace(/^\*/, ''));
+    });
+}
+
+async function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number> {
+    const row = await db.queryFirst<CheckpointRow>(
+        'SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?',
+        source,
+        sourceFile,
+    );
+    return row?.last_imported_line ?? 0;
+}
+
+async function resetCheckpoints(db: ImportOptions['db'], source: string, files: readonly string[]): Promise<void> {
+    for (const file of files) {
+        await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
+    }
+}
+
+async function writeCheckpoint(
+    db: ImportOptions['db'],
+    source: string,
+    sourceFile: string,
+    line: number,
+    now: ImportOptions['now'],
+): Promise<void> {
+    await db.run(
+        `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
+         VALUES (?, ?, ?, ?)
+         ON CONFLICT(source, source_file) DO UPDATE SET
+            last_imported_line = excluded.last_imported_line,
+            updated_at = excluded.updated_at`,
+        source,
+        sourceFile,
+        line,
+        timestamp(now),
+    );
+}
+
+async function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean> {
+    const row = await db.queryFirst<{ record_hash: string }>(
+        'SELECT record_hash FROM history_import_ledger WHERE record_hash = ?',
+        recordHash,
+    );
+    return row !== undefined && row !== null;
+}
+
+async function insertRecord(
+    db: ImportOptions['db'],
+    targetTable: string,
+    recordHash: string,
+    sourceFile: string,
+    sourceLine: number,
+    splitIndex: number,
+    payload: JsonObject,
+    now: ImportOptions['now'],
+): Promise<void> {
+    const table = targetTableFor(targetTable);
+    await db.run(
+        `INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        recordHash,
+        sourceFile,
+        sourceLine,
+        splitIndex,
+        JSON.stringify(payload),
+        timestamp(now),
+    );
+}
+
+async function insertLedger(
+    db: ImportOptions['db'],
+    recordHash: string,
+    source: string,
+    sourceFile: string,
+    sourceLine: number,
+    splitIndex: number,
+    targetTable: string,
+    now: ImportOptions['now'],
+): Promise<void> {
+    await db.run(
+        `INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?)`,
+        recordHash,
+        source,
+        sourceFile,
+        sourceLine,
+        splitIndex,
+        targetTable,
+        timestamp(now),
+    );
+}
+
+function targetTableFor(table: string): string {
+    if (!VALID_TABLE_NAME.test(table)) {
+        throw new Error(`Invalid history ETL target table: ${table}`);
+    }
+    return table;
+}
+
+function timestamp(now: ImportOptions['now']): string {
+    return (now?.() ?? new Date()).toISOString();
+}

package/src/index.ts

@@ -0,0 +1,18 @@
+export { sha256, stableJson } from './hash';
+export { applyHistoryImportSchema, runJsonlImport } from './importer';
+export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
+export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
+export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources';
+export type {
+    FieldTransform,
+    ImportIssue,
+    ImportMode,
+    ImportOptions,
+    ImportResult,
+    JsonObject,
+    LlmJsonlSource,
+    RedactionRule,
+    SourceDefinition,
+    SplitConfig,
+    TransformContext,
+} from './types';

package/src/redaction.ts

@@ -0,0 +1,44 @@
+import type { JsonObject, RedactionRule } from './types';
+
+/** Default redaction rules for common token, key, and email shapes in agent logs. */
+export const DEFAULT_REDACTION_RULES: readonly RedactionRule[] = [
+    {
+        name: 'api-key',
+        pattern: /\b(?:sk|pk|ghp|github_pat|xox[baprs])-[-_a-zA-Z0-9]{12,}\b/g,
+        replacement: '[REDACTED:token]',
+    },
+    {
+        name: 'assignment-secret',
+        pattern: /\b(?:api[_-]?key|token|secret|password)\s*[:=]\s*["']?[^"',\s}]+/gi,
+        replacement: '[REDACTED:secret]',
+    },
+    {
+        name: 'email',
+        pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
+        replacement: '[REDACTED:email]',
+    },
+];
+
+/** Redact supported scalar and composite JSON values recursively. */
+export function redactValue(value: unknown, rules: readonly RedactionRule[] = DEFAULT_REDACTION_RULES): unknown {
+    if (typeof value === 'string') {
+        return rules.reduce((current, rule) => current.replace(rule.pattern, rule.replacement), value);
+    }
+    if (Array.isArray(value)) {
+        return value.map((entry) => redactValue(entry, rules));
+    }
+    if (value !== null && typeof value === 'object') {
+        return Object.fromEntries(
+            Object.entries(value as JsonObject).map(([key, entry]) => [key, redactValue(entry, rules)]),
+        );
+    }
+    return value;
+}
+
+/** Redact a normalized ETL record before hashing or writing it to the database. */
+export function redactRecord<TRecord extends JsonObject>(
+    record: TRecord,
+    rules: readonly RedactionRule[] = DEFAULT_REDACTION_RULES,
+): TRecord {
+    return redactValue(record, rules) as TRecord;
+}

package/src/schema-sql.ts

@@ -0,0 +1,82 @@
+export const HISTORY_IMPORT_SCHEMA_SQL = `
+CREATE TABLE IF NOT EXISTS history_import_checkpoint (
+    source TEXT NOT NULL,
+    source_file TEXT NOT NULL,
+    last_imported_line INTEGER NOT NULL DEFAULT 0,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (source, source_file)
+);
+
+CREATE TABLE IF NOT EXISTS history_import_ledger (
+    record_hash TEXT PRIMARY KEY,
+    source TEXT NOT NULL,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    target_table TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_pi (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_claude (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_codex (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_gemini (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_opencode (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_antigravity (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS history_etl_openclaw (
+    record_hash TEXT PRIMARY KEY,
+    source_file TEXT NOT NULL,
+    source_line INTEGER NOT NULL,
+    split_index INTEGER NOT NULL,
+    payload_json TEXT NOT NULL,
+    imported_at TEXT NOT NULL
+);
+`.trim();

package/src/sources.ts

@@ -0,0 +1,111 @@
+import { z } from 'zod';
+import type { FieldTransform, JsonObject, LlmJsonlSource, SourceDefinition, TransformContext } from './types';
+
+const sourceRecordSchema = z
+    .object({
+        source_record_id: z.string().min(1),
+        created_at: z.string().min(1),
+        content: z.string().min(1),
+        role: z.string().optional(),
+        model: z.string().optional(),
+    })
+    .passthrough();
+
+function firstString(...values: readonly unknown[]): string | undefined {
+    for (const value of values) {
+        if (typeof value === 'string' && value.length > 0) return value;
+        if (typeof value === 'number' && Number.isFinite(value)) return String(value);
+    }
+    return undefined;
+}
+
+function defaultCreatedAt(value: unknown): string {
+    return firstString(value) ?? new Date(0).toISOString();
+}
+
+function defaultRecordId(value: unknown, raw: JsonObject, context: TransformContext): string {
+    return (
+        firstString(
+            value,
+            raw.id,
+            raw.uuid,
+            raw.message_id,
+            raw.messageId,
+            raw.session_id,
+            raw.sessionId,
+            raw.conversation_id,
+        ) ?? `${context.source}:${context.sourceFile}:${context.sourceLine}:${context.splitIndex}`
+    );
+}
+
+function defaultContent(value: unknown, raw: JsonObject): string | undefined {
+    return firstString(value, raw.content, raw.text, raw.message, raw.prompt, raw.response, raw.output);
+}
+
+function transforms(): Readonly<Record<string, FieldTransform>> {
+    return {
+        source_record_id: defaultRecordId,
+        created_at: defaultCreatedAt,
+        content: defaultContent,
+        role: (value, raw) => firstString(value, raw.role, raw.type),
+        model: (value, raw) => firstString(value, raw.model, raw.model_name, raw.modelName),
+    };
+}
+
+function sourceDefinition(
+    source: LlmJsonlSource,
+    displayName: string,
+    defaultRoots: readonly string[],
+    filePatterns: readonly string[],
+): SourceDefinition {
+    return {
+        source,
+        displayName,
+        defaultRoots,
+        filePatterns,
+        targetTable: `history_etl_${source}`,
+        splitConfig: source === 'pi' ? { mode: 'one-to-many', field: 'messages' } : { mode: 'one-to-one' },
+        fieldMap: {
+            id: 'source_record_id',
+            uuid: 'source_record_id',
+            message_id: 'source_record_id',
+            messageId: 'source_record_id',
+            session_id: 'source_record_id',
+            sessionId: 'source_record_id',
+            conversation_id: 'source_record_id',
+            timestamp: 'created_at',
+            created_at: 'created_at',
+            createdAt: 'created_at',
+            time: 'created_at',
+            content: 'content',
+            text: 'content',
+            message: 'content',
+            prompt: 'content',
+            response: 'content',
+            output: 'content',
+            role: 'role',
+            type: 'role',
+            model: 'model',
+            model_name: 'model',
+            modelName: 'model',
+        },
+        fieldTransforms: transforms(),
+        schema: sourceRecordSchema,
+    };
+}
+
+/** Built-in source definitions keyed by source identifier. */
+export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinition>> = {
+    pi: sourceDefinition('pi', 'Pi', ['.pi/history', '.pi'], ['*.jsonl', '*.json']),
+    claude: sourceDefinition('claude', 'Claude Code', ['.claude/projects', '.claude'], ['*.jsonl']),
+    codex: sourceDefinition('codex', 'Codex', ['.codex/sessions', '.codex'], ['*.jsonl']),
+    gemini: sourceDefinition('gemini', 'Gemini CLI', ['.gemini', '.config/gemini'], ['*.jsonl']),
+    opencode: sourceDefinition('opencode', 'OpenCode', ['.opencode', '.local/share/opencode'], ['*.jsonl']),
+    antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
+    openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
+};
+
+/** Resolve a built-in source definition by identifier. */
+export function getSourceDefinition(source: LlmJsonlSource): SourceDefinition {
+    return SOURCE_DEFINITIONS[source];
+}