@gobing-ai/ts-llm-jsonl-importer 0.2.7 → 0.2.9

package/README.md

@@ -1,4 +1,135 @@
 # @gobing-ai/ts-llm-jsonl-importer
 
-Generic JSONL importer for LLM agent history files. The package owns the import schema, source definitions, validation, redaction, deduplication, and checkpointing.
+Generic JSONL import pipeline for AI-agent history-style files: discover files, parse rows, normalize source fields, redact sensitive values, deduplicate by ledger hash, and persist ETL rows.
 
+## What It Provides
+
+`ts-llm-jsonl-importer` is a common JSONL importer. It is intentionally not a conversation-history domain model. Built-in source definitions cover common agent file shapes, but downstream systems consume normalized ETL rows and ledger metadata.
+
+| Export | Purpose |
+|--------|---------|
+| `runJsonlImport()` | Runs discovery, parsing, validation, redaction, dedupe, and persistence |
+| `applyHistoryImportSchema()` | Installs importer-owned checkpoint, ledger, and ETL tables |
+| `SOURCE_DEFINITIONS` | Built-in source definitions |
+| `getSourceDefinition()` | Returns one source definition by key |
+| `redactRecord()` / `redactValue()` | Applies redaction rules before persistence |
+| `sha256()` / `stableJson()` | Stable hash helpers used by the ledger |
+| `HISTORY_IMPORT_SCHEMA_SQL` | SQL schema string for explicit migration flows |
+
+Built-in source keys are `claude`, `codex`, `gemini`, `pi`, `opencode`, `antigravity`, and `openclaw`.
+
+## Installation
+
+```bash
+bun add @gobing-ai/ts-llm-jsonl-importer @gobing-ai/ts-db
+```
+
+The importer expects a `DbAdapter`-compatible object from `@gobing-ai/ts-db`.
+
+## Basic Import
+
+```ts
+import { createDbAdapter } from '@gobing-ai/ts-db';
+import { runJsonlImport } from '@gobing-ai/ts-llm-jsonl-importer';
+
+const db = await createDbAdapter({ driver: 'bun-sqlite', url: './history-import.db' });
+
+const result = await runJsonlImport('codex', {
+    db,
+    roots: ['./agent-history'],
+    mode: 'incremental',
+});
+
+result.importedRecords;
+result.skippedDuplicates;
+```
+
+`runJsonlImport()` applies the package-owned schema automatically before processing. Use `applyHistoryImportSchema(db)` directly when your application has an explicit migration step.
+
+## Import Modes
+
+| Mode | Behavior |
+|------|----------|
+| `incremental` | Reads each file from the last imported line checkpoint |
+| `full` | Clears checkpoints for selected files, scans all lines, and still deduplicates by ledger hash |
+| `force-file` | Scans selected files without using the checkpoint, while preserving ledger-based duplicate protection |
+
+All modes preserve parse and validation issues in the returned `ImportResult`; malformed rows are not inserted.
+
+## Import Specific Files
+
+```ts
+const result = await runJsonlImport('pi', {
+    db,
+    files: ['/tmp/session-1.jsonl', '/tmp/session-2.jsonl'],
+    mode: 'full',
+    now: () => new Date(),
+});
+```
+
+When `files` is provided, roots are ignored. When roots are provided, the importer walks each root and selects files matching the source definition's patterns.
+
+## Redaction
+
+Redaction runs before hashing and persistence, so the ledger hash represents the persisted redacted payload.
+
+```ts
+import { DEFAULT_REDACTION_RULES, runJsonlImport } from '@gobing-ai/ts-llm-jsonl-importer';
+
+await runJsonlImport('openclaw', {
+    db,
+    roots: ['./history'],
+    redactionRules: [
+        ...DEFAULT_REDACTION_RULES,
+        { name: 'account-id', pattern: /acct_[a-z0-9]+/gi, replacement: '[REDACTED:account]' },
+    ],
+});
+```
+
+Rules are applied recursively to string fields in the normalized record.
+
+## Result Shape
+
+```ts
+interface ImportResult {
+    source: string;
+    mode: 'incremental' | 'full' | 'force-file';
+    scannedFiles: number;
+    processedLines: number;
+    importedRecords: number;
+    skippedDuplicates: number;
+    parseErrors: ImportIssue[];
+    validationErrors: ImportIssue[];
+    checkpointUpdates: number;
+}
+```
+
+Use `parseErrors` for invalid JSON or non-object rows. Use `validationErrors` for source rows that parse but fail the source definition schema.
+
+## Stored Tables
+
+The schema contains:
+
+| Table | Purpose |
+|-------|---------|
+| `history_import_checkpoint` | Per-source/per-file last imported line |
+| `history_import_ledger` | Stable hash ledger for dedupe and provenance |
+| `history_etl_<source>` | Redacted normalized payloads for each built-in source |
+
+ETL tables store the normalized payload JSON plus source file, source line, split index, hash, and timestamps.
+
+## Split Records
+
+Most sources are one input row to one ETL row. A source can also split one JSONL row into multiple ETL records. The built-in `pi` definition splits nested `messages` into one row per message when present.
+
+```ts
+const result = await runJsonlImport('pi', { db, files: ['session.jsonl'], mode: 'full' });
+```
+
+Downstream consumers should treat `source_file`, `source_line`, and `split_index` as the stable provenance tuple.
+
+## Boundary Notes
+
+- This package imports JSONL files and writes importer-owned tables; it does not model conversations, turns, tool calls, or analytics semantics.
+- Source definitions are the normalization boundary. Downstream applications own domain-specific interpretation of ETL rows.
+- The importer never stores raw malformed rows. Parse and validation failures are reported in memory through `ImportResult`.

package/dist/errors.d.ts

@@ -0,0 +1,6 @@
+/** Error raised for invalid importer configuration or unsafe generated SQL identifiers. */
+export declare class HistoryImportError extends Error {
+    readonly details?: unknown | undefined;
+    constructor(message: string, details?: unknown | undefined);
+}
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,qBAAa,kBAAmB,SAAQ,KAAK;IAGrC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO;gBAD1B,OAAO,EAAE,MAAM,EACN,OAAO,CAAC,EAAE,OAAO,YAAA;CAKjC"}

package/dist/errors.js

@@ -0,0 +1,9 @@
+/** Error raised for invalid importer configuration or unsafe generated SQL identifiers. */
+export class HistoryImportError extends Error {
+    details;
+    constructor(message, details) {
+        super(message);
+        this.details = details;
+        this.name = 'HistoryImportError';
+    }
+}

package/dist/hash.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAEA,qFAAqF;AACrF,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAYjD;AAED,+DAA+D;AAC/D,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAE7C"}
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAEA,qFAAqF;AACrF,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAajD;AAED,+DAA+D;AAC/D,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAE7C"}

package/dist/hash.js

@@ -2,16 +2,17 @@ import { createHash } from 'node:crypto';
 /** Serialize JSON with sorted object keys so equivalent records hash identically. */
 export function stableJson(value) {
     if (Array.isArray(value)) {
-        return `[${value.map((entry) => stableJson(entry)).join(',')}]`;
+        return `[${value.map((entry) => (entry === undefined ? 'null' : stableJson(entry))).join(',')}]`;
     }
     if (value !== null && typeof value === 'object') {
         const record = value;
         return `{${Object.keys(record)
+            .filter((key) => record[key] !== undefined)
             .sort()
             .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
             .join(',')}}`;
     }
-    return JSON.stringify(value);
+    return JSON.stringify(value) ?? 'null';
 }
 /** Compute a SHA-256 hash for an already-normalized record. */
 export function sha256(value) {

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAEZ,cAAc,EAGjB,MAAM,SAAS,CAAC;AAajB,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,+DAA+D;AAC/D,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA4G1G"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAEZ,cAAc,EAGjB,MAAM,SAAS,CAAC;AAajB,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,+DAA+D;AAC/D,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA4G1G"}

package/dist/importer.js

@@ -1,5 +1,6 @@
 import { resolve } from 'node:path';
 import { getFs, walkDir } from '@gobing-ai/ts-runtime';
+import { HistoryImportError } from './errors.js';
 import { sha256 } from './hash.js';
 import { redactRecord } from './redaction.js';
 import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
@@ -217,7 +218,7 @@ async function insertLedger(db, recordHash, source, sourceFile, sourceLine, spli
 }
 function targetTableFor(table) {
     if (!VALID_TABLE_NAME.test(table)) {
-        throw new Error(`Invalid history ETL target table: ${table}`);
+        throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
     }
     return table;
 }

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+export { HistoryImportError } from './errors';
 export { sha256, stableJson } from './hash';
 export { applyHistoryImportSchema, runJsonlImport } from './importer';
 export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACpE,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACpE,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -1,3 +1,4 @@
+export { HistoryImportError } from './errors.js';
 export { sha256, stableJson } from './hash.js';
 export { applyHistoryImportSchema, runJsonlImport } from './importer.js';
 export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gobing-ai/ts-llm-jsonl-importer",
-    "version": "0.2.7",
+    "version": "0.2.9",
     "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
     "keywords": [
         "typescript",
@@ -47,8 +47,9 @@
         "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
     },
     "dependencies": {
-        "@gobing-ai/ts-db": "^0.2.7",
-        "@gobing-ai/ts-runtime": "^0.2.7",
+        "@gobing-ai/ts-db": "^0.2.9",
+        "@gobing-ai/ts-runtime": "^0.2.9",
+        "@gobing-ai/ts-utils": "^0.2.9",
         "zod": "^4.1.0"
     },
     "devDependencies": {

package/src/errors.ts

@@ -0,0 +1,10 @@
+/** Error raised for invalid importer configuration or unsafe generated SQL identifiers. */
+export class HistoryImportError extends Error {
+    constructor(
+        message: string,
+        readonly details?: unknown,
+    ) {
+        super(message);
+        this.name = 'HistoryImportError';
+    }
+}

package/src/hash.ts

@@ -3,16 +3,17 @@ import { createHash } from 'node:crypto';
 /** Serialize JSON with sorted object keys so equivalent records hash identically. */
 export function stableJson(value: unknown): string {
     if (Array.isArray(value)) {
-        return `[${value.map((entry) => stableJson(entry)).join(',')}]`;
+        return `[${value.map((entry) => (entry === undefined ? 'null' : stableJson(entry))).join(',')}]`;
     }
     if (value !== null && typeof value === 'object') {
         const record = value as Record<string, unknown>;
         return `{${Object.keys(record)
+            .filter((key) => record[key] !== undefined)
             .sort()
             .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
             .join(',')}}`;
     }
-    return JSON.stringify(value);
+    return JSON.stringify(value) ?? 'null';
 }
 
 /** Compute a SHA-256 hash for an already-normalized record. */

package/src/importer.ts

@@ -1,5 +1,6 @@
 import { resolve } from 'node:path';
 import { getFs, walkDir } from '@gobing-ai/ts-runtime';
+import { HistoryImportError } from './errors';
 import { sha256 } from './hash';
 import { redactRecord } from './redaction';
 import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
@@ -332,7 +333,7 @@ async function insertLedger(
 
 function targetTableFor(table: string): string {
     if (!VALID_TABLE_NAME.test(table)) {
-        throw new Error(`Invalid history ETL target table: ${table}`);
+        throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
     }
     return table;
 }

package/src/index.ts

@@ -1,3 +1,4 @@
+export { HistoryImportError } from './errors';
 export { sha256, stableJson } from './hash';
 export { applyHistoryImportSchema, runJsonlImport } from './importer';
 export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';