@gobing-ai/ts-llm-jsonl-importer 0.4.28 → 0.4.30

package/README.md

@@ -145,7 +145,8 @@ Downstream consumers should treat `source_file`, `source_line`, and `split_index
 
 Beyond the built-in source keys, `runJsonlImport()` accepts a fully-specified `SourceDefinition`
 for any custom agent history format. Unknown strings throw `HistoryImportError`; custom definitions
-are validated before any I/O (target-table names must match `VALID_TABLE_NAME` = `/^history_etl_[a-z_]+$/`,
+are validated before any I/O (target-table names must match `VALID_TABLE_NAME` = `/^history_[a-z_]+$/`,
+covering `history_etl_*` blob tables and the typed `history_message` / `history_tool_call` tables,
 and required fields must be present).
 
 ```ts

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAuBA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAOjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA4KrH"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAOjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAgMrH"}

package/dist/importer.js

@@ -1,6 +1,6 @@
 import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
 import { sha256 } from './hash.js';
-import { applyHistoryImportSchema, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, writeCheckpoint, } from './jsonl-importer-dao.js';
+import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, } from './jsonl-importer-dao.js';
 import { redactRecord } from './redaction.js';
 import { resolveSourceDefinition } from './sources.js';
 /** Run the JSONL import pipeline for one source.
@@ -86,19 +86,26 @@ export async function runJsonlImport(source, options) {
                     record: redacted,
                 });
                 desiredHashes.add(recordHash);
-                if (await ledgerExists(options.db, recordHash)) {
-                    skippedDuplicates += 1;
-                    continue;
-                }
                 prepared.push({ split, splitIndex, normalized: redacted, recordHash });
             }
             // A rejected line is reported (validationErrors above) but never persisted — no
             // second pass, no checkpoint advance, no partial rows from this record.
             if (lineRejected)
                 continue;
-            // Second pass: resolve _messageSplitIndex → message_hash, then insert.
+            // Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
+            // instead of a SELECT per record; duplicates are dropped before the write pass.
+            const existing = await ledgerExistingHashes(options.db, prepared.map((e) => e.recordHash));
+            const accepted = prepared.filter((entry) => {
+                if (existing.has(entry.recordHash)) {
+                    skippedDuplicates += 1;
+                    return false;
+                }
+                return true;
+            });
+            // Second pass: resolve _messageSplitIndex → message_hash, then batch.
             const recordHashBySplitIndex = new Map(prepared.map((e) => [e.splitIndex, e.recordHash]));
-            for (const entry of prepared) {
+            const ops = [];
+            for (const entry of accepted) {
                 const { split, splitIndex, normalized, recordHash } = entry;
                 if (normalized.disposition === 'unknown') {
                     unknownRecords += 1;
@@ -113,14 +120,21 @@ export async function runJsonlImport(source, options) {
                     }
                 }
                 if (!options.dryRun) {
-                    await insertRecord(options.db, split.targetTable, recordHash, file, lineNumber, splitIndex, normalized, options.now);
-                    await insertLedger(options.db, recordHash, resolvedSource, file, lineNumber, splitIndex, split.targetTable, options.now);
+                    // Record row + ledger row join one batch op pair — a crash between the two
+                    // writes is impossible (task 0060 F9).
+                    ops.push(recordInsertOp(split.targetTable, recordHash, file, lineNumber, splitIndex, normalized, options.now));
+                    ops.push(ledgerInsertOp(recordHash, resolvedSource, file, lineNumber, splitIndex, split.targetTable, options.now));
                 }
                 importedRecords += 1;
             }
-            if (lineSucceeded && !options.dryRun) {
-                await writeCheckpoint(options.db, resolvedSource, file, lineNumber, options.now);
-                checkpointUpdates += 1;
+            if (!options.dryRun && lineSucceeded) {
+                // Checkpoint upsert joins the same batch so an accepted line can never be
+                // re-imported after a mid-batch crash.
+                ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
+                if (ops.length > 0) {
+                    await options.db.batch(ops);
+                    checkpointUpdates += 1;
+                }
             }
         }
     }

package/dist/jsonl-importer-dao.d.ts

@@ -6,24 +6,49 @@ interface CheckpointRow {
 declare function targetTableFor(table: string): string;
 declare function timestamp(now: ImportOptions['now']): string;
 declare function ETL_TABLE_DDL(table: string): string;
-/** Apply importer-owned schema to the target database. */
+/**
+ * Apply importer-owned schema to the target database.
+ *
+ * Creates the checkpoint, ledger, and typed contract tables from the static SQL, then materializes
+ * every built-in `history_etl_*` table by looping {@link SOURCE_DEFINITIONS} through
+ * {@link ensureTargetTables}. This keeps the migration contract "schema applied ⇒ all built-in ETL
+ * tables exist" for callers who apply the schema and then raw-insert without a prior import.
+ * Adding a new built-in source to {@link SOURCE_DEFINITIONS} is now the only edit needed.
+ */
 export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void>;
 /**
  * Ensure the ETL table(s) for a source definition exist.
  *
- * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
- * `history_etl_*` tables and the typed contract tables. Custom source definitions
- * (and built-in definitions with a `splitConfig.targetTable` override) need their
- * target table(s) created on demand. The table name is already gated by
+ * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} creates only the checkpoint, ledger, and
+ * typed contract tables. Every `history_etl_*` blob table — built-in and custom — is created here
+ * (and by {@link applyHistoryImportSchema} looping {@link SOURCE_DEFINITIONS}). Built-in definitions
+ * with a `splitConfig.targetTable` override also need their target table(s) created on demand. The
+ * table name is already gated by
  * {@link VALID_TABLE_NAME} in {@link validateSourceDefinition} / {@link targetTableFor},
- * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this
- * idempotent for built-in tables that the static schema already created.
+ * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` is
+ * idempotent, so apply-then-import and a second apply are both safe.
  */
 declare function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void>;
 declare function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number>;
 declare function resetCheckpoints(db: ImportOptions['db'], source: string, files: readonly string[]): Promise<void>;
 declare function writeCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string, line: number, now: ImportOptions['now']): Promise<void>;
 declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean>;
+/**
+ * INSERT op for one accepted record row (task 0060 F9). Pure builder so the
+ * per-record batch path and the single-write `insertRecord` can never drift.
+ * Typed tables validate unknown keys here; the caller must have ensured the
+ * target table exists (`ensureTargetTables` runs at import start).
+ */
+export declare function recordInsertOp(targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): DbBatchOp;
+/** INSERT op for one ledger row (task 0060 F9). */
+export declare function ledgerInsertOp(recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): DbBatchOp;
+/** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
+export declare function checkpointUpsertOp(source: string, sourceFile: string, line: number, now: ImportOptions['now']): DbBatchOp;
+/**
+ * Query which of the given hashes already exist in the ledger, in chunks of at
+ * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
+ */
+export declare function ledgerExistingHashes(db: ImportOptions['db'], hashes: readonly string[]): Promise<Set<string>>;
 declare function insertRecord(db: ImportOptions['db'], targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): Promise<void>;
 declare function insertLedger(db: ImportOptions['db'], recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): Promise<void>;
 /**

package/dist/jsonl-importer-dao.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
+{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}

package/dist/jsonl-importer-dao.js

@@ -1,6 +1,6 @@
 import { HistoryImportError } from './errors.js';
 import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
-import { VALID_TABLE_NAME } from './sources.js';
+import { SOURCE_DEFINITIONS, VALID_TABLE_NAME } from './sources.js';
 /** Column allowlist per typed contract table; order is the INSERT column order. */
 const TYPED_TABLE_COLUMNS = {
     history_message: [
@@ -69,7 +69,15 @@ function ETL_TABLE_DDL(table) {
     imported_at TEXT NOT NULL
 );`;
 }
-/** Apply importer-owned schema to the target database. */
+/**
+ * Apply importer-owned schema to the target database.
+ *
+ * Creates the checkpoint, ledger, and typed contract tables from the static SQL, then materializes
+ * every built-in `history_etl_*` table by looping {@link SOURCE_DEFINITIONS} through
+ * {@link ensureTargetTables}. This keeps the migration contract "schema applied ⇒ all built-in ETL
+ * tables exist" for callers who apply the schema and then raw-insert without a prior import.
+ * Adding a new built-in source to {@link SOURCE_DEFINITIONS} is now the only edit needed.
+ */
 export async function applyHistoryImportSchema(db) {
     for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
         const sql = statement.trim();
@@ -77,17 +85,21 @@ export async function applyHistoryImportSchema(db) {
             await db.exec(sql);
         }
     }
+    for (const definition of Object.values(SOURCE_DEFINITIONS)) {
+        await ensureTargetTables(db, definition);
+    }
 }
 /**
  * Ensure the ETL table(s) for a source definition exist.
  *
- * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
- * `history_etl_*` tables and the typed contract tables. Custom source definitions
- * (and built-in definitions with a `splitConfig.targetTable` override) need their
- * target table(s) created on demand. The table name is already gated by
+ * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} creates only the checkpoint, ledger, and
+ * typed contract tables. Every `history_etl_*` blob table — built-in and custom — is created here
+ * (and by {@link applyHistoryImportSchema} looping {@link SOURCE_DEFINITIONS}). Built-in definitions
+ * with a `splitConfig.targetTable` override also need their target table(s) created on demand. The
+ * table name is already gated by
  * {@link VALID_TABLE_NAME} in {@link validateSourceDefinition} / {@link targetTableFor},
- * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this
- * idempotent for built-in tables that the static schema already created.
+ * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` is
+ * idempotent, so apply-then-import and a second apply are both safe.
  */
 async function ensureTargetTables(db, definition) {
     const tables = new Set([targetTableFor(definition.targetTable)]);
@@ -121,8 +133,13 @@ async function ledgerExists(db, recordHash) {
     const row = await db.queryFirst('SELECT record_hash FROM history_import_ledger WHERE record_hash = ?', recordHash);
     return row !== undefined && row !== null;
 }
-const _ensuredTables = new Set();
-async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
+/**
+ * INSERT op for one accepted record row (task 0060 F9). Pure builder so the
+ * per-record batch path and the single-write `insertRecord` can never drift.
+ * Typed tables validate unknown keys here; the caller must have ensured the
+ * target table exists (`ensureTargetTables` runs at import start).
+ */
+export function recordInsertOp(targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
     const table = targetTableFor(targetTable);
     const typedColumns = TYPED_TABLE_COLUMNS[table];
     if (typedColumns !== undefined) {
@@ -142,24 +159,63 @@ async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine,
             const v = payload[col];
             return v === undefined ? null : v;
         });
-        if (!_ensuredTables.has(table)) {
-            // Typed tables are created by the static schema in applyHistoryImportSchema.
-            _ensuredTables.add(table);
-        }
-        await db.run(`INSERT INTO ${table} (${typedColumns.join(', ')})
-             VALUES (${typedColumns.map(() => '?').join(', ')})
-             ON CONFLICT(record_hash) DO NOTHING`, ...values);
+        return {
+            sql: `INSERT INTO ${table} (${typedColumns.join(', ')})
+                 VALUES (${typedColumns.map(() => '?').join(', ')})
+                 ON CONFLICT(record_hash) DO NOTHING`,
+            params: values,
+        };
     }
-    else {
-        // Generic ETL insert path: store as payload_json blob.
-        if (!_ensuredTables.has(table)) {
-            await db.exec(ETL_TABLE_DDL(table));
-            _ensuredTables.add(table);
-        }
-        await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
+    // Generic ETL insert path: store as payload_json blob.
+    return {
+        sql: `INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
              VALUES (?, ?, ?, ?, ?, ?)
-             ON CONFLICT(record_hash) DO NOTHING`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
+             ON CONFLICT(record_hash) DO NOTHING`,
+        params: [recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now)],
+    };
+}
+/** INSERT op for one ledger row (task 0060 F9). */
+export function ledgerInsertOp(recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
+    return {
+        sql: `INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
+             VALUES (?, ?, ?, ?, ?, ?, ?)`,
+        params: [recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now)],
+    };
+}
+/** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
+export function checkpointUpsertOp(source, sourceFile, line, now) {
+    return {
+        sql: `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
+             VALUES (?, ?, ?, ?)
+             ON CONFLICT(source, source_file) DO UPDATE SET
+                last_imported_line = excluded.last_imported_line,
+                updated_at = excluded.updated_at`,
+        params: [source, sourceFile, line, timestamp(now)],
+    };
+}
+/**
+ * Query which of the given hashes already exist in the ledger, in chunks of at
+ * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
+ */
+export async function ledgerExistingHashes(db, hashes) {
+    const existing = new Set();
+    for (let i = 0; i < hashes.length; i += 200) {
+        const chunk = hashes.slice(i, i + 200);
+        const placeholders = chunk.map(() => '?').join(', ');
+        const rows = await db.queryAll(`SELECT record_hash FROM history_import_ledger WHERE record_hash IN (${placeholders})`, ...chunk);
+        for (const row of rows)
+            existing.add(row.record_hash);
+    }
+    return existing;
+}
+async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
+    const op = recordInsertOp(targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now);
+    if (TYPED_TABLE_COLUMNS[targetTableFor(targetTable)] === undefined) {
+        // Custom ETL tables are created on demand; `CREATE TABLE IF NOT EXISTS` is idempotent,
+        // so there is no process-global cache to go stale across adapter instances (task 0060 F8).
+        await db.exec(ETL_TABLE_DDL(targetTableFor(targetTable)));
     }
+    await db.run(op.sql, ...op.params);
 }
 async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
     await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)

package/dist/redaction.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"redaction.d.ts","sourceRoot":"","sources":["../src/redaction.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEzD,qFAAqF;AACrF,eAAO,MAAM,uBAAuB,EAAE,SAAS,aAAa,EAgB3D,CAAC;AAEF,qEAAqE;AACrE,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,GAAE,SAAS,aAAa,EAA4B,GAAG,OAAO,CAa9G;AAED,mFAAmF;AACnF,wBAAgB,YAAY,CAAC,OAAO,SAAS,UAAU,EACnD,MAAM,EAAE,OAAO,EACf,KAAK,GAAE,SAAS,aAAa,EAA4B,GAC1D,OAAO,CAET"}
+{"version":3,"file":"redaction.d.ts","sourceRoot":"","sources":["../src/redaction.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEzD,qFAAqF;AACrF,eAAO,MAAM,uBAAuB,EAAE,SAAS,aAAa,EA+B3D,CAAC;AAEF,qEAAqE;AACrE,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,GAAE,SAAS,aAAa,EAA4B,GAAG,OAAO,CAa9G;AAED,mFAAmF;AACnF,wBAAgB,YAAY,CAAC,OAAO,SAAS,UAAU,EACnD,MAAM,EAAE,OAAO,EACf,KAAK,GAAE,SAAS,aAAa,EAA4B,GAC1D,OAAO,CAET"}

package/dist/redaction.js

@@ -15,6 +15,21 @@ export const DEFAULT_REDACTION_RULES = [
         pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
         replacement: '[REDACTED:email]',
     },
+    {
+        name: 'xai-key',
+        pattern: /\bxai-[A-Za-z0-9_-]{12,}\b/g,
+        replacement: '[REDACTED:token]',
+    },
+    {
+        name: 'aws-access-key-id',
+        pattern: /\bAKIA[0-9A-Z]{16}\b/g,
+        replacement: '[REDACTED:aws-key]',
+    },
+    {
+        name: 'bearer-token',
+        pattern: /\bBearer\s+[A-Za-z0-9._\-=]+\b/g,
+        replacement: '[REDACTED:bearer]',
+    },
 ];
 /** Redact supported scalar and composite JSON values recursively. */
 export function redactValue(value, rules = DEFAULT_REDACTION_RULES) {

package/dist/schema-sql.d.ts

@@ -1,3 +1,11 @@
-/** DDL string that creates the history import checkpoint, ledger, and per-source ETL tables. */
+/**
+ * DDL string that creates the history import checkpoint, ledger, and typed contract tables.
+ *
+ * WHY: per-source `history_etl_*` blob tables are no longer hard-coded here. They are created
+ * dynamically by {@link ensureTargetTables} / {@link ETL_TABLE_DDL} looping every entry of
+ * {@link SOURCE_DEFINITIONS}, so adding a new built-in source needs no edit to this file. This
+ * single-owner rule closes the R16 drift where `omp` / `grok` / `agy` existed only via import-time
+ * DDL while the seven other sources were duplicated in the static string.
+ */
 export declare const HISTORY_IMPORT_SCHEMA_SQL: string;
 //# sourceMappingURL=schema-sql.d.ts.map

package/dist/schema-sql.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,eAAO,MAAM,yBAAyB,QA0I9B,CAAC"}
+{"version":3,"file":"schema-sql.d.ts","sourceRoot":"","sources":["../src/schema-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,QA2E9B,CAAC"}

package/dist/schema-sql.js

@@ -1,4 +1,12 @@
-/** DDL string that creates the history import checkpoint, ledger, and per-source ETL tables. */
+/**
+ * DDL string that creates the history import checkpoint, ledger, and typed contract tables.
+ *
+ * WHY: per-source `history_etl_*` blob tables are no longer hard-coded here. They are created
+ * dynamically by {@link ensureTargetTables} / {@link ETL_TABLE_DDL} looping every entry of
+ * {@link SOURCE_DEFINITIONS}, so adding a new built-in source needs no edit to this file. This
+ * single-owner rule closes the R16 drift where `omp` / `grok` / `agy` existed only via import-time
+ * DDL while the seven other sources were duplicated in the static string.
+ */
 export const HISTORY_IMPORT_SCHEMA_SQL = `
 CREATE TABLE IF NOT EXISTS history_import_checkpoint (
     source TEXT NOT NULL,
@@ -18,69 +26,6 @@ CREATE TABLE IF NOT EXISTS history_import_ledger (
     imported_at TEXT NOT NULL
 );
 
-CREATE TABLE IF NOT EXISTS history_etl_pi (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_claude (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_codex (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_gemini (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_opencode (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_antigravity (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_openclaw (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
 CREATE TABLE IF NOT EXISTS history_message (
     record_hash        TEXT PRIMARY KEY,
     source             TEXT NOT NULL,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gobing-ai/ts-llm-jsonl-importer",
-    "version": "0.4.28",
+    "version": "0.4.30",
     "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
     "keywords": [
         "typescript",
@@ -47,9 +47,9 @@
         "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
     },
     "dependencies": {
-        "@gobing-ai/ts-db": "^0.4.28",
-        "@gobing-ai/ts-runtime": "^0.4.28",
-        "@gobing-ai/ts-utils": "^0.4.28",
+        "@gobing-ai/ts-db": "^0.4.30",
+        "@gobing-ai/ts-runtime": "^0.4.30",
+        "@gobing-ai/ts-utils": "^0.4.30",
         "zod": "^4.1.0"
     },
     "devDependencies": {

package/src/importer.ts

@@ -1,3 +1,4 @@
+import type { DbBatchOp } from '@gobing-ai/ts-db';
 import {
     ambientRuntimePaths,
     createNodeFileSystem,
@@ -9,15 +10,15 @@ import {
 import { sha256 } from './hash';
 import {
     applyHistoryImportSchema,
+    checkpointUpsertOp,
     ensureTargetTables,
-    insertLedger,
-    insertRecord,
-    ledgerExists,
+    ledgerExistingHashes,
+    ledgerInsertOp,
     readCheckpoint,
     reconcileFullImport,
+    recordInsertOp,
     resetCheckpoints,
     targetTableFor,
-    writeCheckpoint,
 } from './jsonl-importer-dao';
 import { redactRecord } from './redaction';
 import { resolveSourceDefinition } from './sources';
@@ -130,10 +131,6 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
                     record: redacted,
                 });
                 desiredHashes.add(recordHash);
-                if (await ledgerExists(options.db, recordHash)) {
-                    skippedDuplicates += 1;
-                    continue;
-                }
                 prepared.push({ split, splitIndex, normalized: redacted, recordHash });
             }
 
@@ -141,9 +138,24 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
             // second pass, no checkpoint advance, no partial rows from this record.
             if (lineRejected) continue;
 
-            // Second pass: resolve _messageSplitIndex → message_hash, then insert.
+            // Chunked ledger lookup (task 0060 F9): one `IN (...)` query per ≤200 hashes
+            // instead of a SELECT per record; duplicates are dropped before the write pass.
+            const existing = await ledgerExistingHashes(
+                options.db,
+                prepared.map((e) => e.recordHash),
+            );
+            const accepted = prepared.filter((entry) => {
+                if (existing.has(entry.recordHash)) {
+                    skippedDuplicates += 1;
+                    return false;
+                }
+                return true;
+            });
+
+            // Second pass: resolve _messageSplitIndex → message_hash, then batch.
             const recordHashBySplitIndex = new Map(prepared.map((e) => [e.splitIndex, e.recordHash] as const));
-            for (const entry of prepared) {
+            const ops: DbBatchOp[] = [];
+            for (const entry of accepted) {
                 const { split, splitIndex, normalized, recordHash } = entry;
                 if (normalized.disposition === 'unknown') {
                     unknownRecords += 1;
@@ -158,33 +170,42 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
                     }
                 }
                 if (!options.dryRun) {
-                    await insertRecord(
-                        options.db,
-                        split.targetTable,
-                        recordHash,
-                        file,
-                        lineNumber,
-                        splitIndex,
-                        normalized,
-                        options.now,
+                    // Record row + ledger row join one batch op pair — a crash between the two
+                    // writes is impossible (task 0060 F9).
+                    ops.push(
+                        recordInsertOp(
+                            split.targetTable,
+                            recordHash,
+                            file,
+                            lineNumber,
+                            splitIndex,
+                            normalized,
+                            options.now,
+                        ),
                     );
-                    await insertLedger(
-                        options.db,
-                        recordHash,
-                        resolvedSource,
-                        file,
-                        lineNumber,
-                        splitIndex,
-                        split.targetTable,
-                        options.now,
+                    ops.push(
+                        ledgerInsertOp(
+                            recordHash,
+                            resolvedSource,
+                            file,
+                            lineNumber,
+                            splitIndex,
+                            split.targetTable,
+                            options.now,
+                        ),
                     );
                 }
                 importedRecords += 1;
             }
 
-            if (lineSucceeded && !options.dryRun) {
-                await writeCheckpoint(options.db, resolvedSource, file, lineNumber, options.now);
-                checkpointUpdates += 1;
+            if (!options.dryRun && lineSucceeded) {
+                // Checkpoint upsert joins the same batch so an accepted line can never be
+                // re-imported after a mid-batch crash.
+                ops.push(checkpointUpsertOp(resolvedSource, file, lineNumber, options.now));
+                if (ops.length > 0) {
+                    await options.db.batch(ops);
+                    checkpointUpdates += 1;
+                }
             }
         }
     }

package/src/jsonl-importer-dao.ts

@@ -1,7 +1,7 @@
 import type { DbAdapter, DbBatchOp } from '@gobing-ai/ts-db';
 import { HistoryImportError } from './errors';
 import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
-import { VALID_TABLE_NAME } from './sources';
+import { SOURCE_DEFINITIONS, VALID_TABLE_NAME } from './sources';
 import type { ImportOptions, JsonObject, ReconcileSummary, SourceDefinition } from './types';
 
 interface CheckpointRow {
@@ -81,7 +81,15 @@ function ETL_TABLE_DDL(table: string): string {
 );`;
 }
 
-/** Apply importer-owned schema to the target database. */
+/**
+ * Apply importer-owned schema to the target database.
+ *
+ * Creates the checkpoint, ledger, and typed contract tables from the static SQL, then materializes
+ * every built-in `history_etl_*` table by looping {@link SOURCE_DEFINITIONS} through
+ * {@link ensureTargetTables}. This keeps the migration contract "schema applied ⇒ all built-in ETL
+ * tables exist" for callers who apply the schema and then raw-insert without a prior import.
+ * Adding a new built-in source to {@link SOURCE_DEFINITIONS} is now the only edit needed.
+ */
 export async function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void> {
     for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
         const sql = statement.trim();
@@ -89,18 +97,22 @@ export async function applyHistoryImportSchema(db: ImportOptions['db']): Promise
             await db.exec(sql);
         }
     }
+    for (const definition of Object.values(SOURCE_DEFINITIONS)) {
+        await ensureTargetTables(db, definition);
+    }
 }
 
 /**
  * Ensure the ETL table(s) for a source definition exist.
  *
- * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
- * `history_etl_*` tables and the typed contract tables. Custom source definitions
- * (and built-in definitions with a `splitConfig.targetTable` override) need their
- * target table(s) created on demand. The table name is already gated by
+ * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} creates only the checkpoint, ledger, and
+ * typed contract tables. Every `history_etl_*` blob table — built-in and custom — is created here
+ * (and by {@link applyHistoryImportSchema} looping {@link SOURCE_DEFINITIONS}). Built-in definitions
+ * with a `splitConfig.targetTable` override also need their target table(s) created on demand. The
+ * table name is already gated by
  * {@link VALID_TABLE_NAME} in {@link validateSourceDefinition} / {@link targetTableFor},
- * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this
- * idempotent for built-in tables that the static schema already created.
+ * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` is
+ * idempotent, so apply-then-import and a second apply are both safe.
  */
 async function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void> {
     const tables = new Set<string>([targetTableFor(definition.targetTable)]);
@@ -157,10 +169,13 @@ async function ledgerExists(db: ImportOptions['db'], recordHash: string): Promis
     return row !== undefined && row !== null;
 }
 
-const _ensuredTables = new Set<string>();
-
-async function insertRecord(
-    db: ImportOptions['db'],
+/**
+ * INSERT op for one accepted record row (task 0060 F9). Pure builder so the
+ * per-record batch path and the single-write `insertRecord` can never drift.
+ * Typed tables validate unknown keys here; the caller must have ensured the
+ * target table exists (`ensureTargetTables` runs at import start).
+ */
+export function recordInsertOp(
     targetTable: string,
     recordHash: string,
     sourceFile: string,
@@ -168,7 +183,7 @@ async function insertRecord(
     splitIndex: number,
     payload: JsonObject,
     now: ImportOptions['now'],
-): Promise<void> {
+): DbBatchOp {
     const table = targetTableFor(targetTable);
     const typedColumns = TYPED_TABLE_COLUMNS[table];
     if (typedColumns !== undefined) {
@@ -189,34 +204,91 @@ async function insertRecord(
             const v = payload[col];
             return v === undefined ? null : v;
         });
-        if (!_ensuredTables.has(table)) {
-            // Typed tables are created by the static schema in applyHistoryImportSchema.
-            _ensuredTables.add(table);
-        }
-        await db.run(
-            `INSERT INTO ${table} (${typedColumns.join(', ')})
-             VALUES (${typedColumns.map(() => '?').join(', ')})
-             ON CONFLICT(record_hash) DO NOTHING`,
-            ...values,
-        );
-    } else {
-        // Generic ETL insert path: store as payload_json blob.
-        if (!_ensuredTables.has(table)) {
-            await db.exec(ETL_TABLE_DDL(table));
-            _ensuredTables.add(table);
-        }
-        await db.run(
-            `INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
+        return {
+            sql: `INSERT INTO ${table} (${typedColumns.join(', ')})
+                 VALUES (${typedColumns.map(() => '?').join(', ')})
+                 ON CONFLICT(record_hash) DO NOTHING`,
+            params: values,
+        };
+    }
+    // Generic ETL insert path: store as payload_json blob.
+    return {
+        sql: `INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
              VALUES (?, ?, ?, ?, ?, ?)
              ON CONFLICT(record_hash) DO NOTHING`,
-            recordHash,
-            sourceFile,
-            sourceLine,
-            splitIndex,
-            JSON.stringify(payload),
-            timestamp(now),
+        params: [recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now)],
+    };
+}
+
+/** INSERT op for one ledger row (task 0060 F9). */
+export function ledgerInsertOp(
+    recordHash: string,
+    source: string,
+    sourceFile: string,
+    sourceLine: number,
+    splitIndex: number,
+    targetTable: string,
+    now: ImportOptions['now'],
+): DbBatchOp {
+    return {
+        sql: `INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
+             VALUES (?, ?, ?, ?, ?, ?, ?)`,
+        params: [recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now)],
+    };
+}
+
+/** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
+export function checkpointUpsertOp(
+    source: string,
+    sourceFile: string,
+    line: number,
+    now: ImportOptions['now'],
+): DbBatchOp {
+    return {
+        sql: `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
+             VALUES (?, ?, ?, ?)
+             ON CONFLICT(source, source_file) DO UPDATE SET
+                last_imported_line = excluded.last_imported_line,
+                updated_at = excluded.updated_at`,
+        params: [source, sourceFile, line, timestamp(now)],
+    };
+}
+
+/**
+ * Query which of the given hashes already exist in the ledger, in chunks of at
+ * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
+ */
+export async function ledgerExistingHashes(db: ImportOptions['db'], hashes: readonly string[]): Promise<Set<string>> {
+    const existing = new Set<string>();
+    for (let i = 0; i < hashes.length; i += 200) {
+        const chunk = hashes.slice(i, i + 200);
+        const placeholders = chunk.map(() => '?').join(', ');
+        const rows = await db.queryAll<{ record_hash: string }>(
+            `SELECT record_hash FROM history_import_ledger WHERE record_hash IN (${placeholders})`,
+            ...chunk,
         );
+        for (const row of rows) existing.add(row.record_hash);
+    }
+    return existing;
+}
+
+async function insertRecord(
+    db: ImportOptions['db'],
+    targetTable: string,
+    recordHash: string,
+    sourceFile: string,
+    sourceLine: number,
+    splitIndex: number,
+    payload: JsonObject,
+    now: ImportOptions['now'],
+): Promise<void> {
+    const op = recordInsertOp(targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now);
+    if (TYPED_TABLE_COLUMNS[targetTableFor(targetTable)] === undefined) {
+        // Custom ETL tables are created on demand; `CREATE TABLE IF NOT EXISTS` is idempotent,
+        // so there is no process-global cache to go stale across adapter instances (task 0060 F8).
+        await db.exec(ETL_TABLE_DDL(targetTableFor(targetTable)));
     }
+    await db.run(op.sql, ...op.params);
 }
 
 async function insertLedger(

package/src/redaction.ts

@@ -17,6 +17,21 @@ export const DEFAULT_REDACTION_RULES: readonly RedactionRule[] = [
         pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
         replacement: '[REDACTED:email]',
     },
+    {
+        name: 'xai-key',
+        pattern: /\bxai-[A-Za-z0-9_-]{12,}\b/g,
+        replacement: '[REDACTED:token]',
+    },
+    {
+        name: 'aws-access-key-id',
+        pattern: /\bAKIA[0-9A-Z]{16}\b/g,
+        replacement: '[REDACTED:aws-key]',
+    },
+    {
+        name: 'bearer-token',
+        pattern: /\bBearer\s+[A-Za-z0-9._\-=]+\b/g,
+        replacement: '[REDACTED:bearer]',
+    },
 ];
 
 /** Redact supported scalar and composite JSON values recursively. */

package/src/schema-sql.ts

@@ -1,4 +1,12 @@
-/** DDL string that creates the history import checkpoint, ledger, and per-source ETL tables. */
+/**
+ * DDL string that creates the history import checkpoint, ledger, and typed contract tables.
+ *
+ * WHY: per-source `history_etl_*` blob tables are no longer hard-coded here. They are created
+ * dynamically by {@link ensureTargetTables} / {@link ETL_TABLE_DDL} looping every entry of
+ * {@link SOURCE_DEFINITIONS}, so adding a new built-in source needs no edit to this file. This
+ * single-owner rule closes the R16 drift where `omp` / `grok` / `agy` existed only via import-time
+ * DDL while the seven other sources were duplicated in the static string.
+ */
 export const HISTORY_IMPORT_SCHEMA_SQL = `
 CREATE TABLE IF NOT EXISTS history_import_checkpoint (
     source TEXT NOT NULL,
@@ -18,69 +26,6 @@ CREATE TABLE IF NOT EXISTS history_import_ledger (
     imported_at TEXT NOT NULL
 );
 
-CREATE TABLE IF NOT EXISTS history_etl_pi (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_claude (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_codex (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_gemini (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_opencode (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_antigravity (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS history_etl_openclaw (
-    record_hash TEXT PRIMARY KEY,
-    source_file TEXT NOT NULL,
-    source_line INTEGER NOT NULL,
-    split_index INTEGER NOT NULL,
-    payload_json TEXT NOT NULL,
-    imported_at TEXT NOT NULL
-);
-
 CREATE TABLE IF NOT EXISTS history_message (
     record_hash        TEXT PRIMARY KEY,
     source             TEXT NOT NULL,