@gobing-ai/ts-llm-jsonl-importer 0.4.42 → 0.4.44

package/README.md

@@ -9,7 +9,7 @@ Generic JSONL import pipeline for AI-agent history-style files: discover files,
 | Export | Purpose |
 |--------|---------|
 | `runJsonlImport()` | Runs discovery, parsing, validation, redaction, dedupe, and persistence. Accepts a built-in source string **or** a custom `SourceDefinition`. |
-| `applyHistoryImportSchema()` | Installs importer-owned checkpoint, ledger, and ETL tables |
+| `applyHistoryImportSchema()` | Installs importer-owned checkpoint, ledger, and typed contract tables |
 | `SOURCE_DEFINITIONS` | Built-in source definitions |
 | `getSourceDefinition()` | Returns one built-in definition by key (throws `HistoryImportError` for unknown keys) |
 | `resolveSourceDefinition()` | Resolves a `string \| SourceDefinition` into a validated definition |
@@ -19,7 +19,7 @@ Generic JSONL import pipeline for AI-agent history-style files: discover files,
 | `sha256()` / `stableJson()` | Stable hash helpers used by the ledger |
 | `HISTORY_IMPORT_SCHEMA_SQL` | SQL schema string for explicit migration flows |
 
-Built-in source keys are `claude`, `codex`, `gemini`, `pi`, `opencode`, `antigravity`, and `openclaw`.
+Built-in source keys are `claude`, `codex`, `gemini`, `pi`, `opencode`, `antigravity`, `openclaw`, `omp`, `grok`, and `agy`.
 
 ## Installation
 
@@ -127,13 +127,18 @@ The schema contains:
 |-------|---------|
 | `history_import_checkpoint` | Per-source/per-file last imported line |
 | `history_import_ledger` | Stable hash ledger for dedupe and provenance |
-| `history_etl_<source>` | Redacted normalized payloads for each built-in source |
+| `history_message` | Typed normalized messages from built-in forensic mappers |
+| `history_tool_call` | Typed normalized tool calls from built-in forensic mappers |
+| `history_etl_<source>` | Redacted payloads for generic/custom definitions; created only after an accepted row targets one |
 
-ETL tables store the normalized payload JSON plus source file, source line, split index, hash, and timestamps.
+Generic ETL tables store normalized payload JSON plus source file, source line, split index, hash, and timestamps.
+Schema setup, empty scans, validation failures, and dry runs do not create empty ETL tables.
+Claude tool results copy native `toolUseResult.durationMs` or `durationSeconds` into `duration_ms`; absent timing
+stays `NULL` and is never inferred from message timestamps.
 
 ## Split Records
 
-Most sources are one input row to one ETL row. A source can also split one JSONL row into multiple ETL records. The built-in `pi` definition splits nested `messages` into one row per message when present.
+A source can split one JSONL row into several typed or generic records. Built-in source mappers normalize supported records into `history_message` and `history_tool_call`; custom definitions may target a generic ETL table.
 
 ```ts
 const result = await runJsonlImport('pi', { db, files: ['session.jsonl'], mode: 'full' });

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAsTrH"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AA4BA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAoFjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA2YrH"}

package/dist/importer.js

@@ -1,6 +1,6 @@
 import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
 import { sha256 } from './hash.js';
-import { applyHistoryImportSchema, checkpointUpsertOp, ensureTargetTables, ledgerExistingHashes, ledgerInsertOp, readCheckpoint, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, } from './jsonl-importer-dao.js';
+import { applyHistoryImportSchema, checkpointUpsertOp, codexUsageAttributionUpdateOp, ensureTargetTable, ledgerExistingHashes, ledgerInsertOp, loadSourceCheckpoints, reconcileFullImport, recordInsertOp, resetCheckpoints, targetTableFor, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, } from './jsonl-importer-dao.js';
 import { claudeToolResultTiming, ompToolResultTiming, timestampToEpochMs } from './mappers.js';
 import { redactRecord } from './redaction.js';
 import { resolveSourceDefinition } from './sources.js';
@@ -52,7 +52,6 @@ export async function runJsonlImport(source, options) {
     const resolvedSource = definition.source;
     const fileSystem = options.fileSystem ?? createNodeFileSystem();
     await applyHistoryImportSchema(options.db);
-    await ensureTargetTables(options.db, definition);
     const mode = options.mode ?? 'incremental';
     const files = await discoverFiles(definition, options.roots, options.files, fileSystem, options.paths);
     if (mode === 'full' && !options.dryRun) {
@@ -65,15 +64,33 @@ export async function runJsonlImport(source, options) {
     let skippedDuplicates = 0;
     let unknownRecords = 0;
     let skippedCorruptLines = 0;
+    let skippedUnchangedFiles = 0;
     let checkpointUpdates = 0;
+    const ensuredTargetTables = new Set();
     // Full-mode desired set (R1, task 0504): every record hash the current source produces.
     // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
     const desiredHashes = new Set();
     // Task 0564 R1: tool-call rows emitted by earlier assistant lines, keyed by
     // (source, session_id, call_id) so a later toolResult line can attach its duration.
     const toolCallRows = new Map();
+    // 0678 R3: codex token_count events carry usage on a meta row; the numbers belong
+    // to the most recent assistant message of that session.
+    const codexLastAssistant = new Map();
+    // 0675 R5: one query per source replaces the per-file SELECT in this loop.
+    // Only incremental mode consults checkpoints; full/force-file read everything.
+    const checkpoints = mode === 'incremental' ? await loadSourceCheckpoints(options.db, resolvedSource) : undefined;
     for (const file of files) {
-        const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
+        const entry = checkpoints?.get(file);
+        const checkpoint = entry?.line ?? 0;
+        if (checkpoints !== undefined && entry !== undefined && entry.size !== null && entry.mtimeMs !== null) {
+            // 0675 R2: identity short-circuit — skip the whole file when (size, mtimeMs) both match.
+            // Fail open on any stat problem; never applies to full/force-file (checkpoints is undefined there).
+            const stat = await safeStat(fileSystem, file);
+            if (stat !== null && stat.size === entry.size && stat.mtimeMs === entry.mtimeMs) {
+                skippedUnchangedFiles += 1;
+                continue;
+            }
+        }
         let lineNumber = 0;
         for await (const rawLine of readLines(fileSystem, file)) {
             lineNumber += 1;
@@ -177,6 +194,14 @@ export async function runJsonlImport(source, options) {
                 }
                 return true;
             });
+            if (!options.dryRun) {
+                for (const entry of accepted) {
+                    if (ensuredTargetTables.has(entry.split.targetTable))
+                        continue;
+                    await ensureTargetTable(options.db, entry.split.targetTable);
+                    ensuredTargetTables.add(entry.split.targetTable);
+                }
+            }
             // Second pass: resolve _messageSplitIndex → message_hash, then batch.
             const recordHashBySplitIndex = new Map(prepared.map((e) => [e.splitIndex, e.recordHash]));
             const ops = [];
@@ -195,10 +220,35 @@ export async function runJsonlImport(source, options) {
                     }
                 }
                 if (!options.dryRun) {
+                    // 0678 R3 (codex): a meta row carrying `_codexUsageCarrier` contributes its
+                    // numbers to the latest assistant message instead of itself; with no known
+                    // target the usage stays unattributed — never reattached to a wrong row.
+                    const carrier = normalized._codexUsageCarrier;
+                    delete normalized._codexUsageCarrier;
+                    if (carrier !== undefined &&
+                        typeof normalized.session_id === 'string' &&
+                        codexLastAssistant.has(normalized.session_id)) {
+                        const lastHash = codexLastAssistant.get(normalized.session_id);
+                        if (lastHash !== undefined) {
+                            ops.push(codexUsageAttributionUpdateOp(lastHash, {
+                                input: carrier.input,
+                                output: carrier.output,
+                                cacheRead: carrier.cacheRead,
+                            }));
+                        }
+                    }
                     // Record row + ledger row join one batch op pair — a crash between the two
                     // writes is impossible (task 0060 F9).
                     ops.push(recordInsertOp(split.targetTable, recordHash, file, lineNumber, splitIndex, normalized, options.now));
                     ops.push(ledgerInsertOp(recordHash, resolvedSource, file, lineNumber, splitIndex, split.targetTable, options.now));
+                    // 0678 R3 (codex): remember this session's latest assistant row so a
+                    // following token_count meta row can attribute its usage to it.
+                    if (definition.source === 'codex' &&
+                        split.targetTable === 'history_message' &&
+                        normalized.role === 'assistant' &&
+                        typeof normalized.session_id === 'string') {
+                        codexLastAssistant.set(normalized.session_id, recordHash);
+                    }
                 }
                 importedRecords += 1;
             }
@@ -229,11 +279,10 @@ export async function runJsonlImport(source, options) {
                         }
                     }
                 }
-                // Task 0624 R2 (claude): attach result sizes from this line's
+                // Task 0624 R2 (claude): attach native durations and result sizes from this line's
                 // tool_result blocks. Pairing is by `tool_use_id` → `call_id`
                 // (source ∘ session ∘ call), the same key the omp branch uses.
-                // No duration is written — claude has no native wall time and the
-                // guarded timestamp fallback does not extend to it (0624 Design).
+                // Missing native timing stays NULL; the timestamp fallback does not extend to claude.
                 if (definition.source === 'claude') {
                     const timing = claudeToolResultTiming(raw);
                     if (timing !== null) {
@@ -246,6 +295,9 @@ export async function runJsonlImport(source, options) {
                             }
                             if (pending !== undefined) {
                                 ops.push(toolCallResultBytesUpdateOp(pending.recordHash, timing.resultBytes));
+                                if (timing.durationMs !== undefined) {
+                                    ops.push(toolCallDurationUpdateOp(pending.recordHash, null, null, timing.durationMs));
+                                }
                             }
                         }
                     }
@@ -258,11 +310,31 @@ export async function runJsonlImport(source, options) {
                     checkpointUpdates += 1;
                 }
             }
+            // 0675 R1/R4: stamp file identity once per READ file (never for skipped ones).
+            // This is also what self-heals legacy rows whose stored identity is NULL — without
+            // it a fully-imported database never arms the short-circuit. Skipped when nothing
+            // changed since the stored checkpoint to keep no-op runs write-free. Degraded
+            // entries (pre-migration database without identity columns, 0678 fail-open) keep
+            // the old line-only write shape via checkpointUpsertOp's optional identity.
+            if (!options.dryRun && mode === 'incremental') {
+                const statAfter = await safeStat(fileSystem, file);
+                // Degraded = a STORED entry without identity (pre-migration database, 0678
+                // fail-open). No stored entry means first-ever import: stamp normally.
+                const identityDegraded = entry !== undefined && entry.size === null && entry.mtimeMs === null;
+                const identityChanged = entry !== undefined &&
+                    (entry.size !== (statAfter?.size ?? null) || entry.mtimeMs !== (statAfter?.mtimeMs ?? null));
+                if (statAfter !== null && !identityDegraded && (entry === undefined || identityChanged)) {
+                    await options.db.batch([
+                        checkpointUpsertOp(resolvedSource, file, Math.max(lineNumber, checkpoint), options.now, {
+                            size: statAfter.size,
+                            mtimeMs: statAfter.mtimeMs,
+                        }),
+                    ]);
+                    checkpointUpdates += 1;
+                }
+            }
         }
     }
-    // Full-mode reconciliation (R1, task 0504): diff the desired hash set against the
-    // persisted ledger and retire stale derived rows (target, tool, ledger, checkpoint) in
-    // one source-scoped batch. Dry-run computes the identical counts without mutation.
     let reconciliation;
     if (mode === 'full') {
         reconciliation = await reconcileFullImport(options.db, resolvedSource, desiredHashes, files, options.dryRun ?? false);
@@ -275,6 +347,7 @@ export async function runJsonlImport(source, options) {
         importedRecords,
         skippedDuplicates,
         skippedCorruptLines,
+        skippedUnchangedFiles,
         unknownRecords,
         parseErrors,
         validationErrors,
@@ -416,6 +489,19 @@ function matchesPattern(path, patterns) {
         return path.endsWith(pattern.replace(/^\*/, ''));
     });
 }
+/**
+ * Stat a file for its identity, returning null on any failure (missing file, stat unsupported).
+ * 0675 R2: the short-circuit fails OPEN — any null falls through to the normal read path.
+ */
+async function safeStat(fileSystem, file) {
+    try {
+        const stat = await fileSystem.stat(file);
+        return stat === null ? null : { size: stat.size, mtimeMs: stat.mtimeMs };
+    }
+    catch {
+        return null;
+    }
+}
 /**
  * Yield lines from a file, streaming when the FileSystem supports it.
  *

package/dist/jsonl-importer-dao.d.ts

@@ -2,6 +2,14 @@ import type { DbAdapter, DbBatchOp } from '@gobing-ai/ts-db';
 import type { ImportOptions, JsonObject, ReconcileSummary, SourceDefinition } from './types';
 interface CheckpointRow {
     readonly last_imported_line: number;
+    readonly source_size?: number | null;
+    readonly source_mtime_ms?: number | null;
+}
+/** File identity + line for one checkpoint row (0675 R1/R5). */
+export interface SourceCheckpoint {
+    readonly line: number;
+    readonly size: number | null;
+    readonly mtimeMs: number | null;
 }
 declare function targetTableFor(table: string): string;
 declare function timestamp(now: ImportOptions['now']): string;
@@ -9,20 +17,19 @@ declare function ETL_TABLE_DDL(table: string): string;
 /**
  * Apply importer-owned schema to the target database.
  *
- * Creates the checkpoint, ledger, and typed contract tables from the static SQL, then materializes
- * every built-in `history_etl_*` table by looping {@link SOURCE_DEFINITIONS} through
- * {@link ensureTargetTables}. This keeps the migration contract "schema applied ⇒ all built-in ETL
- * tables exist" for callers who apply the schema and then raw-insert without a prior import.
- * Adding a new built-in source to {@link SOURCE_DEFINITIONS} is now the only edit needed.
+ * Creates the checkpoint, ledger, and typed contract tables from the static SQL.
+ * Generic ETL tables are created lazily with the first accepted row, so applying
+ * the schema or scanning an empty source never leaves vestigial empty tables.
  */
 export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void>;
+/** Ensure one accepted record's target exists; typed targets come from the static schema. */
+declare function ensureTargetTable(db: ImportOptions['db'], targetTable: string): Promise<void>;
 /**
  * Ensure the ETL table(s) for a source definition exist.
  *
  * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} creates only the checkpoint, ledger, and
- * typed contract tables. Every `history_etl_*` blob table — built-in and custom — is created here
- * (and by {@link applyHistoryImportSchema} looping {@link SOURCE_DEFINITIONS}). Built-in definitions
- * with a `splitConfig.targetTable` override also need their target table(s) created on demand. The
+ * typed contract tables. Callers that explicitly request a definition's generic targets use this
+ * helper; the import pipeline instead creates only targets reached by accepted rows. The
  * table name is already gated by
  * {@link VALID_TABLE_NAME} in {@link validateSourceDefinition} / {@link targetTableFor},
  * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` is
@@ -30,6 +37,16 @@ export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promi
  */
 declare function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void>;
 declare function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number>;
+/**
+ * One query per source (0675 R5): all checkpoint rows for `source` keyed by file,
+ * replacing the per-file SELECT in the import loop.
+ *
+ * Fails open when the identity columns are missing (0678): a database created through a
+ * migration path that has not yet applied 0675's ALTERs would otherwise hard-fail every
+ * import. Line-level behavior degrades to pre-identity semantics — the short-circuit
+ * and identity stamping no-op until the columns exist.
+ */
+export declare function loadSourceCheckpoints(db: ImportOptions['db'], source: string): Promise<Map<string, SourceCheckpoint>>;
 declare function resetCheckpoints(db: ImportOptions['db'], source: string, files: readonly string[]): Promise<void>;
 declare function writeCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string, line: number, now: ImportOptions['now']): Promise<void>;
 declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean>;
@@ -37,13 +54,16 @@ declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Prom
  * INSERT op for one accepted record row (task 0060 F9). Pure builder so the
  * per-record batch path and the single-write `insertRecord` can never drift.
  * Typed tables validate unknown keys here; the caller must have ensured the
- * target table exists (`ensureTargetTables` runs at import start).
+ * target table exists (the import loop ensures accepted targets before batching).
  */
 export declare function recordInsertOp(targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): DbBatchOp;
 /** INSERT op for one ledger row (task 0060 F9). */
 export declare function ledgerInsertOp(recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): DbBatchOp;
 /** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
-export declare function checkpointUpsertOp(source: string, sourceFile: string, line: number, now: ImportOptions['now']): DbBatchOp;
+export declare function checkpointUpsertOp(source: string, sourceFile: string, line: number, now: ImportOptions['now'], identity?: {
+    readonly size: number | null;
+    readonly mtimeMs: number | null;
+}): DbBatchOp;
 /**
  * Targeted UPDATE op attaching a tool duration to a previously-inserted
  * `history_tool_call` row (task 0564 R1). Keyed by `record_hash` (PK), never an
@@ -59,6 +79,13 @@ declare function toolCallDurationUpdateOp(recordHash: string, startedAt: string
  * `history_tool_call` row (task 0624 R2). Keyed by `record_hash` (PK). Idempotent.
  */
 declare function toolCallResultBytesUpdateOp(recordHash: string, resultBytes: number): DbBatchOp;
+/** Targeted UPDATE op attributing a codex usage-carrier row's token counts to the
+ *  latest assistant message of its session (task 0678 R3). Keyed by `record_hash` (PK). */
+export declare function codexUsageAttributionUpdateOp(recordHash: string, tokens: {
+    readonly input: number | null;
+    readonly output: number | null;
+    readonly cacheRead: number | null;
+}): DbBatchOp;
 /**
  * Query which of the given hashes already exist in the ledger, in chunks of at
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -141,5 +168,5 @@ export declare function deleteCheckpointOperation(source: string, sourceFile: st
 /** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
 export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
 export type { CheckpointRow };
-export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };
+export { ETL_TABLE_DDL, ensureTargetTable, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };
 //# sourceMappingURL=jsonl-importer-dao.d.ts.map

package/dist/jsonl-importer-dao.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAwDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUrF;AAED;;;;;;;;;;;GAWG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CASX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,CAOvF;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,2BAA2B,EAC3B,eAAe,GAClB,CAAC"}
+{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,gEAAgE;AAChE,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAwDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,6FAA6F;AAC7F,iBAAe,iBAAiB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG5F;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAQtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACvC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CA0BxC;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC1B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAmCX;AAED,mDAAmD;AACnD,wBAAgB,cAAc,CAC1B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,CAMX;AAED,0FAA0F;AAC1F,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,EACzB,QAAQ,CAAC,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7E,SAAS,CAWX;AAED;;;;;;;;GAQG;AACH,iBAAS,wBAAwB,CAC7B,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,GAC1B,SAAS,CAOX;AAED;;;GAGG;AACH,iBAAS,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,CAOvF;AAED;2FAC2F;AAC3F,wBAAgB,6BAA6B,CACzC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC7G,SAAS,CAOX;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAYnH;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAIf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA4GD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,6DAA6D;AAC7D,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAKvF;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,wBAAwB,EACxB,2BAA2B,EAC3B,eAAe,GAClB,CAAC"}

package/dist/jsonl-importer-dao.js

@@ -1,6 +1,6 @@
 import { HistoryImportError } from './errors.js';
 import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
-import { SOURCE_DEFINITIONS, VALID_TABLE_NAME } from './sources.js';
+import { VALID_TABLE_NAME } from './sources.js';
 /** Column allowlist per typed contract table; order is the INSERT column order. */
 const TYPED_TABLE_COLUMNS = {
     history_message: [
@@ -75,11 +75,9 @@ function ETL_TABLE_DDL(table) {
 /**
  * Apply importer-owned schema to the target database.
  *
- * Creates the checkpoint, ledger, and typed contract tables from the static SQL, then materializes
- * every built-in `history_etl_*` table by looping {@link SOURCE_DEFINITIONS} through
- * {@link ensureTargetTables}. This keeps the migration contract "schema applied ⇒ all built-in ETL
- * tables exist" for callers who apply the schema and then raw-insert without a prior import.
- * Adding a new built-in source to {@link SOURCE_DEFINITIONS} is now the only edit needed.
+ * Creates the checkpoint, ledger, and typed contract tables from the static SQL.
+ * Generic ETL tables are created lazily with the first accepted row, so applying
+ * the schema or scanning an empty source never leaves vestigial empty tables.
  */
 export async function applyHistoryImportSchema(db) {
     for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
@@ -88,17 +86,19 @@ export async function applyHistoryImportSchema(db) {
             await db.exec(sql);
         }
     }
-    for (const definition of Object.values(SOURCE_DEFINITIONS)) {
-        await ensureTargetTables(db, definition);
-    }
+}
+/** Ensure one accepted record's target exists; typed targets come from the static schema. */
+async function ensureTargetTable(db, targetTable) {
+    const table = targetTableFor(targetTable);
+    if (TYPED_TABLE_COLUMNS[table] === undefined)
+        await db.exec(ETL_TABLE_DDL(table));
 }
 /**
  * Ensure the ETL table(s) for a source definition exist.
  *
  * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} creates only the checkpoint, ledger, and
- * typed contract tables. Every `history_etl_*` blob table — built-in and custom — is created here
- * (and by {@link applyHistoryImportSchema} looping {@link SOURCE_DEFINITIONS}). Built-in definitions
- * with a `splitConfig.targetTable` override also need their target table(s) created on demand. The
+ * typed contract tables. Callers that explicitly request a definition's generic targets use this
+ * helper; the import pipeline instead creates only targets reached by accepted rows. The
  * table name is already gated by
  * {@link VALID_TABLE_NAME} in {@link validateSourceDefinition} / {@link targetTableFor},
  * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` is
@@ -110,16 +110,43 @@ async function ensureTargetTables(db, definition) {
         tables.add(targetTableFor(definition.splitConfig.targetTable));
     }
     for (const table of tables) {
-        // Typed tables are created by the static schema; use ETL DDL for custom tables.
-        if (TYPED_TABLE_COLUMNS[table] !== undefined)
-            continue;
-        await db.exec(ETL_TABLE_DDL(table));
+        await ensureTargetTable(db, table);
     }
 }
 async function readCheckpoint(db, source, sourceFile) {
     const row = await db.queryFirst('SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, sourceFile);
     return row?.last_imported_line ?? 0;
 }
+/**
+ * One query per source (0675 R5): all checkpoint rows for `source` keyed by file,
+ * replacing the per-file SELECT in the import loop.
+ *
+ * Fails open when the identity columns are missing (0678): a database created through a
+ * migration path that has not yet applied 0675's ALTERs would otherwise hard-fail every
+ * import. Line-level behavior degrades to pre-identity semantics — the short-circuit
+ * and identity stamping no-op until the columns exist.
+ */
+export async function loadSourceCheckpoints(db, source) {
+    let rows;
+    try {
+        rows = await db.queryAll('SELECT source_file, last_imported_line, source_size, source_mtime_ms FROM history_import_checkpoint WHERE source = ?', source);
+    }
+    catch (err) {
+        if (!/no such column|source_size|source_mtime_ms/i.test(err.message))
+            throw err;
+        rows = (await db.queryAll('SELECT source_file, last_imported_line FROM history_import_checkpoint WHERE source = ?', source));
+        return new Map(rows.map((row) => [row.source_file, { line: row.last_imported_line, size: null, mtimeMs: null }]));
+    }
+    const map = new Map();
+    for (const row of rows) {
+        map.set(row.source_file, {
+            line: row.last_imported_line,
+            size: row.source_size ?? null,
+            mtimeMs: row.source_mtime_ms ?? null,
+        });
+    }
+    return map;
+}
 async function resetCheckpoints(db, source, files) {
     for (const file of files) {
         await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
@@ -140,7 +167,7 @@ async function ledgerExists(db, recordHash) {
  * INSERT op for one accepted record row (task 0060 F9). Pure builder so the
  * per-record batch path and the single-write `insertRecord` can never drift.
  * Typed tables validate unknown keys here; the caller must have ensured the
- * target table exists (`ensureTargetTables` runs at import start).
+ * target table exists (the import loop ensures accepted targets before batching).
  */
 export function recordInsertOp(targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
     const table = targetTableFor(targetTable);
@@ -186,14 +213,16 @@ export function ledgerInsertOp(recordHash, source, sourceFile, sourceLine, split
     };
 }
 /** UPSERT op for one checkpoint row (task 0060 F9: checkpoint joins the record batch). */
-export function checkpointUpsertOp(source, sourceFile, line, now) {
+export function checkpointUpsertOp(source, sourceFile, line, now, identity) {
     return {
-        sql: `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
-             VALUES (?, ?, ?, ?)
+        sql: `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, source_size, source_mtime_ms, updated_at)
+             VALUES (?, ?, ?, ?, ?, ?)
              ON CONFLICT(source, source_file) DO UPDATE SET
                 last_imported_line = excluded.last_imported_line,
+                source_size = excluded.source_size,
+                source_mtime_ms = excluded.source_mtime_ms,
                 updated_at = excluded.updated_at`,
-        params: [source, sourceFile, line, timestamp(now)],
+        params: [source, sourceFile, line, identity?.size ?? null, identity?.mtimeMs ?? null, timestamp(now)],
     };
 }
 /**
@@ -225,6 +254,16 @@ function toolCallResultBytesUpdateOp(recordHash, resultBytes) {
         params: [resultBytes, recordHash],
     };
 }
+/** Targeted UPDATE op attributing a codex usage-carrier row's token counts to the
+ *  latest assistant message of its session (task 0678 R3). Keyed by `record_hash` (PK). */
+export function codexUsageAttributionUpdateOp(recordHash, tokens) {
+    return {
+        sql: `UPDATE history_message
+              SET input_tokens = ?, output_tokens = ?, cache_read_tokens = ?
+              WHERE record_hash = ?`,
+        params: [tokens.input, tokens.output, tokens.cacheRead, recordHash],
+    };
+}
 /**
  * Query which of the given hashes already exist in the ledger, in chunks of at
  * most 200 per `IN (...)` (task 0060 F9 — replaces the per-record SELECT loop).
@@ -242,11 +281,7 @@ export async function ledgerExistingHashes(db, hashes) {
 }
 async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
     const op = recordInsertOp(targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now);
-    if (TYPED_TABLE_COLUMNS[targetTableFor(targetTable)] === undefined) {
-        // Custom ETL tables are created on demand; `CREATE TABLE IF NOT EXISTS` is idempotent,
-        // so there is no process-global cache to go stale across adapter instances (task 0060 F8).
-        await db.exec(ETL_TABLE_DDL(targetTableFor(targetTable)));
-    }
+    await ensureTargetTable(db, targetTable);
     await db.run(op.sql, ...op.params);
 }
 async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
@@ -300,7 +335,7 @@ async function listEtlTables(db) {
  * `(source, realpath)` by keeping the highest `last_imported_line`.
  */
 async function normalizeCheckpointPaths(db, resolveRealPath) {
-    const rows = await db.queryAll('SELECT source, source_file, last_imported_line, updated_at FROM history_import_checkpoint');
+    const rows = await db.queryAll('SELECT source, source_file, last_imported_line, source_size, source_mtime_ms, updated_at FROM history_import_checkpoint');
     // Collapse per (source, realPath) keeping the highest last_imported_line.
     const collapsed = new Map();
     for (const row of rows) {
@@ -312,14 +347,16 @@ async function normalizeCheckpointPaths(db, resolveRealPath) {
                 source: row.source,
                 source_file: canonical,
                 last_imported_line: row.last_imported_line,
+                source_size: row.source_size,
+                source_mtime_ms: row.source_mtime_ms,
                 updated_at: row.updated_at,
             });
         }
     }
     await db.exec('DELETE FROM history_import_checkpoint');
     for (const row of collapsed.values()) {
-        await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
-             VALUES (?, ?, ?, ?)`, row.source, row.source_file, row.last_imported_line, row.updated_at);
+        await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, source_size, source_mtime_ms, updated_at)
+             VALUES (?, ?, ?, ?, ?, ?)`, row.source, row.source_file, row.last_imported_line, row.source_size, row.source_mtime_ms, row.updated_at);
     }
 }
 /** Normalize `source_file` in a table that stores it as a plain column. */
@@ -499,4 +536,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
     }
     return operations;
 }
-export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };
+export { ETL_TABLE_DDL, ensureTargetTable, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, toolCallDurationUpdateOp, toolCallResultBytesUpdateOp, writeCheckpoint, };

package/dist/mappers.d.ts

@@ -39,6 +39,8 @@ export interface ClaudeToolResultTiming {
     toolCallId: string;
     /** Message timestamp as epoch millis, when parseable. */
     timestampMs: number | undefined;
+    /** Native tool duration, when Claude Code emits one. */
+    durationMs: number | undefined;
     /** Serialized size in bytes of the tool result payload. */
     resultBytes: number;
 }
@@ -48,8 +50,8 @@ export interface ClaudeToolResultTiming {
  * `type: "user"` lines whose `message.content` holds `{type: "tool_result",
  * tool_use_id, content}` blocks; the top-level `toolUseResult` carries the raw
  * envelope. `resultBytes` prefers the model-visible block content and falls
- * back to `toolUseResult`; both are null-safe. No wall time exists natively —
- * claude `duration_ms` intentionally stays NULL (never-fabricate, 0624).
+ * back to `toolUseResult`; both are null-safe. Some tool results carry a native
+ * `durationMs` or `durationSeconds`; absent timing stays unmeasured.
  */
 export declare function claudeToolResultTiming(raw: Record<string, unknown>): ClaudeToolResultTiming | null;
 /** Map a Codex JSONL record into one or more ETL split entries. */

package/dist/mappers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAoExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AAwCD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAMhG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkH/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,wEAAwE;AACxE,MAAM,WAAW,sBAAsB;IACnC,uFAAuF;IACvF,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,sBAAsB,GAAG,IAAI,CAkBlG;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAgL1G;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHxG;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAMD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAgB9F;AAiMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4S7E;AAgFD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
+{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAuExE,kFAAkF;AAClF,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAGhD;AA6CD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAMhG;AAMD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAkH/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHvG;AAmBD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA+GxG;AAoBD,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAS5F;AAMD,wEAAwE;AACxE,MAAM,WAAW,sBAAsB;IACnC,uFAAuF;IACvF,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,wDAAwD;IACxD,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,sBAAsB,GAAG,IAAI,CA8BlG;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CA4L1G;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAmHxG;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAiF3G;AAMD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAgB9F;AAiMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA8S7E;AAgFD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}