@gobing-ai/ts-llm-jsonl-importer 0.4.24 → 0.4.25

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAgJrH"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAuBA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAGZ,gBAAgB,EAEnB,MAAM,SAAS,CAAC;AAOjB;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA4KrH"}

package/dist/importer.js

@@ -1,6 +1,6 @@
 import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
 import { sha256 } from './hash.js';
-import { applyHistoryImportSchema, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, writeCheckpoint, } from './jsonl-importer-dao.js';
+import { applyHistoryImportSchema, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, writeCheckpoint, } from './jsonl-importer-dao.js';
 import { redactRecord } from './redaction.js';
 import { resolveSourceDefinition } from './sources.js';
 /** Run the JSONL import pipeline for one source.
@@ -29,6 +29,9 @@ export async function runJsonlImport(source, options) {
     let skippedDuplicates = 0;
     let unknownRecords = 0;
     let checkpointUpdates = 0;
+    // Full-mode desired set (R1, task 0504): every record hash the current source produces.
+    // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
+    const desiredHashes = new Set();
     for (const file of files) {
         const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
         let lineNumber = 0;
@@ -48,6 +51,10 @@ export async function runJsonlImport(source, options) {
                 splitIndex: 0,
             });
             let lineSucceeded = false;
+            // Atomic per-record acceptance (R2, task 0504): a schema-invalid split rejects the
+            // WHOLE line, so no partially accepted rows (or orphaned tool-call rows with a
+            // dangling message_hash) can be left behind by one bad split.
+            let lineRejected = false;
             const prepared = [];
             for (let splitIndex = 0; splitIndex < splitRecords.length; splitIndex += 1) {
                 const split = splitRecords[splitIndex];
@@ -66,6 +73,7 @@ export async function runJsonlImport(source, options) {
                         sourceLine: lineNumber,
                         reason: parsed.error.issues.map((issue) => issue.message).join('; '),
                     });
+                    lineRejected = true;
                     continue;
                 }
                 const redacted = redactRecord(parsed.data, options.redactionRules);
@@ -77,12 +85,17 @@ export async function runJsonlImport(source, options) {
                     splitIndex,
                     record: redacted,
                 });
+                desiredHashes.add(recordHash);
                 if (await ledgerExists(options.db, recordHash)) {
                     skippedDuplicates += 1;
                     continue;
                 }
                 prepared.push({ split, splitIndex, normalized: redacted, recordHash });
             }
+            // A rejected line is reported (validationErrors above) but never persisted — no
+            // second pass, no checkpoint advance, no partial rows from this record.
+            if (lineRejected)
+                continue;
             // Second pass: resolve _messageSplitIndex → message_hash, then insert.
             const recordHashBySplitIndex = new Map(prepared.map((e) => [e.splitIndex, e.recordHash]));
             for (const entry of prepared) {
@@ -111,6 +124,13 @@ export async function runJsonlImport(source, options) {
             }
         }
     }
+    // Full-mode reconciliation (R1, task 0504): diff the desired hash set against the
+    // persisted ledger and retire stale derived rows (target, tool, ledger, checkpoint) in
+    // one source-scoped batch. Dry-run computes the identical counts without mutation.
+    let reconciliation;
+    if (mode === 'full') {
+        reconciliation = await reconcileFullImport(options.db, resolvedSource, desiredHashes, files, options.dryRun ?? false);
+    }
     return {
         source: resolvedSource,
         mode,
@@ -122,6 +142,7 @@ export async function runJsonlImport(source, options) {
         parseErrors,
         validationErrors,
         checkpointUpdates,
+        reconciliation,
     };
 }
 function parseJsonLine(line, sourceFile, sourceLine, parseErrors) {

package/dist/index.d.ts

@@ -6,5 +6,5 @@ export { type OpenCodeImportOptions, runOpenCodeImport } from './opencode-import
 export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
 export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
 export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
-export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, RedactionRule, SourceDefinition, SplitConfig, SplitEntry, TransformContext, } from './types';
+export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, ReconcileSummary, RedactionRule, SourceDefinition, SplitConfig, SplitEntry, TransformContext, } from './types';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAC1F,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAC1F,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}

package/dist/jsonl-importer-dao.d.ts

@@ -1,5 +1,5 @@
 import type { DbAdapter, DbBatchOp } from '@gobing-ai/ts-db';
-import type { ImportOptions, JsonObject, SourceDefinition } from './types';
+import type { ImportOptions, JsonObject, ReconcileSummary, SourceDefinition } from './types';
 interface CheckpointRow {
     readonly last_imported_line: number;
 }
@@ -47,6 +47,19 @@ declare function insertLedger(db: ImportOptions['db'], recordHash: string, sourc
  *   runtime `FileSystem` seam.
  */
 export declare function normalizeSourceFilePaths(db: ImportOptions['db'], resolveRealPath: (sourceFile: string) => string | null | undefined): Promise<void>;
+/**
+ * Reconcile a source's persisted derived rows against the desired set produced by a full
+ * import (task 0504 R1). Rows whose `record_hash` is no longer reproduced by current source
+ * data or mapper output are stale: their target rows (typed contract tables — including tool
+ * calls — and ETL tables), ledger rows, and checkpoints for vanished source files are removed
+ * in ONE source-scoped batch (atomic). Dry-run returns the exact same counts without mutating
+ * the database, so `--dry-run` is a safe preview and a second full write reports zero changes.
+ *
+ * Deletes are keyed by `record_hash` (PK) and checkpoint `(source, source_file)` (PK) — never
+ * by an unindexed ledger predicate — so reconciliation cost scales with the stale count, not
+ * with ledger size.
+ */
+declare function reconcileFullImport(db: ImportOptions['db'], source: string, desiredHashes: ReadonlySet<string>, discoveredFiles: readonly string[], dryRun: boolean): Promise<ReconcileSummary>;
 /** OpenCode message row projected from its SQLite store. */
 export interface OpenCodeMessageRow {
     id: string;
@@ -86,5 +99,5 @@ export declare function openCodeDeleteOperations(entries: readonly OpenCodeExist
 /** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
 export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
 export type { CheckpointRow };
-export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
+export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
 //# sourceMappingURL=jsonl-importer-dao.d.ts.map

package/dist/jsonl-importer-dao.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA+FD,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
+{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7F,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD;;;;;;;;;;;GAWG;AACH,iBAAe,mBAAmB,CAC9B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,EAClC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,MAAM,EAAE,OAAO,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA6C3B;AAID,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}

package/dist/jsonl-importer-dao.js

@@ -252,6 +252,57 @@ function resolveWithFallback(resolveRealPath, sourceFile) {
         return sourceFile;
     }
 }
+/**
+ * Reconcile a source's persisted derived rows against the desired set produced by a full
+ * import (task 0504 R1). Rows whose `record_hash` is no longer reproduced by current source
+ * data or mapper output are stale: their target rows (typed contract tables — including tool
+ * calls — and ETL tables), ledger rows, and checkpoints for vanished source files are removed
+ * in ONE source-scoped batch (atomic). Dry-run returns the exact same counts without mutating
+ * the database, so `--dry-run` is a safe preview and a second full write reports zero changes.
+ *
+ * Deletes are keyed by `record_hash` (PK) and checkpoint `(source, source_file)` (PK) — never
+ * by an unindexed ledger predicate — so reconciliation cost scales with the stale count, not
+ * with ledger size.
+ */
+async function reconcileFullImport(db, source, desiredHashes, discoveredFiles, dryRun) {
+    const ledgerRows = await db.queryAll('SELECT record_hash, target_table FROM history_import_ledger WHERE source = ?', source);
+    const stale = [];
+    for (const row of ledgerRows) {
+        if (!desiredHashes.has(row.record_hash))
+            stale.push(row);
+    }
+    const checkpointRows = await db.queryAll('SELECT source_file FROM history_import_checkpoint WHERE source = ?', source);
+    const discovered = new Set(discoveredFiles);
+    const staleCheckpoints = checkpointRows
+        .map((row) => row.source_file)
+        .filter((sourceFile) => !discovered.has(sourceFile));
+    const summary = {
+        staleTargetRows: stale.length,
+        staleLedgerRows: stale.length,
+        staleCheckpointRows: staleCheckpoints.length,
+    };
+    if (dryRun || (stale.length === 0 && staleCheckpoints.length === 0))
+        return summary;
+    const operations = [];
+    for (const row of stale) {
+        operations.push({
+            sql: `DELETE FROM ${targetTableFor(row.target_table)} WHERE record_hash = ?`,
+            params: [row.record_hash],
+        });
+        operations.push({
+            sql: 'DELETE FROM history_import_ledger WHERE record_hash = ?',
+            params: [row.record_hash],
+        });
+    }
+    for (const sourceFile of staleCheckpoints) {
+        operations.push({
+            sql: 'DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?',
+            params: [source, sourceFile],
+        });
+    }
+    await db.batch(operations);
+    return summary;
+}
 const OPENCODE_SOURCE = 'opencode';
 /** Read one ordered page of OpenCode messages after the supplied cursor. */
 export async function readOpenCodeMessages(db, lastTime, lastId, limit) {
@@ -353,4 +404,4 @@ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
     }
     return operations;
 }
-export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
+export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, reconcileFullImport, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };

package/dist/opencode-importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAc/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAc,aAAa,EAAE,MAAM,SAAS,CAAC;AAYhG,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CA+F7F"}
+{"version":3,"file":"opencode-importer.d.ts","sourceRoot":"","sources":["../src/opencode-importer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,SAAS,EAAkB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAGH,KAAK,UAAU,EAEf,KAAK,YAAY,EACpB,MAAM,uBAAuB,CAAC;AAc/B,OAAO,KAAK,EAAe,UAAU,EAAE,YAAY,EAAgC,aAAa,EAAE,MAAM,SAAS,CAAC;AAYlH,8EAA8E;AAC9E,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,SAAS,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CACpB;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAuI7F"}

package/dist/opencode-importer.js

@@ -24,7 +24,20 @@ export async function runOpenCodeImport(options) {
     let lastTime = -1;
     let lastId = '';
     const existingBySourceFile = await readOpenCodeExistingEntries(options.db);
+    // Source files still present in the OpenCode store this pass (R1, task 0504). Files with
+    // no valid role produce no entries and are deliberately NOT marked seen, so their old
+    // derived rows are swept as no-longer-reproduced in full mode.
+    const seenSourceFiles = new Set();
     const operations = [];
+    // Full-mode reconciliation counters (R1, task 0504): stale rows from BOTH sources —
+    // per-file hash-diff deletes (changed messages) and the vanished-file sweep below. Dry-run
+    // counts the identical totals the write would delete, so `--dry-run` is an exact preview.
+    // Declared here (outside `try`) because the result is returned after `finally` closes the
+    // source database.
+    let staleTargetRows = 0;
+    let staleLedgerRows = 0;
+    let staleCheckpointRows = 0;
+    let reconciliation;
     try {
         while (true) {
             const messages = await readOpenCodeMessages(sourceDb, lastTime, lastId, PAGE_SIZE);
@@ -48,11 +61,18 @@ export async function runOpenCodeImport(options) {
                     validationErrors.push({ sourceFile, sourceLine: 1, reason: 'OpenCode message has no valid role' });
                     continue;
                 }
+                seenSourceFiles.add(sourceFile);
                 const existing = existingBySourceFile.get(sourceFile) ?? [];
                 if (sameHashes(existing, entries)) {
                     skippedDuplicates += entries.length;
                     continue;
                 }
+                if (mode === 'full') {
+                    // Per-file mapper drift: the old derived rows for this source file are
+                    // stale and are deleted before the new entries are written.
+                    staleTargetRows += existing.length;
+                    staleLedgerRows += existing.length;
+                }
                 if (!options.dryRun) {
                     operations.push(...openCodeDeleteOperations(existing));
                     for (const entry of entries) {
@@ -73,6 +93,26 @@ export async function runOpenCodeImport(options) {
             lastTime = tail.time_created;
             lastId = tail.id;
         }
+        // Full-mode reconciliation (R1, task 0504): messages deleted from the OpenCode store
+        // leave stale derived rows behind. Sweep ledger/target rows and checkpoints for source
+        // files never seen this pass, merged into the SAME source-scoped batch as the writes.
+        if (mode === 'full') {
+            for (const [sourceFile, entries] of existingBySourceFile) {
+                if (seenSourceFiles.has(sourceFile))
+                    continue;
+                staleLedgerRows += entries.length;
+                staleTargetRows += entries.length;
+                staleCheckpointRows += 1;
+                if (!options.dryRun) {
+                    operations.push(...openCodeDeleteOperations(entries));
+                    operations.push({
+                        sql: 'DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?',
+                        params: [SOURCE, sourceFile],
+                    });
+                }
+            }
+            reconciliation = { staleTargetRows, staleLedgerRows, staleCheckpointRows };
+        }
         if (operations.length > 0)
             await options.db.batch(operations);
     }
@@ -90,6 +130,7 @@ export async function runOpenCodeImport(options) {
         parseErrors,
         validationErrors,
         checkpointUpdates,
+        reconciliation,
     };
 }
 function groupParts(parts) {

package/dist/types.d.ts

@@ -76,6 +76,20 @@ export interface ImportIssue {
     readonly sourceLine: number;
     readonly reason: string;
 }
+/**
+ * Source-scoped full-mode reconciliation outcome (task 0504 R1). Computed by diffing the
+ * current source's desired record hashes against the persisted ledger; rows no longer
+ * reproduced by current source data or mapper output are stale and are removed in one
+ * source-scoped batch (dry-run reports the same counts without mutating the database).
+ */
+export interface ReconcileSummary {
+    /** Rows deleted from typed/ETL target tables (history_message / history_tool_call / history_etl_*). */
+    readonly staleTargetRows: number;
+    /** Rows deleted from `history_import_ledger`. */
+    readonly staleLedgerRows: number;
+    /** Checkpoint rows deleted for source files no longer discovered. */
+    readonly staleCheckpointRows: number;
+}
 /** Summary produced by the importer control function. */
 export interface ImportResult {
     /** Source identifier — the built-in name or custom definition name that produced this result. */
@@ -89,5 +103,10 @@ export interface ImportResult {
     readonly parseErrors: readonly ImportIssue[];
     readonly validationErrors: readonly ImportIssue[];
     readonly checkpointUpdates: number;
+    /**
+     * Full-mode reconciliation counts (only set when `mode === 'full'`). A second full run
+     * reports zero stale rows once the database matches the current source (task 0504 R1).
+     */
+    readonly reconciliation?: ReconcileSummary;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACtC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GACpB,IAAI,GACJ,QAAQ,GACR,OAAO,GACP,QAAQ,GACR,UAAU,GACV,aAAa,GACb,UAAU,GACV,KAAK,GACL,MAAM,GACN,KAAK,CAAC;AAEZ,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC/B;AAED,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,gBAAgB,KAAK,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IACtG,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,iDAAiD;IACjD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;CAC9C"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gobing-ai/ts-llm-jsonl-importer",
-    "version": "0.4.24",
+    "version": "0.4.25",
     "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
     "keywords": [
         "typescript",
@@ -47,9 +47,9 @@
         "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
     },
     "dependencies": {
-        "@gobing-ai/ts-db": "^0.4.24",
-        "@gobing-ai/ts-runtime": "^0.4.24",
-        "@gobing-ai/ts-utils": "^0.4.24",
+        "@gobing-ai/ts-db": "^0.4.25",
+        "@gobing-ai/ts-runtime": "^0.4.25",
+        "@gobing-ai/ts-utils": "^0.4.25",
         "zod": "^4.1.0"
     },
     "devDependencies": {

package/src/importer.ts

@@ -14,13 +14,22 @@ import {
     insertRecord,
     ledgerExists,
     readCheckpoint,
+    reconcileFullImport,
     resetCheckpoints,
     targetTableFor,
     writeCheckpoint,
 } from './jsonl-importer-dao';
 import { redactRecord } from './redaction';
 import { resolveSourceDefinition } from './sources';
-import type { ImportIssue, ImportOptions, ImportResult, JsonObject, SourceDefinition, TransformContext } from './types';
+import type {
+    ImportIssue,
+    ImportOptions,
+    ImportResult,
+    JsonObject,
+    ReconcileSummary,
+    SourceDefinition,
+    TransformContext,
+} from './types';
 
 interface SplitRecord {
     readonly targetTable: string;
@@ -55,6 +64,9 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
     let skippedDuplicates = 0;
     let unknownRecords = 0;
     let checkpointUpdates = 0;
+    // Full-mode desired set (R1, task 0504): every record hash the current source produces.
+    // ReconcileFullImport diffs this against the persisted ledger to retire stale rows.
+    const desiredHashes = new Set<string>();
 
     for (const file of files) {
         const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
@@ -75,6 +87,10 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
                 splitIndex: 0,
             });
             let lineSucceeded = false;
+            // Atomic per-record acceptance (R2, task 0504): a schema-invalid split rejects the
+            // WHOLE line, so no partially accepted rows (or orphaned tool-call rows with a
+            // dangling message_hash) can be left behind by one bad split.
+            let lineRejected = false;
 
             // First pass: normalize, validate, redact, compute recordHash for every entry.
             interface PreparedEntry {
@@ -100,6 +116,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
                         sourceLine: lineNumber,
                         reason: parsed.error.issues.map((issue) => issue.message).join('; '),
                     });
+                    lineRejected = true;
                     continue;
                 }
 
@@ -112,6 +129,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
                     splitIndex,
                     record: redacted,
                 });
+                desiredHashes.add(recordHash);
                 if (await ledgerExists(options.db, recordHash)) {
                     skippedDuplicates += 1;
                     continue;
@@ -119,6 +137,10 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
                 prepared.push({ split, splitIndex, normalized: redacted, recordHash });
             }
 
+            // A rejected line is reported (validationErrors above) but never persisted — no
+            // second pass, no checkpoint advance, no partial rows from this record.
+            if (lineRejected) continue;
+
             // Second pass: resolve _messageSplitIndex → message_hash, then insert.
             const recordHashBySplitIndex = new Map(prepared.map((e) => [e.splitIndex, e.recordHash] as const));
             for (const entry of prepared) {
@@ -167,6 +189,20 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
         }
     }
 
+    // Full-mode reconciliation (R1, task 0504): diff the desired hash set against the
+    // persisted ledger and retire stale derived rows (target, tool, ledger, checkpoint) in
+    // one source-scoped batch. Dry-run computes the identical counts without mutation.
+    let reconciliation: ReconcileSummary | undefined;
+    if (mode === 'full') {
+        reconciliation = await reconcileFullImport(
+            options.db,
+            resolvedSource,
+            desiredHashes,
+            files,
+            options.dryRun ?? false,
+        );
+    }
+
     return {
         source: resolvedSource,
         mode,
@@ -178,6 +214,7 @@ export async function runJsonlImport(source: string | SourceDefinition, options:
         parseErrors,
         validationErrors,
         checkpointUpdates,
+        reconciliation,
     };
 }
 

package/src/index.ts

@@ -20,6 +20,7 @@ export type {
     ImportResult,
     JsonObject,
     LlmJsonlSource,
+    ReconcileSummary,
     RedactionRule,
     SourceDefinition,
     SplitConfig,

package/src/jsonl-importer-dao.ts

@@ -2,7 +2,7 @@ import type { DbAdapter, DbBatchOp } from '@gobing-ai/ts-db';
 import { HistoryImportError } from './errors';
 import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
 import { VALID_TABLE_NAME } from './sources';
-import type { ImportOptions, JsonObject, SourceDefinition } from './types';
+import type { ImportOptions, JsonObject, ReconcileSummary, SourceDefinition } from './types';
 
 interface CheckpointRow {
     readonly last_imported_line: number;
@@ -369,6 +369,71 @@ function resolveWithFallback(
     }
 }
 
+/**
+ * Reconcile a source's persisted derived rows against the desired set produced by a full
+ * import (task 0504 R1). Rows whose `record_hash` is no longer reproduced by current source
+ * data or mapper output are stale: their target rows (typed contract tables — including tool
+ * calls — and ETL tables), ledger rows, and checkpoints for vanished source files are removed
+ * in ONE source-scoped batch (atomic). Dry-run returns the exact same counts without mutating
+ * the database, so `--dry-run` is a safe preview and a second full write reports zero changes.
+ *
+ * Deletes are keyed by `record_hash` (PK) and checkpoint `(source, source_file)` (PK) — never
+ * by an unindexed ledger predicate — so reconciliation cost scales with the stale count, not
+ * with ledger size.
+ */
+async function reconcileFullImport(
+    db: ImportOptions['db'],
+    source: string,
+    desiredHashes: ReadonlySet<string>,
+    discoveredFiles: readonly string[],
+    dryRun: boolean,
+): Promise<ReconcileSummary> {
+    const ledgerRows = await db.queryAll<{ record_hash: string; target_table: string }>(
+        'SELECT record_hash, target_table FROM history_import_ledger WHERE source = ?',
+        source,
+    );
+    const stale: { record_hash: string; target_table: string }[] = [];
+    for (const row of ledgerRows) {
+        if (!desiredHashes.has(row.record_hash)) stale.push(row);
+    }
+
+    const checkpointRows = await db.queryAll<{ source_file: string }>(
+        'SELECT source_file FROM history_import_checkpoint WHERE source = ?',
+        source,
+    );
+    const discovered = new Set(discoveredFiles);
+    const staleCheckpoints = checkpointRows
+        .map((row) => row.source_file)
+        .filter((sourceFile) => !discovered.has(sourceFile));
+
+    const summary: ReconcileSummary = {
+        staleTargetRows: stale.length,
+        staleLedgerRows: stale.length,
+        staleCheckpointRows: staleCheckpoints.length,
+    };
+    if (dryRun || (stale.length === 0 && staleCheckpoints.length === 0)) return summary;
+
+    const operations: DbBatchOp[] = [];
+    for (const row of stale) {
+        operations.push({
+            sql: `DELETE FROM ${targetTableFor(row.target_table)} WHERE record_hash = ?`,
+            params: [row.record_hash],
+        });
+        operations.push({
+            sql: 'DELETE FROM history_import_ledger WHERE record_hash = ?',
+            params: [row.record_hash],
+        });
+    }
+    for (const sourceFile of staleCheckpoints) {
+        operations.push({
+            sql: 'DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?',
+            params: [source, sourceFile],
+        });
+    }
+    await db.batch(operations);
+    return summary;
+}
+
 const OPENCODE_SOURCE = 'opencode';
 
 /** OpenCode message row projected from its SQLite store. */
@@ -538,6 +603,7 @@ export {
     insertRecord,
     ledgerExists,
     readCheckpoint,
+    reconcileFullImport,
     resetCheckpoints,
     targetTableFor,
     timestamp,

package/src/opencode-importer.ts

@@ -19,7 +19,7 @@ import {
     readOpenCodeParts,
 } from './jsonl-importer-dao';
 import { redactRecord } from './redaction';
-import type { ImportIssue, ImportMode, ImportResult, JsonObject, RedactionRule } from './types';
+import type { ImportIssue, ImportMode, ImportResult, JsonObject, ReconcileSummary, RedactionRule } from './types';
 
 const SOURCE = 'opencode';
 const PAGE_SIZE = 250;
@@ -63,7 +63,20 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
     let lastTime = -1;
     let lastId = '';
     const existingBySourceFile = await readOpenCodeExistingEntries(options.db);
+    // Source files still present in the OpenCode store this pass (R1, task 0504). Files with
+    // no valid role produce no entries and are deliberately NOT marked seen, so their old
+    // derived rows are swept as no-longer-reproduced in full mode.
+    const seenSourceFiles = new Set<string>();
     const operations: DbBatchOp[] = [];
+    // Full-mode reconciliation counters (R1, task 0504): stale rows from BOTH sources —
+    // per-file hash-diff deletes (changed messages) and the vanished-file sweep below. Dry-run
+    // counts the identical totals the write would delete, so `--dry-run` is an exact preview.
+    // Declared here (outside `try`) because the result is returned after `finally` closes the
+    // source database.
+    let staleTargetRows = 0;
+    let staleLedgerRows = 0;
+    let staleCheckpointRows = 0;
+    let reconciliation: ReconcileSummary | undefined;
 
     try {
         while (true) {
@@ -91,6 +104,7 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
                     validationErrors.push({ sourceFile, sourceLine: 1, reason: 'OpenCode message has no valid role' });
                     continue;
                 }
+                seenSourceFiles.add(sourceFile);
 
                 const existing = existingBySourceFile.get(sourceFile) ?? [];
                 if (sameHashes(existing, entries)) {
@@ -98,6 +112,12 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
                     continue;
                 }
 
+                if (mode === 'full') {
+                    // Per-file mapper drift: the old derived rows for this source file are
+                    // stale and are deleted before the new entries are written.
+                    staleTargetRows += existing.length;
+                    staleLedgerRows += existing.length;
+                }
                 if (!options.dryRun) {
                     operations.push(...openCodeDeleteOperations(existing));
                     for (const entry of entries) {
@@ -122,6 +142,25 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
             lastTime = tail.time_created;
             lastId = tail.id;
         }
+        // Full-mode reconciliation (R1, task 0504): messages deleted from the OpenCode store
+        // leave stale derived rows behind. Sweep ledger/target rows and checkpoints for source
+        // files never seen this pass, merged into the SAME source-scoped batch as the writes.
+        if (mode === 'full') {
+            for (const [sourceFile, entries] of existingBySourceFile) {
+                if (seenSourceFiles.has(sourceFile)) continue;
+                staleLedgerRows += entries.length;
+                staleTargetRows += entries.length;
+                staleCheckpointRows += 1;
+                if (!options.dryRun) {
+                    operations.push(...openCodeDeleteOperations(entries));
+                    operations.push({
+                        sql: 'DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?',
+                        params: [SOURCE, sourceFile],
+                    });
+                }
+            }
+            reconciliation = { staleTargetRows, staleLedgerRows, staleCheckpointRows };
+        }
         if (operations.length > 0) await options.db.batch(operations);
     } finally {
         sourceDb.close();
@@ -138,6 +177,7 @@ export async function runOpenCodeImport(options: OpenCodeImportOptions): Promise
         parseErrors,
         validationErrors,
         checkpointUpdates,
+        reconciliation,
     };
 }
 

package/src/types.ts

@@ -101,6 +101,21 @@ export interface ImportIssue {
     readonly reason: string;
 }
 
+/**
+ * Source-scoped full-mode reconciliation outcome (task 0504 R1). Computed by diffing the
+ * current source's desired record hashes against the persisted ledger; rows no longer
+ * reproduced by current source data or mapper output are stale and are removed in one
+ * source-scoped batch (dry-run reports the same counts without mutating the database).
+ */
+export interface ReconcileSummary {
+    /** Rows deleted from typed/ETL target tables (history_message / history_tool_call / history_etl_*). */
+    readonly staleTargetRows: number;
+    /** Rows deleted from `history_import_ledger`. */
+    readonly staleLedgerRows: number;
+    /** Checkpoint rows deleted for source files no longer discovered. */
+    readonly staleCheckpointRows: number;
+}
+
 /** Summary produced by the importer control function. */
 export interface ImportResult {
     /** Source identifier — the built-in name or custom definition name that produced this result. */
@@ -114,4 +129,9 @@ export interface ImportResult {
     readonly parseErrors: readonly ImportIssue[];
     readonly validationErrors: readonly ImportIssue[];
     readonly checkpointUpdates: number;
+    /**
+     * Full-mode reconciliation counts (only set when `mode === 'full'`). A second full run
+     * reports zero stale rows once the database matches the current source (task 0504 R1).
+     */
+    readonly reconciliation?: ReconcileSummary;
 }