@gobing-ai/ts-llm-jsonl-importer 0.4.21 → 0.4.23

package/dist/importer.js

@@ -180,7 +180,10 @@ function normalizeRecord(definition, raw, context) {
 }
 async function discoverFiles(definition, roots, files, fileSystem, paths) {
     if (files !== undefined && files.length > 0) {
-        return files.map((file) => resolvePath(file)).sort();
+        return files
+            .map((file) => resolvePath(file))
+            .map((file) => normalizeSourceFilePath(file, fileSystem))
+            .sort();
     }
     // Root resolution splits by provenance (ADR-023 A1 / task 0042):
     //  - explicit caller-supplied `roots` → cwd semantics: resolve against `paths.cwd` when
@@ -201,16 +204,37 @@ async function discoverFiles(definition, roots, files, fileSystem, paths) {
             continue;
         if (stat.isFile()) {
             if (matchesPattern(root, definition.filePatterns))
-                found.add(root);
+                found.add(normalizeSourceFilePath(root, fileSystem));
             continue;
         }
         for (const file of await walkDir(root, fileSystem)) {
             if (matchesPattern(file, definition.filePatterns))
-                found.add(file);
+                found.add(normalizeSourceFilePath(file, fileSystem));
         }
     }
     return [...found].sort();
 }
+/**
+ * Normalize a discovered source file path to its canonical real-path identity.
+ *
+ * WHY: the same physical session file is reachable via a symlinked path and via its
+ * real path (every agent history dir under `$HOME` is a symlink into the dotfiles
+ * tree here). Without normalization, the checkpoint, ledger, and record_hash keys
+ * diverge, yielding duplicate checkpoint rows and silent full-corpus re-imports.
+ * `realPath` is optional on `FileSystem`: injected and in-memory test doubles may
+ * omit it, and a path that does not exist on disk has no realpath. Fall back to the
+ * original path in either case — never throw, never bypass the injected FileSystem.
+ */
+function normalizeSourceFilePath(path, fileSystem) {
+    if (!fileSystem.realPath)
+        return path;
+    try {
+        return fileSystem.realPath(path);
+    }
+    catch {
+        return path;
+    }
+}
 function matchesPattern(path, patterns) {
     return patterns.some((pattern) => {
         if (pattern === '*.jsonl')

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 export { HistoryImportError } from './errors';
 export { sha256, stableJson } from './hash';
 export { runJsonlImport } from './importer';
-export { applyHistoryImportSchema } from './jsonl-importer-dao';
+export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
 export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
 export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
 export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAC1F,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -1,7 +1,7 @@
 export { HistoryImportError } from './errors.js';
 export { sha256, stableJson } from './hash.js';
 export { runJsonlImport } from './importer.js';
-export { applyHistoryImportSchema } from './jsonl-importer-dao.js';
+export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao.js';
 export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
 export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
 export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources.js';

package/dist/jsonl-importer-dao.d.ts

@@ -25,6 +25,27 @@ declare function writeCheckpoint(db: ImportOptions['db'], source: string, source
 declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean>;
 declare function insertRecord(db: ImportOptions['db'], targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): Promise<void>;
 declare function insertLedger(db: ImportOptions['db'], recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): Promise<void>;
+/**
+ * Idempotent migration that rewrites `source_file` to its realpath across the checkpoint,
+ * ledger, and any contract table present (R4).
+ *
+ * WHY: `record_hash` is `sha256({source, sourceFile, sourceLine, splitIndex, record})`, so
+ * `sourceFile` is inside the hash. Normalizing its representation at discovery (R1) changes
+ * every future hash; pre-normalization rows carry unnormalized paths and their old hashes
+ * remain valid as dedupe keys. This migration rewrites the `source_file` *column* so old and
+ * new rows agree on path identity, and collapses duplicate checkpoint rows produced when one
+ * physical file was imported via both a symlinked and a real path — keeping the highest
+ * `last_imported_line` so an incremental resume does not re-import already-seen content.
+ *
+ * `record_hash` is intentionally NOT touched: it is path-representation dependent by
+ * construction, pre-migration rows are grandfathered, and recomputing it would require the
+ * original record payload that the ledger does not store.
+ *
+ * @param resolveRealPath - resolves a `source_file` to its canonical real path. Falls back to
+ *   the original value when it returns null/undefined or throws. Decouples the DAO from the
+ *   runtime `FileSystem` seam.
+ */
+export declare function normalizeSourceFilePaths(db: ImportOptions['db'], resolveRealPath: (sourceFile: string) => string | null | undefined): Promise<void>;
 export type { CheckpointRow };
 export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
 //# sourceMappingURL=jsonl-importer-dao.d.ts.map

package/dist/jsonl-importer-dao.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
+{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA6FD,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}

package/dist/jsonl-importer-dao.js

@@ -165,4 +165,91 @@ async function insertLedger(db, recordHash, source, sourceFile, sourceLine, spli
     await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
          VALUES (?, ?, ?, ?, ?, ?, ?)`, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now));
 }
+/**
+ * Idempotent migration that rewrites `source_file` to its realpath across the checkpoint,
+ * ledger, and any contract table present (R4).
+ *
+ * WHY: `record_hash` is `sha256({source, sourceFile, sourceLine, splitIndex, record})`, so
+ * `sourceFile` is inside the hash. Normalizing its representation at discovery (R1) changes
+ * every future hash; pre-normalization rows carry unnormalized paths and their old hashes
+ * remain valid as dedupe keys. This migration rewrites the `source_file` *column* so old and
+ * new rows agree on path identity, and collapses duplicate checkpoint rows produced when one
+ * physical file was imported via both a symlinked and a real path — keeping the highest
+ * `last_imported_line` so an incremental resume does not re-import already-seen content.
+ *
+ * `record_hash` is intentionally NOT touched: it is path-representation dependent by
+ * construction, pre-migration rows are grandfathered, and recomputing it would require the
+ * original record payload that the ledger does not store.
+ *
+ * @param resolveRealPath - resolves a `source_file` to its canonical real path. Falls back to
+ *   the original value when it returns null/undefined or throws. Decouples the DAO from the
+ *   runtime `FileSystem` seam.
+ */
+export async function normalizeSourceFilePaths(db, resolveRealPath) {
+    await normalizeCheckpointPaths(db, resolveRealPath);
+    await normalizeColumnPaths(db, resolveRealPath, 'history_import_ledger');
+    for (const table of TYPED_TABLE_COLUMNS_SOURCE_FILE) {
+        if (await tableExists(db, table)) {
+            await normalizeColumnPaths(db, resolveRealPath, table);
+        }
+    }
+    for (const table of await listEtlTables(db)) {
+        await normalizeColumnPaths(db, resolveRealPath, table);
+    }
+}
+/** Typed contract tables that carry a `source_file` column (0466 forensic contract). */
+const TYPED_TABLE_COLUMNS_SOURCE_FILE = ['history_message', 'history_tool_call'];
+async function tableExists(db, table) {
+    const row = await db.queryFirst("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", table);
+    return row !== undefined && row !== null;
+}
+async function listEtlTables(db) {
+    const rows = await db.queryAll("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'history_etl_%'");
+    return rows.map((row) => row.name);
+}
+/**
+ * Normalize `source_file` in the checkpoint table, collapsing duplicate rows per
+ * `(source, realpath)` by keeping the highest `last_imported_line`.
+ */
+async function normalizeCheckpointPaths(db, resolveRealPath) {
+    const rows = await db.queryAll('SELECT source, source_file, last_imported_line, updated_at FROM history_import_checkpoint');
+    // Collapse per (source, realPath) keeping the highest last_imported_line.
+    const collapsed = new Map();
+    for (const row of rows) {
+        const canonical = resolveWithFallback(resolveRealPath, row.source_file);
+        const key = `${row.source}\u0000${canonical}`;
+        const existing = collapsed.get(key);
+        if (existing === undefined || row.last_imported_line > existing.last_imported_line) {
+            collapsed.set(key, {
+                source: row.source,
+                source_file: canonical,
+                last_imported_line: row.last_imported_line,
+                updated_at: row.updated_at,
+            });
+        }
+    }
+    await db.exec('DELETE FROM history_import_checkpoint');
+    for (const row of collapsed.values()) {
+        await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
+             VALUES (?, ?, ?, ?)`, row.source, row.source_file, row.last_imported_line, row.updated_at);
+    }
+}
+/** Normalize `source_file` in a table that stores it as a plain column. */
+async function normalizeColumnPaths(db, resolveRealPath, table) {
+    const rows = await db.queryAll(`SELECT rowid AS rowid, source_file FROM ${table} WHERE source_file IS NOT NULL`);
+    for (const row of rows) {
+        const canonical = resolveWithFallback(resolveRealPath, row.source_file);
+        if (canonical !== row.source_file) {
+            await db.run(`UPDATE ${table} SET source_file = ? WHERE rowid = ?`, canonical, row.rowid);
+        }
+    }
+}
+function resolveWithFallback(resolveRealPath, sourceFile) {
+    try {
+        return resolveRealPath(sourceFile) ?? sourceFile;
+    }
+    catch {
+        return sourceFile;
+    }
+}
 export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gobing-ai/ts-llm-jsonl-importer",
-    "version": "0.4.21",
+    "version": "0.4.23",
     "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
     "keywords": [
         "typescript",
@@ -47,9 +47,9 @@
         "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
     },
     "dependencies": {
-        "@gobing-ai/ts-db": "^0.4.21",
-        "@gobing-ai/ts-runtime": "^0.4.21",
-        "@gobing-ai/ts-utils": "^0.4.21",
+        "@gobing-ai/ts-db": "^0.4.23",
+        "@gobing-ai/ts-runtime": "^0.4.23",
+        "@gobing-ai/ts-utils": "^0.4.23",
         "zod": "^4.1.0"
     },
     "devDependencies": {

package/src/importer.ts

@@ -251,7 +251,10 @@ async function discoverFiles(
     paths: RuntimePaths | undefined,
 ): Promise<readonly string[]> {
     if (files !== undefined && files.length > 0) {
-        return files.map((file) => resolvePath(file)).sort();
+        return files
+            .map((file) => resolvePath(file))
+            .map((file) => normalizeSourceFilePath(file, fileSystem))
+            .sort();
     }
 
     // Root resolution splits by provenance (ADR-023 A1 / task 0042):
@@ -271,16 +274,36 @@ async function discoverFiles(
         const stat = await fileSystem.stat(root);
         if (stat === null) continue;
         if (stat.isFile()) {
-            if (matchesPattern(root, definition.filePatterns)) found.add(root);
+            if (matchesPattern(root, definition.filePatterns)) found.add(normalizeSourceFilePath(root, fileSystem));
             continue;
         }
         for (const file of await walkDir(root, fileSystem)) {
-            if (matchesPattern(file, definition.filePatterns)) found.add(file);
+            if (matchesPattern(file, definition.filePatterns)) found.add(normalizeSourceFilePath(file, fileSystem));
         }
     }
     return [...found].sort();
 }
 
+/**
+ * Normalize a discovered source file path to its canonical real-path identity.
+ *
+ * WHY: the same physical session file is reachable via a symlinked path and via its
+ * real path (every agent history dir under `$HOME` is a symlink into the dotfiles
+ * tree here). Without normalization, the checkpoint, ledger, and record_hash keys
+ * diverge, yielding duplicate checkpoint rows and silent full-corpus re-imports.
+ * `realPath` is optional on `FileSystem`: injected and in-memory test doubles may
+ * omit it, and a path that does not exist on disk has no realpath. Fall back to the
+ * original path in either case — never throw, never bypass the injected FileSystem.
+ */
+function normalizeSourceFilePath(path: string, fileSystem: FileSystem): string {
+    if (!fileSystem.realPath) return path;
+    try {
+        return fileSystem.realPath(path);
+    } catch {
+        return path;
+    }
+}
+
 function matchesPattern(path: string, patterns: readonly string[]): boolean {
     return patterns.some((pattern) => {
         if (pattern === '*.jsonl') return path.endsWith('.jsonl');

package/src/index.ts

@@ -1,7 +1,7 @@
 export { HistoryImportError } from './errors';
 export { sha256, stableJson } from './hash';
 export { runJsonlImport } from './importer';
-export { applyHistoryImportSchema } from './jsonl-importer-dao';
+export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
 export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
 export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
 export {

package/src/jsonl-importer-dao.ts

@@ -241,6 +241,133 @@ async function insertLedger(
     );
 }
 
+/**
+ * Idempotent migration that rewrites `source_file` to its realpath across the checkpoint,
+ * ledger, and any contract table present (R4).
+ *
+ * WHY: `record_hash` is `sha256({source, sourceFile, sourceLine, splitIndex, record})`, so
+ * `sourceFile` is inside the hash. Normalizing its representation at discovery (R1) changes
+ * every future hash; pre-normalization rows carry unnormalized paths and their old hashes
+ * remain valid as dedupe keys. This migration rewrites the `source_file` *column* so old and
+ * new rows agree on path identity, and collapses duplicate checkpoint rows produced when one
+ * physical file was imported via both a symlinked and a real path — keeping the highest
+ * `last_imported_line` so an incremental resume does not re-import already-seen content.
+ *
+ * `record_hash` is intentionally NOT touched: it is path-representation dependent by
+ * construction, pre-migration rows are grandfathered, and recomputing it would require the
+ * original record payload that the ledger does not store.
+ *
+ * @param resolveRealPath - resolves a `source_file` to its canonical real path. Falls back to
+ *   the original value when it returns null/undefined or throws. Decouples the DAO from the
+ *   runtime `FileSystem` seam.
+ */
+export async function normalizeSourceFilePaths(
+    db: ImportOptions['db'],
+    resolveRealPath: (sourceFile: string) => string | null | undefined,
+): Promise<void> {
+    await normalizeCheckpointPaths(db, resolveRealPath);
+    await normalizeColumnPaths(db, resolveRealPath, 'history_import_ledger');
+    for (const table of TYPED_TABLE_COLUMNS_SOURCE_FILE) {
+        if (await tableExists(db, table)) {
+            await normalizeColumnPaths(db, resolveRealPath, table);
+        }
+    }
+    for (const table of await listEtlTables(db)) {
+        await normalizeColumnPaths(db, resolveRealPath, table);
+    }
+}
+
+/** Typed contract tables that carry a `source_file` column (0466 forensic contract). */
+const TYPED_TABLE_COLUMNS_SOURCE_FILE = ['history_message', 'history_tool_call'] as const;
+
+async function tableExists(db: ImportOptions['db'], table: string): Promise<boolean> {
+    const row = await db.queryFirst<{ name: string }>(
+        "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
+        table,
+    );
+    return row !== undefined && row !== null;
+}
+
+async function listEtlTables(db: ImportOptions['db']): Promise<string[]> {
+    const rows = await db.queryAll<{ name: string }>(
+        "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'history_etl_%'",
+    );
+    return rows.map((row) => row.name);
+}
+
+/**
+ * Normalize `source_file` in the checkpoint table, collapsing duplicate rows per
+ * `(source, realpath)` by keeping the highest `last_imported_line`.
+ */
+async function normalizeCheckpointPaths(
+    db: ImportOptions['db'],
+    resolveRealPath: (sourceFile: string) => string | null | undefined,
+): Promise<void> {
+    const rows = await db.queryAll<{
+        source: string;
+        source_file: string;
+        last_imported_line: number;
+        updated_at: string;
+    }>('SELECT source, source_file, last_imported_line, updated_at FROM history_import_checkpoint');
+    // Collapse per (source, realPath) keeping the highest last_imported_line.
+    const collapsed = new Map<
+        string,
+        { source: string; source_file: string; last_imported_line: number; updated_at: string }
+    >();
+    for (const row of rows) {
+        const canonical = resolveWithFallback(resolveRealPath, row.source_file);
+        const key = `${row.source}\u0000${canonical}`;
+        const existing = collapsed.get(key);
+        if (existing === undefined || row.last_imported_line > existing.last_imported_line) {
+            collapsed.set(key, {
+                source: row.source,
+                source_file: canonical,
+                last_imported_line: row.last_imported_line,
+                updated_at: row.updated_at,
+            });
+        }
+    }
+    await db.exec('DELETE FROM history_import_checkpoint');
+    for (const row of collapsed.values()) {
+        await db.run(
+            `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
+             VALUES (?, ?, ?, ?)`,
+            row.source,
+            row.source_file,
+            row.last_imported_line,
+            row.updated_at,
+        );
+    }
+}
+
+/** Normalize `source_file` in a table that stores it as a plain column. */
+async function normalizeColumnPaths(
+    db: ImportOptions['db'],
+    resolveRealPath: (sourceFile: string) => string | null | undefined,
+    table: string,
+): Promise<void> {
+    const rows = await db.queryAll<{ rowid: number; source_file: string }>(
+        `SELECT rowid AS rowid, source_file FROM ${table} WHERE source_file IS NOT NULL`,
+    );
+    for (const row of rows) {
+        const canonical = resolveWithFallback(resolveRealPath, row.source_file);
+        if (canonical !== row.source_file) {
+            await db.run(`UPDATE ${table} SET source_file = ? WHERE rowid = ?`, canonical, row.rowid);
+        }
+    }
+}
+
+function resolveWithFallback(
+    resolveRealPath: (sourceFile: string) => string | null | undefined,
+    sourceFile: string,
+): string {
+    try {
+        return resolveRealPath(sourceFile) ?? sourceFile;
+    } catch {
+        return sourceFile;
+    }
+}
+
 export type { CheckpointRow };
 export {
     ETL_TABLE_DDL,