@gobing-ai/ts-llm-jsonl-importer 0.4.22 → 0.4.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA2IrH"}
1
+ {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAgJrH"}
package/dist/importer.js CHANGED
@@ -41,7 +41,12 @@ export async function runJsonlImport(source, options) {
41
41
  const raw = parseJsonLine(line, file, lineNumber, parseErrors);
42
42
  if (raw === undefined)
43
43
  continue;
44
- const splitRecords = splitRawRecord(definition, raw);
44
+ const splitRecords = splitRawRecord(definition, raw, {
45
+ source: resolvedSource,
46
+ sourceFile: file,
47
+ sourceLine: lineNumber,
48
+ splitIndex: 0,
49
+ });
45
50
  let lineSucceeded = false;
46
51
  const prepared = [];
47
52
  for (let splitIndex = 0; splitIndex < splitRecords.length; splitIndex += 1) {
@@ -133,7 +138,7 @@ function parseJsonLine(line, sourceFile, sourceLine, parseErrors) {
133
138
  return undefined;
134
139
  }
135
140
  }
136
- function splitRawRecord(definition, raw) {
141
+ function splitRawRecord(definition, raw, context) {
137
142
  const targetTable = targetTableFor(definition.targetTable);
138
143
  if (definition.splitConfig.mode === 'one-to-one') {
139
144
  return [{ targetTable, raw }];
@@ -141,7 +146,7 @@ function splitRawRecord(definition, raw) {
141
146
  if (definition.splitConfig.mode === 'custom') {
142
147
  const config = definition.splitConfig;
143
148
  const configTable = config.targetTable !== undefined ? targetTableFor(config.targetTable) : undefined;
144
- return config.split(raw).map((entry) => {
149
+ return config.split(raw, context).map((entry) => {
145
150
  // Normalize: SplitEntry has a `record` property, bare object is the record itself.
146
151
  const hasTargetTable = 'targetTable' in entry && typeof entry.targetTable === 'string';
147
152
  const record = hasTargetTable ? entry.record : entry;
@@ -180,7 +185,10 @@ function normalizeRecord(definition, raw, context) {
180
185
  }
181
186
  async function discoverFiles(definition, roots, files, fileSystem, paths) {
182
187
  if (files !== undefined && files.length > 0) {
183
- return files.map((file) => resolvePath(file)).sort();
188
+ return files
189
+ .map((file) => resolvePath(file))
190
+ .map((file) => normalizeSourceFilePath(file, fileSystem))
191
+ .sort();
184
192
  }
185
193
  // Root resolution splits by provenance (ADR-023 A1 / task 0042):
186
194
  // - explicit caller-supplied `roots` → cwd semantics: resolve against `paths.cwd` when
@@ -201,16 +209,37 @@ async function discoverFiles(definition, roots, files, fileSystem, paths) {
201
209
  continue;
202
210
  if (stat.isFile()) {
203
211
  if (matchesPattern(root, definition.filePatterns))
204
- found.add(root);
212
+ found.add(normalizeSourceFilePath(root, fileSystem));
205
213
  continue;
206
214
  }
207
215
  for (const file of await walkDir(root, fileSystem)) {
208
216
  if (matchesPattern(file, definition.filePatterns))
209
- found.add(file);
217
+ found.add(normalizeSourceFilePath(file, fileSystem));
210
218
  }
211
219
  }
212
220
  return [...found].sort();
213
221
  }
222
+ /**
223
+ * Normalize a discovered source file path to its canonical real-path identity.
224
+ *
225
+ * WHY: the same physical session file is reachable via a symlinked path and via its
226
+ * real path (every agent history dir under `$HOME` is a symlink into the dotfiles
227
+ * tree here). Without normalization, the checkpoint, ledger, and record_hash keys
228
+ * diverge, yielding duplicate checkpoint rows and silent full-corpus re-imports.
229
+ * `realPath` is optional on `FileSystem`: injected and in-memory test doubles may
230
+ * omit it, and a path that does not exist on disk has no realpath. Fall back to the
231
+ * original path in either case — never throw, never bypass the injected FileSystem.
232
+ */
233
+ function normalizeSourceFilePath(path, fileSystem) {
234
+ if (!fileSystem.realPath)
235
+ return path;
236
+ try {
237
+ return fileSystem.realPath(path);
238
+ }
239
+ catch {
240
+ return path;
241
+ }
242
+ }
214
243
  function matchesPattern(path, patterns) {
215
244
  return patterns.some((pattern) => {
216
245
  if (pattern === '*.jsonl')
package/dist/index.d.ts CHANGED
@@ -1,7 +1,8 @@
1
1
  export { HistoryImportError } from './errors';
2
2
  export { sha256, stableJson } from './hash';
3
3
  export { runJsonlImport } from './importer';
4
- export { applyHistoryImportSchema } from './jsonl-importer-dao';
4
+ export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
5
+ export { type OpenCodeImportOptions, runOpenCodeImport } from './opencode-importer';
5
6
  export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
6
7
  export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
7
8
  export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAC1F,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
package/dist/index.js CHANGED
@@ -1,7 +1,8 @@
1
1
  export { HistoryImportError } from './errors.js';
2
2
  export { sha256, stableJson } from './hash.js';
3
3
  export { runJsonlImport } from './importer.js';
4
- export { applyHistoryImportSchema } from './jsonl-importer-dao.js';
4
+ export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao.js';
5
+ export { runOpenCodeImport } from './opencode-importer.js';
5
6
  export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
6
7
  export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
7
8
  export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources.js';
@@ -1,3 +1,4 @@
1
+ import type { DbAdapter, DbBatchOp } from '@gobing-ai/ts-db';
1
2
  import type { ImportOptions, JsonObject, SourceDefinition } from './types';
2
3
  interface CheckpointRow {
3
4
  readonly last_imported_line: number;
@@ -25,6 +26,65 @@ declare function writeCheckpoint(db: ImportOptions['db'], source: string, source
25
26
  declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean>;
26
27
  declare function insertRecord(db: ImportOptions['db'], targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): Promise<void>;
27
28
  declare function insertLedger(db: ImportOptions['db'], recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): Promise<void>;
29
+ /**
30
+ * Idempotent migration that rewrites `source_file` to its realpath across the checkpoint,
31
+ * ledger, and any contract table present (R4).
32
+ *
33
+ * WHY: `record_hash` is `sha256({source, sourceFile, sourceLine, splitIndex, record})`, so
34
+ * `sourceFile` is inside the hash. Normalizing its representation at discovery (R1) changes
35
+ * every future hash; pre-normalization rows carry unnormalized paths and their old hashes
36
+ * remain valid as dedupe keys. This migration rewrites the `source_file` *column* so old and
37
+ * new rows agree on path identity, and collapses duplicate checkpoint rows produced when one
38
+ * physical file was imported via both a symlinked and a real path — keeping the highest
39
+ * `last_imported_line` so an incremental resume does not re-import already-seen content.
40
+ *
41
+ * `record_hash` is intentionally NOT touched: it is path-representation dependent by
42
+ * construction, pre-migration rows are grandfathered, and recomputing it would require the
43
+ * original record payload that the ledger does not store.
44
+ *
45
+ * @param resolveRealPath - resolves a `source_file` to its canonical real path. Falls back to
46
+ * the original value when it returns null/undefined or throws. Decouples the DAO from the
47
+ * runtime `FileSystem` seam.
48
+ */
49
+ export declare function normalizeSourceFilePaths(db: ImportOptions['db'], resolveRealPath: (sourceFile: string) => string | null | undefined): Promise<void>;
50
+ /** OpenCode message row projected from its SQLite store. */
51
+ export interface OpenCodeMessageRow {
52
+ id: string;
53
+ session_id: string;
54
+ time_created: number;
55
+ data: string;
56
+ directory: string;
57
+ }
58
+ /** OpenCode message-part row projected from its SQLite store. */
59
+ export interface OpenCodePartRow {
60
+ id: string;
61
+ message_id: string;
62
+ time_created: number;
63
+ data: string;
64
+ }
65
+ /** Previously imported OpenCode ledger entry. */
66
+ export interface OpenCodeExistingEntry {
67
+ record_hash: string;
68
+ target_table: string;
69
+ }
70
+ /** Normalized OpenCode entry awaiting a history-table write. */
71
+ export interface OpenCodeQueuedEntry {
72
+ targetTable: 'history_message' | 'history_tool_call';
73
+ splitIndex: number;
74
+ record: JsonObject;
75
+ recordHash: string;
76
+ sourceFile: string;
77
+ }
78
+ /** Read one ordered page of OpenCode messages after the supplied cursor. */
79
+ export declare function readOpenCodeMessages(db: DbAdapter, lastTime: number, lastId: string, limit: number): Promise<OpenCodeMessageRow[]>;
80
+ /** Read all parts belonging to the supplied OpenCode message IDs. */
81
+ export declare function readOpenCodeParts(db: DbAdapter, messageIds: readonly string[]): Promise<OpenCodePartRow[]>;
82
+ /** Load existing OpenCode ledger entries grouped by source message ID. */
83
+ export declare function readOpenCodeExistingEntries(db: DbAdapter): Promise<Map<string, OpenCodeExistingEntry[]>>;
84
+ /** Build deletion operations for records superseded by a forced import. */
85
+ export declare function openCodeDeleteOperations(entries: readonly OpenCodeExistingEntry[]): DbBatchOp[];
86
+ /** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
87
+ export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
28
88
  export type { CheckpointRow };
29
89
  export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
30
90
  //# sourceMappingURL=jsonl-importer-dao.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
1
+ {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA+FD,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
@@ -165,4 +165,192 @@ async function insertLedger(db, recordHash, source, sourceFile, sourceLine, spli
165
165
  await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
166
166
  VALUES (?, ?, ?, ?, ?, ?, ?)`, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now));
167
167
  }
168
+ /**
169
+ * Idempotent migration that rewrites `source_file` to its realpath across the checkpoint,
170
+ * ledger, and any contract table present (R4).
171
+ *
172
+ * WHY: `record_hash` is `sha256({source, sourceFile, sourceLine, splitIndex, record})`, so
173
+ * `sourceFile` is inside the hash. Normalizing its representation at discovery (R1) changes
174
+ * every future hash; pre-normalization rows carry unnormalized paths and their old hashes
175
+ * remain valid as dedupe keys. This migration rewrites the `source_file` *column* so old and
176
+ * new rows agree on path identity, and collapses duplicate checkpoint rows produced when one
177
+ * physical file was imported via both a symlinked and a real path — keeping the highest
178
+ * `last_imported_line` so an incremental resume does not re-import already-seen content.
179
+ *
180
+ * `record_hash` is intentionally NOT touched: it is path-representation dependent by
181
+ * construction, pre-migration rows are grandfathered, and recomputing it would require the
182
+ * original record payload that the ledger does not store.
183
+ *
184
+ * @param resolveRealPath - resolves a `source_file` to its canonical real path. Falls back to
185
+ * the original value when it returns null/undefined or throws. Decouples the DAO from the
186
+ * runtime `FileSystem` seam.
187
+ */
188
+ export async function normalizeSourceFilePaths(db, resolveRealPath) {
189
+ await normalizeCheckpointPaths(db, resolveRealPath);
190
+ await normalizeColumnPaths(db, resolveRealPath, 'history_import_ledger');
191
+ for (const table of TYPED_TABLE_COLUMNS_SOURCE_FILE) {
192
+ if (await tableExists(db, table)) {
193
+ await normalizeColumnPaths(db, resolveRealPath, table);
194
+ }
195
+ }
196
+ for (const table of await listEtlTables(db)) {
197
+ await normalizeColumnPaths(db, resolveRealPath, table);
198
+ }
199
+ }
200
+ /** Typed contract tables that carry a `source_file` column (0466 forensic contract). */
201
+ const TYPED_TABLE_COLUMNS_SOURCE_FILE = ['history_message', 'history_tool_call'];
202
+ async function tableExists(db, table) {
203
+ const row = await db.queryFirst("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", table);
204
+ return row !== undefined && row !== null;
205
+ }
206
+ async function listEtlTables(db) {
207
+ const rows = await db.queryAll("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'history_etl_%'");
208
+ return rows.map((row) => row.name);
209
+ }
210
+ /**
211
+ * Normalize `source_file` in the checkpoint table, collapsing duplicate rows per
212
+ * `(source, realpath)` by keeping the highest `last_imported_line`.
213
+ */
214
+ async function normalizeCheckpointPaths(db, resolveRealPath) {
215
+ const rows = await db.queryAll('SELECT source, source_file, last_imported_line, updated_at FROM history_import_checkpoint');
216
+ // Collapse per (source, realPath) keeping the highest last_imported_line.
217
+ const collapsed = new Map();
218
+ for (const row of rows) {
219
+ const canonical = resolveWithFallback(resolveRealPath, row.source_file);
220
+ const key = `${row.source}\u0000${canonical}`;
221
+ const existing = collapsed.get(key);
222
+ if (existing === undefined || row.last_imported_line > existing.last_imported_line) {
223
+ collapsed.set(key, {
224
+ source: row.source,
225
+ source_file: canonical,
226
+ last_imported_line: row.last_imported_line,
227
+ updated_at: row.updated_at,
228
+ });
229
+ }
230
+ }
231
+ await db.exec('DELETE FROM history_import_checkpoint');
232
+ for (const row of collapsed.values()) {
233
+ await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
234
+ VALUES (?, ?, ?, ?)`, row.source, row.source_file, row.last_imported_line, row.updated_at);
235
+ }
236
+ }
237
+ /** Normalize `source_file` in a table that stores it as a plain column. */
238
+ async function normalizeColumnPaths(db, resolveRealPath, table) {
239
+ const rows = await db.queryAll(`SELECT rowid AS rowid, source_file FROM ${table} WHERE source_file IS NOT NULL`);
240
+ for (const row of rows) {
241
+ const canonical = resolveWithFallback(resolveRealPath, row.source_file);
242
+ if (canonical !== row.source_file) {
243
+ await db.run(`UPDATE ${table} SET source_file = ? WHERE rowid = ?`, canonical, row.rowid);
244
+ }
245
+ }
246
+ }
247
+ function resolveWithFallback(resolveRealPath, sourceFile) {
248
+ try {
249
+ return resolveRealPath(sourceFile) ?? sourceFile;
250
+ }
251
+ catch {
252
+ return sourceFile;
253
+ }
254
+ }
255
+ const OPENCODE_SOURCE = 'opencode';
256
+ /** Read one ordered page of OpenCode messages after the supplied cursor. */
257
+ export async function readOpenCodeMessages(db, lastTime, lastId, limit) {
258
+ return db.queryAll(`SELECT m.id, m.session_id, m.time_created, m.data, s.directory
259
+ FROM message m
260
+ JOIN session s ON s.id = m.session_id
261
+ WHERE m.time_created > ? OR (m.time_created = ? AND m.id > ?)
262
+ ORDER BY m.time_created, m.id
263
+ LIMIT ?`, lastTime, lastTime, lastId, limit);
264
+ }
265
+ /** Read all parts belonging to the supplied OpenCode message IDs. */
266
+ export async function readOpenCodeParts(db, messageIds) {
267
+ if (messageIds.length === 0)
268
+ return [];
269
+ return db.queryAll(`SELECT id, message_id, time_created, data
270
+ FROM part
271
+ WHERE message_id IN (${messageIds.map(() => '?').join(', ')})
272
+ ORDER BY time_created, id`, ...messageIds);
273
+ }
274
+ /** Load existing OpenCode ledger entries grouped by source message ID. */
275
+ export async function readOpenCodeExistingEntries(db) {
276
+ const rows = await db.queryAll('SELECT source_file, record_hash, target_table FROM history_import_ledger WHERE source = ?', OPENCODE_SOURCE);
277
+ const bySourceFile = new Map();
278
+ for (const row of rows) {
279
+ const entries = bySourceFile.get(row.source_file) ?? [];
280
+ entries.push({ record_hash: row.record_hash, target_table: row.target_table });
281
+ bySourceFile.set(row.source_file, entries);
282
+ }
283
+ return bySourceFile;
284
+ }
285
+ /** Build deletion operations for records superseded by a forced import. */
286
+ export function openCodeDeleteOperations(entries) {
287
+ return entries.flatMap((entry) => [
288
+ {
289
+ sql: `DELETE FROM ${targetTableFor(entry.target_table)} WHERE record_hash = ?`,
290
+ params: [entry.record_hash],
291
+ },
292
+ {
293
+ sql: 'DELETE FROM history_import_ledger WHERE record_hash = ?',
294
+ params: [entry.record_hash],
295
+ },
296
+ ]);
297
+ }
298
+ /** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
299
+ export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
300
+ const importedAt = timestamp(now);
301
+ const operations = [];
302
+ for (const table of ['history_message', 'history_tool_call']) {
303
+ const tableEntries = entries.filter((entry) => entry.targetTable === table);
304
+ for (let offset = 0; offset < tableEntries.length; offset += 100) {
305
+ const chunk = tableEntries.slice(offset, offset + 100);
306
+ const columns = TYPED_TABLE_COLUMNS[table] ?? [];
307
+ operations.push({
308
+ sql: `WITH input(record_hash, payload_json, imported_at) AS (
309
+ VALUES ${chunk.map(() => '(?, ?, ?)').join(', ')}
310
+ )
311
+ INSERT INTO ${table} (${columns.join(', ')})
312
+ SELECT ${columns
313
+ .map((column) => column === 'record_hash'
314
+ ? 'record_hash'
315
+ : column === 'imported_at'
316
+ ? 'imported_at'
317
+ : `json_extract(payload_json, '$.${column}')`)
318
+ .join(', ')}
319
+ FROM input
320
+ WHERE true
321
+ ON CONFLICT(record_hash) DO NOTHING`,
322
+ params: chunk.flatMap((entry) => [entry.recordHash, JSON.stringify(entry.record), importedAt]),
323
+ });
324
+ }
325
+ }
326
+ for (let offset = 0; offset < entries.length; offset += 200) {
327
+ const chunk = entries.slice(offset, offset + 200);
328
+ operations.push({
329
+ sql: `INSERT INTO history_import_ledger
330
+ (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
331
+ VALUES ${chunk.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ')}`,
332
+ params: chunk.flatMap((entry) => [
333
+ entry.recordHash,
334
+ OPENCODE_SOURCE,
335
+ entry.sourceFile,
336
+ 1,
337
+ entry.splitIndex,
338
+ entry.targetTable,
339
+ importedAt,
340
+ ]),
341
+ });
342
+ }
343
+ for (let offset = 0; offset < checkpointFiles.length; offset += 200) {
344
+ const chunk = checkpointFiles.slice(offset, offset + 200);
345
+ operations.push({
346
+ sql: `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
347
+ VALUES ${chunk.map(() => '(?, ?, ?, ?)').join(', ')}
348
+ ON CONFLICT(source, source_file) DO UPDATE SET
349
+ last_imported_line = excluded.last_imported_line,
350
+ updated_at = excluded.updated_at`,
351
+ params: chunk.flatMap((sourceFile) => [OPENCODE_SOURCE, sourceFile, 1, importedAt]),
352
+ });
353
+ }
354
+ return operations;
355
+ }
168
356
  export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
package/dist/mappers.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { z } from 'zod';
2
- import type { JsonObject, SplitEntry } from './types';
2
+ import type { JsonObject, SplitEntry, TransformContext } from './types';
3
3
  /** Map a Claude Code JSONL record into one or more ETL split entries. */
4
4
  export declare function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[];
5
5
  /** Map a Pi JSONL record into one or more ETL split entries. */
@@ -10,6 +10,8 @@ export declare function ompSplit(raw: Record<string, unknown>): readonly SplitEn
10
10
  export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
11
11
  /** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
12
12
  export declare function agySplit(raw: Record<string, unknown>): readonly SplitEntry[];
13
+ /** Map Gemini CLI message, tool, and session-state events into the forensic contract tables. */
14
+ export declare function geminiSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
13
15
  /** Map a Grok JSONL record into one or more ETL split entries. */
14
16
  export declare function grokSplit(raw: Record<string, unknown>): readonly SplitEntry[];
15
17
  /** Compute a stable field-shape key from a JSON object: sorted, lowercased, +-joined top-level keys. */
@@ -26,6 +28,8 @@ export declare const CODEX_FIELD_MAP: Record<string, string>;
26
28
  export declare const AGY_FIELD_MAP: Record<string, string>;
27
29
  /** Identity field map for the Grok mapper. */
28
30
  export declare const GROK_FIELD_MAP: Record<string, string>;
31
+ /** Identity field map for the Gemini mapper. */
32
+ export declare const GEMINI_FIELD_MAP: Record<string, string>;
29
33
  /** Passthrough Zod schema for the Claude mapper. */
30
34
  export declare const CLAUDE_SCHEMA: z.ZodObject<{}, z.core.$loose>;
31
35
  /** Passthrough Zod schema for the Pi mapper. */
@@ -38,4 +42,6 @@ export declare const CODEX_SCHEMA: z.ZodObject<{}, z.core.$loose>;
38
42
  export declare const AGY_SCHEMA: z.ZodObject<{}, z.core.$loose>;
39
43
  /** Passthrough Zod schema for the Grok mapper. */
40
44
  export declare const GROK_SCHEMA: z.ZodObject<{}, z.core.$loose>;
45
+ /** Passthrough Zod schema for the Gemini mapper. */
46
+ export declare const GEMINI_SCHEMA: z.ZodObject<{}, z.core.$loose>;
41
47
  //# sourceMappingURL=mappers.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AA2FtD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAsE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4F5E;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAgH5E;AAyJD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAoR7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAQlG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC"}
1
+ {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AA2FxE,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAsE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4F5E;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAgH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAgF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAyS7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
package/dist/mappers.js CHANGED
@@ -366,9 +366,9 @@ export function codexSplit(raw) {
366
366
  record: {
367
367
  session_id: sessionId,
368
368
  seq,
369
- role: 'unknown',
369
+ role: 'meta',
370
370
  record_type: 'short_format',
371
- disposition: 'unknown',
371
+ disposition: 'meta',
372
372
  ts,
373
373
  provenance: 'ambient',
374
374
  },
@@ -552,6 +552,109 @@ export function agySplit(raw) {
552
552
  return entries;
553
553
  }
554
554
  // ---------------------------------------------------------------------------
555
+ // Gemini mapper
556
+ // ---------------------------------------------------------------------------
557
+ /** Map Gemini CLI message, tool, and session-state events into the forensic contract tables. */
558
+ export function geminiSplit(raw, context) {
559
+ const recordType = String(raw.type ?? (raw.kind !== undefined ? 'session' : raw.$set !== undefined ? 'state' : ''));
560
+ const state = o(raw.$set);
561
+ const sessionId = sessionIdFromContext(context, raw);
562
+ const seq = context?.sourceLine ?? 0;
563
+ const ts = s(raw.timestamp, raw.startTime, raw.lastUpdated, state.lastUpdated) ?? new Date(0).toISOString();
564
+ if (recordType === 'session' || recordType === 'state') {
565
+ return [
566
+ {
567
+ targetTable: 'history_message',
568
+ record: {
569
+ session_id: sessionId,
570
+ seq,
571
+ role: 'meta',
572
+ record_type: recordType,
573
+ disposition: 'meta',
574
+ ts,
575
+ content_text: s(state.summary) ?? null,
576
+ provenance: detectProvenance(context?.sourceFile),
577
+ },
578
+ },
579
+ ];
580
+ }
581
+ const role = recordType === 'gemini' ? 'assistant' : recordType === 'user' ? 'user' : 'system';
582
+ const tokens = o(raw.tokens);
583
+ const content = geminiContent(raw);
584
+ const entries = [
585
+ {
586
+ targetTable: 'history_message',
587
+ record: {
588
+ session_id: sessionId,
589
+ seq,
590
+ role,
591
+ record_type: recordType,
592
+ disposition: 'keep',
593
+ ts,
594
+ duration_ms: null,
595
+ model: s(raw.model) ?? null,
596
+ input_tokens: numberOrNull(tokens.input),
597
+ output_tokens: numberOrNull(tokens.output),
598
+ cache_read_tokens: numberOrNull(tokens.cached),
599
+ cache_write_tokens: null,
600
+ cost_usd: null,
601
+ content_text: content,
602
+ cwd: null,
603
+ provenance: detectProvenance(context?.sourceFile),
604
+ },
605
+ },
606
+ ];
607
+ if (Array.isArray(raw.toolCalls)) {
608
+ for (const value of raw.toolCalls) {
609
+ if (value === null || typeof value !== 'object' || Array.isArray(value))
610
+ continue;
611
+ const tool = value;
612
+ const status = s(tool.status) ?? 'unknown';
613
+ const timestamp = s(tool.timestamp);
614
+ const result = tool.result ?? tool.resultDisplay;
615
+ entries.push({
616
+ targetTable: 'history_tool_call',
617
+ record: {
618
+ _messageSplitIndex: 0,
619
+ session_id: sessionId,
620
+ seq,
621
+ tool_name: s(tool.name, tool.displayName) ?? 'unknown',
622
+ args_digest: argsDigest(tool.args),
623
+ status,
624
+ started_at: timestamp,
625
+ completed_at: timestamp,
626
+ duration_ms: null,
627
+ result_bytes: result === undefined ? null : new TextEncoder().encode(JSON.stringify(result)).byteLength,
628
+ error_text: status === 'error' ? (s(tool.resultDisplay, tool.result) ?? null) : null,
629
+ },
630
+ });
631
+ }
632
+ }
633
+ return entries;
634
+ }
635
+ function sessionIdFromContext(context, raw) {
636
+ if (context !== undefined) {
637
+ const file = context.sourceFile.split('/').at(-1);
638
+ if (file !== undefined)
639
+ return file.replace(/\.jsonl$/, '');
640
+ }
641
+ return s(raw.sessionId, raw.id) ?? 'unknown';
642
+ }
643
+ function geminiContent(raw) {
644
+ const content = extractContentText(raw.content);
645
+ const thoughts = Array.isArray(raw.thoughts)
646
+ ? raw.thoughts
647
+ .map((value) => (value !== null && typeof value === 'object' ? s(o(value).description) : undefined))
648
+ .filter((value) => value !== undefined)
649
+ .join('\n')
650
+ : '';
651
+ return ([content, thoughts].filter((value) => value !== undefined && value.length > 0).join('\n') ||
652
+ null);
653
+ }
654
+ function numberOrNull(value) {
655
+ return typeof value === 'number' && Number.isFinite(value) ? value : null;
656
+ }
657
+ // ---------------------------------------------------------------------------
555
658
  // Grok mapper
556
659
  // ---------------------------------------------------------------------------
557
660
  /**
@@ -566,13 +669,29 @@ function normalizeGrokRecord(raw) {
566
669
  const update = params?.update;
567
670
  const isUpdatesShape = typeof raw.method === 'string' && params !== undefined && update !== undefined;
568
671
  const body = isUpdatesShape ? update : raw;
569
- const recordType = String((isUpdatesShape ? update?.sessionUpdate : raw.type) ?? raw.sessionUpdate ?? '');
570
- const sessionId = s(params?.sessionId, raw.session_id, raw.sessionId, raw.params?.sessionId) ?? 'unknown';
571
- const ts = normalizeTs(raw.ts ?? raw.timestamp ?? body.ts ?? body.timestamp);
672
+ const recordType = String((isUpdatesShape ? update?.sessionUpdate : raw.type) ??
673
+ raw.sessionUpdate ??
674
+ raw.eventType ??
675
+ (raw.file_snapshots !== undefined || raw.after_snapshots !== undefined
676
+ ? 'file_snapshot'
677
+ : raw.prompt !== undefined
678
+ ? 'user_message'
679
+ : raw.answer !== undefined || raw.question !== undefined
680
+ ? 'assistant'
681
+ : ''));
682
+ const sessionId = s(params?.sessionId, raw.session_id, raw.sessionId, raw.parentSessionId, raw.btwSessionId, raw.params?.sessionId) ?? 'unknown';
683
+ const ts = normalizeTs(raw.ts ?? raw.timestamp ?? raw.created_at ?? raw.askedAt ?? body.ts ?? body.timestamp);
572
684
  const seq = typeof raw.seq === 'number' ? raw.seq : typeof body.seq === 'number' ? body.seq : 0;
573
685
  const meta = (body._meta ?? params?._meta ?? raw._meta);
574
686
  const model = s(meta?.modelId, body.model, raw.model);
575
- const contentText = extractGrokContent(body.content ?? body.text ?? raw.content ?? raw.text);
687
+ const contentText = extractGrokContent(body.content ??
688
+ body.text ??
689
+ raw.content ??
690
+ raw.text ??
691
+ raw.prompt ??
692
+ (raw.answer !== undefined || raw.question !== undefined
693
+ ? [raw.question, raw.answer].filter((value) => typeof value === 'string').join('\n')
694
+ : undefined));
576
695
  const usage = (body.usage ?? raw.usage);
577
696
  const toolMeta = meta?.['x.ai/tool'];
578
697
  const toolName = s(body.title, body.tool_name, body.name, toolMeta?.name, toolMeta?.label, raw.tool_name, raw.name) ?? '';
@@ -756,6 +875,27 @@ export function grokSplit(raw) {
756
875
  return entries;
757
876
  }
758
877
  if (recordType === 'tool_completed') {
878
+ entries.push({
879
+ targetTable: 'history_message',
880
+ record: {
881
+ session_id: sessionId,
882
+ seq,
883
+ role: 'assistant',
884
+ record_type: recordType,
885
+ disposition: 'keep',
886
+ ts,
887
+ duration_ms: n.durationMs ?? null,
888
+ model: n.model ?? null,
889
+ input_tokens: null,
890
+ output_tokens: null,
891
+ cache_read_tokens: null,
892
+ cache_write_tokens: null,
893
+ cost_usd: null,
894
+ content_text: n.contentText,
895
+ cwd: null,
896
+ provenance: 'ambient',
897
+ },
898
+ });
759
899
  entries.push({
760
900
  targetTable: 'history_tool_call',
761
901
  record: {
@@ -1017,6 +1157,8 @@ export const CODEX_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_
1017
1157
  export const AGY_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_CALL_MAPPER_KEYS));
1018
1158
  /** Identity field map for the Grok mapper. */
1019
1159
  export const GROK_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_CALL_MAPPER_KEYS));
1160
+ /** Identity field map for the Gemini mapper. */
1161
+ export const GEMINI_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_CALL_MAPPER_KEYS));
1020
1162
  // ---------------------------------------------------------------------------
1021
1163
  // Zod schemas for the six source mappers
1022
1164
  // ---------------------------------------------------------------------------
@@ -1033,3 +1175,5 @@ export const CODEX_SCHEMA = passthroughSchema;
1033
1175
  export const AGY_SCHEMA = passthroughSchema;
1034
1176
  /** Passthrough Zod schema for the Grok mapper. */
1035
1177
  export const GROK_SCHEMA = passthroughSchema;
1178
+ /** Passthrough Zod schema for the Gemini mapper. */
1179
+ export const GEMINI_SCHEMA = passthroughSchema;
@@ -0,0 +1,17 @@
1
+ import { type DbAdapter } from '@gobing-ai/ts-db';
2
+ import { type FileSystem, type RuntimePaths } from '@gobing-ai/ts-runtime';
3
+ import type { ImportMode, ImportResult, RedactionRule } from './types';
4
+ /** Dependencies and mode controls for importing OpenCode's SQLite history. */
5
+ export interface OpenCodeImportOptions {
6
+ db: DbAdapter;
7
+ sourceDatabase?: string;
8
+ fileSystem?: FileSystem;
9
+ paths?: RuntimePaths;
10
+ mode?: ImportMode;
11
+ dryRun?: boolean;
12
+ redactionRules?: readonly RedactionRule[];
13
+ now?: () => Date;
14
+ }
15
+ /** Import OpenCode's current SQLite history store into the forensic contract tables. */
16
+ export declare function runOpenCodeImport(options: OpenCodeImportOptions): Promise<ImportResult>;
17
+ //# sourceMappingURL=opencode-importer.d.ts.map