@gobing-ai/ts-llm-jsonl-importer 0.4.19 → 0.4.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA8GrH"}
1
+ {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA2IrH"}
package/dist/importer.js CHANGED
@@ -27,6 +27,7 @@ export async function runJsonlImport(source, options) {
27
27
  let processedLines = 0;
28
28
  let importedRecords = 0;
29
29
  let skippedDuplicates = 0;
30
+ let unknownRecords = 0;
30
31
  let checkpointUpdates = 0;
31
32
  for (const file of files) {
32
33
  const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
@@ -42,6 +43,7 @@ export async function runJsonlImport(source, options) {
42
43
  continue;
43
44
  const splitRecords = splitRawRecord(definition, raw);
44
45
  let lineSucceeded = false;
46
+ const prepared = [];
45
47
  for (let splitIndex = 0; splitIndex < splitRecords.length; splitIndex += 1) {
46
48
  const split = splitRecords[splitIndex];
47
49
  if (split === undefined)
@@ -74,8 +76,26 @@ export async function runJsonlImport(source, options) {
74
76
  skippedDuplicates += 1;
75
77
  continue;
76
78
  }
79
+ prepared.push({ split, splitIndex, normalized: redacted, recordHash });
80
+ }
81
+ // Second pass: resolve _messageSplitIndex → message_hash, then insert.
82
+ const recordHashBySplitIndex = new Map(prepared.map((e) => [e.splitIndex, e.recordHash]));
83
+ for (const entry of prepared) {
84
+ const { split, splitIndex, normalized, recordHash } = entry;
85
+ if (normalized.disposition === 'unknown') {
86
+ unknownRecords += 1;
87
+ }
88
+ // Resolve _messageSplitIndex if present (tool call → parent message linkage).
89
+ const msgSplitIdx = normalized._messageSplitIndex;
90
+ delete normalized._messageSplitIndex;
91
+ if (msgSplitIdx !== undefined && typeof msgSplitIdx === 'number') {
92
+ const msgHash = recordHashBySplitIndex.get(msgSplitIdx);
93
+ if (msgHash !== undefined) {
94
+ normalized.message_hash = msgHash;
95
+ }
96
+ }
77
97
  if (!options.dryRun) {
78
- await insertRecord(options.db, split.targetTable, recordHash, file, lineNumber, splitIndex, redacted, options.now);
98
+ await insertRecord(options.db, split.targetTable, recordHash, file, lineNumber, splitIndex, normalized, options.now);
79
99
  await insertLedger(options.db, recordHash, resolvedSource, file, lineNumber, splitIndex, split.targetTable, options.now);
80
100
  }
81
101
  importedRecords += 1;
@@ -93,6 +113,7 @@ export async function runJsonlImport(source, options) {
93
113
  processedLines,
94
114
  importedRecords,
95
115
  skippedDuplicates,
116
+ unknownRecords,
96
117
  parseErrors,
97
118
  validationErrors,
98
119
  checkpointUpdates,
@@ -119,10 +140,16 @@ function splitRawRecord(definition, raw) {
119
140
  }
120
141
  if (definition.splitConfig.mode === 'custom') {
121
142
  const config = definition.splitConfig;
122
- return config.split(raw).map((entry) => ({
123
- targetTable: targetTableFor(config.targetTable ?? definition.targetTable),
124
- raw: entry,
125
- }));
143
+ const configTable = config.targetTable !== undefined ? targetTableFor(config.targetTable) : undefined;
144
+ return config.split(raw).map((entry) => {
145
+ // Normalize: SplitEntry has a `record` property, bare object is the record itself.
146
+ const hasTargetTable = 'targetTable' in entry && typeof entry.targetTable === 'string';
147
+ const record = hasTargetTable ? entry.record : entry;
148
+ const entryTable = hasTargetTable ? entry.targetTable : undefined;
149
+ // Resolution order: entry → splitConfig → definition
150
+ const table = entryTable ?? configTable ?? targetTable;
151
+ return { targetTable: targetTableFor(table), raw: record };
152
+ });
126
153
  }
127
154
  const config = definition.splitConfig;
128
155
  const nested = raw[config.field];
package/dist/index.d.ts CHANGED
@@ -5,5 +5,5 @@ export { applyHistoryImportSchema } from './jsonl-importer-dao';
5
5
  export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
6
6
  export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
7
7
  export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
8
- export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, RedactionRule, SourceDefinition, SplitConfig, TransformContext, } from './types';
8
+ export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, RedactionRule, SourceDefinition, SplitConfig, SplitEntry, TransformContext, } from './types';
9
9
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
@@ -11,12 +11,12 @@ export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promi
11
11
  * Ensure the ETL table(s) for a source definition exist.
12
12
  *
13
13
  * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
14
- * `history_etl_*` tables. Custom source definitions (and built-in definitions
15
- * with a `splitConfig.targetTable` override) need their target table(s) created
16
- * on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
17
- * {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
18
- * interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
19
- * built-in tables that the static schema already created.
14
+ * `history_etl_*` tables and the typed contract tables. Custom source definitions
15
+ * (and built-in definitions with a `splitConfig.targetTable` override) need their
16
+ * target table(s) created on demand. The table name is already gated by
17
+ * {@link VALID_TABLE_NAME} in {@link validateSourceDefinition} / {@link targetTableFor},
18
+ * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this
19
+ * idempotent for built-in tables that the static schema already created.
20
20
  */
21
21
  declare function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void>;
22
22
  declare function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number>;
@@ -1 +1 @@
1
- {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAED,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAQtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
1
+ {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
@@ -1,6 +1,55 @@
1
1
  import { HistoryImportError } from './errors.js';
2
2
  import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
3
3
  import { VALID_TABLE_NAME } from './sources.js';
4
+ /** Column allowlist per typed contract table; order is the INSERT column order. */
5
+ const TYPED_TABLE_COLUMNS = {
6
+ history_message: [
7
+ 'record_hash',
8
+ 'source',
9
+ 'source_file',
10
+ 'source_line',
11
+ 'session_id',
12
+ 'seq',
13
+ 'turn_index',
14
+ 'role',
15
+ 'record_type',
16
+ 'disposition',
17
+ 'ts',
18
+ 'duration_ms',
19
+ 'model',
20
+ 'input_tokens',
21
+ 'output_tokens',
22
+ 'cache_read_tokens',
23
+ 'cache_write_tokens',
24
+ 'cost_usd',
25
+ 'content_text',
26
+ 'cwd',
27
+ 'provenance',
28
+ 'run_id',
29
+ 'task_wbs',
30
+ 'imported_at',
31
+ ],
32
+ history_tool_call: [
33
+ 'record_hash',
34
+ 'message_hash',
35
+ 'source',
36
+ 'source_file',
37
+ 'source_line',
38
+ 'session_id',
39
+ 'seq',
40
+ 'tool_name',
41
+ 'args_digest',
42
+ 'status',
43
+ 'started_at',
44
+ 'completed_at',
45
+ 'duration_ms',
46
+ 'result_bytes',
47
+ 'error_text',
48
+ 'imported_at',
49
+ ],
50
+ };
51
+ /** Keys that a typed table mapper may produce that are not columns. */
52
+ const TYPED_IGNORED_KEYS = new Set(['_meta', 'split_index']);
4
53
  function targetTableFor(table) {
5
54
  if (!VALID_TABLE_NAME.test(table)) {
6
55
  throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
@@ -33,12 +82,12 @@ export async function applyHistoryImportSchema(db) {
33
82
  * Ensure the ETL table(s) for a source definition exist.
34
83
  *
35
84
  * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
36
- * `history_etl_*` tables. Custom source definitions (and built-in definitions
37
- * with a `splitConfig.targetTable` override) need their target table(s) created
38
- * on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
39
- * {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
40
- * interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
41
- * built-in tables that the static schema already created.
85
+ * `history_etl_*` tables and the typed contract tables. Custom source definitions
86
+ * (and built-in definitions with a `splitConfig.targetTable` override) need their
87
+ * target table(s) created on demand. The table name is already gated by
88
+ * {@link VALID_TABLE_NAME} in {@link validateSourceDefinition} / {@link targetTableFor},
89
+ * so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this
90
+ * idempotent for built-in tables that the static schema already created.
42
91
  */
43
92
  async function ensureTargetTables(db, definition) {
44
93
  const tables = new Set([targetTableFor(definition.targetTable)]);
@@ -46,6 +95,9 @@ async function ensureTargetTables(db, definition) {
46
95
  tables.add(targetTableFor(definition.splitConfig.targetTable));
47
96
  }
48
97
  for (const table of tables) {
98
+ // Typed tables are created by the static schema; use ETL DDL for custom tables.
99
+ if (TYPED_TABLE_COLUMNS[table] !== undefined)
100
+ continue;
49
101
  await db.exec(ETL_TABLE_DDL(table));
50
102
  }
51
103
  }
@@ -69,11 +121,45 @@ async function ledgerExists(db, recordHash) {
69
121
  const row = await db.queryFirst('SELECT record_hash FROM history_import_ledger WHERE record_hash = ?', recordHash);
70
122
  return row !== undefined && row !== null;
71
123
  }
124
+ const _ensuredTables = new Set();
72
125
  async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
73
126
  const table = targetTableFor(targetTable);
74
- await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
75
- VALUES (?, ?, ?, ?, ?, ?)
76
- ON CONFLICT(record_hash) DO NOTHING`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
127
+ const typedColumns = TYPED_TABLE_COLUMNS[table];
128
+ if (typedColumns !== undefined) {
129
+ // Typed insert path: map each column from the payload, assert no unknown keys.
130
+ const unknownKeys = Object.keys(payload).filter((k) => !typedColumns.includes(k) && !TYPED_IGNORED_KEYS.has(k));
131
+ if (unknownKeys.length > 0) {
132
+ throw new HistoryImportError(`Typed table "${table}" has unknown columns: ${unknownKeys.join(', ')}. ` +
133
+ `Expected one of: ${typedColumns.join(', ')}.`, { table, unknownKeys, knownColumns: typedColumns });
134
+ }
135
+ // record_hash and imported_at come from the function parameters, not the payload.
136
+ const ts = timestamp(now);
137
+ const values = typedColumns.map((col) => {
138
+ if (col === 'record_hash')
139
+ return recordHash;
140
+ if (col === 'imported_at')
141
+ return ts;
142
+ const v = payload[col];
143
+ return v === undefined ? null : v;
144
+ });
145
+ if (!_ensuredTables.has(table)) {
146
+ // Typed tables are created by the static schema in applyHistoryImportSchema.
147
+ _ensuredTables.add(table);
148
+ }
149
+ await db.run(`INSERT INTO ${table} (${typedColumns.join(', ')})
150
+ VALUES (${typedColumns.map(() => '?').join(', ')})
151
+ ON CONFLICT(record_hash) DO NOTHING`, ...values);
152
+ }
153
+ else {
154
+ // Generic ETL insert path: store as payload_json blob.
155
+ if (!_ensuredTables.has(table)) {
156
+ await db.exec(ETL_TABLE_DDL(table));
157
+ _ensuredTables.add(table);
158
+ }
159
+ await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
160
+ VALUES (?, ?, ?, ?, ?, ?)
161
+ ON CONFLICT(record_hash) DO NOTHING`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
162
+ }
77
163
  }
78
164
  async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
79
165
  await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
@@ -0,0 +1,41 @@
1
+ import { z } from 'zod';
2
+ import type { JsonObject, SplitEntry } from './types';
3
+ /** Map a Claude Code JSONL record into one or more ETL split entries. */
4
+ export declare function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[];
5
+ /** Map a Pi JSONL record into one or more ETL split entries. */
6
+ export declare function piSplit(raw: unknown): readonly SplitEntry[];
7
+ /** Map an OMP JSONL record into one or more ETL split entries. */
8
+ export declare function ompSplit(raw: Record<string, unknown>): readonly SplitEntry[];
9
+ /** Map a Codex JSONL record into one or more ETL split entries. */
10
+ export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
11
+ /** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
12
+ export declare function agySplit(raw: Record<string, unknown>): readonly SplitEntry[];
13
+ /** Map a Grok JSONL record into one or more ETL split entries. */
14
+ export declare function grokSplit(raw: Record<string, unknown>): readonly SplitEntry[];
15
+ /** Compute a stable field-shape key from a JSON object: sorted, lowercased, +-joined top-level keys. */
16
+ export declare function stableFieldShape(raw: JsonObject): string;
17
+ /** Identity field map for the Claude mapper. */
18
+ export declare const CLAUDE_FIELD_MAP: Record<string, string>;
19
+ /** Identity field map for the Pi mapper. */
20
+ export declare const PI_FIELD_MAP: Record<string, string>;
21
+ /** Identity field map for the OMP mapper. */
22
+ export declare const OMP_FIELD_MAP: Record<string, string>;
23
+ /** Identity field map for the Codex mapper. */
24
+ export declare const CODEX_FIELD_MAP: Record<string, string>;
25
+ /** Identity field map for the AGY mapper. */
26
+ export declare const AGY_FIELD_MAP: Record<string, string>;
27
+ /** Identity field map for the Grok mapper. */
28
+ export declare const GROK_FIELD_MAP: Record<string, string>;
29
+ /** Passthrough Zod schema for the Claude mapper. */
30
+ export declare const CLAUDE_SCHEMA: z.ZodObject<{}, z.core.$loose>;
31
+ /** Passthrough Zod schema for the Pi mapper. */
32
+ export declare const PI_SCHEMA: z.ZodObject<{}, z.core.$loose>;
33
+ /** Passthrough Zod schema for the OMP mapper. */
34
+ export declare const OMP_SCHEMA: z.ZodObject<{}, z.core.$loose>;
35
+ /** Passthrough Zod schema for the Codex mapper. */
36
+ export declare const CODEX_SCHEMA: z.ZodObject<{}, z.core.$loose>;
37
+ /** Passthrough Zod schema for the AGY mapper. */
38
+ export declare const AGY_SCHEMA: z.ZodObject<{}, z.core.$loose>;
39
+ /** Passthrough Zod schema for the Grok mapper. */
40
+ export declare const GROK_SCHEMA: z.ZodObject<{}, z.core.$loose>;
41
+ //# sourceMappingURL=mappers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AA2FtD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAsE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4F5E;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAgH5E;AAyJD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAoR7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAQlG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC"}