@gobing-ai/ts-llm-jsonl-importer 0.4.19 → 0.4.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +32 -5
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.d.ts +6 -6
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +95 -9
- package/dist/mappers.d.ts +41 -0
- package/dist/mappers.d.ts.map +1 -0
- package/dist/mappers.js +1035 -0
- package/dist/schema-sql.d.ts.map +1 -1
- package/dist/schema-sql.js +57 -0
- package/dist/sources.d.ts.map +1 -1
- package/dist/sources.js +23 -5
- package/dist/types.d.ts +8 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/importer.ts +41 -5
- package/src/index.ts +1 -0
- package/src/jsonl-importer-dao.ts +108 -17
- package/src/mappers.ts +1130 -0
- package/src/schema-sql.ts +57 -0
- package/src/sources.ts +92 -5
- package/src/types.ts +19 -2
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA2IrH"}
|
package/dist/importer.js
CHANGED
|
@@ -27,6 +27,7 @@ export async function runJsonlImport(source, options) {
|
|
|
27
27
|
let processedLines = 0;
|
|
28
28
|
let importedRecords = 0;
|
|
29
29
|
let skippedDuplicates = 0;
|
|
30
|
+
let unknownRecords = 0;
|
|
30
31
|
let checkpointUpdates = 0;
|
|
31
32
|
for (const file of files) {
|
|
32
33
|
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
|
|
@@ -42,6 +43,7 @@ export async function runJsonlImport(source, options) {
|
|
|
42
43
|
continue;
|
|
43
44
|
const splitRecords = splitRawRecord(definition, raw);
|
|
44
45
|
let lineSucceeded = false;
|
|
46
|
+
const prepared = [];
|
|
45
47
|
for (let splitIndex = 0; splitIndex < splitRecords.length; splitIndex += 1) {
|
|
46
48
|
const split = splitRecords[splitIndex];
|
|
47
49
|
if (split === undefined)
|
|
@@ -74,8 +76,26 @@ export async function runJsonlImport(source, options) {
|
|
|
74
76
|
skippedDuplicates += 1;
|
|
75
77
|
continue;
|
|
76
78
|
}
|
|
79
|
+
prepared.push({ split, splitIndex, normalized: redacted, recordHash });
|
|
80
|
+
}
|
|
81
|
+
// Second pass: resolve _messageSplitIndex → message_hash, then insert.
|
|
82
|
+
const recordHashBySplitIndex = new Map(prepared.map((e) => [e.splitIndex, e.recordHash]));
|
|
83
|
+
for (const entry of prepared) {
|
|
84
|
+
const { split, splitIndex, normalized, recordHash } = entry;
|
|
85
|
+
if (normalized.disposition === 'unknown') {
|
|
86
|
+
unknownRecords += 1;
|
|
87
|
+
}
|
|
88
|
+
// Resolve _messageSplitIndex if present (tool call → parent message linkage).
|
|
89
|
+
const msgSplitIdx = normalized._messageSplitIndex;
|
|
90
|
+
delete normalized._messageSplitIndex;
|
|
91
|
+
if (msgSplitIdx !== undefined && typeof msgSplitIdx === 'number') {
|
|
92
|
+
const msgHash = recordHashBySplitIndex.get(msgSplitIdx);
|
|
93
|
+
if (msgHash !== undefined) {
|
|
94
|
+
normalized.message_hash = msgHash;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
77
97
|
if (!options.dryRun) {
|
|
78
|
-
await insertRecord(options.db, split.targetTable, recordHash, file, lineNumber, splitIndex,
|
|
98
|
+
await insertRecord(options.db, split.targetTable, recordHash, file, lineNumber, splitIndex, normalized, options.now);
|
|
79
99
|
await insertLedger(options.db, recordHash, resolvedSource, file, lineNumber, splitIndex, split.targetTable, options.now);
|
|
80
100
|
}
|
|
81
101
|
importedRecords += 1;
|
|
@@ -93,6 +113,7 @@ export async function runJsonlImport(source, options) {
|
|
|
93
113
|
processedLines,
|
|
94
114
|
importedRecords,
|
|
95
115
|
skippedDuplicates,
|
|
116
|
+
unknownRecords,
|
|
96
117
|
parseErrors,
|
|
97
118
|
validationErrors,
|
|
98
119
|
checkpointUpdates,
|
|
@@ -119,10 +140,16 @@ function splitRawRecord(definition, raw) {
|
|
|
119
140
|
}
|
|
120
141
|
if (definition.splitConfig.mode === 'custom') {
|
|
121
142
|
const config = definition.splitConfig;
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
143
|
+
const configTable = config.targetTable !== undefined ? targetTableFor(config.targetTable) : undefined;
|
|
144
|
+
return config.split(raw).map((entry) => {
|
|
145
|
+
// Normalize: SplitEntry has a `record` property, bare object is the record itself.
|
|
146
|
+
const hasTargetTable = 'targetTable' in entry && typeof entry.targetTable === 'string';
|
|
147
|
+
const record = hasTargetTable ? entry.record : entry;
|
|
148
|
+
const entryTable = hasTargetTable ? entry.targetTable : undefined;
|
|
149
|
+
// Resolution order: entry → splitConfig → definition
|
|
150
|
+
const table = entryTable ?? configTable ?? targetTable;
|
|
151
|
+
return { targetTable: targetTableFor(table), raw: record };
|
|
152
|
+
});
|
|
126
153
|
}
|
|
127
154
|
const config = definition.splitConfig;
|
|
128
155
|
const nested = raw[config.field];
|
package/dist/index.d.ts
CHANGED
|
@@ -5,5 +5,5 @@ export { applyHistoryImportSchema } from './jsonl-importer-dao';
|
|
|
5
5
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
6
6
|
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
7
7
|
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
|
|
8
|
-
export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, RedactionRule, SourceDefinition, SplitConfig, TransformContext, } from './types';
|
|
8
|
+
export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, RedactionRule, SourceDefinition, SplitConfig, SplitEntry, TransformContext, } from './types';
|
|
9
9
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
|
@@ -11,12 +11,12 @@ export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promi
|
|
|
11
11
|
* Ensure the ETL table(s) for a source definition exist.
|
|
12
12
|
*
|
|
13
13
|
* WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
|
|
14
|
-
* `history_etl_*` tables. Custom source definitions
|
|
15
|
-
* with a `splitConfig.targetTable` override) need their
|
|
16
|
-
* on demand. The table name is already gated by
|
|
17
|
-
* {@link validateSourceDefinition} / {@link targetTableFor},
|
|
18
|
-
* interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this
|
|
19
|
-
* built-in tables that the static schema already created.
|
|
14
|
+
* `history_etl_*` tables and the typed contract tables. Custom source definitions
|
|
15
|
+
* (and built-in definitions with a `splitConfig.targetTable` override) need their
|
|
16
|
+
* target table(s) created on demand. The table name is already gated by
|
|
17
|
+
* {@link VALID_TABLE_NAME} in {@link validateSourceDefinition} / {@link targetTableFor},
|
|
18
|
+
* so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this
|
|
19
|
+
* idempotent for built-in tables that the static schema already created.
|
|
20
20
|
*/
|
|
21
21
|
declare function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void>;
|
|
22
22
|
declare function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
|
|
@@ -1,6 +1,55 @@
|
|
|
1
1
|
import { HistoryImportError } from './errors.js';
|
|
2
2
|
import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
|
|
3
3
|
import { VALID_TABLE_NAME } from './sources.js';
|
|
4
|
+
/** Column allowlist per typed contract table; order is the INSERT column order. */
|
|
5
|
+
const TYPED_TABLE_COLUMNS = {
|
|
6
|
+
history_message: [
|
|
7
|
+
'record_hash',
|
|
8
|
+
'source',
|
|
9
|
+
'source_file',
|
|
10
|
+
'source_line',
|
|
11
|
+
'session_id',
|
|
12
|
+
'seq',
|
|
13
|
+
'turn_index',
|
|
14
|
+
'role',
|
|
15
|
+
'record_type',
|
|
16
|
+
'disposition',
|
|
17
|
+
'ts',
|
|
18
|
+
'duration_ms',
|
|
19
|
+
'model',
|
|
20
|
+
'input_tokens',
|
|
21
|
+
'output_tokens',
|
|
22
|
+
'cache_read_tokens',
|
|
23
|
+
'cache_write_tokens',
|
|
24
|
+
'cost_usd',
|
|
25
|
+
'content_text',
|
|
26
|
+
'cwd',
|
|
27
|
+
'provenance',
|
|
28
|
+
'run_id',
|
|
29
|
+
'task_wbs',
|
|
30
|
+
'imported_at',
|
|
31
|
+
],
|
|
32
|
+
history_tool_call: [
|
|
33
|
+
'record_hash',
|
|
34
|
+
'message_hash',
|
|
35
|
+
'source',
|
|
36
|
+
'source_file',
|
|
37
|
+
'source_line',
|
|
38
|
+
'session_id',
|
|
39
|
+
'seq',
|
|
40
|
+
'tool_name',
|
|
41
|
+
'args_digest',
|
|
42
|
+
'status',
|
|
43
|
+
'started_at',
|
|
44
|
+
'completed_at',
|
|
45
|
+
'duration_ms',
|
|
46
|
+
'result_bytes',
|
|
47
|
+
'error_text',
|
|
48
|
+
'imported_at',
|
|
49
|
+
],
|
|
50
|
+
};
|
|
51
|
+
/** Keys that a typed table mapper may produce that are not columns. */
|
|
52
|
+
const TYPED_IGNORED_KEYS = new Set(['_meta', 'split_index']);
|
|
4
53
|
function targetTableFor(table) {
|
|
5
54
|
if (!VALID_TABLE_NAME.test(table)) {
|
|
6
55
|
throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
|
|
@@ -33,12 +82,12 @@ export async function applyHistoryImportSchema(db) {
|
|
|
33
82
|
* Ensure the ETL table(s) for a source definition exist.
|
|
34
83
|
*
|
|
35
84
|
* WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
|
|
36
|
-
* `history_etl_*` tables. Custom source definitions
|
|
37
|
-
* with a `splitConfig.targetTable` override) need their
|
|
38
|
-
* on demand. The table name is already gated by
|
|
39
|
-
* {@link validateSourceDefinition} / {@link targetTableFor},
|
|
40
|
-
* interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this
|
|
41
|
-
* built-in tables that the static schema already created.
|
|
85
|
+
* `history_etl_*` tables and the typed contract tables. Custom source definitions
|
|
86
|
+
* (and built-in definitions with a `splitConfig.targetTable` override) need their
|
|
87
|
+
* target table(s) created on demand. The table name is already gated by
|
|
88
|
+
* {@link VALID_TABLE_NAME} in {@link validateSourceDefinition} / {@link targetTableFor},
|
|
89
|
+
* so it is safe to interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this
|
|
90
|
+
* idempotent for built-in tables that the static schema already created.
|
|
42
91
|
*/
|
|
43
92
|
async function ensureTargetTables(db, definition) {
|
|
44
93
|
const tables = new Set([targetTableFor(definition.targetTable)]);
|
|
@@ -46,6 +95,9 @@ async function ensureTargetTables(db, definition) {
|
|
|
46
95
|
tables.add(targetTableFor(definition.splitConfig.targetTable));
|
|
47
96
|
}
|
|
48
97
|
for (const table of tables) {
|
|
98
|
+
// Typed tables are created by the static schema; use ETL DDL for custom tables.
|
|
99
|
+
if (TYPED_TABLE_COLUMNS[table] !== undefined)
|
|
100
|
+
continue;
|
|
49
101
|
await db.exec(ETL_TABLE_DDL(table));
|
|
50
102
|
}
|
|
51
103
|
}
|
|
@@ -69,11 +121,45 @@ async function ledgerExists(db, recordHash) {
|
|
|
69
121
|
const row = await db.queryFirst('SELECT record_hash FROM history_import_ledger WHERE record_hash = ?', recordHash);
|
|
70
122
|
return row !== undefined && row !== null;
|
|
71
123
|
}
|
|
124
|
+
const _ensuredTables = new Set();
|
|
72
125
|
async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
|
|
73
126
|
const table = targetTableFor(targetTable);
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
127
|
+
const typedColumns = TYPED_TABLE_COLUMNS[table];
|
|
128
|
+
if (typedColumns !== undefined) {
|
|
129
|
+
// Typed insert path: map each column from the payload, assert no unknown keys.
|
|
130
|
+
const unknownKeys = Object.keys(payload).filter((k) => !typedColumns.includes(k) && !TYPED_IGNORED_KEYS.has(k));
|
|
131
|
+
if (unknownKeys.length > 0) {
|
|
132
|
+
throw new HistoryImportError(`Typed table "${table}" has unknown columns: ${unknownKeys.join(', ')}. ` +
|
|
133
|
+
`Expected one of: ${typedColumns.join(', ')}.`, { table, unknownKeys, knownColumns: typedColumns });
|
|
134
|
+
}
|
|
135
|
+
// record_hash and imported_at come from the function parameters, not the payload.
|
|
136
|
+
const ts = timestamp(now);
|
|
137
|
+
const values = typedColumns.map((col) => {
|
|
138
|
+
if (col === 'record_hash')
|
|
139
|
+
return recordHash;
|
|
140
|
+
if (col === 'imported_at')
|
|
141
|
+
return ts;
|
|
142
|
+
const v = payload[col];
|
|
143
|
+
return v === undefined ? null : v;
|
|
144
|
+
});
|
|
145
|
+
if (!_ensuredTables.has(table)) {
|
|
146
|
+
// Typed tables are created by the static schema in applyHistoryImportSchema.
|
|
147
|
+
_ensuredTables.add(table);
|
|
148
|
+
}
|
|
149
|
+
await db.run(`INSERT INTO ${table} (${typedColumns.join(', ')})
|
|
150
|
+
VALUES (${typedColumns.map(() => '?').join(', ')})
|
|
151
|
+
ON CONFLICT(record_hash) DO NOTHING`, ...values);
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
// Generic ETL insert path: store as payload_json blob.
|
|
155
|
+
if (!_ensuredTables.has(table)) {
|
|
156
|
+
await db.exec(ETL_TABLE_DDL(table));
|
|
157
|
+
_ensuredTables.add(table);
|
|
158
|
+
}
|
|
159
|
+
await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
|
|
160
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
161
|
+
ON CONFLICT(record_hash) DO NOTHING`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
|
|
162
|
+
}
|
|
77
163
|
}
|
|
78
164
|
async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
|
|
79
165
|
await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { JsonObject, SplitEntry } from './types';
|
|
3
|
+
/** Map a Claude Code JSONL record into one or more ETL split entries. */
|
|
4
|
+
export declare function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
5
|
+
/** Map a Pi JSONL record into one or more ETL split entries. */
|
|
6
|
+
export declare function piSplit(raw: unknown): readonly SplitEntry[];
|
|
7
|
+
/** Map an OMP JSONL record into one or more ETL split entries. */
|
|
8
|
+
export declare function ompSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
9
|
+
/** Map a Codex JSONL record into one or more ETL split entries. */
|
|
10
|
+
export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
11
|
+
/** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
|
|
12
|
+
export declare function agySplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
13
|
+
/** Map a Grok JSONL record into one or more ETL split entries. */
|
|
14
|
+
export declare function grokSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
15
|
+
/** Compute a stable field-shape key from a JSON object: sorted, lowercased, +-joined top-level keys. */
|
|
16
|
+
export declare function stableFieldShape(raw: JsonObject): string;
|
|
17
|
+
/** Identity field map for the Claude mapper. */
|
|
18
|
+
export declare const CLAUDE_FIELD_MAP: Record<string, string>;
|
|
19
|
+
/** Identity field map for the Pi mapper. */
|
|
20
|
+
export declare const PI_FIELD_MAP: Record<string, string>;
|
|
21
|
+
/** Identity field map for the OMP mapper. */
|
|
22
|
+
export declare const OMP_FIELD_MAP: Record<string, string>;
|
|
23
|
+
/** Identity field map for the Codex mapper. */
|
|
24
|
+
export declare const CODEX_FIELD_MAP: Record<string, string>;
|
|
25
|
+
/** Identity field map for the AGY mapper. */
|
|
26
|
+
export declare const AGY_FIELD_MAP: Record<string, string>;
|
|
27
|
+
/** Identity field map for the Grok mapper. */
|
|
28
|
+
export declare const GROK_FIELD_MAP: Record<string, string>;
|
|
29
|
+
/** Passthrough Zod schema for the Claude mapper. */
|
|
30
|
+
export declare const CLAUDE_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
31
|
+
/** Passthrough Zod schema for the Pi mapper. */
|
|
32
|
+
export declare const PI_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
33
|
+
/** Passthrough Zod schema for the OMP mapper. */
|
|
34
|
+
export declare const OMP_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
35
|
+
/** Passthrough Zod schema for the Codex mapper. */
|
|
36
|
+
export declare const CODEX_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
37
|
+
/** Passthrough Zod schema for the AGY mapper. */
|
|
38
|
+
export declare const AGY_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
39
|
+
/** Passthrough Zod schema for the Grok mapper. */
|
|
40
|
+
export declare const GROK_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
41
|
+
//# sourceMappingURL=mappers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AA2FtD,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAsE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4F5E;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAgH5E;AAyJD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAoR7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAQlG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC"}
|