@gobing-ai/ts-llm-jsonl-importer 0.4.14 → 0.4.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -8,10 +8,13 @@ Generic JSONL import pipeline for AI-agent history-style files: discover files,
8
8
 
9
9
  | Export | Purpose |
10
10
  |--------|---------|
11
- | `runJsonlImport()` | Runs discovery, parsing, validation, redaction, dedupe, and persistence |
11
+ | `runJsonlImport()` | Runs discovery, parsing, validation, redaction, dedupe, and persistence. Accepts a built-in source string **or** a custom `SourceDefinition`. |
12
12
  | `applyHistoryImportSchema()` | Installs importer-owned checkpoint, ledger, and ETL tables |
13
13
  | `SOURCE_DEFINITIONS` | Built-in source definitions |
14
- | `getSourceDefinition()` | Returns one source definition by key |
14
+ | `getSourceDefinition()` | Returns one built-in definition by key (throws `HistoryImportError` for unknown keys) |
15
+ | `resolveSourceDefinition()` | Resolves a `string \| SourceDefinition` into a validated definition |
16
+ | `validateSourceDefinition()` | Validates a custom definition (target-table names, required fields) before pipeline entry |
17
+ | `VALID_TABLE_NAME` | Regex governing every importer-owned ETL table name |
15
18
  | `redactRecord()` / `redactValue()` | Applies redaction rules before persistence |
16
19
  | `sha256()` / `stableJson()` | Stable hash helpers used by the ledger |
17
20
  | `HISTORY_IMPORT_SCHEMA_SQL` | SQL schema string for explicit migration flows |
@@ -138,6 +141,42 @@ const result = await runJsonlImport('pi', { db, files: ['session.jsonl'], mode:
138
141
 
139
142
  Downstream consumers should treat `source_file`, `source_line`, and `split_index` as the stable provenance tuple.
140
143
 
144
+ ## Custom Source Definitions
145
+
146
+ Beyond the built-in source keys, `runJsonlImport()` accepts a fully-specified `SourceDefinition`
147
+ for any custom agent history format. Unknown strings throw `HistoryImportError`; custom definitions
148
+ are validated before any I/O (target-table names must match `VALID_TABLE_NAME` = `/^history_etl_[a-z_]+$/`,
149
+ and required fields must be present).
150
+
151
+ ```ts
152
+ import { z } from 'zod';
153
+ import { createDbAdapter } from '@gobing-ai/ts-db';
154
+ import { runJsonlImport, type SourceDefinition } from '@gobing-ai/ts-llm-jsonl-importer';
155
+
156
+ const acme: SourceDefinition = {
157
+ source: 'acme',
158
+ displayName: 'Acme Assistant',
159
+ defaultRoots: ['.acme'],
160
+ filePatterns: ['*.jsonl'],
161
+ targetTable: 'history_etl_acme',
162
+ splitConfig: { mode: 'one-to-one' },
163
+ fieldMap: { id: 'source_record_id', timestamp: 'created_at', content: 'content' },
164
+ fieldTransforms: {},
165
+ schema: z.object({
166
+ source_record_id: z.string().min(1),
167
+ created_at: z.string().min(1),
168
+ content: z.string().min(1),
169
+ }),
170
+ };
171
+
172
+ const db = await createDbAdapter({ driver: 'bun-sqlite', url: './history.db' });
173
+ const result = await runJsonlImport(acme, { db, roots: ['~/.acme'], mode: 'incremental' });
174
+ ```
175
+
176
+ The custom `source` name flows through checkpoints, the ledger, and `ImportResult.source` exactly
177
+ like a built-in key. The importer creates the target ETL table on demand if it does not exist
178
+ (idempotent `CREATE TABLE IF NOT EXISTS`), so built-in tables are unaffected.
179
+
141
180
  ## Boundary Notes
142
181
 
143
182
  - This package imports JSONL files and writes importer-owned tables; it does not model conversations, turns, tool calls, or analytics semantics.
@@ -1,6 +1,11 @@
1
- import type { ImportOptions, ImportResult, LlmJsonlSource } from './types';
2
- /** Apply importer-owned schema to the target database. */
3
- export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void>;
4
- /** Run the JSONL import pipeline for one source definition. */
5
- export declare function runJsonlImport(source: LlmJsonlSource, options: ImportOptions): Promise<ImportResult>;
1
+ import type { ImportOptions, ImportResult, SourceDefinition } from './types';
2
+ /** Run the JSONL import pipeline for one source.
3
+ *
4
+ * @param source - Either a built-in source identifier resolved from the
5
+ * registry, or a fully-specified {@link SourceDefinition} for a custom
6
+ * source. Unknown strings throw {@link HistoryImportError}; custom
7
+ * definitions are validated before any I/O.
8
+ * @param options - Importer options (database, roots/files, mode, etc.).
9
+ */
10
+ export declare function runJsonlImport(source: string | SourceDefinition, options: ImportOptions): Promise<ImportResult>;
6
11
  //# sourceMappingURL=importer.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAEZ,cAAc,EAGjB,MAAM,SAAS,CAAC;AAajB,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,+DAA+D;AAC/D,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA4G1G"}
1
+ {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA8GrH"}
package/dist/importer.js CHANGED
@@ -1,28 +1,26 @@
1
- import { createNodeFileSystem, resolvePath, walkDir } from '@gobing-ai/ts-runtime';
2
- import { HistoryImportError } from './errors.js';
1
+ import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
3
2
  import { sha256 } from './hash.js';
3
+ import { applyHistoryImportSchema, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, writeCheckpoint, } from './jsonl-importer-dao.js';
4
4
  import { redactRecord } from './redaction.js';
5
- import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
6
- import { getSourceDefinition } from './sources.js';
7
- const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
8
- /** Apply importer-owned schema to the target database. */
9
- export async function applyHistoryImportSchema(db) {
10
- for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
11
- const sql = statement.trim();
12
- if (sql.length > 0) {
13
- await db.exec(sql);
14
- }
15
- }
16
- }
17
- /** Run the JSONL import pipeline for one source definition. */
5
+ import { resolveSourceDefinition } from './sources.js';
6
+ /** Run the JSONL import pipeline for one source.
7
+ *
8
+ * @param source - Either a built-in source identifier resolved from the
9
+ * registry, or a fully-specified {@link SourceDefinition} for a custom
10
+ * source. Unknown strings throw {@link HistoryImportError}; custom
11
+ * definitions are validated before any I/O.
12
+ * @param options - Importer options (database, roots/files, mode, etc.).
13
+ */
18
14
  export async function runJsonlImport(source, options) {
19
- const definition = getSourceDefinition(source);
15
+ const definition = resolveSourceDefinition(source);
16
+ const resolvedSource = definition.source;
20
17
  const fileSystem = options.fileSystem ?? createNodeFileSystem();
21
18
  await applyHistoryImportSchema(options.db);
19
+ await ensureTargetTables(options.db, definition);
22
20
  const mode = options.mode ?? 'incremental';
23
- const files = await discoverFiles(definition, options.roots, options.files, fileSystem);
21
+ const files = await discoverFiles(definition, options.roots, options.files, fileSystem, options.paths);
24
22
  if (mode === 'full' && !options.dryRun) {
25
- await resetCheckpoints(options.db, source, files);
23
+ await resetCheckpoints(options.db, resolvedSource, files);
26
24
  }
27
25
  const parseErrors = [];
28
26
  const validationErrors = [];
@@ -31,7 +29,7 @@ export async function runJsonlImport(source, options) {
31
29
  let skippedDuplicates = 0;
32
30
  let checkpointUpdates = 0;
33
31
  for (const file of files) {
34
- const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, source, file) : 0;
32
+ const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
35
33
  let lineNumber = 0;
36
34
  for await (const rawLine of readLines(fileSystem, file)) {
37
35
  lineNumber += 1;
@@ -49,7 +47,7 @@ export async function runJsonlImport(source, options) {
49
47
  if (split === undefined)
50
48
  continue;
51
49
  const normalized = normalizeRecord(definition, split.raw, {
52
- source,
50
+ source: resolvedSource,
53
51
  sourceFile: file,
54
52
  sourceLine: lineNumber,
55
53
  splitIndex,
@@ -66,7 +64,7 @@ export async function runJsonlImport(source, options) {
66
64
  const redacted = redactRecord(parsed.data, options.redactionRules);
67
65
  lineSucceeded = true;
68
66
  const recordHash = sha256({
69
- source,
67
+ source: resolvedSource,
70
68
  sourceFile: file,
71
69
  sourceLine: lineNumber,
72
70
  splitIndex,
@@ -78,18 +76,18 @@ export async function runJsonlImport(source, options) {
78
76
  }
79
77
  if (!options.dryRun) {
80
78
  await insertRecord(options.db, split.targetTable, recordHash, file, lineNumber, splitIndex, redacted, options.now);
81
- await insertLedger(options.db, recordHash, source, file, lineNumber, splitIndex, split.targetTable, options.now);
79
+ await insertLedger(options.db, recordHash, resolvedSource, file, lineNumber, splitIndex, split.targetTable, options.now);
82
80
  }
83
81
  importedRecords += 1;
84
82
  }
85
83
  if (lineSucceeded && !options.dryRun) {
86
- await writeCheckpoint(options.db, source, file, lineNumber, options.now);
84
+ await writeCheckpoint(options.db, resolvedSource, file, lineNumber, options.now);
87
85
  checkpointUpdates += 1;
88
86
  }
89
87
  }
90
88
  }
91
89
  return {
92
- source,
90
+ source: resolvedSource,
93
91
  mode,
94
92
  scannedFiles: files.length,
95
93
  processedLines,
@@ -153,11 +151,20 @@ function normalizeRecord(definition, raw, context) {
153
151
  normalized.split_index = context.splitIndex;
154
152
  return normalized;
155
153
  }
156
- async function discoverFiles(definition, roots, files, fileSystem) {
154
+ async function discoverFiles(definition, roots, files, fileSystem, paths) {
157
155
  if (files !== undefined && files.length > 0) {
158
156
  return files.map((file) => resolvePath(file)).sort();
159
157
  }
160
- const resolvedRoots = (roots ?? definition.defaultRoots).map((root) => resolvePath(root));
158
+ // Root resolution splits by provenance (ADR-023 A1 / task 0042):
159
+ // - explicit caller-supplied `roots` → cwd semantics: resolve against `paths.cwd` when
160
+ // injected, else ambient cwd (the pre-seam behaviour — R4 additive-only).
161
+ // - registry `defaultRoots` → home-relative: resolve against `paths.home`.
162
+ // Previously both kinds anchored to ambient cwd, so a cwd ≠ $HOME silently skipped every
163
+ // registry source because the home-relative paths never existed relative to cwd.
164
+ const ambient = paths ?? ambientRuntimePaths();
165
+ const resolvedRoots = roots !== undefined
166
+ ? roots.map((root) => resolvePath(ambient.cwd, root))
167
+ : definition.defaultRoots.map((root) => resolvePath(ambient.home, root));
161
168
  const found = new Set();
162
169
  for (const root of resolvedRoots) {
163
170
  if (!(await fileSystem.exists(root)))
@@ -186,45 +193,6 @@ function matchesPattern(path, patterns) {
186
193
  return path.endsWith(pattern.replace(/^\*/, ''));
187
194
  });
188
195
  }
189
- async function readCheckpoint(db, source, sourceFile) {
190
- const row = await db.queryFirst('SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, sourceFile);
191
- return row?.last_imported_line ?? 0;
192
- }
193
- async function resetCheckpoints(db, source, files) {
194
- for (const file of files) {
195
- await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
196
- }
197
- }
198
- async function writeCheckpoint(db, source, sourceFile, line, now) {
199
- await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
200
- VALUES (?, ?, ?, ?)
201
- ON CONFLICT(source, source_file) DO UPDATE SET
202
- last_imported_line = excluded.last_imported_line,
203
- updated_at = excluded.updated_at`, source, sourceFile, line, timestamp(now));
204
- }
205
- async function ledgerExists(db, recordHash) {
206
- const row = await db.queryFirst('SELECT record_hash FROM history_import_ledger WHERE record_hash = ?', recordHash);
207
- return row !== undefined && row !== null;
208
- }
209
- async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
210
- const table = targetTableFor(targetTable);
211
- await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
212
- VALUES (?, ?, ?, ?, ?, ?)
213
- ON CONFLICT(record_hash) DO NOTHING`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
214
- }
215
- async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
216
- await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
217
- VALUES (?, ?, ?, ?, ?, ?, ?)`, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now));
218
- }
219
- function targetTableFor(table) {
220
- if (!VALID_TABLE_NAME.test(table)) {
221
- throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
222
- }
223
- return table;
224
- }
225
- function timestamp(now) {
226
- return (now?.() ?? new Date()).toISOString();
227
- }
228
196
  /**
229
197
  * Yield lines from a file, streaming when the FileSystem supports it.
230
198
  *
package/dist/index.d.ts CHANGED
@@ -1,8 +1,9 @@
1
1
  export { HistoryImportError } from './errors';
2
2
  export { sha256, stableJson } from './hash';
3
- export { applyHistoryImportSchema, runJsonlImport } from './importer';
3
+ export { runJsonlImport } from './importer';
4
+ export { applyHistoryImportSchema } from './jsonl-importer-dao';
4
5
  export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
5
6
  export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
6
- export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources';
7
+ export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
7
8
  export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, RedactionRule, SourceDefinition, SplitConfig, TransformContext, } from './types';
8
9
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACpE,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
package/dist/index.js CHANGED
@@ -1,6 +1,7 @@
1
1
  export { HistoryImportError } from './errors.js';
2
2
  export { sha256, stableJson } from './hash.js';
3
- export { applyHistoryImportSchema, runJsonlImport } from './importer.js';
3
+ export { runJsonlImport } from './importer.js';
4
+ export { applyHistoryImportSchema } from './jsonl-importer-dao.js';
4
5
  export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
5
6
  export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
6
- export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources.js';
7
+ export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources.js';
@@ -0,0 +1,30 @@
1
+ import type { ImportOptions, JsonObject, SourceDefinition } from './types';
2
+ interface CheckpointRow {
3
+ readonly last_imported_line: number;
4
+ }
5
+ declare function targetTableFor(table: string): string;
6
+ declare function timestamp(now: ImportOptions['now']): string;
7
+ declare function ETL_TABLE_DDL(table: string): string;
8
+ /** Apply importer-owned schema to the target database. */
9
+ export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void>;
10
+ /**
11
+ * Ensure the ETL table(s) for a source definition exist.
12
+ *
13
+ * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
14
+ * `history_etl_*` tables. Custom source definitions (and built-in definitions
15
+ * with a `splitConfig.targetTable` override) need their target table(s) created
16
+ * on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
17
+ * {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
18
+ * interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
19
+ * built-in tables that the static schema already created.
20
+ */
21
+ declare function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void>;
22
+ declare function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number>;
23
+ declare function resetCheckpoints(db: ImportOptions['db'], source: string, files: readonly string[]): Promise<void>;
24
+ declare function writeCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string, line: number, now: ImportOptions['now']): Promise<void>;
25
+ declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean>;
26
+ declare function insertRecord(db: ImportOptions['db'], targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): Promise<void>;
27
+ declare function insertLedger(db: ImportOptions['db'], recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): Promise<void>;
28
+ export type { CheckpointRow };
29
+ export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
30
+ //# sourceMappingURL=jsonl-importer-dao.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAED,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAQtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
@@ -0,0 +1,82 @@
1
+ import { HistoryImportError } from './errors.js';
2
+ import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
3
+ import { VALID_TABLE_NAME } from './sources.js';
4
+ function targetTableFor(table) {
5
+ if (!VALID_TABLE_NAME.test(table)) {
6
+ throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
7
+ }
8
+ return table;
9
+ }
10
+ function timestamp(now) {
11
+ return (now?.() ?? new Date()).toISOString();
12
+ }
13
+ function ETL_TABLE_DDL(table) {
14
+ return `CREATE TABLE IF NOT EXISTS ${table} (
15
+ record_hash TEXT PRIMARY KEY,
16
+ source_file TEXT NOT NULL,
17
+ source_line INTEGER NOT NULL,
18
+ split_index INTEGER NOT NULL,
19
+ payload_json TEXT NOT NULL,
20
+ imported_at TEXT NOT NULL
21
+ );`;
22
+ }
23
+ /** Apply importer-owned schema to the target database. */
24
+ export async function applyHistoryImportSchema(db) {
25
+ for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
26
+ const sql = statement.trim();
27
+ if (sql.length > 0) {
28
+ await db.exec(sql);
29
+ }
30
+ }
31
+ }
32
+ /**
33
+ * Ensure the ETL table(s) for a source definition exist.
34
+ *
35
+ * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
36
+ * `history_etl_*` tables. Custom source definitions (and built-in definitions
37
+ * with a `splitConfig.targetTable` override) need their target table(s) created
38
+ * on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
39
+ * {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
40
+ * interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
41
+ * built-in tables that the static schema already created.
42
+ */
43
+ async function ensureTargetTables(db, definition) {
44
+ const tables = new Set([targetTableFor(definition.targetTable)]);
45
+ if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
46
+ tables.add(targetTableFor(definition.splitConfig.targetTable));
47
+ }
48
+ for (const table of tables) {
49
+ await db.exec(ETL_TABLE_DDL(table));
50
+ }
51
+ }
52
+ async function readCheckpoint(db, source, sourceFile) {
53
+ const row = await db.queryFirst('SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, sourceFile);
54
+ return row?.last_imported_line ?? 0;
55
+ }
56
+ async function resetCheckpoints(db, source, files) {
57
+ for (const file of files) {
58
+ await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
59
+ }
60
+ }
61
+ async function writeCheckpoint(db, source, sourceFile, line, now) {
62
+ await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
63
+ VALUES (?, ?, ?, ?)
64
+ ON CONFLICT(source, source_file) DO UPDATE SET
65
+ last_imported_line = excluded.last_imported_line,
66
+ updated_at = excluded.updated_at`, source, sourceFile, line, timestamp(now));
67
+ }
68
+ async function ledgerExists(db, recordHash) {
69
+ const row = await db.queryFirst('SELECT record_hash FROM history_import_ledger WHERE record_hash = ?', recordHash);
70
+ return row !== undefined && row !== null;
71
+ }
72
+ async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
73
+ const table = targetTableFor(targetTable);
74
+ await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
75
+ VALUES (?, ?, ?, ?, ?, ?)
76
+ ON CONFLICT(record_hash) DO NOTHING`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
77
+ }
78
+ async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
79
+ await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
80
+ VALUES (?, ?, ?, ?, ?, ?, ?)`, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now));
81
+ }
82
+ export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
package/dist/sources.d.ts CHANGED
@@ -1,6 +1,36 @@
1
1
  import type { LlmJsonlSource, SourceDefinition } from './types';
2
+ /**
3
+ * Pattern every importer-owned table must match.
4
+ *
5
+ * WHY: the table name is interpolated into generated SQL (it is not a bind
6
+ * parameter), so it must be a safe identifier. The `history_etl_` prefix also
7
+ * namespaces importer tables away from consumer schemas. Both built-in and
8
+ * custom source definitions are held to this rule.
9
+ */
10
+ export declare const VALID_TABLE_NAME: RegExp;
2
11
  /** Built-in source definitions keyed by source identifier. */
3
12
  export declare const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinition>>;
4
- /** Resolve a built-in source definition by identifier. */
13
+ /**
14
+ * Resolve a built-in source definition by identifier.
15
+ *
16
+ * Throws {@link HistoryImportError} for unknown identifiers so misconfigured
17
+ * callers fail fast at the seam instead of producing empty imports.
18
+ */
5
19
  export declare function getSourceDefinition(source: LlmJsonlSource): SourceDefinition;
20
+ /**
21
+ * Validate a caller-supplied source definition before it enters the pipeline.
22
+ *
23
+ * WHY: target table names are interpolated into SQL, so both the primary
24
+ * `targetTable` and any `splitConfig.targetTable` override must match
25
+ * {@link VALID_TABLE_NAME}. Required structural fields are checked here so the
26
+ * importer can assume they are present. Built-in definitions skip this — they
27
+ * are validated by construction.
28
+ */
29
+ export declare function validateSourceDefinition(definition: SourceDefinition): SourceDefinition;
30
+ /**
31
+ * Resolve a `string | SourceDefinition` source argument into a validated
32
+ * {@link SourceDefinition}. Strings resolve from {@link SOURCE_DEFINITIONS}
33
+ * (unknown strings throw); objects are validated via {@link validateSourceDefinition}.
34
+ */
35
+ export declare function resolveSourceDefinition(input: string | SourceDefinition): SourceDefinition;
6
36
  //# sourceMappingURL=sources.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AA+F9G,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAQjF,CAAC;AAEF,0DAA0D;AAC1D,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAE5E"}
1
+ {"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AA8F9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAA0B,CAAC;AAExD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAQjF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAM5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,GAAG,gBAAgB,CA+BvF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,gBAAgB,CAK1F"}
package/dist/sources.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { z } from 'zod';
2
+ import { HistoryImportError } from './errors.js';
2
3
  const sourceRecordSchema = z
3
4
  .object({
4
5
  source_record_id: z.string().min(1),
@@ -71,6 +72,15 @@ function sourceDefinition(source, displayName, defaultRoots, filePatterns) {
71
72
  schema: sourceRecordSchema,
72
73
  };
73
74
  }
75
+ /**
76
+ * Pattern every importer-owned table must match.
77
+ *
78
+ * WHY: the table name is interpolated into generated SQL (it is not a bind
79
+ * parameter), so it must be a safe identifier. The `history_etl_` prefix also
80
+ * namespaces importer tables away from consumer schemas. Both built-in and
81
+ * custom source definitions are held to this rule.
82
+ */
83
+ export const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
74
84
  /** Built-in source definitions keyed by source identifier. */
75
85
  export const SOURCE_DEFINITIONS = {
76
86
  pi: sourceDefinition('pi', 'Pi', ['.pi/history', '.pi'], ['*.jsonl', '*.json']),
@@ -81,7 +91,73 @@ export const SOURCE_DEFINITIONS = {
81
91
  antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
82
92
  openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
83
93
  };
84
- /** Resolve a built-in source definition by identifier. */
94
+ /**
95
+ * Resolve a built-in source definition by identifier.
96
+ *
97
+ * Throws {@link HistoryImportError} for unknown identifiers so misconfigured
98
+ * callers fail fast at the seam instead of producing empty imports.
99
+ */
85
100
  export function getSourceDefinition(source) {
86
- return SOURCE_DEFINITIONS[source];
101
+ const definition = SOURCE_DEFINITIONS[source];
102
+ if (definition === undefined) {
103
+ throw new HistoryImportError(`Unknown LLM JSONL source: ${source}`, { source });
104
+ }
105
+ return definition;
106
+ }
107
+ /**
108
+ * Validate a caller-supplied source definition before it enters the pipeline.
109
+ *
110
+ * WHY: target table names are interpolated into SQL, so both the primary
111
+ * `targetTable` and any `splitConfig.targetTable` override must match
112
+ * {@link VALID_TABLE_NAME}. Required structural fields are checked here so the
113
+ * importer can assume they are present. Built-in definitions skip this — they
114
+ * are validated by construction.
115
+ */
116
+ export function validateSourceDefinition(definition) {
117
+ const requiredFields = [
118
+ 'source',
119
+ 'displayName',
120
+ 'defaultRoots',
121
+ 'filePatterns',
122
+ 'targetTable',
123
+ 'splitConfig',
124
+ 'fieldMap',
125
+ 'fieldTransforms',
126
+ 'schema',
127
+ ];
128
+ for (const field of requiredFields) {
129
+ if (definition[field] === undefined || definition[field] === null) {
130
+ throw new HistoryImportError(`Source definition is missing required field: ${String(field)}`, {
131
+ source: definition.source,
132
+ field: String(field),
133
+ });
134
+ }
135
+ }
136
+ if (definition.filePatterns.length === 0) {
137
+ throw new HistoryImportError('Source definition must declare at least one file pattern', {
138
+ source: definition.source,
139
+ field: 'filePatterns',
140
+ });
141
+ }
142
+ assertValidTable(definition.targetTable, definition.source);
143
+ if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
144
+ assertValidTable(definition.splitConfig.targetTable, definition.source);
145
+ }
146
+ return definition;
147
+ }
148
+ /**
149
+ * Resolve a `string | SourceDefinition` source argument into a validated
150
+ * {@link SourceDefinition}. Strings resolve from {@link SOURCE_DEFINITIONS}
151
+ * (unknown strings throw); objects are validated via {@link validateSourceDefinition}.
152
+ */
153
+ export function resolveSourceDefinition(input) {
154
+ if (typeof input === 'string') {
155
+ return getSourceDefinition(input);
156
+ }
157
+ return validateSourceDefinition(input);
158
+ }
159
+ function assertValidTable(table, source) {
160
+ if (!VALID_TABLE_NAME.test(table)) {
161
+ throw new HistoryImportError(`Invalid history ETL target table for source "${source}": ${table}. Must match ${VALID_TABLE_NAME.source}.`, { source, table });
162
+ }
87
163
  }
package/dist/types.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { DbAdapter } from '@gobing-ai/ts-db';
2
- import type { FileSystem } from '@gobing-ai/ts-runtime';
2
+ import type { FileSystem, RuntimePaths } from '@gobing-ai/ts-runtime';
3
3
  import type { z } from 'zod';
4
4
  /** Built-in source identifiers supported by the importer. */
5
5
  export type LlmJsonlSource = 'pi' | 'claude' | 'codex' | 'gemini' | 'opencode' | 'antigravity' | 'openclaw';
@@ -23,14 +23,16 @@ export type JsonObject = Record<string, unknown>;
23
23
  export type FieldTransform = (value: unknown, raw: JsonObject, context: TransformContext) => unknown;
24
24
  /** Context available while normalizing a split record. */
25
25
  export interface TransformContext {
26
- readonly source: LlmJsonlSource;
26
+ /** Source identifier — a built-in {@link LlmJsonlSource} or a custom definition's name. */
27
+ readonly source: string;
27
28
  readonly sourceFile: string;
28
29
  readonly sourceLine: number;
29
30
  readonly splitIndex: number;
30
31
  }
31
32
  /** Declarative importer configuration for one LLM history source. */
32
33
  export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
33
- readonly source: LlmJsonlSource;
34
+ /** Source identifier — a built-in {@link LlmJsonlSource} or a custom name. Becomes the checkpoint/ledger key. */
35
+ readonly source: string;
34
36
  readonly displayName: string;
35
37
  readonly defaultRoots: readonly string[];
36
38
  readonly filePatterns: readonly string[];
@@ -56,6 +58,12 @@ export interface ImportOptions {
56
58
  readonly dryRun?: boolean;
57
59
  readonly redactionRules?: readonly RedactionRule[];
58
60
  readonly now?: () => Date;
61
+ /**
62
+ * Optional cwd/home anchor (ADR-023 A1 / task 0042). When set, registry `defaultRoots`
63
+ * resolve against `paths.home` instead of the ambient working directory; explicit
64
+ * {@link ImportOptions.roots} keep cwd semantics unchanged. Defaults to ambient cwd/home.
65
+ */
66
+ readonly paths?: RuntimePaths;
59
67
  }
60
68
  /** Structured importer error that can be reported without persisting raw input. */
61
69
  export interface ImportIssue {
@@ -65,7 +73,8 @@ export interface ImportIssue {
65
73
  }
66
74
  /** Summary produced by the importer control function. */
67
75
  export interface ImportResult {
68
- readonly source: LlmJsonlSource;
76
+ /** Source identifier — the built-in name or custom definition name that produced this result. */
77
+ readonly source: string;
69
78
  readonly mode: ImportMode;
70
79
  readonly scannedFiles: number;
71
80
  readonly processedLines: number;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC;AAE5G,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,SAAS,UAAU,EAAE,CAAC;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAC7B;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACtC"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC;AAE5G,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,SAAS,UAAU,EAAE,CAAC;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACtC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gobing-ai/ts-llm-jsonl-importer",
3
- "version": "0.4.14",
3
+ "version": "0.4.16",
4
4
  "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
5
5
  "keywords": [
6
6
  "typescript",
@@ -47,9 +47,9 @@
47
47
  "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
48
48
  },
49
49
  "dependencies": {
50
- "@gobing-ai/ts-db": "^0.4.14",
51
- "@gobing-ai/ts-runtime": "^0.4.14",
52
- "@gobing-ai/ts-utils": "^0.4.14",
50
+ "@gobing-ai/ts-db": "^0.4.16",
51
+ "@gobing-ai/ts-runtime": "^0.4.16",
52
+ "@gobing-ai/ts-utils": "^0.4.16",
53
53
  "zod": "^4.1.0"
54
54
  },
55
55
  "devDependencies": {
package/src/importer.ts CHANGED
@@ -1,50 +1,51 @@
1
- import { createNodeFileSystem, type FileSystem, resolvePath, walkDir } from '@gobing-ai/ts-runtime';
2
- import { HistoryImportError } from './errors';
1
+ import {
2
+ ambientRuntimePaths,
3
+ createNodeFileSystem,
4
+ type FileSystem,
5
+ type RuntimePaths,
6
+ resolvePath,
7
+ walkDir,
8
+ } from '@gobing-ai/ts-runtime';
3
9
  import { sha256 } from './hash';
10
+ import {
11
+ applyHistoryImportSchema,
12
+ ensureTargetTables,
13
+ insertLedger,
14
+ insertRecord,
15
+ ledgerExists,
16
+ readCheckpoint,
17
+ resetCheckpoints,
18
+ targetTableFor,
19
+ writeCheckpoint,
20
+ } from './jsonl-importer-dao';
4
21
  import { redactRecord } from './redaction';
5
- import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
6
- import { getSourceDefinition } from './sources';
7
- import type {
8
- ImportIssue,
9
- ImportOptions,
10
- ImportResult,
11
- JsonObject,
12
- LlmJsonlSource,
13
- SourceDefinition,
14
- TransformContext,
15
- } from './types';
22
+ import { resolveSourceDefinition } from './sources';
23
+ import type { ImportIssue, ImportOptions, ImportResult, JsonObject, SourceDefinition, TransformContext } from './types';
16
24
 
17
25
  interface SplitRecord {
18
26
  readonly targetTable: string;
19
27
  readonly raw: JsonObject;
20
28
  }
21
29
 
22
- interface CheckpointRow {
23
- readonly last_imported_line: number;
24
- }
25
-
26
- const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
27
-
28
- /** Apply importer-owned schema to the target database. */
29
- export async function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void> {
30
- for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
31
- const sql = statement.trim();
32
- if (sql.length > 0) {
33
- await db.exec(sql);
34
- }
35
- }
36
- }
37
-
38
- /** Run the JSONL import pipeline for one source definition. */
39
- export async function runJsonlImport(source: LlmJsonlSource, options: ImportOptions): Promise<ImportResult> {
40
- const definition = getSourceDefinition(source);
30
+ /** Run the JSONL import pipeline for one source.
31
+ *
32
+ * @param source - Either a built-in source identifier resolved from the
33
+ * registry, or a fully-specified {@link SourceDefinition} for a custom
34
+ * source. Unknown strings throw {@link HistoryImportError}; custom
35
+ * definitions are validated before any I/O.
36
+ * @param options - Importer options (database, roots/files, mode, etc.).
37
+ */
38
+ export async function runJsonlImport(source: string | SourceDefinition, options: ImportOptions): Promise<ImportResult> {
39
+ const definition = resolveSourceDefinition(source);
40
+ const resolvedSource = definition.source;
41
41
  const fileSystem = options.fileSystem ?? createNodeFileSystem();
42
42
  await applyHistoryImportSchema(options.db);
43
+ await ensureTargetTables(options.db, definition);
43
44
 
44
45
  const mode = options.mode ?? 'incremental';
45
- const files = await discoverFiles(definition, options.roots, options.files, fileSystem);
46
+ const files = await discoverFiles(definition, options.roots, options.files, fileSystem, options.paths);
46
47
  if (mode === 'full' && !options.dryRun) {
47
- await resetCheckpoints(options.db, source, files);
48
+ await resetCheckpoints(options.db, resolvedSource, files);
48
49
  }
49
50
 
50
51
  const parseErrors: ImportIssue[] = [];
@@ -55,7 +56,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
55
56
  let checkpointUpdates = 0;
56
57
 
57
58
  for (const file of files) {
58
- const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, source, file) : 0;
59
+ const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
59
60
  let lineNumber = 0;
60
61
  for await (const rawLine of readLines(fileSystem, file)) {
61
62
  lineNumber += 1;
@@ -73,7 +74,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
73
74
  const split = splitRecords[splitIndex];
74
75
  if (split === undefined) continue;
75
76
  const normalized = normalizeRecord(definition, split.raw, {
76
- source,
77
+ source: resolvedSource,
77
78
  sourceFile: file,
78
79
  sourceLine: lineNumber,
79
80
  splitIndex,
@@ -91,7 +92,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
91
92
  const redacted = redactRecord(parsed.data, options.redactionRules);
92
93
  lineSucceeded = true;
93
94
  const recordHash = sha256({
94
- source,
95
+ source: resolvedSource,
95
96
  sourceFile: file,
96
97
  sourceLine: lineNumber,
97
98
  splitIndex,
@@ -115,7 +116,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
115
116
  await insertLedger(
116
117
  options.db,
117
118
  recordHash,
118
- source,
119
+ resolvedSource,
119
120
  file,
120
121
  lineNumber,
121
122
  splitIndex,
@@ -127,14 +128,14 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
127
128
  }
128
129
 
129
130
  if (lineSucceeded && !options.dryRun) {
130
- await writeCheckpoint(options.db, source, file, lineNumber, options.now);
131
+ await writeCheckpoint(options.db, resolvedSource, file, lineNumber, options.now);
131
132
  checkpointUpdates += 1;
132
133
  }
133
134
  }
134
135
  }
135
136
 
136
137
  return {
137
- source,
138
+ source: resolvedSource,
138
139
  mode,
139
140
  scannedFiles: files.length,
140
141
  processedLines,
@@ -211,12 +212,23 @@ async function discoverFiles(
211
212
  roots: readonly string[] | undefined,
212
213
  files: readonly string[] | undefined,
213
214
  fileSystem: FileSystem,
215
+ paths: RuntimePaths | undefined,
214
216
  ): Promise<readonly string[]> {
215
217
  if (files !== undefined && files.length > 0) {
216
218
  return files.map((file) => resolvePath(file)).sort();
217
219
  }
218
220
 
219
- const resolvedRoots = (roots ?? definition.defaultRoots).map((root) => resolvePath(root));
221
+ // Root resolution splits by provenance (ADR-023 A1 / task 0042):
222
+ // - explicit caller-supplied `roots` → cwd semantics: resolve against `paths.cwd` when
223
+ // injected, else ambient cwd (the pre-seam behaviour — R4 additive-only).
224
+ // - registry `defaultRoots` → home-relative: resolve against `paths.home`.
225
+ // Previously both kinds anchored to ambient cwd, so a cwd ≠ $HOME silently skipped every
226
+ // registry source because the home-relative paths never existed relative to cwd.
227
+ const ambient = paths ?? ambientRuntimePaths();
228
+ const resolvedRoots =
229
+ roots !== undefined
230
+ ? roots.map((root) => resolvePath(ambient.cwd, root))
231
+ : definition.defaultRoots.map((root) => resolvePath(ambient.home, root));
220
232
  const found = new Set<string>();
221
233
  for (const root of resolvedRoots) {
222
234
  if (!(await fileSystem.exists(root))) continue;
@@ -241,107 +253,6 @@ function matchesPattern(path: string, patterns: readonly string[]): boolean {
241
253
  });
242
254
  }
243
255
 
244
- async function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number> {
245
- const row = await db.queryFirst<CheckpointRow>(
246
- 'SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?',
247
- source,
248
- sourceFile,
249
- );
250
- return row?.last_imported_line ?? 0;
251
- }
252
-
253
- async function resetCheckpoints(db: ImportOptions['db'], source: string, files: readonly string[]): Promise<void> {
254
- for (const file of files) {
255
- await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
256
- }
257
- }
258
-
259
- async function writeCheckpoint(
260
- db: ImportOptions['db'],
261
- source: string,
262
- sourceFile: string,
263
- line: number,
264
- now: ImportOptions['now'],
265
- ): Promise<void> {
266
- await db.run(
267
- `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
268
- VALUES (?, ?, ?, ?)
269
- ON CONFLICT(source, source_file) DO UPDATE SET
270
- last_imported_line = excluded.last_imported_line,
271
- updated_at = excluded.updated_at`,
272
- source,
273
- sourceFile,
274
- line,
275
- timestamp(now),
276
- );
277
- }
278
-
279
- async function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean> {
280
- const row = await db.queryFirst<{ record_hash: string }>(
281
- 'SELECT record_hash FROM history_import_ledger WHERE record_hash = ?',
282
- recordHash,
283
- );
284
- return row !== undefined && row !== null;
285
- }
286
-
287
- async function insertRecord(
288
- db: ImportOptions['db'],
289
- targetTable: string,
290
- recordHash: string,
291
- sourceFile: string,
292
- sourceLine: number,
293
- splitIndex: number,
294
- payload: JsonObject,
295
- now: ImportOptions['now'],
296
- ): Promise<void> {
297
- const table = targetTableFor(targetTable);
298
- await db.run(
299
- `INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
300
- VALUES (?, ?, ?, ?, ?, ?)
301
- ON CONFLICT(record_hash) DO NOTHING`,
302
- recordHash,
303
- sourceFile,
304
- sourceLine,
305
- splitIndex,
306
- JSON.stringify(payload),
307
- timestamp(now),
308
- );
309
- }
310
-
311
- async function insertLedger(
312
- db: ImportOptions['db'],
313
- recordHash: string,
314
- source: string,
315
- sourceFile: string,
316
- sourceLine: number,
317
- splitIndex: number,
318
- targetTable: string,
319
- now: ImportOptions['now'],
320
- ): Promise<void> {
321
- await db.run(
322
- `INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
323
- VALUES (?, ?, ?, ?, ?, ?, ?)`,
324
- recordHash,
325
- source,
326
- sourceFile,
327
- sourceLine,
328
- splitIndex,
329
- targetTable,
330
- timestamp(now),
331
- );
332
- }
333
-
334
- function targetTableFor(table: string): string {
335
- if (!VALID_TABLE_NAME.test(table)) {
336
- throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
337
- }
338
- return table;
339
- }
340
-
341
- function timestamp(now: ImportOptions['now']): string {
342
- return (now?.() ?? new Date()).toISOString();
343
- }
344
-
345
256
  /**
346
257
  * Yield lines from a file, streaming when the FileSystem supports it.
347
258
  *
package/src/index.ts CHANGED
@@ -1,9 +1,16 @@
1
1
  export { HistoryImportError } from './errors';
2
2
  export { sha256, stableJson } from './hash';
3
- export { applyHistoryImportSchema, runJsonlImport } from './importer';
3
+ export { runJsonlImport } from './importer';
4
+ export { applyHistoryImportSchema } from './jsonl-importer-dao';
4
5
  export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
5
6
  export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
6
- export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources';
7
+ export {
8
+ getSourceDefinition,
9
+ resolveSourceDefinition,
10
+ SOURCE_DEFINITIONS,
11
+ VALID_TABLE_NAME,
12
+ validateSourceDefinition,
13
+ } from './sources';
7
14
  export type {
8
15
  FieldTransform,
9
16
  ImportIssue,
@@ -0,0 +1,165 @@
1
+ import { HistoryImportError } from './errors';
2
+ import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
3
+ import { VALID_TABLE_NAME } from './sources';
4
+ import type { ImportOptions, JsonObject, SourceDefinition } from './types';
5
+
6
+ interface CheckpointRow {
7
+ readonly last_imported_line: number;
8
+ }
9
+
10
+ function targetTableFor(table: string): string {
11
+ if (!VALID_TABLE_NAME.test(table)) {
12
+ throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
13
+ }
14
+ return table;
15
+ }
16
+
17
+ function timestamp(now: ImportOptions['now']): string {
18
+ return (now?.() ?? new Date()).toISOString();
19
+ }
20
+
21
+ function ETL_TABLE_DDL(table: string): string {
22
+ return `CREATE TABLE IF NOT EXISTS ${table} (
23
+ record_hash TEXT PRIMARY KEY,
24
+ source_file TEXT NOT NULL,
25
+ source_line INTEGER NOT NULL,
26
+ split_index INTEGER NOT NULL,
27
+ payload_json TEXT NOT NULL,
28
+ imported_at TEXT NOT NULL
29
+ );`;
30
+ }
31
+
32
+ /** Apply importer-owned schema to the target database. */
33
+ export async function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void> {
34
+ for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
35
+ const sql = statement.trim();
36
+ if (sql.length > 0) {
37
+ await db.exec(sql);
38
+ }
39
+ }
40
+ }
41
+
42
+ /**
43
+ * Ensure the ETL table(s) for a source definition exist.
44
+ *
45
+ * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
46
+ * `history_etl_*` tables. Custom source definitions (and built-in definitions
47
+ * with a `splitConfig.targetTable` override) need their target table(s) created
48
+ * on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
49
+ * {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
50
+ * interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
51
+ * built-in tables that the static schema already created.
52
+ */
53
+ async function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void> {
54
+ const tables = new Set<string>([targetTableFor(definition.targetTable)]);
55
+ if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
56
+ tables.add(targetTableFor(definition.splitConfig.targetTable));
57
+ }
58
+ for (const table of tables) {
59
+ await db.exec(ETL_TABLE_DDL(table));
60
+ }
61
+ }
62
+
63
+ async function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number> {
64
+ const row = await db.queryFirst<CheckpointRow>(
65
+ 'SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?',
66
+ source,
67
+ sourceFile,
68
+ );
69
+ return row?.last_imported_line ?? 0;
70
+ }
71
+
72
+ async function resetCheckpoints(db: ImportOptions['db'], source: string, files: readonly string[]): Promise<void> {
73
+ for (const file of files) {
74
+ await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
75
+ }
76
+ }
77
+
78
+ async function writeCheckpoint(
79
+ db: ImportOptions['db'],
80
+ source: string,
81
+ sourceFile: string,
82
+ line: number,
83
+ now: ImportOptions['now'],
84
+ ): Promise<void> {
85
+ await db.run(
86
+ `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
87
+ VALUES (?, ?, ?, ?)
88
+ ON CONFLICT(source, source_file) DO UPDATE SET
89
+ last_imported_line = excluded.last_imported_line,
90
+ updated_at = excluded.updated_at`,
91
+ source,
92
+ sourceFile,
93
+ line,
94
+ timestamp(now),
95
+ );
96
+ }
97
+
98
+ async function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean> {
99
+ const row = await db.queryFirst<{ record_hash: string }>(
100
+ 'SELECT record_hash FROM history_import_ledger WHERE record_hash = ?',
101
+ recordHash,
102
+ );
103
+ return row !== undefined && row !== null;
104
+ }
105
+
106
+ async function insertRecord(
107
+ db: ImportOptions['db'],
108
+ targetTable: string,
109
+ recordHash: string,
110
+ sourceFile: string,
111
+ sourceLine: number,
112
+ splitIndex: number,
113
+ payload: JsonObject,
114
+ now: ImportOptions['now'],
115
+ ): Promise<void> {
116
+ const table = targetTableFor(targetTable);
117
+ await db.run(
118
+ `INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
119
+ VALUES (?, ?, ?, ?, ?, ?)
120
+ ON CONFLICT(record_hash) DO NOTHING`,
121
+ recordHash,
122
+ sourceFile,
123
+ sourceLine,
124
+ splitIndex,
125
+ JSON.stringify(payload),
126
+ timestamp(now),
127
+ );
128
+ }
129
+
130
+ async function insertLedger(
131
+ db: ImportOptions['db'],
132
+ recordHash: string,
133
+ source: string,
134
+ sourceFile: string,
135
+ sourceLine: number,
136
+ splitIndex: number,
137
+ targetTable: string,
138
+ now: ImportOptions['now'],
139
+ ): Promise<void> {
140
+ await db.run(
141
+ `INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
142
+ VALUES (?, ?, ?, ?, ?, ?, ?)`,
143
+ recordHash,
144
+ source,
145
+ sourceFile,
146
+ sourceLine,
147
+ splitIndex,
148
+ targetTable,
149
+ timestamp(now),
150
+ );
151
+ }
152
+
153
+ export type { CheckpointRow };
154
+ export {
155
+ ETL_TABLE_DDL,
156
+ ensureTargetTables,
157
+ insertLedger,
158
+ insertRecord,
159
+ ledgerExists,
160
+ readCheckpoint,
161
+ resetCheckpoints,
162
+ targetTableFor,
163
+ timestamp,
164
+ writeCheckpoint,
165
+ };
package/src/sources.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { z } from 'zod';
2
+ import { HistoryImportError } from './errors';
2
3
  import type { FieldTransform, JsonObject, LlmJsonlSource, SourceDefinition, TransformContext } from './types';
3
4
 
4
5
  const sourceRecordSchema = z
@@ -93,6 +94,15 @@ function sourceDefinition(
93
94
  schema: sourceRecordSchema,
94
95
  };
95
96
  }
97
+ /**
98
+ * Pattern every importer-owned table must match.
99
+ *
100
+ * WHY: the table name is interpolated into generated SQL (it is not a bind
101
+ * parameter), so it must be a safe identifier. The `history_etl_` prefix also
102
+ * namespaces importer tables away from consumer schemas. Both built-in and
103
+ * custom source definitions are held to this rule.
104
+ */
105
+ export const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
96
106
 
97
107
  /** Built-in source definitions keyed by source identifier. */
98
108
  export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinition>> = {
@@ -105,7 +115,79 @@ export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinitio
105
115
  openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
106
116
  };
107
117
 
108
- /** Resolve a built-in source definition by identifier. */
118
+ /**
119
+ * Resolve a built-in source definition by identifier.
120
+ *
121
+ * Throws {@link HistoryImportError} for unknown identifiers so misconfigured
122
+ * callers fail fast at the seam instead of producing empty imports.
123
+ */
109
124
  export function getSourceDefinition(source: LlmJsonlSource): SourceDefinition {
110
- return SOURCE_DEFINITIONS[source];
125
+ const definition = SOURCE_DEFINITIONS[source];
126
+ if (definition === undefined) {
127
+ throw new HistoryImportError(`Unknown LLM JSONL source: ${source}`, { source });
128
+ }
129
+ return definition;
130
+ }
131
+
132
+ /**
133
+ * Validate a caller-supplied source definition before it enters the pipeline.
134
+ *
135
+ * WHY: target table names are interpolated into SQL, so both the primary
136
+ * `targetTable` and any `splitConfig.targetTable` override must match
137
+ * {@link VALID_TABLE_NAME}. Required structural fields are checked here so the
138
+ * importer can assume they are present. Built-in definitions skip this — they
139
+ * are validated by construction.
140
+ */
141
+ export function validateSourceDefinition(definition: SourceDefinition): SourceDefinition {
142
+ const requiredFields: ReadonlyArray<keyof SourceDefinition> = [
143
+ 'source',
144
+ 'displayName',
145
+ 'defaultRoots',
146
+ 'filePatterns',
147
+ 'targetTable',
148
+ 'splitConfig',
149
+ 'fieldMap',
150
+ 'fieldTransforms',
151
+ 'schema',
152
+ ];
153
+ for (const field of requiredFields) {
154
+ if (definition[field] === undefined || definition[field] === null) {
155
+ throw new HistoryImportError(`Source definition is missing required field: ${String(field)}`, {
156
+ source: definition.source,
157
+ field: String(field),
158
+ });
159
+ }
160
+ }
161
+ if (definition.filePatterns.length === 0) {
162
+ throw new HistoryImportError('Source definition must declare at least one file pattern', {
163
+ source: definition.source,
164
+ field: 'filePatterns',
165
+ });
166
+ }
167
+ assertValidTable(definition.targetTable, definition.source);
168
+ if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
169
+ assertValidTable(definition.splitConfig.targetTable, definition.source);
170
+ }
171
+ return definition;
172
+ }
173
+
174
+ /**
175
+ * Resolve a `string | SourceDefinition` source argument into a validated
176
+ * {@link SourceDefinition}. Strings resolve from {@link SOURCE_DEFINITIONS}
177
+ * (unknown strings throw); objects are validated via {@link validateSourceDefinition}.
178
+ */
179
+ export function resolveSourceDefinition(input: string | SourceDefinition): SourceDefinition {
180
+ if (typeof input === 'string') {
181
+ return getSourceDefinition(input as LlmJsonlSource);
182
+ }
183
+ return validateSourceDefinition(input);
184
+ }
185
+
186
+ function assertValidTable(table: string, source: string): void {
187
+ if (!VALID_TABLE_NAME.test(table)) {
188
+ throw new HistoryImportError(
189
+ `Invalid history ETL target table for source "${source}": ${table}. Must match ${VALID_TABLE_NAME.source}.`,
190
+ { source, table },
191
+ );
192
+ }
111
193
  }
package/src/types.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { DbAdapter } from '@gobing-ai/ts-db';
2
- import type { FileSystem } from '@gobing-ai/ts-runtime';
2
+ import type { FileSystem, RuntimePaths } from '@gobing-ai/ts-runtime';
3
3
  import type { z } from 'zod';
4
4
 
5
5
  /** Built-in source identifiers supported by the importer. */
@@ -32,7 +32,8 @@ export type FieldTransform = (value: unknown, raw: JsonObject, context: Transfor
32
32
 
33
33
  /** Context available while normalizing a split record. */
34
34
  export interface TransformContext {
35
- readonly source: LlmJsonlSource;
35
+ /** Source identifier — a built-in {@link LlmJsonlSource} or a custom definition's name. */
36
+ readonly source: string;
36
37
  readonly sourceFile: string;
37
38
  readonly sourceLine: number;
38
39
  readonly splitIndex: number;
@@ -40,7 +41,8 @@ export interface TransformContext {
40
41
 
41
42
  /** Declarative importer configuration for one LLM history source. */
42
43
  export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
43
- readonly source: LlmJsonlSource;
44
+ /** Source identifier — a built-in {@link LlmJsonlSource} or a custom name. Becomes the checkpoint/ledger key. */
45
+ readonly source: string;
44
46
  readonly displayName: string;
45
47
  readonly defaultRoots: readonly string[];
46
48
  readonly filePatterns: readonly string[];
@@ -68,6 +70,12 @@ export interface ImportOptions {
68
70
  readonly dryRun?: boolean;
69
71
  readonly redactionRules?: readonly RedactionRule[];
70
72
  readonly now?: () => Date;
73
+ /**
74
+ * Optional cwd/home anchor (ADR-023 A1 / task 0042). When set, registry `defaultRoots`
75
+ * resolve against `paths.home` instead of the ambient working directory; explicit
76
+ * {@link ImportOptions.roots} keep cwd semantics unchanged. Defaults to ambient cwd/home.
77
+ */
78
+ readonly paths?: RuntimePaths;
71
79
  }
72
80
 
73
81
  /** Structured importer error that can be reported without persisting raw input. */
@@ -79,7 +87,8 @@ export interface ImportIssue {
79
87
 
80
88
  /** Summary produced by the importer control function. */
81
89
  export interface ImportResult {
82
- readonly source: LlmJsonlSource;
90
+ /** Source identifier — the built-in name or custom definition name that produced this result. */
91
+ readonly source: string;
83
92
  readonly mode: ImportMode;
84
93
  readonly scannedFiles: number;
85
94
  readonly processedLines: number;