@gobing-ai/ts-llm-jsonl-importer 0.4.14 → 0.4.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -8,10 +8,13 @@ Generic JSONL import pipeline for AI-agent history-style files: discover files,
8
8
 
9
9
  | Export | Purpose |
10
10
  |--------|---------|
11
- | `runJsonlImport()` | Runs discovery, parsing, validation, redaction, dedupe, and persistence |
11
+ | `runJsonlImport()` | Runs discovery, parsing, validation, redaction, dedupe, and persistence. Accepts a built-in source string **or** a custom `SourceDefinition`. |
12
12
  | `applyHistoryImportSchema()` | Installs importer-owned checkpoint, ledger, and ETL tables |
13
13
  | `SOURCE_DEFINITIONS` | Built-in source definitions |
14
- | `getSourceDefinition()` | Returns one source definition by key |
14
+ | `getSourceDefinition()` | Returns one built-in definition by key (throws `HistoryImportError` for unknown keys) |
15
+ | `resolveSourceDefinition()` | Resolves a `string \| SourceDefinition` into a validated definition |
16
+ | `validateSourceDefinition()` | Validates a custom definition (target-table names, required fields) before pipeline entry |
17
+ | `VALID_TABLE_NAME` | Regex governing every importer-owned ETL table name |
15
18
  | `redactRecord()` / `redactValue()` | Applies redaction rules before persistence |
16
19
  | `sha256()` / `stableJson()` | Stable hash helpers used by the ledger |
17
20
  | `HISTORY_IMPORT_SCHEMA_SQL` | SQL schema string for explicit migration flows |
@@ -138,6 +141,42 @@ const result = await runJsonlImport('pi', { db, files: ['session.jsonl'], mode:
138
141
 
139
142
  Downstream consumers should treat `source_file`, `source_line`, and `split_index` as the stable provenance tuple.
140
143
 
144
+ ## Custom Source Definitions
145
+
146
+ Beyond the built-in source keys, `runJsonlImport()` accepts a fully-specified `SourceDefinition`
147
+ for any custom agent history format. Unknown strings throw `HistoryImportError`; custom definitions
148
+ are validated before any I/O (target-table names must match `VALID_TABLE_NAME` = `/^history_etl_[a-z_]+$/`,
149
+ and required fields must be present).
150
+
151
+ ```ts
152
+ import { z } from 'zod';
153
+ import { createDbAdapter } from '@gobing-ai/ts-db';
154
+ import { runJsonlImport, type SourceDefinition } from '@gobing-ai/ts-llm-jsonl-importer';
155
+
156
+ const acme: SourceDefinition = {
157
+ source: 'acme',
158
+ displayName: 'Acme Assistant',
159
+ defaultRoots: ['.acme'],
160
+ filePatterns: ['*.jsonl'],
161
+ targetTable: 'history_etl_acme',
162
+ splitConfig: { mode: 'one-to-one' },
163
+ fieldMap: { id: 'source_record_id', timestamp: 'created_at', content: 'content' },
164
+ fieldTransforms: {},
165
+ schema: z.object({
166
+ source_record_id: z.string().min(1),
167
+ created_at: z.string().min(1),
168
+ content: z.string().min(1),
169
+ }),
170
+ };
171
+
172
+ const db = await createDbAdapter({ driver: 'bun-sqlite', url: './history.db' });
173
+ const result = await runJsonlImport(acme, { db, roots: ['~/.acme'], mode: 'incremental' });
174
+ ```
175
+
176
+ The custom `source` name flows through checkpoints, the ledger, and `ImportResult.source` exactly
177
+ like a built-in key. The importer creates the target ETL table on demand if it does not exist
178
+ (idempotent `CREATE TABLE IF NOT EXISTS`), so built-in tables are unaffected.
179
+
141
180
  ## Boundary Notes
142
181
 
143
182
  - This package imports JSONL files and writes importer-owned tables; it does not model conversations, turns, tool calls, or analytics semantics.
@@ -1,6 +1,14 @@
1
- import type { ImportOptions, ImportResult, LlmJsonlSource } from './types';
1
+ import type { ImportOptions, ImportResult, SourceDefinition } from './types';
2
2
  /** Apply importer-owned schema to the target database. */
3
3
  export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void>;
4
- /** Run the JSONL import pipeline for one source definition. */
5
- export declare function runJsonlImport(source: LlmJsonlSource, options: ImportOptions): Promise<ImportResult>;
4
+ /**
5
+ * Run the JSONL import pipeline for one source.
6
+ *
7
+ * @param source - Either a built-in source identifier resolved from the
8
+ * registry, or a fully-specified {@link SourceDefinition} for a custom
9
+ * source. Unknown strings throw {@link HistoryImportError}; custom
10
+ * definitions are validated before any I/O.
11
+ * @param options - Importer options (database, roots/files, mode, etc.).
12
+ */
13
+ export declare function runJsonlImport(source: string | SourceDefinition, options: ImportOptions): Promise<ImportResult>;
6
14
  //# sourceMappingURL=importer.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAER,aAAa,EACb,YAAY,EAEZ,cAAc,EAGjB,MAAM,SAAS,CAAC;AAajB,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED,+DAA+D;AAC/D,wBAAsB,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA4G1G"}
1
+ {"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAWxH,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAiCD;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA8GrH"}
package/dist/importer.js CHANGED
@@ -1,10 +1,9 @@
1
- import { createNodeFileSystem, resolvePath, walkDir } from '@gobing-ai/ts-runtime';
1
+ import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
2
2
  import { HistoryImportError } from './errors.js';
3
3
  import { sha256 } from './hash.js';
4
4
  import { redactRecord } from './redaction.js';
5
5
  import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
6
- import { getSourceDefinition } from './sources.js';
7
- const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
6
+ import { resolveSourceDefinition, VALID_TABLE_NAME } from './sources.js';
8
7
  /** Apply importer-owned schema to the target database. */
9
8
  export async function applyHistoryImportSchema(db) {
10
9
  for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
@@ -14,15 +13,55 @@ export async function applyHistoryImportSchema(db) {
14
13
  }
15
14
  }
16
15
  }
17
- /** Run the JSONL import pipeline for one source definition. */
16
+ /**
17
+ * Ensure the ETL table(s) for a source definition exist.
18
+ *
19
+ * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
20
+ * `history_etl_*` tables. Custom source definitions (and built-in definitions
21
+ * with a `splitConfig.targetTable` override) need their target table(s) created
22
+ * on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
23
+ * {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
24
+ * interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
25
+ * built-in tables that the static schema already created.
26
+ */
27
+ async function ensureTargetTables(db, definition) {
28
+ const tables = new Set([targetTableFor(definition.targetTable)]);
29
+ if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
30
+ tables.add(targetTableFor(definition.splitConfig.targetTable));
31
+ }
32
+ for (const table of tables) {
33
+ await db.exec(ETL_TABLE_DDL(table));
34
+ }
35
+ }
36
+ function ETL_TABLE_DDL(table) {
37
+ return `CREATE TABLE IF NOT EXISTS ${table} (
38
+ record_hash TEXT PRIMARY KEY,
39
+ source_file TEXT NOT NULL,
40
+ source_line INTEGER NOT NULL,
41
+ split_index INTEGER NOT NULL,
42
+ payload_json TEXT NOT NULL,
43
+ imported_at TEXT NOT NULL
44
+ );`;
45
+ }
46
+ /**
47
+ * Run the JSONL import pipeline for one source.
48
+ *
49
+ * @param source - Either a built-in source identifier resolved from the
50
+ * registry, or a fully-specified {@link SourceDefinition} for a custom
51
+ * source. Unknown strings throw {@link HistoryImportError}; custom
52
+ * definitions are validated before any I/O.
53
+ * @param options - Importer options (database, roots/files, mode, etc.).
54
+ */
18
55
  export async function runJsonlImport(source, options) {
19
- const definition = getSourceDefinition(source);
56
+ const definition = resolveSourceDefinition(source);
57
+ const resolvedSource = definition.source;
20
58
  const fileSystem = options.fileSystem ?? createNodeFileSystem();
21
59
  await applyHistoryImportSchema(options.db);
60
+ await ensureTargetTables(options.db, definition);
22
61
  const mode = options.mode ?? 'incremental';
23
- const files = await discoverFiles(definition, options.roots, options.files, fileSystem);
62
+ const files = await discoverFiles(definition, options.roots, options.files, fileSystem, options.paths);
24
63
  if (mode === 'full' && !options.dryRun) {
25
- await resetCheckpoints(options.db, source, files);
64
+ await resetCheckpoints(options.db, resolvedSource, files);
26
65
  }
27
66
  const parseErrors = [];
28
67
  const validationErrors = [];
@@ -31,7 +70,7 @@ export async function runJsonlImport(source, options) {
31
70
  let skippedDuplicates = 0;
32
71
  let checkpointUpdates = 0;
33
72
  for (const file of files) {
34
- const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, source, file) : 0;
73
+ const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
35
74
  let lineNumber = 0;
36
75
  for await (const rawLine of readLines(fileSystem, file)) {
37
76
  lineNumber += 1;
@@ -49,7 +88,7 @@ export async function runJsonlImport(source, options) {
49
88
  if (split === undefined)
50
89
  continue;
51
90
  const normalized = normalizeRecord(definition, split.raw, {
52
- source,
91
+ source: resolvedSource,
53
92
  sourceFile: file,
54
93
  sourceLine: lineNumber,
55
94
  splitIndex,
@@ -66,7 +105,7 @@ export async function runJsonlImport(source, options) {
66
105
  const redacted = redactRecord(parsed.data, options.redactionRules);
67
106
  lineSucceeded = true;
68
107
  const recordHash = sha256({
69
- source,
108
+ source: resolvedSource,
70
109
  sourceFile: file,
71
110
  sourceLine: lineNumber,
72
111
  splitIndex,
@@ -78,18 +117,18 @@ export async function runJsonlImport(source, options) {
78
117
  }
79
118
  if (!options.dryRun) {
80
119
  await insertRecord(options.db, split.targetTable, recordHash, file, lineNumber, splitIndex, redacted, options.now);
81
- await insertLedger(options.db, recordHash, source, file, lineNumber, splitIndex, split.targetTable, options.now);
120
+ await insertLedger(options.db, recordHash, resolvedSource, file, lineNumber, splitIndex, split.targetTable, options.now);
82
121
  }
83
122
  importedRecords += 1;
84
123
  }
85
124
  if (lineSucceeded && !options.dryRun) {
86
- await writeCheckpoint(options.db, source, file, lineNumber, options.now);
125
+ await writeCheckpoint(options.db, resolvedSource, file, lineNumber, options.now);
87
126
  checkpointUpdates += 1;
88
127
  }
89
128
  }
90
129
  }
91
130
  return {
92
- source,
131
+ source: resolvedSource,
93
132
  mode,
94
133
  scannedFiles: files.length,
95
134
  processedLines,
@@ -153,11 +192,20 @@ function normalizeRecord(definition, raw, context) {
153
192
  normalized.split_index = context.splitIndex;
154
193
  return normalized;
155
194
  }
156
- async function discoverFiles(definition, roots, files, fileSystem) {
195
+ async function discoverFiles(definition, roots, files, fileSystem, paths) {
157
196
  if (files !== undefined && files.length > 0) {
158
197
  return files.map((file) => resolvePath(file)).sort();
159
198
  }
160
- const resolvedRoots = (roots ?? definition.defaultRoots).map((root) => resolvePath(root));
199
+ // Root resolution splits by provenance (ADR-023 A1 / task 0042):
200
+ // - explicit caller-supplied `roots` → cwd semantics: resolve against `paths.cwd` when
201
+ // injected, else ambient cwd (the pre-seam behaviour — R4 additive-only).
202
+ // - registry `defaultRoots` → home-relative: resolve against `paths.home`.
203
+ // Previously both kinds anchored to ambient cwd, so a cwd ≠ $HOME silently skipped every
204
+ // registry source because the home-relative paths never existed relative to cwd.
205
+ const ambient = paths ?? ambientRuntimePaths();
206
+ const resolvedRoots = roots !== undefined
207
+ ? roots.map((root) => resolvePath(ambient.cwd, root))
208
+ : definition.defaultRoots.map((root) => resolvePath(ambient.home, root));
161
209
  const found = new Set();
162
210
  for (const root of resolvedRoots) {
163
211
  if (!(await fileSystem.exists(root)))
package/dist/index.d.ts CHANGED
@@ -3,6 +3,6 @@ export { sha256, stableJson } from './hash';
3
3
  export { applyHistoryImportSchema, runJsonlImport } from './importer';
4
4
  export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
5
5
  export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
6
- export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources';
6
+ export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
7
7
  export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, RedactionRule, SourceDefinition, SplitConfig, TransformContext, } from './types';
8
8
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACpE,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
package/dist/index.js CHANGED
@@ -3,4 +3,4 @@ export { sha256, stableJson } from './hash.js';
3
3
  export { applyHistoryImportSchema, runJsonlImport } from './importer.js';
4
4
  export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
5
5
  export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
6
- export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources.js';
6
+ export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources.js';
package/dist/sources.d.ts CHANGED
@@ -1,6 +1,36 @@
1
1
  import type { LlmJsonlSource, SourceDefinition } from './types';
2
+ /**
3
+ * Pattern every importer-owned table must match.
4
+ *
5
+ * WHY: the table name is interpolated into generated SQL (it is not a bind
6
+ * parameter), so it must be a safe identifier. The `history_etl_` prefix also
7
+ * namespaces importer tables away from consumer schemas. Both built-in and
8
+ * custom source definitions are held to this rule.
9
+ */
10
+ export declare const VALID_TABLE_NAME: RegExp;
2
11
  /** Built-in source definitions keyed by source identifier. */
3
12
  export declare const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinition>>;
4
- /** Resolve a built-in source definition by identifier. */
13
+ /**
14
+ * Resolve a built-in source definition by identifier.
15
+ *
16
+ * Throws {@link HistoryImportError} for unknown identifiers so misconfigured
17
+ * callers fail fast at the seam instead of producing empty imports.
18
+ */
5
19
  export declare function getSourceDefinition(source: LlmJsonlSource): SourceDefinition;
20
+ /**
21
+ * Validate a caller-supplied source definition before it enters the pipeline.
22
+ *
23
+ * WHY: target table names are interpolated into SQL, so both the primary
24
+ * `targetTable` and any `splitConfig.targetTable` override must match
25
+ * {@link VALID_TABLE_NAME}. Required structural fields are checked here so the
26
+ * importer can assume they are present. Built-in definitions skip this — they
27
+ * are validated by construction.
28
+ */
29
+ export declare function validateSourceDefinition(definition: SourceDefinition): SourceDefinition;
30
+ /**
31
+ * Resolve a `string | SourceDefinition` source argument into a validated
32
+ * {@link SourceDefinition}. Strings resolve from {@link SOURCE_DEFINITIONS}
33
+ * (unknown strings throw); objects are validated via {@link validateSourceDefinition}.
34
+ */
35
+ export declare function resolveSourceDefinition(input: string | SourceDefinition): SourceDefinition;
6
36
  //# sourceMappingURL=sources.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AA+F9G,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAQjF,CAAC;AAEF,0DAA0D;AAC1D,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAE5E"}
1
+ {"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AA8F9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAA0B,CAAC;AAExD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAQjF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAM5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,GAAG,gBAAgB,CA+BvF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,gBAAgB,CAK1F"}
package/dist/sources.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { z } from 'zod';
2
+ import { HistoryImportError } from './errors.js';
2
3
  const sourceRecordSchema = z
3
4
  .object({
4
5
  source_record_id: z.string().min(1),
@@ -71,6 +72,15 @@ function sourceDefinition(source, displayName, defaultRoots, filePatterns) {
71
72
  schema: sourceRecordSchema,
72
73
  };
73
74
  }
75
+ /**
76
+ * Pattern every importer-owned table must match.
77
+ *
78
+ * WHY: the table name is interpolated into generated SQL (it is not a bind
79
+ * parameter), so it must be a safe identifier. The `history_etl_` prefix also
80
+ * namespaces importer tables away from consumer schemas. Both built-in and
81
+ * custom source definitions are held to this rule.
82
+ */
83
+ export const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
74
84
  /** Built-in source definitions keyed by source identifier. */
75
85
  export const SOURCE_DEFINITIONS = {
76
86
  pi: sourceDefinition('pi', 'Pi', ['.pi/history', '.pi'], ['*.jsonl', '*.json']),
@@ -81,7 +91,73 @@ export const SOURCE_DEFINITIONS = {
81
91
  antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
82
92
  openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
83
93
  };
84
- /** Resolve a built-in source definition by identifier. */
94
+ /**
95
+ * Resolve a built-in source definition by identifier.
96
+ *
97
+ * Throws {@link HistoryImportError} for unknown identifiers so misconfigured
98
+ * callers fail fast at the seam instead of producing empty imports.
99
+ */
85
100
  export function getSourceDefinition(source) {
86
- return SOURCE_DEFINITIONS[source];
101
+ const definition = SOURCE_DEFINITIONS[source];
102
+ if (definition === undefined) {
103
+ throw new HistoryImportError(`Unknown LLM JSONL source: ${source}`, { source });
104
+ }
105
+ return definition;
106
+ }
107
+ /**
108
+ * Validate a caller-supplied source definition before it enters the pipeline.
109
+ *
110
+ * WHY: target table names are interpolated into SQL, so both the primary
111
+ * `targetTable` and any `splitConfig.targetTable` override must match
112
+ * {@link VALID_TABLE_NAME}. Required structural fields are checked here so the
113
+ * importer can assume they are present. Built-in definitions skip this — they
114
+ * are validated by construction.
115
+ */
116
+ export function validateSourceDefinition(definition) {
117
+ const requiredFields = [
118
+ 'source',
119
+ 'displayName',
120
+ 'defaultRoots',
121
+ 'filePatterns',
122
+ 'targetTable',
123
+ 'splitConfig',
124
+ 'fieldMap',
125
+ 'fieldTransforms',
126
+ 'schema',
127
+ ];
128
+ for (const field of requiredFields) {
129
+ if (definition[field] === undefined || definition[field] === null) {
130
+ throw new HistoryImportError(`Source definition is missing required field: ${String(field)}`, {
131
+ source: definition.source,
132
+ field: String(field),
133
+ });
134
+ }
135
+ }
136
+ if (definition.filePatterns.length === 0) {
137
+ throw new HistoryImportError('Source definition must declare at least one file pattern', {
138
+ source: definition.source,
139
+ field: 'filePatterns',
140
+ });
141
+ }
142
+ assertValidTable(definition.targetTable, definition.source);
143
+ if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
144
+ assertValidTable(definition.splitConfig.targetTable, definition.source);
145
+ }
146
+ return definition;
147
+ }
148
+ /**
149
+ * Resolve a `string | SourceDefinition` source argument into a validated
150
+ * {@link SourceDefinition}. Strings resolve from {@link SOURCE_DEFINITIONS}
151
+ * (unknown strings throw); objects are validated via {@link validateSourceDefinition}.
152
+ */
153
+ export function resolveSourceDefinition(input) {
154
+ if (typeof input === 'string') {
155
+ return getSourceDefinition(input);
156
+ }
157
+ return validateSourceDefinition(input);
158
+ }
159
+ function assertValidTable(table, source) {
160
+ if (!VALID_TABLE_NAME.test(table)) {
161
+ throw new HistoryImportError(`Invalid history ETL target table for source "${source}": ${table}. Must match ${VALID_TABLE_NAME.source}.`, { source, table });
162
+ }
87
163
  }
package/dist/types.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { DbAdapter } from '@gobing-ai/ts-db';
2
- import type { FileSystem } from '@gobing-ai/ts-runtime';
2
+ import type { FileSystem, RuntimePaths } from '@gobing-ai/ts-runtime';
3
3
  import type { z } from 'zod';
4
4
  /** Built-in source identifiers supported by the importer. */
5
5
  export type LlmJsonlSource = 'pi' | 'claude' | 'codex' | 'gemini' | 'opencode' | 'antigravity' | 'openclaw';
@@ -23,14 +23,16 @@ export type JsonObject = Record<string, unknown>;
23
23
  export type FieldTransform = (value: unknown, raw: JsonObject, context: TransformContext) => unknown;
24
24
  /** Context available while normalizing a split record. */
25
25
  export interface TransformContext {
26
- readonly source: LlmJsonlSource;
26
+ /** Source identifier — a built-in {@link LlmJsonlSource} or a custom definition's name. */
27
+ readonly source: string;
27
28
  readonly sourceFile: string;
28
29
  readonly sourceLine: number;
29
30
  readonly splitIndex: number;
30
31
  }
31
32
  /** Declarative importer configuration for one LLM history source. */
32
33
  export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
33
- readonly source: LlmJsonlSource;
34
+ /** Source identifier — a built-in {@link LlmJsonlSource} or a custom name. Becomes the checkpoint/ledger key. */
35
+ readonly source: string;
34
36
  readonly displayName: string;
35
37
  readonly defaultRoots: readonly string[];
36
38
  readonly filePatterns: readonly string[];
@@ -56,6 +58,12 @@ export interface ImportOptions {
56
58
  readonly dryRun?: boolean;
57
59
  readonly redactionRules?: readonly RedactionRule[];
58
60
  readonly now?: () => Date;
61
+ /**
62
+ * Optional cwd/home anchor (ADR-023 A1 / task 0042). When set, registry `defaultRoots`
63
+ * resolve against `paths.home` instead of the ambient working directory; explicit
64
+ * {@link ImportOptions.roots} keep cwd semantics unchanged. Defaults to ambient cwd/home.
65
+ */
66
+ readonly paths?: RuntimePaths;
59
67
  }
60
68
  /** Structured importer error that can be reported without persisting raw input. */
61
69
  export interface ImportIssue {
@@ -65,7 +73,8 @@ export interface ImportIssue {
65
73
  }
66
74
  /** Summary produced by the importer control function. */
67
75
  export interface ImportResult {
68
- readonly source: LlmJsonlSource;
76
+ /** Source identifier — the built-in name or custom definition name that produced this result. */
77
+ readonly source: string;
69
78
  readonly mode: ImportMode;
70
79
  readonly scannedFiles: number;
71
80
  readonly processedLines: number;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC;AAE5G,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,SAAS,UAAU,EAAE,CAAC;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAC7B;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACtC"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC;AAE5G,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,SAAS,UAAU,EAAE,CAAC;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACtC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gobing-ai/ts-llm-jsonl-importer",
3
- "version": "0.4.14",
3
+ "version": "0.4.15",
4
4
  "description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
5
5
  "keywords": [
6
6
  "typescript",
@@ -47,9 +47,9 @@
47
47
  "release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
48
48
  },
49
49
  "dependencies": {
50
- "@gobing-ai/ts-db": "^0.4.14",
51
- "@gobing-ai/ts-runtime": "^0.4.14",
52
- "@gobing-ai/ts-utils": "^0.4.14",
50
+ "@gobing-ai/ts-db": "^0.4.15",
51
+ "@gobing-ai/ts-runtime": "^0.4.15",
52
+ "@gobing-ai/ts-utils": "^0.4.15",
53
53
  "zod": "^4.1.0"
54
54
  },
55
55
  "devDependencies": {
package/src/importer.ts CHANGED
@@ -1,18 +1,17 @@
1
- import { createNodeFileSystem, type FileSystem, resolvePath, walkDir } from '@gobing-ai/ts-runtime';
1
+ import {
2
+ ambientRuntimePaths,
3
+ createNodeFileSystem,
4
+ type FileSystem,
5
+ type RuntimePaths,
6
+ resolvePath,
7
+ walkDir,
8
+ } from '@gobing-ai/ts-runtime';
2
9
  import { HistoryImportError } from './errors';
3
10
  import { sha256 } from './hash';
4
11
  import { redactRecord } from './redaction';
5
12
  import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
6
- import { getSourceDefinition } from './sources';
7
- import type {
8
- ImportIssue,
9
- ImportOptions,
10
- ImportResult,
11
- JsonObject,
12
- LlmJsonlSource,
13
- SourceDefinition,
14
- TransformContext,
15
- } from './types';
13
+ import { resolveSourceDefinition, VALID_TABLE_NAME } from './sources';
14
+ import type { ImportIssue, ImportOptions, ImportResult, JsonObject, SourceDefinition, TransformContext } from './types';
16
15
 
17
16
  interface SplitRecord {
18
17
  readonly targetTable: string;
@@ -23,8 +22,6 @@ interface CheckpointRow {
23
22
  readonly last_imported_line: number;
24
23
  }
25
24
 
26
- const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
27
-
28
25
  /** Apply importer-owned schema to the target database. */
29
26
  export async function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void> {
30
27
  for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
@@ -35,16 +32,57 @@ export async function applyHistoryImportSchema(db: ImportOptions['db']): Promise
35
32
  }
36
33
  }
37
34
 
38
- /** Run the JSONL import pipeline for one source definition. */
39
- export async function runJsonlImport(source: LlmJsonlSource, options: ImportOptions): Promise<ImportResult> {
40
- const definition = getSourceDefinition(source);
35
+ /**
36
+ * Ensure the ETL table(s) for a source definition exist.
37
+ *
38
+ * WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
39
+ * `history_etl_*` tables. Custom source definitions (and built-in definitions
40
+ * with a `splitConfig.targetTable` override) need their target table(s) created
41
+ * on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
42
+ * {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
43
+ * interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
44
+ * built-in tables that the static schema already created.
45
+ */
46
+ async function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void> {
47
+ const tables = new Set<string>([targetTableFor(definition.targetTable)]);
48
+ if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
49
+ tables.add(targetTableFor(definition.splitConfig.targetTable));
50
+ }
51
+ for (const table of tables) {
52
+ await db.exec(ETL_TABLE_DDL(table));
53
+ }
54
+ }
55
+
56
+ function ETL_TABLE_DDL(table: string): string {
57
+ return `CREATE TABLE IF NOT EXISTS ${table} (
58
+ record_hash TEXT PRIMARY KEY,
59
+ source_file TEXT NOT NULL,
60
+ source_line INTEGER NOT NULL,
61
+ split_index INTEGER NOT NULL,
62
+ payload_json TEXT NOT NULL,
63
+ imported_at TEXT NOT NULL
64
+ );`;
65
+ }
66
+ /**
67
+ * Run the JSONL import pipeline for one source.
68
+ *
69
+ * @param source - Either a built-in source identifier resolved from the
70
+ * registry, or a fully-specified {@link SourceDefinition} for a custom
71
+ * source. Unknown strings throw {@link HistoryImportError}; custom
72
+ * definitions are validated before any I/O.
73
+ * @param options - Importer options (database, roots/files, mode, etc.).
74
+ */
75
+ export async function runJsonlImport(source: string | SourceDefinition, options: ImportOptions): Promise<ImportResult> {
76
+ const definition = resolveSourceDefinition(source);
77
+ const resolvedSource = definition.source;
41
78
  const fileSystem = options.fileSystem ?? createNodeFileSystem();
42
79
  await applyHistoryImportSchema(options.db);
80
+ await ensureTargetTables(options.db, definition);
43
81
 
44
82
  const mode = options.mode ?? 'incremental';
45
- const files = await discoverFiles(definition, options.roots, options.files, fileSystem);
83
+ const files = await discoverFiles(definition, options.roots, options.files, fileSystem, options.paths);
46
84
  if (mode === 'full' && !options.dryRun) {
47
- await resetCheckpoints(options.db, source, files);
85
+ await resetCheckpoints(options.db, resolvedSource, files);
48
86
  }
49
87
 
50
88
  const parseErrors: ImportIssue[] = [];
@@ -55,7 +93,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
55
93
  let checkpointUpdates = 0;
56
94
 
57
95
  for (const file of files) {
58
- const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, source, file) : 0;
96
+ const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
59
97
  let lineNumber = 0;
60
98
  for await (const rawLine of readLines(fileSystem, file)) {
61
99
  lineNumber += 1;
@@ -73,7 +111,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
73
111
  const split = splitRecords[splitIndex];
74
112
  if (split === undefined) continue;
75
113
  const normalized = normalizeRecord(definition, split.raw, {
76
- source,
114
+ source: resolvedSource,
77
115
  sourceFile: file,
78
116
  sourceLine: lineNumber,
79
117
  splitIndex,
@@ -91,7 +129,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
91
129
  const redacted = redactRecord(parsed.data, options.redactionRules);
92
130
  lineSucceeded = true;
93
131
  const recordHash = sha256({
94
- source,
132
+ source: resolvedSource,
95
133
  sourceFile: file,
96
134
  sourceLine: lineNumber,
97
135
  splitIndex,
@@ -115,7 +153,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
115
153
  await insertLedger(
116
154
  options.db,
117
155
  recordHash,
118
- source,
156
+ resolvedSource,
119
157
  file,
120
158
  lineNumber,
121
159
  splitIndex,
@@ -127,14 +165,14 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
127
165
  }
128
166
 
129
167
  if (lineSucceeded && !options.dryRun) {
130
- await writeCheckpoint(options.db, source, file, lineNumber, options.now);
168
+ await writeCheckpoint(options.db, resolvedSource, file, lineNumber, options.now);
131
169
  checkpointUpdates += 1;
132
170
  }
133
171
  }
134
172
  }
135
173
 
136
174
  return {
137
- source,
175
+ source: resolvedSource,
138
176
  mode,
139
177
  scannedFiles: files.length,
140
178
  processedLines,
@@ -211,12 +249,23 @@ async function discoverFiles(
211
249
  roots: readonly string[] | undefined,
212
250
  files: readonly string[] | undefined,
213
251
  fileSystem: FileSystem,
252
+ paths: RuntimePaths | undefined,
214
253
  ): Promise<readonly string[]> {
215
254
  if (files !== undefined && files.length > 0) {
216
255
  return files.map((file) => resolvePath(file)).sort();
217
256
  }
218
257
 
219
- const resolvedRoots = (roots ?? definition.defaultRoots).map((root) => resolvePath(root));
258
+ // Root resolution splits by provenance (ADR-023 A1 / task 0042):
259
+ // - explicit caller-supplied `roots` → cwd semantics: resolve against `paths.cwd` when
260
+ // injected, else ambient cwd (the pre-seam behaviour — R4 additive-only).
261
+ // - registry `defaultRoots` → home-relative: resolve against `paths.home`.
262
+ // Previously both kinds anchored to ambient cwd, so a cwd ≠ $HOME silently skipped every
263
+ // registry source because the home-relative paths never existed relative to cwd.
264
+ const ambient = paths ?? ambientRuntimePaths();
265
+ const resolvedRoots =
266
+ roots !== undefined
267
+ ? roots.map((root) => resolvePath(ambient.cwd, root))
268
+ : definition.defaultRoots.map((root) => resolvePath(ambient.home, root));
220
269
  const found = new Set<string>();
221
270
  for (const root of resolvedRoots) {
222
271
  if (!(await fileSystem.exists(root))) continue;
package/src/index.ts CHANGED
@@ -3,7 +3,13 @@ export { sha256, stableJson } from './hash';
3
3
  export { applyHistoryImportSchema, runJsonlImport } from './importer';
4
4
  export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
5
5
  export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
6
- export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources';
6
+ export {
7
+ getSourceDefinition,
8
+ resolveSourceDefinition,
9
+ SOURCE_DEFINITIONS,
10
+ VALID_TABLE_NAME,
11
+ validateSourceDefinition,
12
+ } from './sources';
7
13
  export type {
8
14
  FieldTransform,
9
15
  ImportIssue,
package/src/sources.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { z } from 'zod';
2
+ import { HistoryImportError } from './errors';
2
3
  import type { FieldTransform, JsonObject, LlmJsonlSource, SourceDefinition, TransformContext } from './types';
3
4
 
4
5
  const sourceRecordSchema = z
@@ -93,6 +94,15 @@ function sourceDefinition(
93
94
  schema: sourceRecordSchema,
94
95
  };
95
96
  }
97
+ /**
98
+ * Pattern every importer-owned table must match.
99
+ *
100
+ * WHY: the table name is interpolated into generated SQL (it is not a bind
101
+ * parameter), so it must be a safe identifier. The `history_etl_` prefix also
102
+ * namespaces importer tables away from consumer schemas. Both built-in and
103
+ * custom source definitions are held to this rule.
104
+ */
105
+ export const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
96
106
 
97
107
  /** Built-in source definitions keyed by source identifier. */
98
108
  export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinition>> = {
@@ -105,7 +115,79 @@ export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinitio
105
115
  openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
106
116
  };
107
117
 
108
- /** Resolve a built-in source definition by identifier. */
118
+ /**
119
+ * Resolve a built-in source definition by identifier.
120
+ *
121
+ * Throws {@link HistoryImportError} for unknown identifiers so misconfigured
122
+ * callers fail fast at the seam instead of producing empty imports.
123
+ */
109
124
  export function getSourceDefinition(source: LlmJsonlSource): SourceDefinition {
110
- return SOURCE_DEFINITIONS[source];
125
+ const definition = SOURCE_DEFINITIONS[source];
126
+ if (definition === undefined) {
127
+ throw new HistoryImportError(`Unknown LLM JSONL source: ${source}`, { source });
128
+ }
129
+ return definition;
130
+ }
131
+
132
+ /**
133
+ * Validate a caller-supplied source definition before it enters the pipeline.
134
+ *
135
+ * WHY: target table names are interpolated into SQL, so both the primary
136
+ * `targetTable` and any `splitConfig.targetTable` override must match
137
+ * {@link VALID_TABLE_NAME}. Required structural fields are checked here so the
138
+ * importer can assume they are present. Built-in definitions skip this — they
139
+ * are validated by construction.
140
+ */
141
+ export function validateSourceDefinition(definition: SourceDefinition): SourceDefinition {
142
+ const requiredFields: ReadonlyArray<keyof SourceDefinition> = [
143
+ 'source',
144
+ 'displayName',
145
+ 'defaultRoots',
146
+ 'filePatterns',
147
+ 'targetTable',
148
+ 'splitConfig',
149
+ 'fieldMap',
150
+ 'fieldTransforms',
151
+ 'schema',
152
+ ];
153
+ for (const field of requiredFields) {
154
+ if (definition[field] === undefined || definition[field] === null) {
155
+ throw new HistoryImportError(`Source definition is missing required field: ${String(field)}`, {
156
+ source: definition.source,
157
+ field: String(field),
158
+ });
159
+ }
160
+ }
161
+ if (definition.filePatterns.length === 0) {
162
+ throw new HistoryImportError('Source definition must declare at least one file pattern', {
163
+ source: definition.source,
164
+ field: 'filePatterns',
165
+ });
166
+ }
167
+ assertValidTable(definition.targetTable, definition.source);
168
+ if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
169
+ assertValidTable(definition.splitConfig.targetTable, definition.source);
170
+ }
171
+ return definition;
172
+ }
173
+
174
+ /**
175
+ * Resolve a `string | SourceDefinition` source argument into a validated
176
+ * {@link SourceDefinition}. Strings resolve from {@link SOURCE_DEFINITIONS}
177
+ * (unknown strings throw); objects are validated via {@link validateSourceDefinition}.
178
+ */
179
+ export function resolveSourceDefinition(input: string | SourceDefinition): SourceDefinition {
180
+ if (typeof input === 'string') {
181
+ return getSourceDefinition(input as LlmJsonlSource);
182
+ }
183
+ return validateSourceDefinition(input);
184
+ }
185
+
186
+ function assertValidTable(table: string, source: string): void {
187
+ if (!VALID_TABLE_NAME.test(table)) {
188
+ throw new HistoryImportError(
189
+ `Invalid history ETL target table for source "${source}": ${table}. Must match ${VALID_TABLE_NAME.source}.`,
190
+ { source, table },
191
+ );
192
+ }
111
193
  }
package/src/types.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { DbAdapter } from '@gobing-ai/ts-db';
2
- import type { FileSystem } from '@gobing-ai/ts-runtime';
2
+ import type { FileSystem, RuntimePaths } from '@gobing-ai/ts-runtime';
3
3
  import type { z } from 'zod';
4
4
 
5
5
  /** Built-in source identifiers supported by the importer. */
@@ -32,7 +32,8 @@ export type FieldTransform = (value: unknown, raw: JsonObject, context: Transfor
32
32
 
33
33
  /** Context available while normalizing a split record. */
34
34
  export interface TransformContext {
35
- readonly source: LlmJsonlSource;
35
+ /** Source identifier — a built-in {@link LlmJsonlSource} or a custom definition's name. */
36
+ readonly source: string;
36
37
  readonly sourceFile: string;
37
38
  readonly sourceLine: number;
38
39
  readonly splitIndex: number;
@@ -40,7 +41,8 @@ export interface TransformContext {
40
41
 
41
42
  /** Declarative importer configuration for one LLM history source. */
42
43
  export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
43
- readonly source: LlmJsonlSource;
44
+ /** Source identifier — a built-in {@link LlmJsonlSource} or a custom name. Becomes the checkpoint/ledger key. */
45
+ readonly source: string;
44
46
  readonly displayName: string;
45
47
  readonly defaultRoots: readonly string[];
46
48
  readonly filePatterns: readonly string[];
@@ -68,6 +70,12 @@ export interface ImportOptions {
68
70
  readonly dryRun?: boolean;
69
71
  readonly redactionRules?: readonly RedactionRule[];
70
72
  readonly now?: () => Date;
73
+ /**
74
+ * Optional cwd/home anchor (ADR-023 A1 / task 0042). When set, registry `defaultRoots`
75
+ * resolve against `paths.home` instead of the ambient working directory; explicit
76
+ * {@link ImportOptions.roots} keep cwd semantics unchanged. Defaults to ambient cwd/home.
77
+ */
78
+ readonly paths?: RuntimePaths;
71
79
  }
72
80
 
73
81
  /** Structured importer error that can be reported without persisting raw input. */
@@ -79,7 +87,8 @@ export interface ImportIssue {
79
87
 
80
88
  /** Summary produced by the importer control function. */
81
89
  export interface ImportResult {
82
- readonly source: LlmJsonlSource;
90
+ /** Source identifier — the built-in name or custom definition name that produced this result. */
91
+ readonly source: string;
83
92
  readonly mode: ImportMode;
84
93
  readonly scannedFiles: number;
85
94
  readonly processedLines: number;