@gobing-ai/ts-llm-jsonl-importer 0.4.14 → 0.4.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -2
- package/dist/importer.d.ts +10 -5
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +33 -65
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/jsonl-importer-dao.d.ts +30 -0
- package/dist/jsonl-importer-dao.d.ts.map +1 -0
- package/dist/jsonl-importer-dao.js +82 -0
- package/dist/sources.d.ts +31 -1
- package/dist/sources.d.ts.map +1 -1
- package/dist/sources.js +78 -2
- package/dist/types.d.ts +13 -4
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/importer.ts +53 -142
- package/src/index.ts +9 -2
- package/src/jsonl-importer-dao.ts +165 -0
- package/src/sources.ts +84 -2
- package/src/types.ts +13 -4
package/README.md
CHANGED
|
@@ -8,10 +8,13 @@ Generic JSONL import pipeline for AI-agent history-style files: discover files,
|
|
|
8
8
|
|
|
9
9
|
| Export | Purpose |
|
|
10
10
|
|--------|---------|
|
|
11
|
-
| `runJsonlImport()` | Runs discovery, parsing, validation, redaction, dedupe, and persistence |
|
|
11
|
+
| `runJsonlImport()` | Runs discovery, parsing, validation, redaction, dedupe, and persistence. Accepts a built-in source string **or** a custom `SourceDefinition`. |
|
|
12
12
|
| `applyHistoryImportSchema()` | Installs importer-owned checkpoint, ledger, and ETL tables |
|
|
13
13
|
| `SOURCE_DEFINITIONS` | Built-in source definitions |
|
|
14
|
-
| `getSourceDefinition()` | Returns one
|
|
14
|
+
| `getSourceDefinition()` | Returns one built-in definition by key (throws `HistoryImportError` for unknown keys) |
|
|
15
|
+
| `resolveSourceDefinition()` | Resolves a `string \| SourceDefinition` into a validated definition |
|
|
16
|
+
| `validateSourceDefinition()` | Validates a custom definition (target-table names, required fields) before pipeline entry |
|
|
17
|
+
| `VALID_TABLE_NAME` | Regex governing every importer-owned ETL table name |
|
|
15
18
|
| `redactRecord()` / `redactValue()` | Applies redaction rules before persistence |
|
|
16
19
|
| `sha256()` / `stableJson()` | Stable hash helpers used by the ledger |
|
|
17
20
|
| `HISTORY_IMPORT_SCHEMA_SQL` | SQL schema string for explicit migration flows |
|
|
@@ -138,6 +141,42 @@ const result = await runJsonlImport('pi', { db, files: ['session.jsonl'], mode:
|
|
|
138
141
|
|
|
139
142
|
Downstream consumers should treat `source_file`, `source_line`, and `split_index` as the stable provenance tuple.
|
|
140
143
|
|
|
144
|
+
## Custom Source Definitions
|
|
145
|
+
|
|
146
|
+
Beyond the built-in source keys, `runJsonlImport()` accepts a fully-specified `SourceDefinition`
|
|
147
|
+
for any custom agent history format. Unknown strings throw `HistoryImportError`; custom definitions
|
|
148
|
+
are validated before any I/O (target-table names must match `VALID_TABLE_NAME` = `/^history_etl_[a-z_]+$/`,
|
|
149
|
+
and required fields must be present).
|
|
150
|
+
|
|
151
|
+
```ts
|
|
152
|
+
import { z } from 'zod';
|
|
153
|
+
import { createDbAdapter } from '@gobing-ai/ts-db';
|
|
154
|
+
import { runJsonlImport, type SourceDefinition } from '@gobing-ai/ts-llm-jsonl-importer';
|
|
155
|
+
|
|
156
|
+
const acme: SourceDefinition = {
|
|
157
|
+
source: 'acme',
|
|
158
|
+
displayName: 'Acme Assistant',
|
|
159
|
+
defaultRoots: ['.acme'],
|
|
160
|
+
filePatterns: ['*.jsonl'],
|
|
161
|
+
targetTable: 'history_etl_acme',
|
|
162
|
+
splitConfig: { mode: 'one-to-one' },
|
|
163
|
+
fieldMap: { id: 'source_record_id', timestamp: 'created_at', content: 'content' },
|
|
164
|
+
fieldTransforms: {},
|
|
165
|
+
schema: z.object({
|
|
166
|
+
source_record_id: z.string().min(1),
|
|
167
|
+
created_at: z.string().min(1),
|
|
168
|
+
content: z.string().min(1),
|
|
169
|
+
}),
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
const db = await createDbAdapter({ driver: 'bun-sqlite', url: './history.db' });
|
|
173
|
+
const result = await runJsonlImport(acme, { db, roots: ['~/.acme'], mode: 'incremental' });
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
The custom `source` name flows through checkpoints, the ledger, and `ImportResult.source` exactly
|
|
177
|
+
like a built-in key. The importer creates the target ETL table on demand if it does not exist
|
|
178
|
+
(idempotent `CREATE TABLE IF NOT EXISTS`), so built-in tables are unaffected.
|
|
179
|
+
|
|
141
180
|
## Boundary Notes
|
|
142
181
|
|
|
143
182
|
- This package imports JSONL files and writes importer-owned tables; it does not model conversations, turns, tool calls, or analytics semantics.
|
package/dist/importer.d.ts
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
|
-
import type { ImportOptions, ImportResult,
|
|
2
|
-
/**
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
import type { ImportOptions, ImportResult, SourceDefinition } from './types';
|
|
2
|
+
/** Run the JSONL import pipeline for one source.
|
|
3
|
+
*
|
|
4
|
+
* @param source - Either a built-in source identifier resolved from the
|
|
5
|
+
* registry, or a fully-specified {@link SourceDefinition} for a custom
|
|
6
|
+
* source. Unknown strings throw {@link HistoryImportError}; custom
|
|
7
|
+
* definitions are validated before any I/O.
|
|
8
|
+
* @param options - Importer options (database, roots/files, mode, etc.).
|
|
9
|
+
*/
|
|
10
|
+
export declare function runJsonlImport(source: string | SourceDefinition, options: ImportOptions): Promise<ImportResult>;
|
|
6
11
|
//# sourceMappingURL=importer.d.ts.map
|
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CA8GrH"}
|
package/dist/importer.js
CHANGED
|
@@ -1,28 +1,26 @@
|
|
|
1
|
-
import { createNodeFileSystem, resolvePath, walkDir } from '@gobing-ai/ts-runtime';
|
|
2
|
-
import { HistoryImportError } from './errors.js';
|
|
1
|
+
import { ambientRuntimePaths, createNodeFileSystem, resolvePath, walkDir, } from '@gobing-ai/ts-runtime';
|
|
3
2
|
import { sha256 } from './hash.js';
|
|
3
|
+
import { applyHistoryImportSchema, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, writeCheckpoint, } from './jsonl-importer-dao.js';
|
|
4
4
|
import { redactRecord } from './redaction.js';
|
|
5
|
-
import {
|
|
6
|
-
import
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
/** Run the JSONL import pipeline for one source definition. */
|
|
5
|
+
import { resolveSourceDefinition } from './sources.js';
|
|
6
|
+
/** Run the JSONL import pipeline for one source.
|
|
7
|
+
*
|
|
8
|
+
* @param source - Either a built-in source identifier resolved from the
|
|
9
|
+
* registry, or a fully-specified {@link SourceDefinition} for a custom
|
|
10
|
+
* source. Unknown strings throw {@link HistoryImportError}; custom
|
|
11
|
+
* definitions are validated before any I/O.
|
|
12
|
+
* @param options - Importer options (database, roots/files, mode, etc.).
|
|
13
|
+
*/
|
|
18
14
|
export async function runJsonlImport(source, options) {
|
|
19
|
-
const definition =
|
|
15
|
+
const definition = resolveSourceDefinition(source);
|
|
16
|
+
const resolvedSource = definition.source;
|
|
20
17
|
const fileSystem = options.fileSystem ?? createNodeFileSystem();
|
|
21
18
|
await applyHistoryImportSchema(options.db);
|
|
19
|
+
await ensureTargetTables(options.db, definition);
|
|
22
20
|
const mode = options.mode ?? 'incremental';
|
|
23
|
-
const files = await discoverFiles(definition, options.roots, options.files, fileSystem);
|
|
21
|
+
const files = await discoverFiles(definition, options.roots, options.files, fileSystem, options.paths);
|
|
24
22
|
if (mode === 'full' && !options.dryRun) {
|
|
25
|
-
await resetCheckpoints(options.db,
|
|
23
|
+
await resetCheckpoints(options.db, resolvedSource, files);
|
|
26
24
|
}
|
|
27
25
|
const parseErrors = [];
|
|
28
26
|
const validationErrors = [];
|
|
@@ -31,7 +29,7 @@ export async function runJsonlImport(source, options) {
|
|
|
31
29
|
let skippedDuplicates = 0;
|
|
32
30
|
let checkpointUpdates = 0;
|
|
33
31
|
for (const file of files) {
|
|
34
|
-
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db,
|
|
32
|
+
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
|
|
35
33
|
let lineNumber = 0;
|
|
36
34
|
for await (const rawLine of readLines(fileSystem, file)) {
|
|
37
35
|
lineNumber += 1;
|
|
@@ -49,7 +47,7 @@ export async function runJsonlImport(source, options) {
|
|
|
49
47
|
if (split === undefined)
|
|
50
48
|
continue;
|
|
51
49
|
const normalized = normalizeRecord(definition, split.raw, {
|
|
52
|
-
source,
|
|
50
|
+
source: resolvedSource,
|
|
53
51
|
sourceFile: file,
|
|
54
52
|
sourceLine: lineNumber,
|
|
55
53
|
splitIndex,
|
|
@@ -66,7 +64,7 @@ export async function runJsonlImport(source, options) {
|
|
|
66
64
|
const redacted = redactRecord(parsed.data, options.redactionRules);
|
|
67
65
|
lineSucceeded = true;
|
|
68
66
|
const recordHash = sha256({
|
|
69
|
-
source,
|
|
67
|
+
source: resolvedSource,
|
|
70
68
|
sourceFile: file,
|
|
71
69
|
sourceLine: lineNumber,
|
|
72
70
|
splitIndex,
|
|
@@ -78,18 +76,18 @@ export async function runJsonlImport(source, options) {
|
|
|
78
76
|
}
|
|
79
77
|
if (!options.dryRun) {
|
|
80
78
|
await insertRecord(options.db, split.targetTable, recordHash, file, lineNumber, splitIndex, redacted, options.now);
|
|
81
|
-
await insertLedger(options.db, recordHash,
|
|
79
|
+
await insertLedger(options.db, recordHash, resolvedSource, file, lineNumber, splitIndex, split.targetTable, options.now);
|
|
82
80
|
}
|
|
83
81
|
importedRecords += 1;
|
|
84
82
|
}
|
|
85
83
|
if (lineSucceeded && !options.dryRun) {
|
|
86
|
-
await writeCheckpoint(options.db,
|
|
84
|
+
await writeCheckpoint(options.db, resolvedSource, file, lineNumber, options.now);
|
|
87
85
|
checkpointUpdates += 1;
|
|
88
86
|
}
|
|
89
87
|
}
|
|
90
88
|
}
|
|
91
89
|
return {
|
|
92
|
-
source,
|
|
90
|
+
source: resolvedSource,
|
|
93
91
|
mode,
|
|
94
92
|
scannedFiles: files.length,
|
|
95
93
|
processedLines,
|
|
@@ -153,11 +151,20 @@ function normalizeRecord(definition, raw, context) {
|
|
|
153
151
|
normalized.split_index = context.splitIndex;
|
|
154
152
|
return normalized;
|
|
155
153
|
}
|
|
156
|
-
async function discoverFiles(definition, roots, files, fileSystem) {
|
|
154
|
+
async function discoverFiles(definition, roots, files, fileSystem, paths) {
|
|
157
155
|
if (files !== undefined && files.length > 0) {
|
|
158
156
|
return files.map((file) => resolvePath(file)).sort();
|
|
159
157
|
}
|
|
160
|
-
|
|
158
|
+
// Root resolution splits by provenance (ADR-023 A1 / task 0042):
|
|
159
|
+
// - explicit caller-supplied `roots` → cwd semantics: resolve against `paths.cwd` when
|
|
160
|
+
// injected, else ambient cwd (the pre-seam behaviour — R4 additive-only).
|
|
161
|
+
// - registry `defaultRoots` → home-relative: resolve against `paths.home`.
|
|
162
|
+
// Previously both kinds anchored to ambient cwd, so a cwd ≠ $HOME silently skipped every
|
|
163
|
+
// registry source because the home-relative paths never existed relative to cwd.
|
|
164
|
+
const ambient = paths ?? ambientRuntimePaths();
|
|
165
|
+
const resolvedRoots = roots !== undefined
|
|
166
|
+
? roots.map((root) => resolvePath(ambient.cwd, root))
|
|
167
|
+
: definition.defaultRoots.map((root) => resolvePath(ambient.home, root));
|
|
161
168
|
const found = new Set();
|
|
162
169
|
for (const root of resolvedRoots) {
|
|
163
170
|
if (!(await fileSystem.exists(root)))
|
|
@@ -186,45 +193,6 @@ function matchesPattern(path, patterns) {
|
|
|
186
193
|
return path.endsWith(pattern.replace(/^\*/, ''));
|
|
187
194
|
});
|
|
188
195
|
}
|
|
189
|
-
async function readCheckpoint(db, source, sourceFile) {
|
|
190
|
-
const row = await db.queryFirst('SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, sourceFile);
|
|
191
|
-
return row?.last_imported_line ?? 0;
|
|
192
|
-
}
|
|
193
|
-
async function resetCheckpoints(db, source, files) {
|
|
194
|
-
for (const file of files) {
|
|
195
|
-
await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
async function writeCheckpoint(db, source, sourceFile, line, now) {
|
|
199
|
-
await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
|
|
200
|
-
VALUES (?, ?, ?, ?)
|
|
201
|
-
ON CONFLICT(source, source_file) DO UPDATE SET
|
|
202
|
-
last_imported_line = excluded.last_imported_line,
|
|
203
|
-
updated_at = excluded.updated_at`, source, sourceFile, line, timestamp(now));
|
|
204
|
-
}
|
|
205
|
-
async function ledgerExists(db, recordHash) {
|
|
206
|
-
const row = await db.queryFirst('SELECT record_hash FROM history_import_ledger WHERE record_hash = ?', recordHash);
|
|
207
|
-
return row !== undefined && row !== null;
|
|
208
|
-
}
|
|
209
|
-
async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
|
|
210
|
-
const table = targetTableFor(targetTable);
|
|
211
|
-
await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
|
|
212
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
213
|
-
ON CONFLICT(record_hash) DO NOTHING`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
|
|
214
|
-
}
|
|
215
|
-
async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
|
|
216
|
-
await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
|
|
217
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now));
|
|
218
|
-
}
|
|
219
|
-
function targetTableFor(table) {
|
|
220
|
-
if (!VALID_TABLE_NAME.test(table)) {
|
|
221
|
-
throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
|
|
222
|
-
}
|
|
223
|
-
return table;
|
|
224
|
-
}
|
|
225
|
-
function timestamp(now) {
|
|
226
|
-
return (now?.() ?? new Date()).toISOString();
|
|
227
|
-
}
|
|
228
196
|
/**
|
|
229
197
|
* Yield lines from a file, streaming when the FileSystem supports it.
|
|
230
198
|
*
|
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
export { HistoryImportError } from './errors';
|
|
2
2
|
export { sha256, stableJson } from './hash';
|
|
3
|
-
export {
|
|
3
|
+
export { runJsonlImport } from './importer';
|
|
4
|
+
export { applyHistoryImportSchema } from './jsonl-importer-dao';
|
|
4
5
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
5
6
|
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
6
|
-
export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources';
|
|
7
|
+
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
|
|
7
8
|
export type { FieldTransform, ImportIssue, ImportMode, ImportOptions, ImportResult, JsonObject, LlmJsonlSource, RedactionRule, SourceDefinition, SplitConfig, TransformContext, } from './types';
|
|
8
9
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export { HistoryImportError } from './errors.js';
|
|
2
2
|
export { sha256, stableJson } from './hash.js';
|
|
3
|
-
export {
|
|
3
|
+
export { runJsonlImport } from './importer.js';
|
|
4
|
+
export { applyHistoryImportSchema } from './jsonl-importer-dao.js';
|
|
4
5
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
|
|
5
6
|
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
|
|
6
|
-
export { getSourceDefinition, SOURCE_DEFINITIONS } from './sources.js';
|
|
7
|
+
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources.js';
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { ImportOptions, JsonObject, SourceDefinition } from './types';
|
|
2
|
+
interface CheckpointRow {
|
|
3
|
+
readonly last_imported_line: number;
|
|
4
|
+
}
|
|
5
|
+
declare function targetTableFor(table: string): string;
|
|
6
|
+
declare function timestamp(now: ImportOptions['now']): string;
|
|
7
|
+
declare function ETL_TABLE_DDL(table: string): string;
|
|
8
|
+
/** Apply importer-owned schema to the target database. */
|
|
9
|
+
export declare function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void>;
|
|
10
|
+
/**
|
|
11
|
+
* Ensure the ETL table(s) for a source definition exist.
|
|
12
|
+
*
|
|
13
|
+
* WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
|
|
14
|
+
* `history_etl_*` tables. Custom source definitions (and built-in definitions
|
|
15
|
+
* with a `splitConfig.targetTable` override) need their target table(s) created
|
|
16
|
+
* on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
|
|
17
|
+
* {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
|
|
18
|
+
* interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
|
|
19
|
+
* built-in tables that the static schema already created.
|
|
20
|
+
*/
|
|
21
|
+
declare function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void>;
|
|
22
|
+
declare function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number>;
|
|
23
|
+
declare function resetCheckpoints(db: ImportOptions['db'], source: string, files: readonly string[]): Promise<void>;
|
|
24
|
+
declare function writeCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string, line: number, now: ImportOptions['now']): Promise<void>;
|
|
25
|
+
declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean>;
|
|
26
|
+
declare function insertRecord(db: ImportOptions['db'], targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): Promise<void>;
|
|
27
|
+
declare function insertLedger(db: ImportOptions['db'], recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): Promise<void>;
|
|
28
|
+
export type { CheckpointRow };
|
|
29
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
|
30
|
+
//# sourceMappingURL=jsonl-importer-dao.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAED,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAQtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { HistoryImportError } from './errors.js';
|
|
2
|
+
import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
|
|
3
|
+
import { VALID_TABLE_NAME } from './sources.js';
|
|
4
|
+
function targetTableFor(table) {
|
|
5
|
+
if (!VALID_TABLE_NAME.test(table)) {
|
|
6
|
+
throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
|
|
7
|
+
}
|
|
8
|
+
return table;
|
|
9
|
+
}
|
|
10
|
+
function timestamp(now) {
|
|
11
|
+
return (now?.() ?? new Date()).toISOString();
|
|
12
|
+
}
|
|
13
|
+
function ETL_TABLE_DDL(table) {
|
|
14
|
+
return `CREATE TABLE IF NOT EXISTS ${table} (
|
|
15
|
+
record_hash TEXT PRIMARY KEY,
|
|
16
|
+
source_file TEXT NOT NULL,
|
|
17
|
+
source_line INTEGER NOT NULL,
|
|
18
|
+
split_index INTEGER NOT NULL,
|
|
19
|
+
payload_json TEXT NOT NULL,
|
|
20
|
+
imported_at TEXT NOT NULL
|
|
21
|
+
);`;
|
|
22
|
+
}
|
|
23
|
+
/** Apply importer-owned schema to the target database. */
|
|
24
|
+
export async function applyHistoryImportSchema(db) {
|
|
25
|
+
for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
|
|
26
|
+
const sql = statement.trim();
|
|
27
|
+
if (sql.length > 0) {
|
|
28
|
+
await db.exec(sql);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Ensure the ETL table(s) for a source definition exist.
|
|
34
|
+
*
|
|
35
|
+
* WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
|
|
36
|
+
* `history_etl_*` tables. Custom source definitions (and built-in definitions
|
|
37
|
+
* with a `splitConfig.targetTable` override) need their target table(s) created
|
|
38
|
+
* on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
|
|
39
|
+
* {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
|
|
40
|
+
* interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
|
|
41
|
+
* built-in tables that the static schema already created.
|
|
42
|
+
*/
|
|
43
|
+
async function ensureTargetTables(db, definition) {
|
|
44
|
+
const tables = new Set([targetTableFor(definition.targetTable)]);
|
|
45
|
+
if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
|
|
46
|
+
tables.add(targetTableFor(definition.splitConfig.targetTable));
|
|
47
|
+
}
|
|
48
|
+
for (const table of tables) {
|
|
49
|
+
await db.exec(ETL_TABLE_DDL(table));
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async function readCheckpoint(db, source, sourceFile) {
|
|
53
|
+
const row = await db.queryFirst('SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, sourceFile);
|
|
54
|
+
return row?.last_imported_line ?? 0;
|
|
55
|
+
}
|
|
56
|
+
async function resetCheckpoints(db, source, files) {
|
|
57
|
+
for (const file of files) {
|
|
58
|
+
await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
async function writeCheckpoint(db, source, sourceFile, line, now) {
|
|
62
|
+
await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
|
|
63
|
+
VALUES (?, ?, ?, ?)
|
|
64
|
+
ON CONFLICT(source, source_file) DO UPDATE SET
|
|
65
|
+
last_imported_line = excluded.last_imported_line,
|
|
66
|
+
updated_at = excluded.updated_at`, source, sourceFile, line, timestamp(now));
|
|
67
|
+
}
|
|
68
|
+
async function ledgerExists(db, recordHash) {
|
|
69
|
+
const row = await db.queryFirst('SELECT record_hash FROM history_import_ledger WHERE record_hash = ?', recordHash);
|
|
70
|
+
return row !== undefined && row !== null;
|
|
71
|
+
}
|
|
72
|
+
async function insertRecord(db, targetTable, recordHash, sourceFile, sourceLine, splitIndex, payload, now) {
|
|
73
|
+
const table = targetTableFor(targetTable);
|
|
74
|
+
await db.run(`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
|
|
75
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
76
|
+
ON CONFLICT(record_hash) DO NOTHING`, recordHash, sourceFile, sourceLine, splitIndex, JSON.stringify(payload), timestamp(now));
|
|
77
|
+
}
|
|
78
|
+
async function insertLedger(db, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, now) {
|
|
79
|
+
await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
|
|
80
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now));
|
|
81
|
+
}
|
|
82
|
+
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
package/dist/sources.d.ts
CHANGED
|
@@ -1,6 +1,36 @@
|
|
|
1
1
|
import type { LlmJsonlSource, SourceDefinition } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* Pattern every importer-owned table must match.
|
|
4
|
+
*
|
|
5
|
+
* WHY: the table name is interpolated into generated SQL (it is not a bind
|
|
6
|
+
* parameter), so it must be a safe identifier. The `history_etl_` prefix also
|
|
7
|
+
* namespaces importer tables away from consumer schemas. Both built-in and
|
|
8
|
+
* custom source definitions are held to this rule.
|
|
9
|
+
*/
|
|
10
|
+
export declare const VALID_TABLE_NAME: RegExp;
|
|
2
11
|
/** Built-in source definitions keyed by source identifier. */
|
|
3
12
|
export declare const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinition>>;
|
|
4
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* Resolve a built-in source definition by identifier.
|
|
15
|
+
*
|
|
16
|
+
* Throws {@link HistoryImportError} for unknown identifiers so misconfigured
|
|
17
|
+
* callers fail fast at the seam instead of producing empty imports.
|
|
18
|
+
*/
|
|
5
19
|
export declare function getSourceDefinition(source: LlmJsonlSource): SourceDefinition;
|
|
20
|
+
/**
|
|
21
|
+
* Validate a caller-supplied source definition before it enters the pipeline.
|
|
22
|
+
*
|
|
23
|
+
* WHY: target table names are interpolated into SQL, so both the primary
|
|
24
|
+
* `targetTable` and any `splitConfig.targetTable` override must match
|
|
25
|
+
* {@link VALID_TABLE_NAME}. Required structural fields are checked here so the
|
|
26
|
+
* importer can assume they are present. Built-in definitions skip this — they
|
|
27
|
+
* are validated by construction.
|
|
28
|
+
*/
|
|
29
|
+
export declare function validateSourceDefinition(definition: SourceDefinition): SourceDefinition;
|
|
30
|
+
/**
|
|
31
|
+
* Resolve a `string | SourceDefinition` source argument into a validated
|
|
32
|
+
* {@link SourceDefinition}. Strings resolve from {@link SOURCE_DEFINITIONS}
|
|
33
|
+
* (unknown strings throw); objects are validated via {@link validateSourceDefinition}.
|
|
34
|
+
*/
|
|
35
|
+
export declare function resolveSourceDefinition(input: string | SourceDefinition): SourceDefinition;
|
|
6
36
|
//# sourceMappingURL=sources.d.ts.map
|
package/dist/sources.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAA8B,cAAc,EAAE,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AA8F9G;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,QAA0B,CAAC;AAExD,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAQjF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAM5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,GAAG,gBAAgB,CA+BvF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,gBAAgB,CAK1F"}
|
package/dist/sources.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { HistoryImportError } from './errors.js';
|
|
2
3
|
const sourceRecordSchema = z
|
|
3
4
|
.object({
|
|
4
5
|
source_record_id: z.string().min(1),
|
|
@@ -71,6 +72,15 @@ function sourceDefinition(source, displayName, defaultRoots, filePatterns) {
|
|
|
71
72
|
schema: sourceRecordSchema,
|
|
72
73
|
};
|
|
73
74
|
}
|
|
75
|
+
/**
|
|
76
|
+
* Pattern every importer-owned table must match.
|
|
77
|
+
*
|
|
78
|
+
* WHY: the table name is interpolated into generated SQL (it is not a bind
|
|
79
|
+
* parameter), so it must be a safe identifier. The `history_etl_` prefix also
|
|
80
|
+
* namespaces importer tables away from consumer schemas. Both built-in and
|
|
81
|
+
* custom source definitions are held to this rule.
|
|
82
|
+
*/
|
|
83
|
+
export const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
|
|
74
84
|
/** Built-in source definitions keyed by source identifier. */
|
|
75
85
|
export const SOURCE_DEFINITIONS = {
|
|
76
86
|
pi: sourceDefinition('pi', 'Pi', ['.pi/history', '.pi'], ['*.jsonl', '*.json']),
|
|
@@ -81,7 +91,73 @@ export const SOURCE_DEFINITIONS = {
|
|
|
81
91
|
antigravity: sourceDefinition('antigravity', 'Antigravity', ['.antigravity'], ['*.jsonl']),
|
|
82
92
|
openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
|
|
83
93
|
};
|
|
84
|
-
/**
|
|
94
|
+
/**
|
|
95
|
+
* Resolve a built-in source definition by identifier.
|
|
96
|
+
*
|
|
97
|
+
* Throws {@link HistoryImportError} for unknown identifiers so misconfigured
|
|
98
|
+
* callers fail fast at the seam instead of producing empty imports.
|
|
99
|
+
*/
|
|
85
100
|
export function getSourceDefinition(source) {
|
|
86
|
-
|
|
101
|
+
const definition = SOURCE_DEFINITIONS[source];
|
|
102
|
+
if (definition === undefined) {
|
|
103
|
+
throw new HistoryImportError(`Unknown LLM JSONL source: ${source}`, { source });
|
|
104
|
+
}
|
|
105
|
+
return definition;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Validate a caller-supplied source definition before it enters the pipeline.
|
|
109
|
+
*
|
|
110
|
+
* WHY: target table names are interpolated into SQL, so both the primary
|
|
111
|
+
* `targetTable` and any `splitConfig.targetTable` override must match
|
|
112
|
+
* {@link VALID_TABLE_NAME}. Required structural fields are checked here so the
|
|
113
|
+
* importer can assume they are present. Built-in definitions skip this — they
|
|
114
|
+
* are validated by construction.
|
|
115
|
+
*/
|
|
116
|
+
export function validateSourceDefinition(definition) {
|
|
117
|
+
const requiredFields = [
|
|
118
|
+
'source',
|
|
119
|
+
'displayName',
|
|
120
|
+
'defaultRoots',
|
|
121
|
+
'filePatterns',
|
|
122
|
+
'targetTable',
|
|
123
|
+
'splitConfig',
|
|
124
|
+
'fieldMap',
|
|
125
|
+
'fieldTransforms',
|
|
126
|
+
'schema',
|
|
127
|
+
];
|
|
128
|
+
for (const field of requiredFields) {
|
|
129
|
+
if (definition[field] === undefined || definition[field] === null) {
|
|
130
|
+
throw new HistoryImportError(`Source definition is missing required field: ${String(field)}`, {
|
|
131
|
+
source: definition.source,
|
|
132
|
+
field: String(field),
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
if (definition.filePatterns.length === 0) {
|
|
137
|
+
throw new HistoryImportError('Source definition must declare at least one file pattern', {
|
|
138
|
+
source: definition.source,
|
|
139
|
+
field: 'filePatterns',
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
assertValidTable(definition.targetTable, definition.source);
|
|
143
|
+
if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
|
|
144
|
+
assertValidTable(definition.splitConfig.targetTable, definition.source);
|
|
145
|
+
}
|
|
146
|
+
return definition;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Resolve a `string | SourceDefinition` source argument into a validated
|
|
150
|
+
* {@link SourceDefinition}. Strings resolve from {@link SOURCE_DEFINITIONS}
|
|
151
|
+
* (unknown strings throw); objects are validated via {@link validateSourceDefinition}.
|
|
152
|
+
*/
|
|
153
|
+
export function resolveSourceDefinition(input) {
|
|
154
|
+
if (typeof input === 'string') {
|
|
155
|
+
return getSourceDefinition(input);
|
|
156
|
+
}
|
|
157
|
+
return validateSourceDefinition(input);
|
|
158
|
+
}
|
|
159
|
+
function assertValidTable(table, source) {
|
|
160
|
+
if (!VALID_TABLE_NAME.test(table)) {
|
|
161
|
+
throw new HistoryImportError(`Invalid history ETL target table for source "${source}": ${table}. Must match ${VALID_TABLE_NAME.source}.`, { source, table });
|
|
162
|
+
}
|
|
87
163
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { DbAdapter } from '@gobing-ai/ts-db';
|
|
2
|
-
import type { FileSystem } from '@gobing-ai/ts-runtime';
|
|
2
|
+
import type { FileSystem, RuntimePaths } from '@gobing-ai/ts-runtime';
|
|
3
3
|
import type { z } from 'zod';
|
|
4
4
|
/** Built-in source identifiers supported by the importer. */
|
|
5
5
|
export type LlmJsonlSource = 'pi' | 'claude' | 'codex' | 'gemini' | 'opencode' | 'antigravity' | 'openclaw';
|
|
@@ -23,14 +23,16 @@ export type JsonObject = Record<string, unknown>;
|
|
|
23
23
|
export type FieldTransform = (value: unknown, raw: JsonObject, context: TransformContext) => unknown;
|
|
24
24
|
/** Context available while normalizing a split record. */
|
|
25
25
|
export interface TransformContext {
|
|
26
|
-
|
|
26
|
+
/** Source identifier — a built-in {@link LlmJsonlSource} or a custom definition's name. */
|
|
27
|
+
readonly source: string;
|
|
27
28
|
readonly sourceFile: string;
|
|
28
29
|
readonly sourceLine: number;
|
|
29
30
|
readonly splitIndex: number;
|
|
30
31
|
}
|
|
31
32
|
/** Declarative importer configuration for one LLM history source. */
|
|
32
33
|
export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
|
|
33
|
-
|
|
34
|
+
/** Source identifier — a built-in {@link LlmJsonlSource} or a custom name. Becomes the checkpoint/ledger key. */
|
|
35
|
+
readonly source: string;
|
|
34
36
|
readonly displayName: string;
|
|
35
37
|
readonly defaultRoots: readonly string[];
|
|
36
38
|
readonly filePatterns: readonly string[];
|
|
@@ -56,6 +58,12 @@ export interface ImportOptions {
|
|
|
56
58
|
readonly dryRun?: boolean;
|
|
57
59
|
readonly redactionRules?: readonly RedactionRule[];
|
|
58
60
|
readonly now?: () => Date;
|
|
61
|
+
/**
|
|
62
|
+
* Optional cwd/home anchor (ADR-023 A1 / task 0042). When set, registry `defaultRoots`
|
|
63
|
+
* resolve against `paths.home` instead of the ambient working directory; explicit
|
|
64
|
+
* {@link ImportOptions.roots} keep cwd semantics unchanged. Defaults to ambient cwd/home.
|
|
65
|
+
*/
|
|
66
|
+
readonly paths?: RuntimePaths;
|
|
59
67
|
}
|
|
60
68
|
/** Structured importer error that can be reported without persisting raw input. */
|
|
61
69
|
export interface ImportIssue {
|
|
@@ -65,7 +73,8 @@ export interface ImportIssue {
|
|
|
65
73
|
}
|
|
66
74
|
/** Summary produced by the importer control function. */
|
|
67
75
|
export interface ImportResult {
|
|
68
|
-
|
|
76
|
+
/** Source identifier — the built-in name or custom definition name that produced this result. */
|
|
77
|
+
readonly source: string;
|
|
69
78
|
readonly mode: ImportMode;
|
|
70
79
|
readonly scannedFiles: number;
|
|
71
80
|
readonly processedLines: number;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,6DAA6D;AAC7D,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC;AAE5G,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;AAE/D,kFAAkF;AAClF,MAAM,MAAM,WAAW,GACjB;IACI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;CAC/B,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,GACD;IACI,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,SAAS,UAAU,EAAE,CAAC;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAER,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEjD,sDAAsD;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAErG,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,UAAU,GAAG,UAAU;IACrE,iHAAiH;IACjH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;CACvC;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAChC;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CACjC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,iGAAiG;IACjG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,WAAW,EAAE,CAAC;IAClD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACtC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-llm-jsonl-importer",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.16",
|
|
4
4
|
"description": "@gobing-ai/ts-llm-jsonl-importer — Generic JSONL importer for LLM agent history files.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-llm-jsonl-importer-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
52
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-db": "^0.4.16",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.16",
|
|
52
|
+
"@gobing-ai/ts-utils": "^0.4.16",
|
|
53
53
|
"zod": "^4.1.0"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
package/src/importer.ts
CHANGED
|
@@ -1,50 +1,51 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
ambientRuntimePaths,
|
|
3
|
+
createNodeFileSystem,
|
|
4
|
+
type FileSystem,
|
|
5
|
+
type RuntimePaths,
|
|
6
|
+
resolvePath,
|
|
7
|
+
walkDir,
|
|
8
|
+
} from '@gobing-ai/ts-runtime';
|
|
3
9
|
import { sha256 } from './hash';
|
|
10
|
+
import {
|
|
11
|
+
applyHistoryImportSchema,
|
|
12
|
+
ensureTargetTables,
|
|
13
|
+
insertLedger,
|
|
14
|
+
insertRecord,
|
|
15
|
+
ledgerExists,
|
|
16
|
+
readCheckpoint,
|
|
17
|
+
resetCheckpoints,
|
|
18
|
+
targetTableFor,
|
|
19
|
+
writeCheckpoint,
|
|
20
|
+
} from './jsonl-importer-dao';
|
|
4
21
|
import { redactRecord } from './redaction';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import type {
|
|
8
|
-
ImportIssue,
|
|
9
|
-
ImportOptions,
|
|
10
|
-
ImportResult,
|
|
11
|
-
JsonObject,
|
|
12
|
-
LlmJsonlSource,
|
|
13
|
-
SourceDefinition,
|
|
14
|
-
TransformContext,
|
|
15
|
-
} from './types';
|
|
22
|
+
import { resolveSourceDefinition } from './sources';
|
|
23
|
+
import type { ImportIssue, ImportOptions, ImportResult, JsonObject, SourceDefinition, TransformContext } from './types';
|
|
16
24
|
|
|
17
25
|
interface SplitRecord {
|
|
18
26
|
readonly targetTable: string;
|
|
19
27
|
readonly raw: JsonObject;
|
|
20
28
|
}
|
|
21
29
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
await db.exec(sql);
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
/** Run the JSONL import pipeline for one source definition. */
|
|
39
|
-
export async function runJsonlImport(source: LlmJsonlSource, options: ImportOptions): Promise<ImportResult> {
|
|
40
|
-
const definition = getSourceDefinition(source);
|
|
30
|
+
/** Run the JSONL import pipeline for one source.
|
|
31
|
+
*
|
|
32
|
+
* @param source - Either a built-in source identifier resolved from the
|
|
33
|
+
* registry, or a fully-specified {@link SourceDefinition} for a custom
|
|
34
|
+
* source. Unknown strings throw {@link HistoryImportError}; custom
|
|
35
|
+
* definitions are validated before any I/O.
|
|
36
|
+
* @param options - Importer options (database, roots/files, mode, etc.).
|
|
37
|
+
*/
|
|
38
|
+
export async function runJsonlImport(source: string | SourceDefinition, options: ImportOptions): Promise<ImportResult> {
|
|
39
|
+
const definition = resolveSourceDefinition(source);
|
|
40
|
+
const resolvedSource = definition.source;
|
|
41
41
|
const fileSystem = options.fileSystem ?? createNodeFileSystem();
|
|
42
42
|
await applyHistoryImportSchema(options.db);
|
|
43
|
+
await ensureTargetTables(options.db, definition);
|
|
43
44
|
|
|
44
45
|
const mode = options.mode ?? 'incremental';
|
|
45
|
-
const files = await discoverFiles(definition, options.roots, options.files, fileSystem);
|
|
46
|
+
const files = await discoverFiles(definition, options.roots, options.files, fileSystem, options.paths);
|
|
46
47
|
if (mode === 'full' && !options.dryRun) {
|
|
47
|
-
await resetCheckpoints(options.db,
|
|
48
|
+
await resetCheckpoints(options.db, resolvedSource, files);
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
const parseErrors: ImportIssue[] = [];
|
|
@@ -55,7 +56,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
|
|
|
55
56
|
let checkpointUpdates = 0;
|
|
56
57
|
|
|
57
58
|
for (const file of files) {
|
|
58
|
-
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db,
|
|
59
|
+
const checkpoint = mode === 'incremental' ? await readCheckpoint(options.db, resolvedSource, file) : 0;
|
|
59
60
|
let lineNumber = 0;
|
|
60
61
|
for await (const rawLine of readLines(fileSystem, file)) {
|
|
61
62
|
lineNumber += 1;
|
|
@@ -73,7 +74,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
|
|
|
73
74
|
const split = splitRecords[splitIndex];
|
|
74
75
|
if (split === undefined) continue;
|
|
75
76
|
const normalized = normalizeRecord(definition, split.raw, {
|
|
76
|
-
source,
|
|
77
|
+
source: resolvedSource,
|
|
77
78
|
sourceFile: file,
|
|
78
79
|
sourceLine: lineNumber,
|
|
79
80
|
splitIndex,
|
|
@@ -91,7 +92,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
|
|
|
91
92
|
const redacted = redactRecord(parsed.data, options.redactionRules);
|
|
92
93
|
lineSucceeded = true;
|
|
93
94
|
const recordHash = sha256({
|
|
94
|
-
source,
|
|
95
|
+
source: resolvedSource,
|
|
95
96
|
sourceFile: file,
|
|
96
97
|
sourceLine: lineNumber,
|
|
97
98
|
splitIndex,
|
|
@@ -115,7 +116,7 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
|
|
|
115
116
|
await insertLedger(
|
|
116
117
|
options.db,
|
|
117
118
|
recordHash,
|
|
118
|
-
|
|
119
|
+
resolvedSource,
|
|
119
120
|
file,
|
|
120
121
|
lineNumber,
|
|
121
122
|
splitIndex,
|
|
@@ -127,14 +128,14 @@ export async function runJsonlImport(source: LlmJsonlSource, options: ImportOpti
|
|
|
127
128
|
}
|
|
128
129
|
|
|
129
130
|
if (lineSucceeded && !options.dryRun) {
|
|
130
|
-
await writeCheckpoint(options.db,
|
|
131
|
+
await writeCheckpoint(options.db, resolvedSource, file, lineNumber, options.now);
|
|
131
132
|
checkpointUpdates += 1;
|
|
132
133
|
}
|
|
133
134
|
}
|
|
134
135
|
}
|
|
135
136
|
|
|
136
137
|
return {
|
|
137
|
-
source,
|
|
138
|
+
source: resolvedSource,
|
|
138
139
|
mode,
|
|
139
140
|
scannedFiles: files.length,
|
|
140
141
|
processedLines,
|
|
@@ -211,12 +212,23 @@ async function discoverFiles(
|
|
|
211
212
|
roots: readonly string[] | undefined,
|
|
212
213
|
files: readonly string[] | undefined,
|
|
213
214
|
fileSystem: FileSystem,
|
|
215
|
+
paths: RuntimePaths | undefined,
|
|
214
216
|
): Promise<readonly string[]> {
|
|
215
217
|
if (files !== undefined && files.length > 0) {
|
|
216
218
|
return files.map((file) => resolvePath(file)).sort();
|
|
217
219
|
}
|
|
218
220
|
|
|
219
|
-
|
|
221
|
+
// Root resolution splits by provenance (ADR-023 A1 / task 0042):
|
|
222
|
+
// - explicit caller-supplied `roots` → cwd semantics: resolve against `paths.cwd` when
|
|
223
|
+
// injected, else ambient cwd (the pre-seam behaviour — R4 additive-only).
|
|
224
|
+
// - registry `defaultRoots` → home-relative: resolve against `paths.home`.
|
|
225
|
+
// Previously both kinds anchored to ambient cwd, so a cwd ≠ $HOME silently skipped every
|
|
226
|
+
// registry source because the home-relative paths never existed relative to cwd.
|
|
227
|
+
const ambient = paths ?? ambientRuntimePaths();
|
|
228
|
+
const resolvedRoots =
|
|
229
|
+
roots !== undefined
|
|
230
|
+
? roots.map((root) => resolvePath(ambient.cwd, root))
|
|
231
|
+
: definition.defaultRoots.map((root) => resolvePath(ambient.home, root));
|
|
220
232
|
const found = new Set<string>();
|
|
221
233
|
for (const root of resolvedRoots) {
|
|
222
234
|
if (!(await fileSystem.exists(root))) continue;
|
|
@@ -241,107 +253,6 @@ function matchesPattern(path: string, patterns: readonly string[]): boolean {
|
|
|
241
253
|
});
|
|
242
254
|
}
|
|
243
255
|
|
|
244
|
-
async function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number> {
|
|
245
|
-
const row = await db.queryFirst<CheckpointRow>(
|
|
246
|
-
'SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?',
|
|
247
|
-
source,
|
|
248
|
-
sourceFile,
|
|
249
|
-
);
|
|
250
|
-
return row?.last_imported_line ?? 0;
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
async function resetCheckpoints(db: ImportOptions['db'], source: string, files: readonly string[]): Promise<void> {
|
|
254
|
-
for (const file of files) {
|
|
255
|
-
await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
async function writeCheckpoint(
|
|
260
|
-
db: ImportOptions['db'],
|
|
261
|
-
source: string,
|
|
262
|
-
sourceFile: string,
|
|
263
|
-
line: number,
|
|
264
|
-
now: ImportOptions['now'],
|
|
265
|
-
): Promise<void> {
|
|
266
|
-
await db.run(
|
|
267
|
-
`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
|
|
268
|
-
VALUES (?, ?, ?, ?)
|
|
269
|
-
ON CONFLICT(source, source_file) DO UPDATE SET
|
|
270
|
-
last_imported_line = excluded.last_imported_line,
|
|
271
|
-
updated_at = excluded.updated_at`,
|
|
272
|
-
source,
|
|
273
|
-
sourceFile,
|
|
274
|
-
line,
|
|
275
|
-
timestamp(now),
|
|
276
|
-
);
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
async function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean> {
|
|
280
|
-
const row = await db.queryFirst<{ record_hash: string }>(
|
|
281
|
-
'SELECT record_hash FROM history_import_ledger WHERE record_hash = ?',
|
|
282
|
-
recordHash,
|
|
283
|
-
);
|
|
284
|
-
return row !== undefined && row !== null;
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
async function insertRecord(
|
|
288
|
-
db: ImportOptions['db'],
|
|
289
|
-
targetTable: string,
|
|
290
|
-
recordHash: string,
|
|
291
|
-
sourceFile: string,
|
|
292
|
-
sourceLine: number,
|
|
293
|
-
splitIndex: number,
|
|
294
|
-
payload: JsonObject,
|
|
295
|
-
now: ImportOptions['now'],
|
|
296
|
-
): Promise<void> {
|
|
297
|
-
const table = targetTableFor(targetTable);
|
|
298
|
-
await db.run(
|
|
299
|
-
`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
|
|
300
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
301
|
-
ON CONFLICT(record_hash) DO NOTHING`,
|
|
302
|
-
recordHash,
|
|
303
|
-
sourceFile,
|
|
304
|
-
sourceLine,
|
|
305
|
-
splitIndex,
|
|
306
|
-
JSON.stringify(payload),
|
|
307
|
-
timestamp(now),
|
|
308
|
-
);
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
async function insertLedger(
|
|
312
|
-
db: ImportOptions['db'],
|
|
313
|
-
recordHash: string,
|
|
314
|
-
source: string,
|
|
315
|
-
sourceFile: string,
|
|
316
|
-
sourceLine: number,
|
|
317
|
-
splitIndex: number,
|
|
318
|
-
targetTable: string,
|
|
319
|
-
now: ImportOptions['now'],
|
|
320
|
-
): Promise<void> {
|
|
321
|
-
await db.run(
|
|
322
|
-
`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
|
|
323
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
324
|
-
recordHash,
|
|
325
|
-
source,
|
|
326
|
-
sourceFile,
|
|
327
|
-
sourceLine,
|
|
328
|
-
splitIndex,
|
|
329
|
-
targetTable,
|
|
330
|
-
timestamp(now),
|
|
331
|
-
);
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
function targetTableFor(table: string): string {
|
|
335
|
-
if (!VALID_TABLE_NAME.test(table)) {
|
|
336
|
-
throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
|
|
337
|
-
}
|
|
338
|
-
return table;
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
function timestamp(now: ImportOptions['now']): string {
|
|
342
|
-
return (now?.() ?? new Date()).toISOString();
|
|
343
|
-
}
|
|
344
|
-
|
|
345
256
|
/**
|
|
346
257
|
* Yield lines from a file, streaming when the FileSystem supports it.
|
|
347
258
|
*
|
package/src/index.ts
CHANGED
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
export { HistoryImportError } from './errors';
|
|
2
2
|
export { sha256, stableJson } from './hash';
|
|
3
|
-
export {
|
|
3
|
+
export { runJsonlImport } from './importer';
|
|
4
|
+
export { applyHistoryImportSchema } from './jsonl-importer-dao';
|
|
4
5
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
5
6
|
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
6
|
-
export {
|
|
7
|
+
export {
|
|
8
|
+
getSourceDefinition,
|
|
9
|
+
resolveSourceDefinition,
|
|
10
|
+
SOURCE_DEFINITIONS,
|
|
11
|
+
VALID_TABLE_NAME,
|
|
12
|
+
validateSourceDefinition,
|
|
13
|
+
} from './sources';
|
|
7
14
|
export type {
|
|
8
15
|
FieldTransform,
|
|
9
16
|
ImportIssue,
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { HistoryImportError } from './errors';
|
|
2
|
+
import { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
3
|
+
import { VALID_TABLE_NAME } from './sources';
|
|
4
|
+
import type { ImportOptions, JsonObject, SourceDefinition } from './types';
|
|
5
|
+
|
|
6
|
+
interface CheckpointRow {
|
|
7
|
+
readonly last_imported_line: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function targetTableFor(table: string): string {
|
|
11
|
+
if (!VALID_TABLE_NAME.test(table)) {
|
|
12
|
+
throw new HistoryImportError(`Invalid history ETL target table: ${table}`, { table });
|
|
13
|
+
}
|
|
14
|
+
return table;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function timestamp(now: ImportOptions['now']): string {
|
|
18
|
+
return (now?.() ?? new Date()).toISOString();
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function ETL_TABLE_DDL(table: string): string {
|
|
22
|
+
return `CREATE TABLE IF NOT EXISTS ${table} (
|
|
23
|
+
record_hash TEXT PRIMARY KEY,
|
|
24
|
+
source_file TEXT NOT NULL,
|
|
25
|
+
source_line INTEGER NOT NULL,
|
|
26
|
+
split_index INTEGER NOT NULL,
|
|
27
|
+
payload_json TEXT NOT NULL,
|
|
28
|
+
imported_at TEXT NOT NULL
|
|
29
|
+
);`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Apply importer-owned schema to the target database. */
|
|
33
|
+
export async function applyHistoryImportSchema(db: ImportOptions['db']): Promise<void> {
|
|
34
|
+
for (const statement of HISTORY_IMPORT_SCHEMA_SQL.split(';')) {
|
|
35
|
+
const sql = statement.trim();
|
|
36
|
+
if (sql.length > 0) {
|
|
37
|
+
await db.exec(sql);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Ensure the ETL table(s) for a source definition exist.
|
|
44
|
+
*
|
|
45
|
+
* WHY: the static {@link HISTORY_IMPORT_SCHEMA_SQL} only creates the built-in
|
|
46
|
+
* `history_etl_*` tables. Custom source definitions (and built-in definitions
|
|
47
|
+
* with a `splitConfig.targetTable` override) need their target table(s) created
|
|
48
|
+
* on demand. The table name is already gated by {@link VALID_TABLE_NAME} in
|
|
49
|
+
* {@link validateSourceDefinition} / {@link targetTableFor}, so it is safe to
|
|
50
|
+
* interpolate into DDL. `CREATE TABLE IF NOT EXISTS` makes this idempotent for
|
|
51
|
+
* built-in tables that the static schema already created.
|
|
52
|
+
*/
|
|
53
|
+
async function ensureTargetTables(db: ImportOptions['db'], definition: SourceDefinition): Promise<void> {
|
|
54
|
+
const tables = new Set<string>([targetTableFor(definition.targetTable)]);
|
|
55
|
+
if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
|
|
56
|
+
tables.add(targetTableFor(definition.splitConfig.targetTable));
|
|
57
|
+
}
|
|
58
|
+
for (const table of tables) {
|
|
59
|
+
await db.exec(ETL_TABLE_DDL(table));
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function readCheckpoint(db: ImportOptions['db'], source: string, sourceFile: string): Promise<number> {
|
|
64
|
+
const row = await db.queryFirst<CheckpointRow>(
|
|
65
|
+
'SELECT last_imported_line FROM history_import_checkpoint WHERE source = ? AND source_file = ?',
|
|
66
|
+
source,
|
|
67
|
+
sourceFile,
|
|
68
|
+
);
|
|
69
|
+
return row?.last_imported_line ?? 0;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function resetCheckpoints(db: ImportOptions['db'], source: string, files: readonly string[]): Promise<void> {
|
|
73
|
+
for (const file of files) {
|
|
74
|
+
await db.run('DELETE FROM history_import_checkpoint WHERE source = ? AND source_file = ?', source, file);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async function writeCheckpoint(
|
|
79
|
+
db: ImportOptions['db'],
|
|
80
|
+
source: string,
|
|
81
|
+
sourceFile: string,
|
|
82
|
+
line: number,
|
|
83
|
+
now: ImportOptions['now'],
|
|
84
|
+
): Promise<void> {
|
|
85
|
+
await db.run(
|
|
86
|
+
`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
|
|
87
|
+
VALUES (?, ?, ?, ?)
|
|
88
|
+
ON CONFLICT(source, source_file) DO UPDATE SET
|
|
89
|
+
last_imported_line = excluded.last_imported_line,
|
|
90
|
+
updated_at = excluded.updated_at`,
|
|
91
|
+
source,
|
|
92
|
+
sourceFile,
|
|
93
|
+
line,
|
|
94
|
+
timestamp(now),
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean> {
|
|
99
|
+
const row = await db.queryFirst<{ record_hash: string }>(
|
|
100
|
+
'SELECT record_hash FROM history_import_ledger WHERE record_hash = ?',
|
|
101
|
+
recordHash,
|
|
102
|
+
);
|
|
103
|
+
return row !== undefined && row !== null;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function insertRecord(
|
|
107
|
+
db: ImportOptions['db'],
|
|
108
|
+
targetTable: string,
|
|
109
|
+
recordHash: string,
|
|
110
|
+
sourceFile: string,
|
|
111
|
+
sourceLine: number,
|
|
112
|
+
splitIndex: number,
|
|
113
|
+
payload: JsonObject,
|
|
114
|
+
now: ImportOptions['now'],
|
|
115
|
+
): Promise<void> {
|
|
116
|
+
const table = targetTableFor(targetTable);
|
|
117
|
+
await db.run(
|
|
118
|
+
`INSERT INTO ${table} (record_hash, source_file, source_line, split_index, payload_json, imported_at)
|
|
119
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
120
|
+
ON CONFLICT(record_hash) DO NOTHING`,
|
|
121
|
+
recordHash,
|
|
122
|
+
sourceFile,
|
|
123
|
+
sourceLine,
|
|
124
|
+
splitIndex,
|
|
125
|
+
JSON.stringify(payload),
|
|
126
|
+
timestamp(now),
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async function insertLedger(
|
|
131
|
+
db: ImportOptions['db'],
|
|
132
|
+
recordHash: string,
|
|
133
|
+
source: string,
|
|
134
|
+
sourceFile: string,
|
|
135
|
+
sourceLine: number,
|
|
136
|
+
splitIndex: number,
|
|
137
|
+
targetTable: string,
|
|
138
|
+
now: ImportOptions['now'],
|
|
139
|
+
): Promise<void> {
|
|
140
|
+
await db.run(
|
|
141
|
+
`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
|
|
142
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
143
|
+
recordHash,
|
|
144
|
+
source,
|
|
145
|
+
sourceFile,
|
|
146
|
+
sourceLine,
|
|
147
|
+
splitIndex,
|
|
148
|
+
targetTable,
|
|
149
|
+
timestamp(now),
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export type { CheckpointRow };
|
|
154
|
+
export {
|
|
155
|
+
ETL_TABLE_DDL,
|
|
156
|
+
ensureTargetTables,
|
|
157
|
+
insertLedger,
|
|
158
|
+
insertRecord,
|
|
159
|
+
ledgerExists,
|
|
160
|
+
readCheckpoint,
|
|
161
|
+
resetCheckpoints,
|
|
162
|
+
targetTableFor,
|
|
163
|
+
timestamp,
|
|
164
|
+
writeCheckpoint,
|
|
165
|
+
};
|
package/src/sources.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { HistoryImportError } from './errors';
|
|
2
3
|
import type { FieldTransform, JsonObject, LlmJsonlSource, SourceDefinition, TransformContext } from './types';
|
|
3
4
|
|
|
4
5
|
const sourceRecordSchema = z
|
|
@@ -93,6 +94,15 @@ function sourceDefinition(
|
|
|
93
94
|
schema: sourceRecordSchema,
|
|
94
95
|
};
|
|
95
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* Pattern every importer-owned table must match.
|
|
99
|
+
*
|
|
100
|
+
* WHY: the table name is interpolated into generated SQL (it is not a bind
|
|
101
|
+
* parameter), so it must be a safe identifier. The `history_etl_` prefix also
|
|
102
|
+
* namespaces importer tables away from consumer schemas. Both built-in and
|
|
103
|
+
* custom source definitions are held to this rule.
|
|
104
|
+
*/
|
|
105
|
+
export const VALID_TABLE_NAME = /^history_etl_[a-z_]+$/;
|
|
96
106
|
|
|
97
107
|
/** Built-in source definitions keyed by source identifier. */
|
|
98
108
|
export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinition>> = {
|
|
@@ -105,7 +115,79 @@ export const SOURCE_DEFINITIONS: Readonly<Record<LlmJsonlSource, SourceDefinitio
|
|
|
105
115
|
openclaw: sourceDefinition('openclaw', 'OpenClaw', ['.openclaw'], ['*.jsonl']),
|
|
106
116
|
};
|
|
107
117
|
|
|
108
|
-
/**
|
|
118
|
+
/**
|
|
119
|
+
* Resolve a built-in source definition by identifier.
|
|
120
|
+
*
|
|
121
|
+
* Throws {@link HistoryImportError} for unknown identifiers so misconfigured
|
|
122
|
+
* callers fail fast at the seam instead of producing empty imports.
|
|
123
|
+
*/
|
|
109
124
|
export function getSourceDefinition(source: LlmJsonlSource): SourceDefinition {
|
|
110
|
-
|
|
125
|
+
const definition = SOURCE_DEFINITIONS[source];
|
|
126
|
+
if (definition === undefined) {
|
|
127
|
+
throw new HistoryImportError(`Unknown LLM JSONL source: ${source}`, { source });
|
|
128
|
+
}
|
|
129
|
+
return definition;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Validate a caller-supplied source definition before it enters the pipeline.
|
|
134
|
+
*
|
|
135
|
+
* WHY: target table names are interpolated into SQL, so both the primary
|
|
136
|
+
* `targetTable` and any `splitConfig.targetTable` override must match
|
|
137
|
+
* {@link VALID_TABLE_NAME}. Required structural fields are checked here so the
|
|
138
|
+
* importer can assume they are present. Built-in definitions skip this — they
|
|
139
|
+
* are validated by construction.
|
|
140
|
+
*/
|
|
141
|
+
export function validateSourceDefinition(definition: SourceDefinition): SourceDefinition {
|
|
142
|
+
const requiredFields: ReadonlyArray<keyof SourceDefinition> = [
|
|
143
|
+
'source',
|
|
144
|
+
'displayName',
|
|
145
|
+
'defaultRoots',
|
|
146
|
+
'filePatterns',
|
|
147
|
+
'targetTable',
|
|
148
|
+
'splitConfig',
|
|
149
|
+
'fieldMap',
|
|
150
|
+
'fieldTransforms',
|
|
151
|
+
'schema',
|
|
152
|
+
];
|
|
153
|
+
for (const field of requiredFields) {
|
|
154
|
+
if (definition[field] === undefined || definition[field] === null) {
|
|
155
|
+
throw new HistoryImportError(`Source definition is missing required field: ${String(field)}`, {
|
|
156
|
+
source: definition.source,
|
|
157
|
+
field: String(field),
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (definition.filePatterns.length === 0) {
|
|
162
|
+
throw new HistoryImportError('Source definition must declare at least one file pattern', {
|
|
163
|
+
source: definition.source,
|
|
164
|
+
field: 'filePatterns',
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
assertValidTable(definition.targetTable, definition.source);
|
|
168
|
+
if (definition.splitConfig.mode !== 'one-to-one' && definition.splitConfig.targetTable !== undefined) {
|
|
169
|
+
assertValidTable(definition.splitConfig.targetTable, definition.source);
|
|
170
|
+
}
|
|
171
|
+
return definition;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Resolve a `string | SourceDefinition` source argument into a validated
|
|
176
|
+
* {@link SourceDefinition}. Strings resolve from {@link SOURCE_DEFINITIONS}
|
|
177
|
+
* (unknown strings throw); objects are validated via {@link validateSourceDefinition}.
|
|
178
|
+
*/
|
|
179
|
+
export function resolveSourceDefinition(input: string | SourceDefinition): SourceDefinition {
|
|
180
|
+
if (typeof input === 'string') {
|
|
181
|
+
return getSourceDefinition(input as LlmJsonlSource);
|
|
182
|
+
}
|
|
183
|
+
return validateSourceDefinition(input);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function assertValidTable(table: string, source: string): void {
|
|
187
|
+
if (!VALID_TABLE_NAME.test(table)) {
|
|
188
|
+
throw new HistoryImportError(
|
|
189
|
+
`Invalid history ETL target table for source "${source}": ${table}. Must match ${VALID_TABLE_NAME.source}.`,
|
|
190
|
+
{ source, table },
|
|
191
|
+
);
|
|
192
|
+
}
|
|
111
193
|
}
|
package/src/types.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { DbAdapter } from '@gobing-ai/ts-db';
|
|
2
|
-
import type { FileSystem } from '@gobing-ai/ts-runtime';
|
|
2
|
+
import type { FileSystem, RuntimePaths } from '@gobing-ai/ts-runtime';
|
|
3
3
|
import type { z } from 'zod';
|
|
4
4
|
|
|
5
5
|
/** Built-in source identifiers supported by the importer. */
|
|
@@ -32,7 +32,8 @@ export type FieldTransform = (value: unknown, raw: JsonObject, context: Transfor
|
|
|
32
32
|
|
|
33
33
|
/** Context available while normalizing a split record. */
|
|
34
34
|
export interface TransformContext {
|
|
35
|
-
|
|
35
|
+
/** Source identifier — a built-in {@link LlmJsonlSource} or a custom definition's name. */
|
|
36
|
+
readonly source: string;
|
|
36
37
|
readonly sourceFile: string;
|
|
37
38
|
readonly sourceLine: number;
|
|
38
39
|
readonly splitIndex: number;
|
|
@@ -40,7 +41,8 @@ export interface TransformContext {
|
|
|
40
41
|
|
|
41
42
|
/** Declarative importer configuration for one LLM history source. */
|
|
42
43
|
export interface SourceDefinition<TRecord extends JsonObject = JsonObject> {
|
|
43
|
-
|
|
44
|
+
/** Source identifier — a built-in {@link LlmJsonlSource} or a custom name. Becomes the checkpoint/ledger key. */
|
|
45
|
+
readonly source: string;
|
|
44
46
|
readonly displayName: string;
|
|
45
47
|
readonly defaultRoots: readonly string[];
|
|
46
48
|
readonly filePatterns: readonly string[];
|
|
@@ -68,6 +70,12 @@ export interface ImportOptions {
|
|
|
68
70
|
readonly dryRun?: boolean;
|
|
69
71
|
readonly redactionRules?: readonly RedactionRule[];
|
|
70
72
|
readonly now?: () => Date;
|
|
73
|
+
/**
|
|
74
|
+
* Optional cwd/home anchor (ADR-023 A1 / task 0042). When set, registry `defaultRoots`
|
|
75
|
+
* resolve against `paths.home` instead of the ambient working directory; explicit
|
|
76
|
+
* {@link ImportOptions.roots} keep cwd semantics unchanged. Defaults to ambient cwd/home.
|
|
77
|
+
*/
|
|
78
|
+
readonly paths?: RuntimePaths;
|
|
71
79
|
}
|
|
72
80
|
|
|
73
81
|
/** Structured importer error that can be reported without persisting raw input. */
|
|
@@ -79,7 +87,8 @@ export interface ImportIssue {
|
|
|
79
87
|
|
|
80
88
|
/** Summary produced by the importer control function. */
|
|
81
89
|
export interface ImportResult {
|
|
82
|
-
|
|
90
|
+
/** Source identifier — the built-in name or custom definition name that produced this result. */
|
|
91
|
+
readonly source: string;
|
|
83
92
|
readonly mode: ImportMode;
|
|
84
93
|
readonly scannedFiles: number;
|
|
85
94
|
readonly processedLines: number;
|