@gobing-ai/ts-llm-jsonl-importer 0.4.22 → 0.4.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +35 -6
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/jsonl-importer-dao.d.ts +60 -0
- package/dist/jsonl-importer-dao.d.ts.map +1 -1
- package/dist/jsonl-importer-dao.js +188 -0
- package/dist/mappers.d.ts +7 -1
- package/dist/mappers.d.ts.map +1 -1
- package/dist/mappers.js +150 -6
- package/dist/opencode-importer.d.ts +17 -0
- package/dist/opencode-importer.d.ts.map +1 -0
- package/dist/opencode-importer.js +231 -0
- package/dist/sources.d.ts.map +1 -1
- package/dist/sources.js +2 -2
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/importer.ts +38 -6
- package/src/index.ts +2 -1
- package/src/jsonl-importer-dao.ts +289 -0
- package/src/mappers.ts +166 -6
- package/src/opencode-importer.ts +299 -0
- package/src/sources.ts +13 -2
- package/src/types.ts +1 -1
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAe,aAAa,EAAE,YAAY,EAAc,gBAAgB,EAAoB,MAAM,SAAS,CAAC;AAOxH;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAgJrH"}
|
package/dist/importer.js
CHANGED
|
@@ -41,7 +41,12 @@ export async function runJsonlImport(source, options) {
|
|
|
41
41
|
const raw = parseJsonLine(line, file, lineNumber, parseErrors);
|
|
42
42
|
if (raw === undefined)
|
|
43
43
|
continue;
|
|
44
|
-
const splitRecords = splitRawRecord(definition, raw
|
|
44
|
+
const splitRecords = splitRawRecord(definition, raw, {
|
|
45
|
+
source: resolvedSource,
|
|
46
|
+
sourceFile: file,
|
|
47
|
+
sourceLine: lineNumber,
|
|
48
|
+
splitIndex: 0,
|
|
49
|
+
});
|
|
45
50
|
let lineSucceeded = false;
|
|
46
51
|
const prepared = [];
|
|
47
52
|
for (let splitIndex = 0; splitIndex < splitRecords.length; splitIndex += 1) {
|
|
@@ -133,7 +138,7 @@ function parseJsonLine(line, sourceFile, sourceLine, parseErrors) {
|
|
|
133
138
|
return undefined;
|
|
134
139
|
}
|
|
135
140
|
}
|
|
136
|
-
function splitRawRecord(definition, raw) {
|
|
141
|
+
function splitRawRecord(definition, raw, context) {
|
|
137
142
|
const targetTable = targetTableFor(definition.targetTable);
|
|
138
143
|
if (definition.splitConfig.mode === 'one-to-one') {
|
|
139
144
|
return [{ targetTable, raw }];
|
|
@@ -141,7 +146,7 @@ function splitRawRecord(definition, raw) {
|
|
|
141
146
|
if (definition.splitConfig.mode === 'custom') {
|
|
142
147
|
const config = definition.splitConfig;
|
|
143
148
|
const configTable = config.targetTable !== undefined ? targetTableFor(config.targetTable) : undefined;
|
|
144
|
-
return config.split(raw).map((entry) => {
|
|
149
|
+
return config.split(raw, context).map((entry) => {
|
|
145
150
|
// Normalize: SplitEntry has a `record` property, bare object is the record itself.
|
|
146
151
|
const hasTargetTable = 'targetTable' in entry && typeof entry.targetTable === 'string';
|
|
147
152
|
const record = hasTargetTable ? entry.record : entry;
|
|
@@ -180,7 +185,10 @@ function normalizeRecord(definition, raw, context) {
|
|
|
180
185
|
}
|
|
181
186
|
async function discoverFiles(definition, roots, files, fileSystem, paths) {
|
|
182
187
|
if (files !== undefined && files.length > 0) {
|
|
183
|
-
return files
|
|
188
|
+
return files
|
|
189
|
+
.map((file) => resolvePath(file))
|
|
190
|
+
.map((file) => normalizeSourceFilePath(file, fileSystem))
|
|
191
|
+
.sort();
|
|
184
192
|
}
|
|
185
193
|
// Root resolution splits by provenance (ADR-023 A1 / task 0042):
|
|
186
194
|
// - explicit caller-supplied `roots` → cwd semantics: resolve against `paths.cwd` when
|
|
@@ -201,16 +209,37 @@ async function discoverFiles(definition, roots, files, fileSystem, paths) {
|
|
|
201
209
|
continue;
|
|
202
210
|
if (stat.isFile()) {
|
|
203
211
|
if (matchesPattern(root, definition.filePatterns))
|
|
204
|
-
found.add(root);
|
|
212
|
+
found.add(normalizeSourceFilePath(root, fileSystem));
|
|
205
213
|
continue;
|
|
206
214
|
}
|
|
207
215
|
for (const file of await walkDir(root, fileSystem)) {
|
|
208
216
|
if (matchesPattern(file, definition.filePatterns))
|
|
209
|
-
found.add(file);
|
|
217
|
+
found.add(normalizeSourceFilePath(file, fileSystem));
|
|
210
218
|
}
|
|
211
219
|
}
|
|
212
220
|
return [...found].sort();
|
|
213
221
|
}
|
|
222
|
+
/**
|
|
223
|
+
* Normalize a discovered source file path to its canonical real-path identity.
|
|
224
|
+
*
|
|
225
|
+
* WHY: the same physical session file is reachable via a symlinked path and via its
|
|
226
|
+
* real path (every agent history dir under `$HOME` is a symlink into the dotfiles
|
|
227
|
+
* tree here). Without normalization, the checkpoint, ledger, and record_hash keys
|
|
228
|
+
* diverge, yielding duplicate checkpoint rows and silent full-corpus re-imports.
|
|
229
|
+
* `realPath` is optional on `FileSystem`: injected and in-memory test doubles may
|
|
230
|
+
* omit it, and a path that does not exist on disk has no realpath. Fall back to the
|
|
231
|
+
* original path in either case — never throw, never bypass the injected FileSystem.
|
|
232
|
+
*/
|
|
233
|
+
function normalizeSourceFilePath(path, fileSystem) {
|
|
234
|
+
if (!fileSystem.realPath)
|
|
235
|
+
return path;
|
|
236
|
+
try {
|
|
237
|
+
return fileSystem.realPath(path);
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
return path;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
214
243
|
function matchesPattern(path, patterns) {
|
|
215
244
|
return patterns.some((pattern) => {
|
|
216
245
|
if (pattern === '*.jsonl')
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
export { HistoryImportError } from './errors';
|
|
2
2
|
export { sha256, stableJson } from './hash';
|
|
3
3
|
export { runJsonlImport } from './importer';
|
|
4
|
-
export { applyHistoryImportSchema } from './jsonl-importer-dao';
|
|
4
|
+
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao';
|
|
5
|
+
export { type OpenCodeImportOptions, runOpenCodeImport } from './opencode-importer';
|
|
5
6
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction';
|
|
6
7
|
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql';
|
|
7
8
|
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAC1F,OAAO,EAAE,KAAK,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EACH,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,GAC3B,MAAM,WAAW,CAAC;AACnB,YAAY,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,gBAAgB,GACnB,MAAM,SAAS,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
export { HistoryImportError } from './errors.js';
|
|
2
2
|
export { sha256, stableJson } from './hash.js';
|
|
3
3
|
export { runJsonlImport } from './importer.js';
|
|
4
|
-
export { applyHistoryImportSchema } from './jsonl-importer-dao.js';
|
|
4
|
+
export { applyHistoryImportSchema, normalizeSourceFilePaths } from './jsonl-importer-dao.js';
|
|
5
|
+
export { runOpenCodeImport } from './opencode-importer.js';
|
|
5
6
|
export { DEFAULT_REDACTION_RULES, redactRecord, redactValue } from './redaction.js';
|
|
6
7
|
export { HISTORY_IMPORT_SCHEMA_SQL } from './schema-sql.js';
|
|
7
8
|
export { getSourceDefinition, resolveSourceDefinition, SOURCE_DEFINITIONS, VALID_TABLE_NAME, validateSourceDefinition, } from './sources.js';
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { DbAdapter, DbBatchOp } from '@gobing-ai/ts-db';
|
|
1
2
|
import type { ImportOptions, JsonObject, SourceDefinition } from './types';
|
|
2
3
|
interface CheckpointRow {
|
|
3
4
|
readonly last_imported_line: number;
|
|
@@ -25,6 +26,65 @@ declare function writeCheckpoint(db: ImportOptions['db'], source: string, source
|
|
|
25
26
|
declare function ledgerExists(db: ImportOptions['db'], recordHash: string): Promise<boolean>;
|
|
26
27
|
declare function insertRecord(db: ImportOptions['db'], targetTable: string, recordHash: string, sourceFile: string, sourceLine: number, splitIndex: number, payload: JsonObject, now: ImportOptions['now']): Promise<void>;
|
|
27
28
|
declare function insertLedger(db: ImportOptions['db'], recordHash: string, source: string, sourceFile: string, sourceLine: number, splitIndex: number, targetTable: string, now: ImportOptions['now']): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Idempotent migration that rewrites `source_file` to its realpath across the checkpoint,
|
|
31
|
+
* ledger, and any contract table present (R4).
|
|
32
|
+
*
|
|
33
|
+
* WHY: `record_hash` is `sha256({source, sourceFile, sourceLine, splitIndex, record})`, so
|
|
34
|
+
* `sourceFile` is inside the hash. Normalizing its representation at discovery (R1) changes
|
|
35
|
+
* every future hash; pre-normalization rows carry unnormalized paths and their old hashes
|
|
36
|
+
* remain valid as dedupe keys. This migration rewrites the `source_file` *column* so old and
|
|
37
|
+
* new rows agree on path identity, and collapses duplicate checkpoint rows produced when one
|
|
38
|
+
* physical file was imported via both a symlinked and a real path — keeping the highest
|
|
39
|
+
* `last_imported_line` so an incremental resume does not re-import already-seen content.
|
|
40
|
+
*
|
|
41
|
+
* `record_hash` is intentionally NOT touched: it is path-representation dependent by
|
|
42
|
+
* construction, pre-migration rows are grandfathered, and recomputing it would require the
|
|
43
|
+
* original record payload that the ledger does not store.
|
|
44
|
+
*
|
|
45
|
+
* @param resolveRealPath - resolves a `source_file` to its canonical real path. Falls back to
|
|
46
|
+
* the original value when it returns null/undefined or throws. Decouples the DAO from the
|
|
47
|
+
* runtime `FileSystem` seam.
|
|
48
|
+
*/
|
|
49
|
+
export declare function normalizeSourceFilePaths(db: ImportOptions['db'], resolveRealPath: (sourceFile: string) => string | null | undefined): Promise<void>;
|
|
50
|
+
/** OpenCode message row projected from its SQLite store. */
|
|
51
|
+
export interface OpenCodeMessageRow {
|
|
52
|
+
id: string;
|
|
53
|
+
session_id: string;
|
|
54
|
+
time_created: number;
|
|
55
|
+
data: string;
|
|
56
|
+
directory: string;
|
|
57
|
+
}
|
|
58
|
+
/** OpenCode message-part row projected from its SQLite store. */
|
|
59
|
+
export interface OpenCodePartRow {
|
|
60
|
+
id: string;
|
|
61
|
+
message_id: string;
|
|
62
|
+
time_created: number;
|
|
63
|
+
data: string;
|
|
64
|
+
}
|
|
65
|
+
/** Previously imported OpenCode ledger entry. */
|
|
66
|
+
export interface OpenCodeExistingEntry {
|
|
67
|
+
record_hash: string;
|
|
68
|
+
target_table: string;
|
|
69
|
+
}
|
|
70
|
+
/** Normalized OpenCode entry awaiting a history-table write. */
|
|
71
|
+
export interface OpenCodeQueuedEntry {
|
|
72
|
+
targetTable: 'history_message' | 'history_tool_call';
|
|
73
|
+
splitIndex: number;
|
|
74
|
+
record: JsonObject;
|
|
75
|
+
recordHash: string;
|
|
76
|
+
sourceFile: string;
|
|
77
|
+
}
|
|
78
|
+
/** Read one ordered page of OpenCode messages after the supplied cursor. */
|
|
79
|
+
export declare function readOpenCodeMessages(db: DbAdapter, lastTime: number, lastId: string, limit: number): Promise<OpenCodeMessageRow[]>;
|
|
80
|
+
/** Read all parts belonging to the supplied OpenCode message IDs. */
|
|
81
|
+
export declare function readOpenCodeParts(db: DbAdapter, messageIds: readonly string[]): Promise<OpenCodePartRow[]>;
|
|
82
|
+
/** Load existing OpenCode ledger entries grouped by source message ID. */
|
|
83
|
+
export declare function readOpenCodeExistingEntries(db: DbAdapter): Promise<Map<string, OpenCodeExistingEntry[]>>;
|
|
84
|
+
/** Build deletion operations for records superseded by a forced import. */
|
|
85
|
+
export declare function openCodeDeleteOperations(entries: readonly OpenCodeExistingEntry[]): DbBatchOp[];
|
|
86
|
+
/** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
|
|
87
|
+
export declare function openCodeBulkWriteOperations(entries: readonly OpenCodeQueuedEntry[], checkpointFiles: readonly string[], now: ImportOptions['now']): DbBatchOp[];
|
|
28
88
|
export type { CheckpointRow };
|
|
29
89
|
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
|
30
90
|
//# sourceMappingURL=jsonl-importer-dao.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"jsonl-importer-dao.d.ts","sourceRoot":"","sources":["../src/jsonl-importer-dao.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3E,UAAU,aAAa;IACnB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACvC;AAqDD,iBAAS,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK7C;AAED,iBAAS,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,MAAM,CAEpD;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS5C;AAED,0DAA0D;AAC1D,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrF;AAED;;;;;;;;;;GAUG;AACH,iBAAe,kBAAkB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAUtG;AAED,iBAAe,cAAc,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO1G;AAED,iBAAe,gBAAgB,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhH;AAED,iBAAe,eAAe,CAC1B,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,iBAAe,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAMzF;AAID,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiDf;AAED,iBAAe,YAAY,CACvB,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,wBAAwB,CAC1C,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,EACvB,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACnE,OAAO,CAAC,IAAI,CAAC,CAWf;AA+FD,4DAA4D;AAC5D,MAAM,WAAW,kBAAkB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,qBAAqB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,mBAAmB;IAChC,WAAW,EAAE,iBAAiB,GAAG,mBAAmB,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACtC,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAED,qEAAqE;AACrE,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAShH;AAED,0EAA0E;AAC1E,wBAAsB,2BAA2B,CAAC,EAAE,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAY9G;AAED,2EAA2E;AAC3E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,SAAS,EAAE,CAW/F;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACvC,OAAO,EAAE,SAAS,mBAAmB,EAAE,EACvC,eAAe,EAAE,SAAS,MAAM,EAAE,EAClC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,GAC1B,SAAS,EAAE,CA0Db;AAED,YAAY,EAAE,aAAa,EAAE,CAAC;AAC9B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,GAClB,CAAC"}
|
|
@@ -165,4 +165,192 @@ async function insertLedger(db, recordHash, source, sourceFile, sourceLine, spli
|
|
|
165
165
|
await db.run(`INSERT INTO history_import_ledger (record_hash, source, source_file, source_line, split_index, target_table, imported_at)
|
|
166
166
|
VALUES (?, ?, ?, ?, ?, ?, ?)`, recordHash, source, sourceFile, sourceLine, splitIndex, targetTable, timestamp(now));
|
|
167
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* Idempotent migration that rewrites `source_file` to its realpath across the checkpoint,
|
|
170
|
+
* ledger, and any contract table present (R4).
|
|
171
|
+
*
|
|
172
|
+
* WHY: `record_hash` is `sha256({source, sourceFile, sourceLine, splitIndex, record})`, so
|
|
173
|
+
* `sourceFile` is inside the hash. Normalizing its representation at discovery (R1) changes
|
|
174
|
+
* every future hash; pre-normalization rows carry unnormalized paths and their old hashes
|
|
175
|
+
* remain valid as dedupe keys. This migration rewrites the `source_file` *column* so old and
|
|
176
|
+
* new rows agree on path identity, and collapses duplicate checkpoint rows produced when one
|
|
177
|
+
* physical file was imported via both a symlinked and a real path — keeping the highest
|
|
178
|
+
* `last_imported_line` so an incremental resume does not re-import already-seen content.
|
|
179
|
+
*
|
|
180
|
+
* `record_hash` is intentionally NOT touched: it is path-representation dependent by
|
|
181
|
+
* construction, pre-migration rows are grandfathered, and recomputing it would require the
|
|
182
|
+
* original record payload that the ledger does not store.
|
|
183
|
+
*
|
|
184
|
+
* @param resolveRealPath - resolves a `source_file` to its canonical real path. Falls back to
|
|
185
|
+
* the original value when it returns null/undefined or throws. Decouples the DAO from the
|
|
186
|
+
* runtime `FileSystem` seam.
|
|
187
|
+
*/
|
|
188
|
+
export async function normalizeSourceFilePaths(db, resolveRealPath) {
|
|
189
|
+
await normalizeCheckpointPaths(db, resolveRealPath);
|
|
190
|
+
await normalizeColumnPaths(db, resolveRealPath, 'history_import_ledger');
|
|
191
|
+
for (const table of TYPED_TABLE_COLUMNS_SOURCE_FILE) {
|
|
192
|
+
if (await tableExists(db, table)) {
|
|
193
|
+
await normalizeColumnPaths(db, resolveRealPath, table);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
for (const table of await listEtlTables(db)) {
|
|
197
|
+
await normalizeColumnPaths(db, resolveRealPath, table);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/** Typed contract tables that carry a `source_file` column (0466 forensic contract). */
|
|
201
|
+
const TYPED_TABLE_COLUMNS_SOURCE_FILE = ['history_message', 'history_tool_call'];
|
|
202
|
+
async function tableExists(db, table) {
|
|
203
|
+
const row = await db.queryFirst("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", table);
|
|
204
|
+
return row !== undefined && row !== null;
|
|
205
|
+
}
|
|
206
|
+
async function listEtlTables(db) {
|
|
207
|
+
const rows = await db.queryAll("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'history_etl_%'");
|
|
208
|
+
return rows.map((row) => row.name);
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Normalize `source_file` in the checkpoint table, collapsing duplicate rows per
|
|
212
|
+
* `(source, realpath)` by keeping the highest `last_imported_line`.
|
|
213
|
+
*/
|
|
214
|
+
async function normalizeCheckpointPaths(db, resolveRealPath) {
|
|
215
|
+
const rows = await db.queryAll('SELECT source, source_file, last_imported_line, updated_at FROM history_import_checkpoint');
|
|
216
|
+
// Collapse per (source, realPath) keeping the highest last_imported_line.
|
|
217
|
+
const collapsed = new Map();
|
|
218
|
+
for (const row of rows) {
|
|
219
|
+
const canonical = resolveWithFallback(resolveRealPath, row.source_file);
|
|
220
|
+
const key = `${row.source}\u0000${canonical}`;
|
|
221
|
+
const existing = collapsed.get(key);
|
|
222
|
+
if (existing === undefined || row.last_imported_line > existing.last_imported_line) {
|
|
223
|
+
collapsed.set(key, {
|
|
224
|
+
source: row.source,
|
|
225
|
+
source_file: canonical,
|
|
226
|
+
last_imported_line: row.last_imported_line,
|
|
227
|
+
updated_at: row.updated_at,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
await db.exec('DELETE FROM history_import_checkpoint');
|
|
232
|
+
for (const row of collapsed.values()) {
|
|
233
|
+
await db.run(`INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
|
|
234
|
+
VALUES (?, ?, ?, ?)`, row.source, row.source_file, row.last_imported_line, row.updated_at);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
/** Normalize `source_file` in a table that stores it as a plain column. */
|
|
238
|
+
async function normalizeColumnPaths(db, resolveRealPath, table) {
|
|
239
|
+
const rows = await db.queryAll(`SELECT rowid AS rowid, source_file FROM ${table} WHERE source_file IS NOT NULL`);
|
|
240
|
+
for (const row of rows) {
|
|
241
|
+
const canonical = resolveWithFallback(resolveRealPath, row.source_file);
|
|
242
|
+
if (canonical !== row.source_file) {
|
|
243
|
+
await db.run(`UPDATE ${table} SET source_file = ? WHERE rowid = ?`, canonical, row.rowid);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
function resolveWithFallback(resolveRealPath, sourceFile) {
|
|
248
|
+
try {
|
|
249
|
+
return resolveRealPath(sourceFile) ?? sourceFile;
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
return sourceFile;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
const OPENCODE_SOURCE = 'opencode';
|
|
256
|
+
/** Read one ordered page of OpenCode messages after the supplied cursor. */
|
|
257
|
+
export async function readOpenCodeMessages(db, lastTime, lastId, limit) {
|
|
258
|
+
return db.queryAll(`SELECT m.id, m.session_id, m.time_created, m.data, s.directory
|
|
259
|
+
FROM message m
|
|
260
|
+
JOIN session s ON s.id = m.session_id
|
|
261
|
+
WHERE m.time_created > ? OR (m.time_created = ? AND m.id > ?)
|
|
262
|
+
ORDER BY m.time_created, m.id
|
|
263
|
+
LIMIT ?`, lastTime, lastTime, lastId, limit);
|
|
264
|
+
}
|
|
265
|
+
/** Read all parts belonging to the supplied OpenCode message IDs. */
|
|
266
|
+
export async function readOpenCodeParts(db, messageIds) {
|
|
267
|
+
if (messageIds.length === 0)
|
|
268
|
+
return [];
|
|
269
|
+
return db.queryAll(`SELECT id, message_id, time_created, data
|
|
270
|
+
FROM part
|
|
271
|
+
WHERE message_id IN (${messageIds.map(() => '?').join(', ')})
|
|
272
|
+
ORDER BY time_created, id`, ...messageIds);
|
|
273
|
+
}
|
|
274
|
+
/** Load existing OpenCode ledger entries grouped by source message ID. */
|
|
275
|
+
export async function readOpenCodeExistingEntries(db) {
|
|
276
|
+
const rows = await db.queryAll('SELECT source_file, record_hash, target_table FROM history_import_ledger WHERE source = ?', OPENCODE_SOURCE);
|
|
277
|
+
const bySourceFile = new Map();
|
|
278
|
+
for (const row of rows) {
|
|
279
|
+
const entries = bySourceFile.get(row.source_file) ?? [];
|
|
280
|
+
entries.push({ record_hash: row.record_hash, target_table: row.target_table });
|
|
281
|
+
bySourceFile.set(row.source_file, entries);
|
|
282
|
+
}
|
|
283
|
+
return bySourceFile;
|
|
284
|
+
}
|
|
285
|
+
/** Build deletion operations for records superseded by a forced import. */
|
|
286
|
+
export function openCodeDeleteOperations(entries) {
|
|
287
|
+
return entries.flatMap((entry) => [
|
|
288
|
+
{
|
|
289
|
+
sql: `DELETE FROM ${targetTableFor(entry.target_table)} WHERE record_hash = ?`,
|
|
290
|
+
params: [entry.record_hash],
|
|
291
|
+
},
|
|
292
|
+
{
|
|
293
|
+
sql: 'DELETE FROM history_import_ledger WHERE record_hash = ?',
|
|
294
|
+
params: [entry.record_hash],
|
|
295
|
+
},
|
|
296
|
+
]);
|
|
297
|
+
}
|
|
298
|
+
/** Build batched history, ledger, and checkpoint writes for OpenCode entries. */
|
|
299
|
+
export function openCodeBulkWriteOperations(entries, checkpointFiles, now) {
|
|
300
|
+
const importedAt = timestamp(now);
|
|
301
|
+
const operations = [];
|
|
302
|
+
for (const table of ['history_message', 'history_tool_call']) {
|
|
303
|
+
const tableEntries = entries.filter((entry) => entry.targetTable === table);
|
|
304
|
+
for (let offset = 0; offset < tableEntries.length; offset += 100) {
|
|
305
|
+
const chunk = tableEntries.slice(offset, offset + 100);
|
|
306
|
+
const columns = TYPED_TABLE_COLUMNS[table] ?? [];
|
|
307
|
+
operations.push({
|
|
308
|
+
sql: `WITH input(record_hash, payload_json, imported_at) AS (
|
|
309
|
+
VALUES ${chunk.map(() => '(?, ?, ?)').join(', ')}
|
|
310
|
+
)
|
|
311
|
+
INSERT INTO ${table} (${columns.join(', ')})
|
|
312
|
+
SELECT ${columns
|
|
313
|
+
.map((column) => column === 'record_hash'
|
|
314
|
+
? 'record_hash'
|
|
315
|
+
: column === 'imported_at'
|
|
316
|
+
? 'imported_at'
|
|
317
|
+
: `json_extract(payload_json, '$.${column}')`)
|
|
318
|
+
.join(', ')}
|
|
319
|
+
FROM input
|
|
320
|
+
WHERE true
|
|
321
|
+
ON CONFLICT(record_hash) DO NOTHING`,
|
|
322
|
+
params: chunk.flatMap((entry) => [entry.recordHash, JSON.stringify(entry.record), importedAt]),
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
for (let offset = 0; offset < entries.length; offset += 200) {
|
|
327
|
+
const chunk = entries.slice(offset, offset + 200);
|
|
328
|
+
operations.push({
|
|
329
|
+
sql: `INSERT INTO history_import_ledger
|
|
330
|
+
(record_hash, source, source_file, source_line, split_index, target_table, imported_at)
|
|
331
|
+
VALUES ${chunk.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ')}`,
|
|
332
|
+
params: chunk.flatMap((entry) => [
|
|
333
|
+
entry.recordHash,
|
|
334
|
+
OPENCODE_SOURCE,
|
|
335
|
+
entry.sourceFile,
|
|
336
|
+
1,
|
|
337
|
+
entry.splitIndex,
|
|
338
|
+
entry.targetTable,
|
|
339
|
+
importedAt,
|
|
340
|
+
]),
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
for (let offset = 0; offset < checkpointFiles.length; offset += 200) {
|
|
344
|
+
const chunk = checkpointFiles.slice(offset, offset + 200);
|
|
345
|
+
operations.push({
|
|
346
|
+
sql: `INSERT INTO history_import_checkpoint (source, source_file, last_imported_line, updated_at)
|
|
347
|
+
VALUES ${chunk.map(() => '(?, ?, ?, ?)').join(', ')}
|
|
348
|
+
ON CONFLICT(source, source_file) DO UPDATE SET
|
|
349
|
+
last_imported_line = excluded.last_imported_line,
|
|
350
|
+
updated_at = excluded.updated_at`,
|
|
351
|
+
params: chunk.flatMap((sourceFile) => [OPENCODE_SOURCE, sourceFile, 1, importedAt]),
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
return operations;
|
|
355
|
+
}
|
|
168
356
|
export { ETL_TABLE_DDL, ensureTargetTables, insertLedger, insertRecord, ledgerExists, readCheckpoint, resetCheckpoints, targetTableFor, timestamp, writeCheckpoint, };
|
package/dist/mappers.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import type { JsonObject, SplitEntry } from './types';
|
|
2
|
+
import type { JsonObject, SplitEntry, TransformContext } from './types';
|
|
3
3
|
/** Map a Claude Code JSONL record into one or more ETL split entries. */
|
|
4
4
|
export declare function claudeSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
5
5
|
/** Map a Pi JSONL record into one or more ETL split entries. */
|
|
@@ -10,6 +10,8 @@ export declare function ompSplit(raw: Record<string, unknown>): readonly SplitEn
|
|
|
10
10
|
export declare function codexSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
11
11
|
/** Map an Antigravity (AGY) JSONL record into one or more ETL split entries. */
|
|
12
12
|
export declare function agySplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
13
|
+
/** Map Gemini CLI message, tool, and session-state events into the forensic contract tables. */
|
|
14
|
+
export declare function geminiSplit(raw: Record<string, unknown>, context?: TransformContext): readonly SplitEntry[];
|
|
13
15
|
/** Map a Grok JSONL record into one or more ETL split entries. */
|
|
14
16
|
export declare function grokSplit(raw: Record<string, unknown>): readonly SplitEntry[];
|
|
15
17
|
/** Compute a stable field-shape key from a JSON object: sorted, lowercased, +-joined top-level keys. */
|
|
@@ -26,6 +28,8 @@ export declare const CODEX_FIELD_MAP: Record<string, string>;
|
|
|
26
28
|
export declare const AGY_FIELD_MAP: Record<string, string>;
|
|
27
29
|
/** Identity field map for the Grok mapper. */
|
|
28
30
|
export declare const GROK_FIELD_MAP: Record<string, string>;
|
|
31
|
+
/** Identity field map for the Gemini mapper. */
|
|
32
|
+
export declare const GEMINI_FIELD_MAP: Record<string, string>;
|
|
29
33
|
/** Passthrough Zod schema for the Claude mapper. */
|
|
30
34
|
export declare const CLAUDE_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
31
35
|
/** Passthrough Zod schema for the Pi mapper. */
|
|
@@ -38,4 +42,6 @@ export declare const CODEX_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
|
38
42
|
export declare const AGY_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
39
43
|
/** Passthrough Zod schema for the Grok mapper. */
|
|
40
44
|
export declare const GROK_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
45
|
+
/** Passthrough Zod schema for the Gemini mapper. */
|
|
46
|
+
export declare const GEMINI_SCHEMA: z.ZodObject<{}, z.core.$loose>;
|
|
41
47
|
//# sourceMappingURL=mappers.d.ts.map
|
package/dist/mappers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"mappers.d.ts","sourceRoot":"","sources":["../src/mappers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AA2FxE,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4G/E;AAMD,gEAAgE;AAChE,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,UAAU,EAAE,CAsE3D;AAMD,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CA4F5E;AAMD,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAiG9E;AAMD,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAgH5E;AAMD,gGAAgG;AAChG,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS,UAAU,EAAE,CAgF3G;AAyMD,kEAAkE;AAClE,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,UAAU,EAAE,CAyS7E;AAwDD,wGAAwG;AACxG,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAKxD;AAMD,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AACpG,4CAA4C;AAC5C,eAAO,MAAM,YAAY,wBAAsE,CAAC;AAChG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,+CAA+C;AAC/C,eAAO,MAAM,eAAe,wBAAsE,CAAC;AACnG,6CAA6C;AAC7C,eAAO,MAAM,aAAa,wBAAsE,CAAC;AACjG,8CAA8C;AAC9C,eAAO,MAAM,cAAc,wBAAsE,CAAC;AAClG,gDAAgD;AAChD,eAAO,MAAM,gBAAgB,wBAAsE,CAAC;AAQpG,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC;AAC/C,gDAAgD;AAChD,eAAO,MAAM,SAAS,gCAAoB,CAAC;AAC3C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,mDAAmD;AACnD,eAAO,MAAM,YAAY,gCAAoB,CAAC;AAC9C,iDAAiD;AACjD,eAAO,MAAM,UAAU,gCAAoB,CAAC;AAC5C,kDAAkD;AAClD,eAAO,MAAM,WAAW,gCAAoB,CAAC;AAC7C,oDAAoD;AACpD,eAAO,MAAM,aAAa,gCAAoB,CAAC"}
|
package/dist/mappers.js
CHANGED
|
@@ -366,9 +366,9 @@ export function codexSplit(raw) {
|
|
|
366
366
|
record: {
|
|
367
367
|
session_id: sessionId,
|
|
368
368
|
seq,
|
|
369
|
-
role: '
|
|
369
|
+
role: 'meta',
|
|
370
370
|
record_type: 'short_format',
|
|
371
|
-
disposition: '
|
|
371
|
+
disposition: 'meta',
|
|
372
372
|
ts,
|
|
373
373
|
provenance: 'ambient',
|
|
374
374
|
},
|
|
@@ -552,6 +552,109 @@ export function agySplit(raw) {
|
|
|
552
552
|
return entries;
|
|
553
553
|
}
|
|
554
554
|
// ---------------------------------------------------------------------------
|
|
555
|
+
// Gemini mapper
|
|
556
|
+
// ---------------------------------------------------------------------------
|
|
557
|
+
/** Map Gemini CLI message, tool, and session-state events into the forensic contract tables. */
|
|
558
|
+
export function geminiSplit(raw, context) {
|
|
559
|
+
const recordType = String(raw.type ?? (raw.kind !== undefined ? 'session' : raw.$set !== undefined ? 'state' : ''));
|
|
560
|
+
const state = o(raw.$set);
|
|
561
|
+
const sessionId = sessionIdFromContext(context, raw);
|
|
562
|
+
const seq = context?.sourceLine ?? 0;
|
|
563
|
+
const ts = s(raw.timestamp, raw.startTime, raw.lastUpdated, state.lastUpdated) ?? new Date(0).toISOString();
|
|
564
|
+
if (recordType === 'session' || recordType === 'state') {
|
|
565
|
+
return [
|
|
566
|
+
{
|
|
567
|
+
targetTable: 'history_message',
|
|
568
|
+
record: {
|
|
569
|
+
session_id: sessionId,
|
|
570
|
+
seq,
|
|
571
|
+
role: 'meta',
|
|
572
|
+
record_type: recordType,
|
|
573
|
+
disposition: 'meta',
|
|
574
|
+
ts,
|
|
575
|
+
content_text: s(state.summary) ?? null,
|
|
576
|
+
provenance: detectProvenance(context?.sourceFile),
|
|
577
|
+
},
|
|
578
|
+
},
|
|
579
|
+
];
|
|
580
|
+
}
|
|
581
|
+
const role = recordType === 'gemini' ? 'assistant' : recordType === 'user' ? 'user' : 'system';
|
|
582
|
+
const tokens = o(raw.tokens);
|
|
583
|
+
const content = geminiContent(raw);
|
|
584
|
+
const entries = [
|
|
585
|
+
{
|
|
586
|
+
targetTable: 'history_message',
|
|
587
|
+
record: {
|
|
588
|
+
session_id: sessionId,
|
|
589
|
+
seq,
|
|
590
|
+
role,
|
|
591
|
+
record_type: recordType,
|
|
592
|
+
disposition: 'keep',
|
|
593
|
+
ts,
|
|
594
|
+
duration_ms: null,
|
|
595
|
+
model: s(raw.model) ?? null,
|
|
596
|
+
input_tokens: numberOrNull(tokens.input),
|
|
597
|
+
output_tokens: numberOrNull(tokens.output),
|
|
598
|
+
cache_read_tokens: numberOrNull(tokens.cached),
|
|
599
|
+
cache_write_tokens: null,
|
|
600
|
+
cost_usd: null,
|
|
601
|
+
content_text: content,
|
|
602
|
+
cwd: null,
|
|
603
|
+
provenance: detectProvenance(context?.sourceFile),
|
|
604
|
+
},
|
|
605
|
+
},
|
|
606
|
+
];
|
|
607
|
+
if (Array.isArray(raw.toolCalls)) {
|
|
608
|
+
for (const value of raw.toolCalls) {
|
|
609
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value))
|
|
610
|
+
continue;
|
|
611
|
+
const tool = value;
|
|
612
|
+
const status = s(tool.status) ?? 'unknown';
|
|
613
|
+
const timestamp = s(tool.timestamp);
|
|
614
|
+
const result = tool.result ?? tool.resultDisplay;
|
|
615
|
+
entries.push({
|
|
616
|
+
targetTable: 'history_tool_call',
|
|
617
|
+
record: {
|
|
618
|
+
_messageSplitIndex: 0,
|
|
619
|
+
session_id: sessionId,
|
|
620
|
+
seq,
|
|
621
|
+
tool_name: s(tool.name, tool.displayName) ?? 'unknown',
|
|
622
|
+
args_digest: argsDigest(tool.args),
|
|
623
|
+
status,
|
|
624
|
+
started_at: timestamp,
|
|
625
|
+
completed_at: timestamp,
|
|
626
|
+
duration_ms: null,
|
|
627
|
+
result_bytes: result === undefined ? null : new TextEncoder().encode(JSON.stringify(result)).byteLength,
|
|
628
|
+
error_text: status === 'error' ? (s(tool.resultDisplay, tool.result) ?? null) : null,
|
|
629
|
+
},
|
|
630
|
+
});
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
return entries;
|
|
634
|
+
}
|
|
635
|
+
function sessionIdFromContext(context, raw) {
|
|
636
|
+
if (context !== undefined) {
|
|
637
|
+
const file = context.sourceFile.split('/').at(-1);
|
|
638
|
+
if (file !== undefined)
|
|
639
|
+
return file.replace(/\.jsonl$/, '');
|
|
640
|
+
}
|
|
641
|
+
return s(raw.sessionId, raw.id) ?? 'unknown';
|
|
642
|
+
}
|
|
643
|
+
function geminiContent(raw) {
|
|
644
|
+
const content = extractContentText(raw.content);
|
|
645
|
+
const thoughts = Array.isArray(raw.thoughts)
|
|
646
|
+
? raw.thoughts
|
|
647
|
+
.map((value) => (value !== null && typeof value === 'object' ? s(o(value).description) : undefined))
|
|
648
|
+
.filter((value) => value !== undefined)
|
|
649
|
+
.join('\n')
|
|
650
|
+
: '';
|
|
651
|
+
return ([content, thoughts].filter((value) => value !== undefined && value.length > 0).join('\n') ||
|
|
652
|
+
null);
|
|
653
|
+
}
|
|
654
|
+
function numberOrNull(value) {
|
|
655
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
656
|
+
}
|
|
657
|
+
// ---------------------------------------------------------------------------
|
|
555
658
|
// Grok mapper
|
|
556
659
|
// ---------------------------------------------------------------------------
|
|
557
660
|
/**
|
|
@@ -566,13 +669,29 @@ function normalizeGrokRecord(raw) {
|
|
|
566
669
|
const update = params?.update;
|
|
567
670
|
const isUpdatesShape = typeof raw.method === 'string' && params !== undefined && update !== undefined;
|
|
568
671
|
const body = isUpdatesShape ? update : raw;
|
|
569
|
-
const recordType = String((isUpdatesShape ? update?.sessionUpdate : raw.type) ??
|
|
570
|
-
|
|
571
|
-
|
|
672
|
+
const recordType = String((isUpdatesShape ? update?.sessionUpdate : raw.type) ??
|
|
673
|
+
raw.sessionUpdate ??
|
|
674
|
+
raw.eventType ??
|
|
675
|
+
(raw.file_snapshots !== undefined || raw.after_snapshots !== undefined
|
|
676
|
+
? 'file_snapshot'
|
|
677
|
+
: raw.prompt !== undefined
|
|
678
|
+
? 'user_message'
|
|
679
|
+
: raw.answer !== undefined || raw.question !== undefined
|
|
680
|
+
? 'assistant'
|
|
681
|
+
: ''));
|
|
682
|
+
const sessionId = s(params?.sessionId, raw.session_id, raw.sessionId, raw.parentSessionId, raw.btwSessionId, raw.params?.sessionId) ?? 'unknown';
|
|
683
|
+
const ts = normalizeTs(raw.ts ?? raw.timestamp ?? raw.created_at ?? raw.askedAt ?? body.ts ?? body.timestamp);
|
|
572
684
|
const seq = typeof raw.seq === 'number' ? raw.seq : typeof body.seq === 'number' ? body.seq : 0;
|
|
573
685
|
const meta = (body._meta ?? params?._meta ?? raw._meta);
|
|
574
686
|
const model = s(meta?.modelId, body.model, raw.model);
|
|
575
|
-
const contentText = extractGrokContent(body.content ??
|
|
687
|
+
const contentText = extractGrokContent(body.content ??
|
|
688
|
+
body.text ??
|
|
689
|
+
raw.content ??
|
|
690
|
+
raw.text ??
|
|
691
|
+
raw.prompt ??
|
|
692
|
+
(raw.answer !== undefined || raw.question !== undefined
|
|
693
|
+
? [raw.question, raw.answer].filter((value) => typeof value === 'string').join('\n')
|
|
694
|
+
: undefined));
|
|
576
695
|
const usage = (body.usage ?? raw.usage);
|
|
577
696
|
const toolMeta = meta?.['x.ai/tool'];
|
|
578
697
|
const toolName = s(body.title, body.tool_name, body.name, toolMeta?.name, toolMeta?.label, raw.tool_name, raw.name) ?? '';
|
|
@@ -756,6 +875,27 @@ export function grokSplit(raw) {
|
|
|
756
875
|
return entries;
|
|
757
876
|
}
|
|
758
877
|
if (recordType === 'tool_completed') {
|
|
878
|
+
entries.push({
|
|
879
|
+
targetTable: 'history_message',
|
|
880
|
+
record: {
|
|
881
|
+
session_id: sessionId,
|
|
882
|
+
seq,
|
|
883
|
+
role: 'assistant',
|
|
884
|
+
record_type: recordType,
|
|
885
|
+
disposition: 'keep',
|
|
886
|
+
ts,
|
|
887
|
+
duration_ms: n.durationMs ?? null,
|
|
888
|
+
model: n.model ?? null,
|
|
889
|
+
input_tokens: null,
|
|
890
|
+
output_tokens: null,
|
|
891
|
+
cache_read_tokens: null,
|
|
892
|
+
cache_write_tokens: null,
|
|
893
|
+
cost_usd: null,
|
|
894
|
+
content_text: n.contentText,
|
|
895
|
+
cwd: null,
|
|
896
|
+
provenance: 'ambient',
|
|
897
|
+
},
|
|
898
|
+
});
|
|
759
899
|
entries.push({
|
|
760
900
|
targetTable: 'history_tool_call',
|
|
761
901
|
record: {
|
|
@@ -1017,6 +1157,8 @@ export const CODEX_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_
|
|
|
1017
1157
|
export const AGY_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_CALL_MAPPER_KEYS));
|
|
1018
1158
|
/** Identity field map for the Grok mapper. */
|
|
1019
1159
|
export const GROK_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_CALL_MAPPER_KEYS));
|
|
1160
|
+
/** Identity field map for the Gemini mapper. */
|
|
1161
|
+
export const GEMINI_FIELD_MAP = identityFieldMap(MESSAGE_MAPPER_KEYS.concat(TOOL_CALL_MAPPER_KEYS));
|
|
1020
1162
|
// ---------------------------------------------------------------------------
|
|
1021
1163
|
// Zod schemas for the six source mappers
|
|
1022
1164
|
// ---------------------------------------------------------------------------
|
|
@@ -1033,3 +1175,5 @@ export const CODEX_SCHEMA = passthroughSchema;
|
|
|
1033
1175
|
export const AGY_SCHEMA = passthroughSchema;
|
|
1034
1176
|
/** Passthrough Zod schema for the Grok mapper. */
|
|
1035
1177
|
export const GROK_SCHEMA = passthroughSchema;
|
|
1178
|
+
/** Passthrough Zod schema for the Gemini mapper. */
|
|
1179
|
+
export const GEMINI_SCHEMA = passthroughSchema;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type DbAdapter } from '@gobing-ai/ts-db';
|
|
2
|
+
import { type FileSystem, type RuntimePaths } from '@gobing-ai/ts-runtime';
|
|
3
|
+
import type { ImportMode, ImportResult, RedactionRule } from './types';
|
|
4
|
+
/** Dependencies and mode controls for importing OpenCode's SQLite history. */
|
|
5
|
+
export interface OpenCodeImportOptions {
|
|
6
|
+
db: DbAdapter;
|
|
7
|
+
sourceDatabase?: string;
|
|
8
|
+
fileSystem?: FileSystem;
|
|
9
|
+
paths?: RuntimePaths;
|
|
10
|
+
mode?: ImportMode;
|
|
11
|
+
dryRun?: boolean;
|
|
12
|
+
redactionRules?: readonly RedactionRule[];
|
|
13
|
+
now?: () => Date;
|
|
14
|
+
}
|
|
15
|
+
/** Import OpenCode's current SQLite history store into the forensic contract tables. */
|
|
16
|
+
export declare function runOpenCodeImport(options: OpenCodeImportOptions): Promise<ImportResult>;
|
|
17
|
+
//# sourceMappingURL=opencode-importer.d.ts.map
|