@toiroakr/lines-db 0.9.2 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/bin/cli.mjs +7172 -125
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +3 -3
- package/dist/index.mjs.map +1 -1
- package/package.json +17 -27
- package/src/cli.ts +112 -111
- package/tsdown.config.ts +9 -5
- package/dist/index.cjs +0 -1463
- package/dist/index.d.cts +0 -604
- package/dist/index.d.cts.map +0 -1
package/dist/index.cjs
DELETED
|
@@ -1,1463 +0,0 @@
|
|
|
1
|
-
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
let node_fs_promises = require("node:fs/promises");
|
|
3
|
-
let node_path = require("node:path");
|
|
4
|
-
let node_url = require("node:url");
|
|
5
|
-
let node_util = require("node:util");
|
|
6
|
-
//#region src/runtime.ts
|
|
7
|
-
function detectRuntime() {
|
|
8
|
-
if (typeof process !== "undefined" && process.versions && process.versions.node) return "node";
|
|
9
|
-
return "unknown";
|
|
10
|
-
}
|
|
11
|
-
const RUNTIME = detectRuntime();
|
|
12
|
-
//#endregion
|
|
13
|
-
//#region src/sqlite-adapter.ts
|
|
14
|
-
/**
|
|
15
|
-
* SQLite adapter for Node.js
|
|
16
|
-
*/
|
|
17
|
-
/**
|
|
18
|
-
* Create a SQLite database instance for Node.js
|
|
19
|
-
*/
|
|
20
|
-
function createDatabase(path = ":memory:") {
|
|
21
|
-
if (RUNTIME === "node") return createNodeDatabase(path);
|
|
22
|
-
else throw new Error(`Unsupported runtime: ${RUNTIME}`);
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* Create a Node.js SQLite database
|
|
26
|
-
*/
|
|
27
|
-
function createNodeDatabase(path) {
|
|
28
|
-
const { DatabaseSync } = require("node:sqlite");
|
|
29
|
-
const db = new DatabaseSync(path);
|
|
30
|
-
db.exec("PRAGMA foreign_keys = ON");
|
|
31
|
-
return {
|
|
32
|
-
prepare(sql) {
|
|
33
|
-
const stmt = db.prepare(sql);
|
|
34
|
-
return {
|
|
35
|
-
run(...params) {
|
|
36
|
-
return stmt.run(...params);
|
|
37
|
-
},
|
|
38
|
-
get(...params) {
|
|
39
|
-
return stmt.get(...params);
|
|
40
|
-
},
|
|
41
|
-
all(...params) {
|
|
42
|
-
return stmt.all(...params);
|
|
43
|
-
}
|
|
44
|
-
};
|
|
45
|
-
},
|
|
46
|
-
exec(sql) {
|
|
47
|
-
db.exec(sql);
|
|
48
|
-
},
|
|
49
|
-
close() {
|
|
50
|
-
db.close();
|
|
51
|
-
}
|
|
52
|
-
};
|
|
53
|
-
}
|
|
54
|
-
//#endregion
|
|
55
|
-
//#region src/jsonl-reader.ts
|
|
56
|
-
var JsonlReader = class {
|
|
57
|
-
static overrides = null;
|
|
58
|
-
/**
|
|
59
|
-
* Temporarily override the data returned for specific JSONL files.
|
|
60
|
-
* Useful for scenarios like migration validation where in-memory data should be used.
|
|
61
|
-
*/
|
|
62
|
-
static async withOverrides(overrides, fn) {
|
|
63
|
-
const normalized = /* @__PURE__ */ new Map();
|
|
64
|
-
for (const [filePath, rows] of overrides) normalized.set((0, node_path.normalize)(filePath), rows);
|
|
65
|
-
const previousOverrides = this.overrides;
|
|
66
|
-
this.overrides = normalized;
|
|
67
|
-
try {
|
|
68
|
-
return await fn();
|
|
69
|
-
} finally {
|
|
70
|
-
this.overrides = previousOverrides;
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* Read JSONL file and parse each line as JSON
|
|
75
|
-
*/
|
|
76
|
-
static async read(filePath) {
|
|
77
|
-
const overrideRows = this.overrides?.get((0, node_path.normalize)(filePath));
|
|
78
|
-
if (overrideRows) return overrideRows.map((row) => JSON.parse(JSON.stringify(row)));
|
|
79
|
-
return (await (0, node_fs_promises.readFile)(filePath, "utf-8")).trim().split("\n").filter((line) => line.trim().length > 0).map((line) => {
|
|
80
|
-
try {
|
|
81
|
-
return JSON.parse(line);
|
|
82
|
-
} catch (error) {
|
|
83
|
-
throw new Error(`Failed to parse JSON line: ${line}`, { cause: error });
|
|
84
|
-
}
|
|
85
|
-
});
|
|
86
|
-
}
|
|
87
|
-
/**
|
|
88
|
-
* Infer schema from JSONL data
|
|
89
|
-
*/
|
|
90
|
-
static inferSchema(tableName, data) {
|
|
91
|
-
if (data.length === 0) throw new Error("Cannot infer schema from empty data");
|
|
92
|
-
const columnTypes = /* @__PURE__ */ new Map();
|
|
93
|
-
const booleanColumns = /* @__PURE__ */ new Set();
|
|
94
|
-
const nonBooleanColumns = /* @__PURE__ */ new Set();
|
|
95
|
-
for (const row of data) for (const [key, value] of Object.entries(row)) {
|
|
96
|
-
if (!columnTypes.has(key)) columnTypes.set(key, /* @__PURE__ */ new Set());
|
|
97
|
-
columnTypes.get(key).add(this.inferType(value));
|
|
98
|
-
if (typeof value === "boolean") booleanColumns.add(key);
|
|
99
|
-
else if (value !== null) nonBooleanColumns.add(key);
|
|
100
|
-
}
|
|
101
|
-
const columns = [];
|
|
102
|
-
for (const [columnName, types] of columnTypes.entries()) {
|
|
103
|
-
const typeArray = Array.from(types);
|
|
104
|
-
let sqlType = "TEXT";
|
|
105
|
-
if (typeArray.length === 1) sqlType = typeArray[0];
|
|
106
|
-
else if (typeArray.every((t) => t === "INTEGER" || t === "REAL")) sqlType = "REAL";
|
|
107
|
-
else if (!typeArray.includes("NULL")) sqlType = "TEXT";
|
|
108
|
-
else if (typeArray.length === 2 && typeArray.includes("NULL")) sqlType = typeArray.find((t) => t !== "NULL");
|
|
109
|
-
const isBooleanColumn = booleanColumns.has(columnName) && !nonBooleanColumns.has(columnName);
|
|
110
|
-
columns.push({
|
|
111
|
-
name: columnName,
|
|
112
|
-
type: sqlType,
|
|
113
|
-
notNull: !typeArray.includes("NULL"),
|
|
114
|
-
valueType: isBooleanColumn ? "boolean" : void 0
|
|
115
|
-
});
|
|
116
|
-
}
|
|
117
|
-
const idColumn = columns.find((col) => col.name === "id");
|
|
118
|
-
if (idColumn) idColumn.primaryKey = true;
|
|
119
|
-
return {
|
|
120
|
-
name: tableName,
|
|
121
|
-
columns
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
|
-
static inferType(value) {
|
|
125
|
-
if (value === null || value === void 0) return "NULL";
|
|
126
|
-
if (typeof value === "number") return Number.isInteger(value) ? "INTEGER" : "REAL";
|
|
127
|
-
if (typeof value === "string") return "TEXT";
|
|
128
|
-
if (typeof value === "boolean") return "INTEGER";
|
|
129
|
-
if (typeof value === "object") return "JSON";
|
|
130
|
-
return "TEXT";
|
|
131
|
-
}
|
|
132
|
-
};
|
|
133
|
-
//#endregion
|
|
134
|
-
//#region src/jsonl-writer.ts
|
|
135
|
-
var JsonlWriter = class {
|
|
136
|
-
/**
|
|
137
|
-
* Write data to JSONL file
|
|
138
|
-
*/
|
|
139
|
-
static async write(filePath, data) {
|
|
140
|
-
await (0, node_fs_promises.writeFile)(filePath, data.map((obj) => JSON.stringify(obj)).join("\n") + "\n", "utf-8");
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Append data to JSONL file
|
|
144
|
-
*/
|
|
145
|
-
static async append(filePath, data) {
|
|
146
|
-
const { readFile, writeFile } = await import("node:fs/promises");
|
|
147
|
-
try {
|
|
148
|
-
const existing = await readFile(filePath, "utf-8");
|
|
149
|
-
const lines = data.map((obj) => JSON.stringify(obj)).join("\n");
|
|
150
|
-
await writeFile(filePath, existing.trim() + "\n" + lines + "\n", "utf-8");
|
|
151
|
-
} catch (error) {
|
|
152
|
-
if (error.code === "ENOENT") await this.write(filePath, data);
|
|
153
|
-
else throw error;
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
};
|
|
157
|
-
//#endregion
|
|
158
|
-
//#region src/schema-extensions.ts
|
|
159
|
-
/**
|
|
160
|
-
* Supported schema file extensions, in priority order.
|
|
161
|
-
* The first match wins when discovering schema files.
|
|
162
|
-
*/
|
|
163
|
-
const SCHEMA_EXTENSIONS = [
|
|
164
|
-
".schema.ts",
|
|
165
|
-
".schema.mts",
|
|
166
|
-
".schema.cts"
|
|
167
|
-
];
|
|
168
|
-
/**
|
|
169
|
-
* Map from schema extensions to their JavaScript import counterparts.
|
|
170
|
-
*/
|
|
171
|
-
const SCHEMA_TO_JS_IMPORT_MAP = {
|
|
172
|
-
".schema.ts": ".schema.js",
|
|
173
|
-
".schema.mts": ".schema.mjs",
|
|
174
|
-
".schema.cts": ".schema.cjs"
|
|
175
|
-
};
|
|
176
|
-
/**
|
|
177
|
-
* Try each supported schema extension and return the full path of the first
|
|
178
|
-
* one that exists on disk. Returns undefined if none is found.
|
|
179
|
-
*/
|
|
180
|
-
async function findSchemaFile(dir, tableName) {
|
|
181
|
-
for (const ext of SCHEMA_EXTENSIONS) {
|
|
182
|
-
const candidate = (0, node_path.join)(dir, `${tableName}${ext}`);
|
|
183
|
-
try {
|
|
184
|
-
await (0, node_fs_promises.access)(candidate);
|
|
185
|
-
return candidate;
|
|
186
|
-
} catch {}
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
/**
|
|
190
|
-
* Synchronously find a schema file among directory entries.
|
|
191
|
-
* Returns the full path of the first match, or undefined.
|
|
192
|
-
*/
|
|
193
|
-
function findSchemaFileInEntries(dataDirPath, tableName, entries) {
|
|
194
|
-
for (const ext of SCHEMA_EXTENSIONS) {
|
|
195
|
-
const candidateName = `${tableName}${ext}`;
|
|
196
|
-
if (entries.some((e) => e.isFile() && e.name === candidateName)) return (0, node_path.join)(dataDirPath, candidateName);
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Check if a filename matches any supported schema file pattern.
|
|
201
|
-
*/
|
|
202
|
-
function isSchemaFile(fileName) {
|
|
203
|
-
return SCHEMA_EXTENSIONS.some((ext) => fileName.endsWith(ext));
|
|
204
|
-
}
|
|
205
|
-
/**
|
|
206
|
-
* Extract table name from a schema filename.
|
|
207
|
-
* e.g., "users.schema.ts" -> "users", "users.schema.mts" -> "users"
|
|
208
|
-
*/
|
|
209
|
-
function extractTableNameFromSchemaFile(fileName) {
|
|
210
|
-
for (const ext of SCHEMA_EXTENSIONS) if (fileName.endsWith(ext)) return fileName.slice(0, -ext.length);
|
|
211
|
-
return null;
|
|
212
|
-
}
|
|
213
|
-
/**
|
|
214
|
-
* Rewrite a TypeScript path to its JavaScript counterpart for ESM imports.
|
|
215
|
-
* ".schema.ts" -> ".schema.js", ".schema.mts" -> ".schema.mjs", ".schema.cts" -> ".schema.cjs"
|
|
216
|
-
*/
|
|
217
|
-
function rewriteExtensionForImport(filePath) {
|
|
218
|
-
for (const [tsExt, jsExt] of Object.entries(SCHEMA_TO_JS_IMPORT_MAP)) if (filePath.endsWith(tsExt)) return filePath.slice(0, -tsExt.length) + jsExt;
|
|
219
|
-
return filePath;
|
|
220
|
-
}
|
|
221
|
-
//#endregion
|
|
222
|
-
//#region src/schema-loader.ts
|
|
223
|
-
var SchemaLoader = class {
|
|
224
|
-
/**
|
|
225
|
-
* Check if a schema file exists for a table
|
|
226
|
-
*/
|
|
227
|
-
static async hasSchema(jsonlPath) {
|
|
228
|
-
return await findSchemaFile((0, node_path.dirname)(jsonlPath), (0, node_path.basename)(jsonlPath, ".jsonl")) !== void 0;
|
|
229
|
-
}
|
|
230
|
-
/**
|
|
231
|
-
* Load a validation schema file for a table
|
|
232
|
-
* Requires ${tableName}.schema.{ts,mts,cts} to exist alongside the JSONL file
|
|
233
|
-
*/
|
|
234
|
-
static async loadSchema(jsonlPath) {
|
|
235
|
-
const dir = (0, node_path.dirname)(jsonlPath);
|
|
236
|
-
const tableName = (0, node_path.basename)(jsonlPath, ".jsonl");
|
|
237
|
-
const schemaPath = await findSchemaFile(dir, tableName);
|
|
238
|
-
if (!schemaPath) throw new Error(`Schema file not found for table '${tableName}'. Expected one of: ${SCHEMA_EXTENSIONS.map((ext) => `${tableName}${ext}`).join(", ")}`);
|
|
239
|
-
try {
|
|
240
|
-
const module = await import(`${(0, node_url.pathToFileURL)(schemaPath).href}?t=${Date.now()}`);
|
|
241
|
-
const schema = module.default || module.schema;
|
|
242
|
-
if (schema && this.isStandardSchema(schema)) return schema;
|
|
243
|
-
throw new Error(`Schema file ${schemaPath} does not export a valid StandardSchema`);
|
|
244
|
-
} catch (error) {
|
|
245
|
-
throw new Error(`Failed to load schema for table '${tableName}' from ${schemaPath}: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : void 0 });
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
/**
|
|
249
|
-
* Check if an object implements the StandardSchema interface
|
|
250
|
-
*/
|
|
251
|
-
static isStandardSchema(obj) {
|
|
252
|
-
if (!obj || typeof obj !== "object") return false;
|
|
253
|
-
const standard = obj["~standard"];
|
|
254
|
-
if (!standard || typeof standard !== "object") return false;
|
|
255
|
-
const standardObj = standard;
|
|
256
|
-
return standardObj.version === 1 && typeof standardObj.vendor === "string" && typeof standardObj.validate === "function";
|
|
257
|
-
}
|
|
258
|
-
};
|
|
259
|
-
//#endregion
|
|
260
|
-
//#region src/directory-scanner.ts
|
|
261
|
-
var DirectoryScanner = class {
|
|
262
|
-
/**
|
|
263
|
-
* Scan directory for JSONL files and create table configurations
|
|
264
|
-
*/
|
|
265
|
-
static async scanDirectory(dataDir) {
|
|
266
|
-
const tables = /* @__PURE__ */ new Map();
|
|
267
|
-
try {
|
|
268
|
-
const files = await (0, node_fs_promises.readdir)(dataDir);
|
|
269
|
-
for (const file of files) if ((0, node_path.extname)(file) === ".jsonl") {
|
|
270
|
-
const tableName = (0, node_path.basename)(file, ".jsonl");
|
|
271
|
-
const jsonlPath = (0, node_path.join)(dataDir, file);
|
|
272
|
-
tables.set(tableName, {
|
|
273
|
-
jsonlPath,
|
|
274
|
-
autoInferSchema: true
|
|
275
|
-
});
|
|
276
|
-
}
|
|
277
|
-
if (tables.size === 0) console.warn(`Warning: No JSONL files found in directory: ${dataDir}`);
|
|
278
|
-
return tables;
|
|
279
|
-
} catch (error) {
|
|
280
|
-
throw new Error(`Failed to scan directory ${dataDir}: ${error instanceof Error ? error.message : String(error)}`);
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
};
|
|
284
|
-
//#endregion
|
|
285
|
-
//#region src/schema.ts
|
|
286
|
-
/**
|
|
287
|
-
* Define a bidirectional schema with optional backward transformation
|
|
288
|
-
*
|
|
289
|
-
* @param schema - Standard Schema for validation
|
|
290
|
-
* @param options - SchemaOptions object. When Input and Output types differ, backward transformation is required
|
|
291
|
-
*
|
|
292
|
-
* @example
|
|
293
|
-
* // No transformation - backward not needed
|
|
294
|
-
* const schema = defineSchema(
|
|
295
|
-
* v.object({ id: v.number(), name: v.string() })
|
|
296
|
-
* );
|
|
297
|
-
*
|
|
298
|
-
* @example
|
|
299
|
-
* // With transformation - backward REQUIRED
|
|
300
|
-
* const schema = defineSchema(
|
|
301
|
-
* v.pipe(v.string(), v.transform(Number)),
|
|
302
|
-
* {
|
|
303
|
-
* backward: (num) => String(num) // backward: number → string (REQUIRED)
|
|
304
|
-
* }
|
|
305
|
-
* );
|
|
306
|
-
*
|
|
307
|
-
* @example
|
|
308
|
-
* // With primary key and foreign key
|
|
309
|
-
* const schema = defineSchema(
|
|
310
|
-
* v.object({ id: v.number(), customerId: v.number() }),
|
|
311
|
-
* {
|
|
312
|
-
* primaryKey: 'id',
|
|
313
|
-
* foreignKeys: [
|
|
314
|
-
* { column: 'customerId', references: { table: 'users', column: 'id' } }
|
|
315
|
-
* ]
|
|
316
|
-
* }
|
|
317
|
-
* );
|
|
318
|
-
*/
|
|
319
|
-
function defineSchema(schema, ...args) {
|
|
320
|
-
const options = args[0];
|
|
321
|
-
const bidirectionalSchema = Object.create(schema);
|
|
322
|
-
if (options) {
|
|
323
|
-
if (options.backward) bidirectionalSchema.backward = options.backward;
|
|
324
|
-
if (options.primaryKey) bidirectionalSchema.primaryKey = options.primaryKey;
|
|
325
|
-
if (options.foreignKeys) bidirectionalSchema.foreignKeys = options.foreignKeys;
|
|
326
|
-
if (options.indexes) bidirectionalSchema.indexes = options.indexes;
|
|
327
|
-
}
|
|
328
|
-
Object.defineProperty(bidirectionalSchema, "~standard", {
|
|
329
|
-
value: schema["~standard"],
|
|
330
|
-
enumerable: true,
|
|
331
|
-
configurable: true
|
|
332
|
-
});
|
|
333
|
-
return bidirectionalSchema;
|
|
334
|
-
}
|
|
335
|
-
/**
|
|
336
|
-
* Check if a schema has backward transformation
|
|
337
|
-
*/
|
|
338
|
-
function hasBackward(schema) {
|
|
339
|
-
return "backward" in schema && typeof schema.backward === "function";
|
|
340
|
-
}
|
|
341
|
-
//#endregion
|
|
342
|
-
//#region src/database.ts
|
|
343
|
-
var LinesDB = class LinesDB {
|
|
344
|
-
db;
|
|
345
|
-
config;
|
|
346
|
-
schemas = /* @__PURE__ */ new Map();
|
|
347
|
-
validationSchemas = /* @__PURE__ */ new Map();
|
|
348
|
-
tables = /* @__PURE__ */ new Map();
|
|
349
|
-
inTransaction = false;
|
|
350
|
-
constructor(config, dbPath) {
|
|
351
|
-
this.config = config;
|
|
352
|
-
this.db = createDatabase(dbPath ?? ":memory:");
|
|
353
|
-
}
|
|
354
|
-
static create(config, dbPath) {
|
|
355
|
-
return new LinesDB(config, dbPath);
|
|
356
|
-
}
|
|
357
|
-
/**
|
|
358
|
-
* Initialize database by loading all JSONL files or a specific table
|
|
359
|
-
* Uses dependency resolution to ensure foreign key references are loaded in correct order
|
|
360
|
-
* @param options Optional configuration for initialization
|
|
361
|
-
* @param options.tableName Optional table name to initialize. If not provided, initializes all tables
|
|
362
|
-
* @param options.detailedValidate If true, performs detailed validation by inserting rows one by one to catch constraint violations
|
|
363
|
-
* @param options.transform Optional transform function to apply to rows before validation (only applied to the specified tableName)
|
|
364
|
-
* @returns ValidationResult containing validation status, errors, and warnings
|
|
365
|
-
*/
|
|
366
|
-
async initialize(options) {
|
|
367
|
-
const allErrors = [];
|
|
368
|
-
const allWarnings = [];
|
|
369
|
-
const allRowCounts = /* @__PURE__ */ new Map();
|
|
370
|
-
const tableName = options?.tableName;
|
|
371
|
-
const detailedValidate = options?.detailedValidate ?? false;
|
|
372
|
-
const transform = options?.transform;
|
|
373
|
-
this.tables = await DirectoryScanner.scanDirectory(this.config.dataDir);
|
|
374
|
-
const tablesToLoad = tableName ? [tableName] : Array.from(this.tables.keys());
|
|
375
|
-
for (const tableNameToLoad of tablesToLoad) if (!this.tables.has(tableNameToLoad)) throw new Error(`Table '${tableNameToLoad}' not found in directory '${this.config.dataDir}'`);
|
|
376
|
-
const loadedTables = /* @__PURE__ */ new Set();
|
|
377
|
-
const loadingTables = /* @__PURE__ */ new Set();
|
|
378
|
-
const attemptedTables = /* @__PURE__ */ new Set();
|
|
379
|
-
const allDeferredForeignKeys = [];
|
|
380
|
-
for (const tableNameToLoad of tablesToLoad) if (!attemptedTables.has(tableNameToLoad)) {
|
|
381
|
-
const tableTransform = tableNameToLoad === tableName ? transform : void 0;
|
|
382
|
-
const { errors, warnings, rowCounts: tableRowCounts, deferredForeignKeys } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
|
|
383
|
-
allErrors.push(...errors);
|
|
384
|
-
allWarnings.push(...warnings);
|
|
385
|
-
allDeferredForeignKeys.push(...deferredForeignKeys);
|
|
386
|
-
for (const [k, v] of tableRowCounts) allRowCounts.set(k, v);
|
|
387
|
-
}
|
|
388
|
-
if (detailedValidate && allDeferredForeignKeys.length > 0) for (const { tableName: tName, foreignKey: fk, filePath } of allDeferredForeignKeys) {
|
|
389
|
-
if (!loadedTables.has(fk.references.table)) continue;
|
|
390
|
-
const deferredErrors = this.validateDeferredForeignKey(tName, fk, filePath);
|
|
391
|
-
allErrors.push(...deferredErrors);
|
|
392
|
-
}
|
|
393
|
-
const tableResults = tablesToLoad.map((name) => {
|
|
394
|
-
const tableErrors = allErrors.filter((e) => e.tableName === name);
|
|
395
|
-
const tableWarnings = allWarnings.filter((w) => w.includes(`'${name}'`));
|
|
396
|
-
return {
|
|
397
|
-
tableName: name,
|
|
398
|
-
valid: tableErrors.length === 0,
|
|
399
|
-
rowCount: allRowCounts.get(name) ?? 0,
|
|
400
|
-
errors: tableErrors,
|
|
401
|
-
warnings: tableWarnings
|
|
402
|
-
};
|
|
403
|
-
});
|
|
404
|
-
return {
|
|
405
|
-
valid: allErrors.length === 0,
|
|
406
|
-
errors: allErrors,
|
|
407
|
-
warnings: allWarnings,
|
|
408
|
-
tableResults
|
|
409
|
-
};
|
|
410
|
-
}
|
|
411
|
-
/**
|
|
412
|
-
* Load a table and its dependencies recursively
|
|
413
|
-
*/
|
|
414
|
-
async loadTableWithDependencies(tableName, loadedTables, loadingTables, attemptedTables, detailedValidate, transform) {
|
|
415
|
-
const errors = [];
|
|
416
|
-
const warnings = [];
|
|
417
|
-
const rowCounts = /* @__PURE__ */ new Map();
|
|
418
|
-
const deferredForeignKeys = [];
|
|
419
|
-
if (attemptedTables.has(tableName)) return {
|
|
420
|
-
errors,
|
|
421
|
-
warnings,
|
|
422
|
-
rowCounts,
|
|
423
|
-
deferredForeignKeys
|
|
424
|
-
};
|
|
425
|
-
attemptedTables.add(tableName);
|
|
426
|
-
if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
|
|
427
|
-
const tableConfig = this.tables.get(tableName);
|
|
428
|
-
if (!tableConfig) throw new Error(`Table configuration not found for '${tableName}'`);
|
|
429
|
-
loadingTables.add(tableName);
|
|
430
|
-
try {
|
|
431
|
-
let foreignKeys;
|
|
432
|
-
try {
|
|
433
|
-
const { pathToFileURL } = await import("node:url");
|
|
434
|
-
const schemaPath = await findSchemaFile((0, node_path.dirname)(tableConfig.jsonlPath), (0, node_path.basename)(tableConfig.jsonlPath, ".jsonl"));
|
|
435
|
-
if (schemaPath) {
|
|
436
|
-
const schemaModule = await import(`${pathToFileURL(schemaPath).href}?t=${Date.now()}`);
|
|
437
|
-
foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
|
|
438
|
-
}
|
|
439
|
-
} catch {}
|
|
440
|
-
if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
|
|
441
|
-
const referencedTable = fk.references.table;
|
|
442
|
-
if (referencedTable === tableName) continue;
|
|
443
|
-
if (!attemptedTables.has(referencedTable)) if (this.tables.has(referencedTable)) {
|
|
444
|
-
const depResult = await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables, attemptedTables, detailedValidate, void 0);
|
|
445
|
-
errors.push(...depResult.errors);
|
|
446
|
-
warnings.push(...depResult.warnings);
|
|
447
|
-
deferredForeignKeys.push(...depResult.deferredForeignKeys);
|
|
448
|
-
for (const [k, v] of depResult.rowCounts) rowCounts.set(k, v);
|
|
449
|
-
} else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
|
|
450
|
-
}
|
|
451
|
-
const failedDependencies = /* @__PURE__ */ new Set();
|
|
452
|
-
const circularDependencies = /* @__PURE__ */ new Set();
|
|
453
|
-
if (foreignKeys && foreignKeys.length > 0) {
|
|
454
|
-
for (const fk of foreignKeys) {
|
|
455
|
-
const referencedTable = fk.references.table;
|
|
456
|
-
if (referencedTable === tableName) continue;
|
|
457
|
-
if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable)) if (loadingTables.has(referencedTable)) circularDependencies.add(referencedTable);
|
|
458
|
-
else failedDependencies.add(referencedTable);
|
|
459
|
-
}
|
|
460
|
-
if (failedDependencies.size > 0) for (const dep of failedDependencies) warnings.push(`Skipping foreign key validation for table '${tableName}': referenced table '${dep}' has validation errors`);
|
|
461
|
-
}
|
|
462
|
-
const allSkippedDependencies = new Set([...failedDependencies, ...circularDependencies]);
|
|
463
|
-
const { loaded, rowCount, errors: loadErrors } = await this.loadTable(tableName, tableConfig, detailedValidate, transform, allSkippedDependencies);
|
|
464
|
-
errors.push(...loadErrors);
|
|
465
|
-
rowCounts.set(tableName, rowCount);
|
|
466
|
-
if (loaded) {
|
|
467
|
-
loadedTables.add(tableName);
|
|
468
|
-
if (foreignKeys && circularDependencies.size > 0) {
|
|
469
|
-
for (const fk of foreignKeys) if (circularDependencies.has(fk.references.table)) deferredForeignKeys.push({
|
|
470
|
-
tableName,
|
|
471
|
-
foreignKey: fk,
|
|
472
|
-
filePath: tableConfig.jsonlPath
|
|
473
|
-
});
|
|
474
|
-
}
|
|
475
|
-
} else {
|
|
476
|
-
warnings.push(`Table '${tableName}' was not loaded (no data or skipped)`);
|
|
477
|
-
this.tables.delete(tableName);
|
|
478
|
-
}
|
|
479
|
-
} finally {
|
|
480
|
-
loadingTables.delete(tableName);
|
|
481
|
-
}
|
|
482
|
-
return {
|
|
483
|
-
errors,
|
|
484
|
-
warnings,
|
|
485
|
-
rowCounts,
|
|
486
|
-
deferredForeignKeys
|
|
487
|
-
};
|
|
488
|
-
}
|
|
489
|
-
/**
|
|
490
|
-
* Load a single table from JSONL file
|
|
491
|
-
* @returns Object with loaded status and validation errors
|
|
492
|
-
*/
|
|
493
|
-
async loadTable(tableName, config, detailedValidate, transform, failedDependencies) {
|
|
494
|
-
let data = await JsonlReader.read(config.jsonlPath);
|
|
495
|
-
if (transform) data = data.map((row) => transform(row));
|
|
496
|
-
let validationSchema = config.validationSchema;
|
|
497
|
-
const schemaMetadata = {};
|
|
498
|
-
if (!validationSchema) try {
|
|
499
|
-
validationSchema = await SchemaLoader.loadSchema(config.jsonlPath);
|
|
500
|
-
} catch (_error) {}
|
|
501
|
-
if (!config.validationSchema) try {
|
|
502
|
-
const { pathToFileURL } = await import("node:url");
|
|
503
|
-
const schemaPath = await findSchemaFile((0, node_path.dirname)(config.jsonlPath), (0, node_path.basename)(config.jsonlPath, ".jsonl"));
|
|
504
|
-
if (!schemaPath) throw new Error("Schema file not found");
|
|
505
|
-
const schemaModule = await import(`${pathToFileURL(schemaPath).href}?t=${Date.now()}`);
|
|
506
|
-
const schemaExport = schemaModule.schema || schemaModule.default;
|
|
507
|
-
if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
|
|
508
|
-
else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
|
|
509
|
-
if (schemaExport?.foreignKeys) schemaMetadata.foreignKeys = schemaExport.foreignKeys;
|
|
510
|
-
else if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
|
|
511
|
-
if (schemaExport?.indexes) schemaMetadata.indexes = schemaExport.indexes;
|
|
512
|
-
else if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
|
|
513
|
-
if (process.env.DEBUG_LINES_DB) {
|
|
514
|
-
console.log(`[lines-db] Schema metadata for ${tableName}:`);
|
|
515
|
-
console.log(` primaryKey: ${schemaMetadata.primaryKey}`);
|
|
516
|
-
console.log(` foreignKeys: ${JSON.stringify(schemaMetadata.foreignKeys)}`);
|
|
517
|
-
console.log(` indexes: ${JSON.stringify(schemaMetadata.indexes)}`);
|
|
518
|
-
}
|
|
519
|
-
} catch (_error) {
|
|
520
|
-
if (process.env.DEBUG_LINES_DB) console.warn(`[lines-db] Failed to load schema metadata for ${tableName}:`, _error instanceof Error ? _error.message : String(_error));
|
|
521
|
-
}
|
|
522
|
-
this.validationSchemas.set(tableName, validationSchema);
|
|
523
|
-
const validationErrors = [];
|
|
524
|
-
const validatedData = [];
|
|
525
|
-
for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
|
|
526
|
-
const row = data[rowIndex];
|
|
527
|
-
try {
|
|
528
|
-
const validatedRow = this.validateAndTransform(tableName, row);
|
|
529
|
-
validatedData.push(validatedRow);
|
|
530
|
-
} catch (error) {
|
|
531
|
-
if (error instanceof Error && error.name === "ValidationError") validationErrors.push({
|
|
532
|
-
rowIndex,
|
|
533
|
-
rowData: row,
|
|
534
|
-
error
|
|
535
|
-
});
|
|
536
|
-
else throw error;
|
|
537
|
-
}
|
|
538
|
-
}
|
|
539
|
-
const validationErrorDetails = validationErrors.map((ve) => ({
|
|
540
|
-
file: config.jsonlPath,
|
|
541
|
-
tableName,
|
|
542
|
-
rowIndex: ve.rowIndex,
|
|
543
|
-
issues: ve.error.issues,
|
|
544
|
-
type: "schema"
|
|
545
|
-
}));
|
|
546
|
-
if (validationErrors.length > 0) return {
|
|
547
|
-
loaded: false,
|
|
548
|
-
rowCount: data.length,
|
|
549
|
-
errors: validationErrorDetails
|
|
550
|
-
};
|
|
551
|
-
let schema;
|
|
552
|
-
let inferredSchema;
|
|
553
|
-
if (validatedData.length > 0) inferredSchema = JsonlReader.inferSchema(tableName, validatedData);
|
|
554
|
-
if (config.schema) {
|
|
555
|
-
schema = config.schema;
|
|
556
|
-
if (inferredSchema) for (const inferredCol of inferredSchema.columns) {
|
|
557
|
-
const schemaCol = schema.columns.find((c) => c.name === inferredCol.name);
|
|
558
|
-
if (schemaCol && inferredCol.valueType && !schemaCol.valueType) schemaCol.valueType = inferredCol.valueType;
|
|
559
|
-
}
|
|
560
|
-
} else if (config.autoInferSchema !== false) {
|
|
561
|
-
if (validatedData.length === 0) return {
|
|
562
|
-
loaded: false,
|
|
563
|
-
rowCount: 0,
|
|
564
|
-
errors: []
|
|
565
|
-
};
|
|
566
|
-
schema = inferredSchema;
|
|
567
|
-
} else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
|
|
568
|
-
const biSchema = validationSchema;
|
|
569
|
-
const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
|
|
570
|
-
const foreignKeys = biSchema?.foreignKeys || schemaMetadata.foreignKeys;
|
|
571
|
-
const indexes = biSchema?.indexes || schemaMetadata.indexes;
|
|
572
|
-
if (primaryKey && !schema.columns.some((col) => col.primaryKey)) {
|
|
573
|
-
const col = schema.columns.find((c) => c.name === primaryKey);
|
|
574
|
-
if (col) col.primaryKey = true;
|
|
575
|
-
} else if (!primaryKey && !schema.columns.some((col) => col.primaryKey)) {
|
|
576
|
-
const idColumn = schema.columns.find((c) => c.name === "id");
|
|
577
|
-
if (idColumn) idColumn.primaryKey = true;
|
|
578
|
-
}
|
|
579
|
-
if (foreignKeys) schema.foreignKeys = failedDependencies && failedDependencies.size > 0 ? foreignKeys.filter((fk) => !failedDependencies.has(fk.references.table)) : foreignKeys;
|
|
580
|
-
if (indexes) {
|
|
581
|
-
schema.indexes = indexes;
|
|
582
|
-
for (const index of indexes) if (index.unique && index.columns.length === 1) {
|
|
583
|
-
const col = schema.columns.find((c) => c.name === index.columns[0]);
|
|
584
|
-
if (col && !col.unique && !col.primaryKey) col.unique = true;
|
|
585
|
-
}
|
|
586
|
-
}
|
|
587
|
-
this.schemas.set(tableName, schema);
|
|
588
|
-
this.createTable(schema);
|
|
589
|
-
if (detailedValidate) {
|
|
590
|
-
const insertErrors = this.insertDataWithDetailedValidation(tableName, schema, validatedData, config.jsonlPath);
|
|
591
|
-
if (insertErrors.length > 0) return {
|
|
592
|
-
loaded: false,
|
|
593
|
-
rowCount: data.length,
|
|
594
|
-
errors: insertErrors
|
|
595
|
-
};
|
|
596
|
-
} else this.insertData(tableName, schema, validatedData);
|
|
597
|
-
return {
|
|
598
|
-
loaded: true,
|
|
599
|
-
rowCount: data.length,
|
|
600
|
-
errors: []
|
|
601
|
-
};
|
|
602
|
-
}
|
|
603
|
-
/**
|
|
604
|
-
* Create table in SQLite with constraints and indexes
|
|
605
|
-
*/
|
|
606
|
-
createTable(schema) {
|
|
607
|
-
const quotedTableName = this.quoteTableName(schema.name);
|
|
608
|
-
const uniqueColumns = /* @__PURE__ */ new Set();
|
|
609
|
-
for (const col of schema.columns) if (col.unique) uniqueColumns.add(col.name);
|
|
610
|
-
if (schema.indexes) {
|
|
611
|
-
for (const index of schema.indexes) if (index.unique && index.columns.length === 1) uniqueColumns.add(index.columns[0]);
|
|
612
|
-
}
|
|
613
|
-
const columnDefs = schema.columns.map((col) => {
|
|
614
|
-
const sqlType = col.type === "JSON" ? "TEXT" : col.type;
|
|
615
|
-
const parts = [this.quoteIdentifier(col.name), sqlType];
|
|
616
|
-
if (col.primaryKey) parts.push("PRIMARY KEY");
|
|
617
|
-
if (col.notNull) parts.push("NOT NULL");
|
|
618
|
-
if (uniqueColumns.has(col.name) && !col.primaryKey) parts.push("UNIQUE");
|
|
619
|
-
return parts.join(" ");
|
|
620
|
-
});
|
|
621
|
-
const foreignKeyDefs = [];
|
|
622
|
-
if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) {
|
|
623
|
-
const fkParts = [`FOREIGN KEY (${this.quoteIdentifier(fk.column)})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`];
|
|
624
|
-
if (fk.onDelete) fkParts.push(`ON DELETE ${fk.onDelete}`);
|
|
625
|
-
if (fk.onUpdate) fkParts.push(`ON UPDATE ${fk.onUpdate}`);
|
|
626
|
-
foreignKeyDefs.push(fkParts.join(" "));
|
|
627
|
-
}
|
|
628
|
-
const sql = `CREATE TABLE IF NOT EXISTS ${quotedTableName} (${[...columnDefs, ...foreignKeyDefs].join(", ")})`;
|
|
629
|
-
this.db.exec(sql);
|
|
630
|
-
if (schema.indexes && schema.indexes.length > 0) for (let i = 0; i < schema.indexes.length; i++) {
|
|
631
|
-
const index = schema.indexes[i];
|
|
632
|
-
const safeTableName = schema.name.replace(/[^a-zA-Z0-9]/g, "_");
|
|
633
|
-
const resolvedIndexName = index.name || `idx_${safeTableName}_${index.columns.join("_")}_${i}`;
|
|
634
|
-
const indexSql = `CREATE ${index.unique ? "UNIQUE " : ""}INDEX IF NOT EXISTS ${this.quoteIdentifier(resolvedIndexName)} ON ${quotedTableName} (${index.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`;
|
|
635
|
-
this.db.exec(indexSql);
|
|
636
|
-
}
|
|
637
|
-
}
|
|
638
|
-
/**
|
|
639
|
-
* Quote table name to handle special characters in SQL
|
|
640
|
-
*/
|
|
641
|
-
quoteTableName(tableName) {
|
|
642
|
-
return this.quoteIdentifier(tableName);
|
|
643
|
-
}
|
|
644
|
-
/**
|
|
645
|
-
* Quote identifier for SQL statements, escaping embedded quotes
|
|
646
|
-
*/
|
|
647
|
-
quoteIdentifier(identifier) {
|
|
648
|
-
return `"${identifier.replace(/"/g, "\"\"")}"`;
|
|
649
|
-
}
|
|
650
|
-
/**
|
|
651
|
-
* Insert data into table using batch insert (multiple rows per SQL)
|
|
652
|
-
* SQLite has a parameter limit (default 999), so we batch rows accordingly
|
|
653
|
-
* Throws exception if any constraint violation occurs
|
|
654
|
-
*/
|
|
655
|
-
insertData(tableName, schema, data) {
|
|
656
|
-
if (data.length === 0) return;
|
|
657
|
-
const columnNames = schema.columns.map((col) => col.name);
|
|
658
|
-
const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
|
|
659
|
-
const columnCount = columnNames.length;
|
|
660
|
-
const maxBatchSize = Math.floor(900 / columnCount);
|
|
661
|
-
const batchSize = Math.max(1, Math.min(maxBatchSize, 100));
|
|
662
|
-
for (let i = 0; i < data.length; i += batchSize) {
|
|
663
|
-
const batch = data.slice(i, i + batchSize);
|
|
664
|
-
const rowPlaceholders = columnNames.map(() => "?").join(", ");
|
|
665
|
-
const valuesPlaceholders = batch.map(() => `(${rowPlaceholders})`).join(", ");
|
|
666
|
-
const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES ${valuesPlaceholders}`;
|
|
667
|
-
const values = [];
|
|
668
|
-
for (const row of batch) for (const col of columnNames) values.push(this.normalizeValue(row[col]));
|
|
669
|
-
this.db.prepare(sql).run(...values);
|
|
670
|
-
}
|
|
671
|
-
}
|
|
672
|
-
/**
|
|
673
|
-
* Insert data into table one row at a time with detailed error reporting
|
|
674
|
-
* This is used for validation to catch constraint violations
|
|
675
|
-
*/
|
|
676
|
-
insertDataWithDetailedValidation(tableName, schema, data, filePath) {
|
|
677
|
-
const errors = [];
|
|
678
|
-
const columnNames = schema.columns.map((col) => col.name);
|
|
679
|
-
const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
|
|
680
|
-
const placeholders = columnNames.map(() => "?").join(", ");
|
|
681
|
-
const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
|
|
682
|
-
const stmt = this.db.prepare(sql);
|
|
683
|
-
for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
|
|
684
|
-
const row = data[rowIndex];
|
|
685
|
-
try {
|
|
686
|
-
const values = columnNames.map((col) => this.normalizeValue(row[col]));
|
|
687
|
-
stmt.run(...values);
|
|
688
|
-
} catch (error) {
|
|
689
|
-
const constraintError = this.analyzeConstraintError(error, filePath, tableName, rowIndex, row, schema.foreignKeys || []);
|
|
690
|
-
if (constraintError) errors.push(constraintError);
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
return errors;
|
|
694
|
-
}
|
|
695
|
-
/**
|
|
696
|
-
* Analyze constraint error and extract detailed information
|
|
697
|
-
*/
|
|
698
|
-
analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys) {
|
|
699
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
700
|
-
if (errorMessage.includes("FOREIGN KEY constraint failed")) for (const fk of foreignKeys) {
|
|
701
|
-
const fkValue = row[fk.column];
|
|
702
|
-
if (fkValue === null || fkValue === void 0) continue;
|
|
703
|
-
try {
|
|
704
|
-
const result = this.query(`SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`, [this.normalizeValue(fkValue)]);
|
|
705
|
-
if (result.length > 0 && result[0].count === 0) return {
|
|
706
|
-
file,
|
|
707
|
-
tableName,
|
|
708
|
-
rowIndex,
|
|
709
|
-
issues: [],
|
|
710
|
-
type: "foreignKey",
|
|
711
|
-
foreignKeyError: {
|
|
712
|
-
column: fk.column,
|
|
713
|
-
value: fkValue,
|
|
714
|
-
referencedTable: fk.references.table,
|
|
715
|
-
referencedColumn: fk.references.column
|
|
716
|
-
}
|
|
717
|
-
};
|
|
718
|
-
} catch (_) {}
|
|
719
|
-
}
|
|
720
|
-
return {
|
|
721
|
-
file,
|
|
722
|
-
tableName,
|
|
723
|
-
rowIndex,
|
|
724
|
-
issues: [{
|
|
725
|
-
message: errorMessage,
|
|
726
|
-
path: []
|
|
727
|
-
}],
|
|
728
|
-
type: "schema"
|
|
729
|
-
};
|
|
730
|
-
}
|
|
731
|
-
/**
|
|
732
|
-
* Validate a deferred foreign key constraint after all tables have been loaded.
|
|
733
|
-
* Used for circular dependency FK validation.
|
|
734
|
-
*/
|
|
735
|
-
validateDeferredForeignKey(tableName, fk, filePath) {
|
|
736
|
-
const errors = [];
|
|
737
|
-
const quotedTable = this.quoteTableName(tableName);
|
|
738
|
-
const quotedColumn = this.quoteIdentifier(fk.column);
|
|
739
|
-
const quotedRefTable = this.quoteTableName(fk.references.table);
|
|
740
|
-
const sql = `SELECT rowid - 1 as idx, ${quotedColumn} as val FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL AND ${quotedColumn} NOT IN (SELECT ${this.quoteIdentifier(fk.references.column)} FROM ${quotedRefTable})`;
|
|
741
|
-
try {
|
|
742
|
-
const rows = this.query(sql);
|
|
743
|
-
for (const row of rows) errors.push({
|
|
744
|
-
file: filePath,
|
|
745
|
-
tableName,
|
|
746
|
-
rowIndex: row.idx,
|
|
747
|
-
issues: [],
|
|
748
|
-
type: "foreignKey",
|
|
749
|
-
foreignKeyError: {
|
|
750
|
-
column: fk.column,
|
|
751
|
-
value: row.val,
|
|
752
|
-
referencedTable: fk.references.table,
|
|
753
|
-
referencedColumn: fk.references.column
|
|
754
|
-
}
|
|
755
|
-
});
|
|
756
|
-
} catch (_) {}
|
|
757
|
-
return errors;
|
|
758
|
-
}
|
|
759
|
-
/**
|
|
760
|
-
* Execute a raw SQL query
|
|
761
|
-
*/
|
|
762
|
-
query(sql, params = []) {
|
|
763
|
-
return this.db.prepare(sql).all(...params);
|
|
764
|
-
}
|
|
765
|
-
/**
|
|
766
|
-
* Execute a SQL query that returns a single row
|
|
767
|
-
*/
|
|
768
|
-
queryOne(sql, params = []) {
|
|
769
|
-
const result = this.db.prepare(sql).get(...params);
|
|
770
|
-
return result === void 0 ? null : result;
|
|
771
|
-
}
|
|
772
|
-
/**
|
|
773
|
-
* Execute a SQL statement (INSERT, UPDATE, DELETE)
|
|
774
|
-
*/
|
|
775
|
-
execute(sql, params = []) {
|
|
776
|
-
return this.db.prepare(sql).run(...params);
|
|
777
|
-
}
|
|
778
|
-
/**
|
|
779
|
-
* Find rows by condition (supports OR/AND with arrays and function filters)
|
|
780
|
-
* If where is not provided, returns all rows
|
|
781
|
-
*/
|
|
782
|
-
find(tableName, where) {
|
|
783
|
-
if (where === void 0) return this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
|
|
784
|
-
if (Array.isArray(where) && where.length === 0) return [];
|
|
785
|
-
const { sql, values, functionFilters, hasOrWithFunctionFilters } = this.buildWhereClause(where);
|
|
786
|
-
let rows;
|
|
787
|
-
if (hasOrWithFunctionFilters) {
|
|
788
|
-
rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
|
|
789
|
-
return this.applyOrConditionWithFilters(rows, where);
|
|
790
|
-
}
|
|
791
|
-
if (sql) rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values).map((row) => this.deserializeRow(tableName, row));
|
|
792
|
-
else rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
|
|
793
|
-
return this.applyFunctionFilters(rows, functionFilters);
|
|
794
|
-
}
|
|
795
|
-
/**
|
|
796
|
-
* Find a single row by condition (supports OR/AND with arrays and function filters)
|
|
797
|
-
*/
|
|
798
|
-
findOne(tableName, where) {
|
|
799
|
-
const { sql, values, functionFilters } = this.buildWhereClause(where);
|
|
800
|
-
let rows;
|
|
801
|
-
if (sql) rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values).map((row) => this.deserializeRow(tableName, row));
|
|
802
|
-
else rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
|
|
803
|
-
const filtered = this.applyFunctionFilters(rows, functionFilters);
|
|
804
|
-
return filtered.length > 0 ? filtered[0] : null;
|
|
805
|
-
}
|
|
806
|
-
/**
|
|
807
|
-
* Deserialize JSON columns in a row
|
|
808
|
-
*/
|
|
809
|
-
deserializeRow(tableName, row) {
|
|
810
|
-
const schema = this.schemas.get(tableName);
|
|
811
|
-
if (!schema) return row;
|
|
812
|
-
const deserializedRow = { ...row };
|
|
813
|
-
for (const column of schema.columns) {
|
|
814
|
-
const colName = column.name;
|
|
815
|
-
if (!(colName in deserializedRow)) continue;
|
|
816
|
-
const value = deserializedRow[colName];
|
|
817
|
-
if (column.type === "JSON" && typeof value === "string") {
|
|
818
|
-
try {
|
|
819
|
-
deserializedRow[colName] = JSON.parse(value);
|
|
820
|
-
} catch (error) {
|
|
821
|
-
console.warn(`Failed to parse JSON column ${colName}:`, error);
|
|
822
|
-
}
|
|
823
|
-
continue;
|
|
824
|
-
}
|
|
825
|
-
if (column.valueType === "boolean") {
|
|
826
|
-
if (typeof value === "number") deserializedRow[colName] = value === 0 ? false : true;
|
|
827
|
-
else if (typeof value === "bigint") deserializedRow[colName] = value === 0n ? false : true;
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
return deserializedRow;
|
|
831
|
-
}
|
|
832
|
-
/**
|
|
833
|
-
* Validate data using StandardSchema and return the transformed value
|
|
834
|
-
* Note: Only synchronous validation is supported
|
|
835
|
-
*/
|
|
836
|
-
validateAndTransform(tableName, data) {
|
|
837
|
-
const schema = this.validationSchemas.get(tableName);
|
|
838
|
-
if (!schema) return data;
|
|
839
|
-
const result = schema["~standard"].validate(data);
|
|
840
|
-
if (result instanceof Promise) throw new Error("Asynchronous validation is not supported. Please use synchronous validation schemas.");
|
|
841
|
-
if (result.issues && result.issues.length > 0) {
|
|
842
|
-
const errorMessage = `Validation failed for table '${tableName}':\n${result.issues.map((issue) => {
|
|
843
|
-
let pathStr = "root";
|
|
844
|
-
if (issue.path && issue.path.length > 0) pathStr = issue.path.map((segment) => {
|
|
845
|
-
if (typeof segment === "object" && segment !== null && "key" in segment) return String(segment.key);
|
|
846
|
-
return String(segment);
|
|
847
|
-
}).join(".");
|
|
848
|
-
return ` - ${pathStr}: ${issue.message}`;
|
|
849
|
-
}).join("\n")}`;
|
|
850
|
-
const error = new Error(errorMessage);
|
|
851
|
-
error.name = "ValidationError";
|
|
852
|
-
error.issues = result.issues;
|
|
853
|
-
throw error;
|
|
854
|
-
}
|
|
855
|
-
const transformedValue = "value" in result ? result.value : data;
|
|
856
|
-
const normalizedValue = {};
|
|
857
|
-
for (const [key, value] of Object.entries(transformedValue)) normalizedValue[key] = value === void 0 ? null : value;
|
|
858
|
-
return normalizedValue;
|
|
859
|
-
}
|
|
860
|
-
/**
|
|
861
|
-
* Validate data using StandardSchema (without returning transformed value)
|
|
862
|
-
* Note: Only synchronous validation is supported
|
|
863
|
-
*/
|
|
864
|
-
validateData(tableName, data) {
|
|
865
|
-
this.validateAndTransform(tableName, data);
|
|
866
|
-
}
|
|
867
|
-
/**
|
|
868
|
-
* Insert a row into a table with validation
|
|
869
|
-
*/
|
|
870
|
-
insert(tableName, data) {
|
|
871
|
-
this.validateData(tableName, data);
|
|
872
|
-
if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
|
|
873
|
-
const columnNames = Object.keys(data);
|
|
874
|
-
const quotedColumns = columnNames.map((col) => this.quoteIdentifier(col));
|
|
875
|
-
const placeholders = columnNames.map(() => "?").join(", ");
|
|
876
|
-
const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
|
|
877
|
-
const values = Object.values(data).map((v) => this.normalizeValue(v));
|
|
878
|
-
const result = this.execute(sql, values);
|
|
879
|
-
if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
|
|
880
|
-
console.error(`Failed to sync table ${tableName}:`, err);
|
|
881
|
-
});
|
|
882
|
-
return result;
|
|
883
|
-
}
|
|
884
|
-
/**
|
|
885
|
-
* Batch insert rows with validation per record.
|
|
886
|
-
*/
|
|
887
|
-
batchInsert(tableName, records) {
|
|
888
|
-
if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
|
|
889
|
-
if (records.length === 0) return {
|
|
890
|
-
changes: 0,
|
|
891
|
-
lastInsertRowid: 0
|
|
892
|
-
};
|
|
893
|
-
let totalChanges = 0n;
|
|
894
|
-
let lastRowid = 0n;
|
|
895
|
-
for (const record of records) {
|
|
896
|
-
this.validateData(tableName, record);
|
|
897
|
-
const columnNames = Object.keys(record);
|
|
898
|
-
const quotedColumns = columnNames.map((col) => this.quoteIdentifier(col));
|
|
899
|
-
const placeholders = columnNames.map(() => "?").join(", ");
|
|
900
|
-
const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
|
|
901
|
-
const values = columnNames.map((col) => this.normalizeValue(record[col]));
|
|
902
|
-
const result = this.execute(sql, values);
|
|
903
|
-
totalChanges += BigInt(result.changes);
|
|
904
|
-
lastRowid = BigInt(result.lastInsertRowid);
|
|
905
|
-
}
|
|
906
|
-
if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
|
|
907
|
-
console.error(`Failed to sync table ${tableName}:`, err);
|
|
908
|
-
});
|
|
909
|
-
return {
|
|
910
|
-
changes: totalChanges,
|
|
911
|
-
lastInsertRowid: lastRowid
|
|
912
|
-
};
|
|
913
|
-
}
|
|
914
|
-
/**
|
|
915
|
-
* Update rows in a table with validation (supports OR/AND with arrays)
|
|
916
|
-
* Note: Function filters are not supported for update operations
|
|
917
|
-
* Note: By default, validation is enabled. For partial updates, existing data is fetched
|
|
918
|
-
* and merged before validation. Set options.validate = false to disable validation.
|
|
919
|
-
*/
|
|
920
|
-
update(tableName, data, where, options) {
|
|
921
|
-
if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
|
|
922
|
-
const shouldValidate = options?.validate !== false;
|
|
923
|
-
const hasValidationSchema = this.validationSchemas.has(tableName);
|
|
924
|
-
if (shouldValidate && hasValidationSchema) {
|
|
925
|
-
const existingRows = this.find(tableName, where);
|
|
926
|
-
for (const existingRow of existingRows) {
|
|
927
|
-
const mergedData = {
|
|
928
|
-
...existingRow,
|
|
929
|
-
...data
|
|
930
|
-
};
|
|
931
|
-
this.validateData(tableName, mergedData);
|
|
932
|
-
}
|
|
933
|
-
}
|
|
934
|
-
const { sql: whereSql, values: whereValues, functionFilters } = this.buildWhereClause(where);
|
|
935
|
-
if (functionFilters.length > 0) throw new Error("Function filters are not supported in update operations");
|
|
936
|
-
const setClauses = Object.keys(data).map((key) => `${this.quoteIdentifier(key)} = ?`).join(", ");
|
|
937
|
-
const sql = `UPDATE ${this.quoteTableName(tableName)} SET ${setClauses} WHERE ${whereSql}`;
|
|
938
|
-
const values = [...Object.values(data).map((v) => this.normalizeValue(v)), ...whereValues];
|
|
939
|
-
const result = this.execute(sql, values);
|
|
940
|
-
if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
|
|
941
|
-
console.error(`Failed to sync table ${tableName}:`, err);
|
|
942
|
-
});
|
|
943
|
-
return result;
|
|
944
|
-
}
|
|
945
|
-
/**
|
|
946
|
-
* Batch update rows with record-specific values and validation.
|
|
947
|
-
* Each record must include the primary key to identify the target row.
|
|
948
|
-
* Validation runs once per merged record unless explicitly disabled.
|
|
949
|
-
*/
|
|
950
|
-
batchUpdate(tableName, records, options) {
|
|
951
|
-
const schema = this.schemas.get(tableName);
|
|
952
|
-
if (!schema) throw new Error(`Table ${tableName} does not exist`);
|
|
953
|
-
if (records.length === 0) return {
|
|
954
|
-
changes: 0,
|
|
955
|
-
lastInsertRowid: 0
|
|
956
|
-
};
|
|
957
|
-
const pkColumn = schema.columns.find((col) => col.primaryKey);
|
|
958
|
-
if (!pkColumn) throw new Error(`Table ${tableName} does not have a primary key`);
|
|
959
|
-
const pkName = pkColumn.name;
|
|
960
|
-
const pkValues = [];
|
|
961
|
-
for (const record of records) {
|
|
962
|
-
const pkValue = record[pkName];
|
|
963
|
-
if (pkValue === void 0) throw new Error(`Record is missing primary key '${String(pkName)}': ${JSON.stringify(record)}`);
|
|
964
|
-
pkValues.push(pkValue);
|
|
965
|
-
}
|
|
966
|
-
const shouldValidate = options?.validate !== false;
|
|
967
|
-
const hasValidationSchema = this.validationSchemas.has(tableName);
|
|
968
|
-
if (shouldValidate && hasValidationSchema) {
|
|
969
|
-
const orCondition = pkValues.map((pkValue) => ({ [pkName]: pkValue }));
|
|
970
|
-
const existingRows = this.find(tableName, orCondition);
|
|
971
|
-
const existingRowsMap = /* @__PURE__ */ new Map();
|
|
972
|
-
for (const row of existingRows) {
|
|
973
|
-
const pkValue = row[pkName];
|
|
974
|
-
existingRowsMap.set(pkValue, row);
|
|
975
|
-
}
|
|
976
|
-
const validationErrors = [];
|
|
977
|
-
for (let i = 0; i < records.length; i++) {
|
|
978
|
-
const record = records[i];
|
|
979
|
-
const pkValue = record[pkName];
|
|
980
|
-
const existingRow = existingRowsMap.get(pkValue);
|
|
981
|
-
if (!existingRow) throw new Error(`No existing row found with ${String(pkName)}=${JSON.stringify(pkValue)}`);
|
|
982
|
-
const mergedData = {
|
|
983
|
-
...existingRow,
|
|
984
|
-
...record
|
|
985
|
-
};
|
|
986
|
-
try {
|
|
987
|
-
this.validateData(tableName, mergedData);
|
|
988
|
-
} catch (error) {
|
|
989
|
-
if (error instanceof Error && error.name === "ValidationError") validationErrors.push({
|
|
990
|
-
rowIndex: i,
|
|
991
|
-
rowData: mergedData,
|
|
992
|
-
pkValue,
|
|
993
|
-
error
|
|
994
|
-
});
|
|
995
|
-
else throw error;
|
|
996
|
-
}
|
|
997
|
-
}
|
|
998
|
-
if (validationErrors.length > 0) {
|
|
999
|
-
const enhancedError = /* @__PURE__ */ new Error(`Validation failed for ${validationErrors.length} row(s)`);
|
|
1000
|
-
enhancedError.name = "ValidationError";
|
|
1001
|
-
enhancedError.validationErrors = validationErrors;
|
|
1002
|
-
enhancedError.issues = validationErrors[0].error.issues;
|
|
1003
|
-
throw enhancedError;
|
|
1004
|
-
}
|
|
1005
|
-
}
|
|
1006
|
-
let totalChanges = 0n;
|
|
1007
|
-
let lastRowid = 0n;
|
|
1008
|
-
for (const record of records) {
|
|
1009
|
-
const pkValue = record[pkName];
|
|
1010
|
-
const where = { [pkName]: pkValue };
|
|
1011
|
-
const result = this.update(tableName, record, where, { validate: false });
|
|
1012
|
-
totalChanges += BigInt(result.changes);
|
|
1013
|
-
lastRowid = BigInt(result.lastInsertRowid);
|
|
1014
|
-
}
|
|
1015
|
-
return {
|
|
1016
|
-
changes: totalChanges,
|
|
1017
|
-
lastInsertRowid: lastRowid
|
|
1018
|
-
};
|
|
1019
|
-
}
|
|
1020
|
-
/**
|
|
1021
|
-
* Delete rows from a table (supports OR/AND with arrays)
|
|
1022
|
-
* Note: Function filters are not supported for delete operations
|
|
1023
|
-
*/
|
|
1024
|
-
delete(tableName, where) {
|
|
1025
|
-
if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
|
|
1026
|
-
const { sql: whereSql, values, functionFilters } = this.buildWhereClause(where);
|
|
1027
|
-
if (functionFilters.length > 0) throw new Error("Function filters are not supported in delete operations");
|
|
1028
|
-
const sql = `DELETE FROM ${this.quoteTableName(tableName)} WHERE ${whereSql}`;
|
|
1029
|
-
const result = this.execute(sql, values);
|
|
1030
|
-
if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
|
|
1031
|
-
console.error(`Failed to sync table ${tableName}:`, err);
|
|
1032
|
-
});
|
|
1033
|
-
return result;
|
|
1034
|
-
}
|
|
1035
|
-
/**
|
|
1036
|
-
* Batch delete rows by primary key.
|
|
1037
|
-
*/
|
|
1038
|
-
batchDelete(tableName, records) {
|
|
1039
|
-
const schema = this.schemas.get(tableName);
|
|
1040
|
-
if (!schema) throw new Error(`Table ${tableName} does not exist`);
|
|
1041
|
-
if (records.length === 0) return {
|
|
1042
|
-
changes: 0,
|
|
1043
|
-
lastInsertRowid: 0
|
|
1044
|
-
};
|
|
1045
|
-
const pkColumn = schema.columns.find((col) => col.primaryKey);
|
|
1046
|
-
if (!pkColumn) throw new Error(`Table ${tableName} does not have a primary key`);
|
|
1047
|
-
const pkName = pkColumn.name;
|
|
1048
|
-
const pkValues = records.map((record, index) => {
|
|
1049
|
-
const pkValue = record[pkName];
|
|
1050
|
-
if (pkValue === void 0) throw new Error(`Record at index ${index} is missing primary key '${String(pkName)}'`);
|
|
1051
|
-
return pkValue;
|
|
1052
|
-
});
|
|
1053
|
-
const placeholders = pkValues.map(() => "?").join(", ");
|
|
1054
|
-
const sql = `DELETE FROM ${this.quoteTableName(tableName)} WHERE ${this.quoteIdentifier(pkName)} IN (${placeholders})`;
|
|
1055
|
-
const values = pkValues.map((value) => this.normalizeValue(value));
|
|
1056
|
-
const result = this.execute(sql, values);
|
|
1057
|
-
if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
|
|
1058
|
-
console.error(`Failed to sync table ${tableName}:`, err);
|
|
1059
|
-
});
|
|
1060
|
-
return {
|
|
1061
|
-
changes: BigInt(result.changes),
|
|
1062
|
-
lastInsertRowid: BigInt(result.lastInsertRowid)
|
|
1063
|
-
};
|
|
1064
|
-
}
|
|
1065
|
-
/**
|
|
1066
|
-
* Normalize value for SQLite
|
|
1067
|
-
*/
|
|
1068
|
-
normalizeValue(value) {
|
|
1069
|
-
if (value === null || value === void 0) return null;
|
|
1070
|
-
if (typeof value === "boolean") return value ? 1 : 0;
|
|
1071
|
-
if (typeof value === "string" || typeof value === "number" || typeof value === "bigint") return value;
|
|
1072
|
-
if (value instanceof Uint8Array) return value;
|
|
1073
|
-
return JSON.stringify(value);
|
|
1074
|
-
}
|
|
1075
|
-
/**
|
|
1076
|
-
* Build WHERE clause from condition (supports OR/AND with arrays and functions)
|
|
1077
|
-
*/
|
|
1078
|
-
buildWhereClause(condition) {
|
|
1079
|
-
const values = [];
|
|
1080
|
-
const functionFilters = [];
|
|
1081
|
-
let hasOrWithFunctionFilters = false;
|
|
1082
|
-
const buildCondition = (cond, isInOr = false) => {
|
|
1083
|
-
if (Array.isArray(cond)) return cond.map((item) => {
|
|
1084
|
-
const clause = Array.isArray(item) ? buildCondition(item, true) : buildCondition(item, true);
|
|
1085
|
-
return clause ? `(${clause})` : "";
|
|
1086
|
-
}).filter((clause) => clause !== "").join(" OR ");
|
|
1087
|
-
const conditions = [];
|
|
1088
|
-
let hasFunctionFilter = false;
|
|
1089
|
-
for (const [key, value] of Object.entries(cond)) if (typeof value === "function") {
|
|
1090
|
-
functionFilters.push({
|
|
1091
|
-
key,
|
|
1092
|
-
fn: value
|
|
1093
|
-
});
|
|
1094
|
-
hasFunctionFilter = true;
|
|
1095
|
-
} else {
|
|
1096
|
-
conditions.push(`${this.quoteIdentifier(key)} = ?`);
|
|
1097
|
-
values.push(this.normalizeValue(value));
|
|
1098
|
-
}
|
|
1099
|
-
if (isInOr && hasFunctionFilter) hasOrWithFunctionFilters = true;
|
|
1100
|
-
return conditions.join(" AND ");
|
|
1101
|
-
};
|
|
1102
|
-
return {
|
|
1103
|
-
sql: buildCondition(condition),
|
|
1104
|
-
values,
|
|
1105
|
-
functionFilters,
|
|
1106
|
-
hasOrWithFunctionFilters
|
|
1107
|
-
};
|
|
1108
|
-
}
|
|
1109
|
-
/**
|
|
1110
|
-
* Apply OR condition with function filters by evaluating each row against the condition
|
|
1111
|
-
*/
|
|
1112
|
-
applyOrConditionWithFilters(rows, condition) {
|
|
1113
|
-
return rows.filter((row) => this.matchesOrCondition(row, condition));
|
|
1114
|
-
}
|
|
1115
|
-
/**
|
|
1116
|
-
* Check if a row matches an OR/AND condition (recursively)
|
|
1117
|
-
*/
|
|
1118
|
-
matchesOrCondition(row, condition) {
|
|
1119
|
-
if (Array.isArray(condition)) return condition.some((item) => this.matchesOrCondition(row, item));
|
|
1120
|
-
return Object.entries(condition).every(([key, value]) => {
|
|
1121
|
-
const rowValue = row[key];
|
|
1122
|
-
if (typeof value === "function") return value(rowValue);
|
|
1123
|
-
return rowValue === value;
|
|
1124
|
-
});
|
|
1125
|
-
}
|
|
1126
|
-
/**
|
|
1127
|
-
* Apply function filters to rows
|
|
1128
|
-
*/
|
|
1129
|
-
applyFunctionFilters(rows, functionFilters) {
|
|
1130
|
-
if (functionFilters.length === 0) return rows;
|
|
1131
|
-
return rows.filter((row) => {
|
|
1132
|
-
return functionFilters.every(({ key, fn }) => {
|
|
1133
|
-
const value = row[key];
|
|
1134
|
-
return fn(value);
|
|
1135
|
-
});
|
|
1136
|
-
});
|
|
1137
|
-
}
|
|
1138
|
-
/**
|
|
1139
|
-
* Get table schema
|
|
1140
|
-
*/
|
|
1141
|
-
getSchema(tableName) {
|
|
1142
|
-
return this.schemas.get(tableName);
|
|
1143
|
-
}
|
|
1144
|
-
/**
|
|
1145
|
-
* Get all table names
|
|
1146
|
-
*/
|
|
1147
|
-
getTableNames() {
|
|
1148
|
-
return Array.from(this.schemas.keys());
|
|
1149
|
-
}
|
|
1150
|
-
/**
|
|
1151
|
-
* Sync a specific table back to its JSONL file
|
|
1152
|
-
* Uses backward transformation when available
|
|
1153
|
-
*/
|
|
1154
|
-
async syncTable(tableName) {
|
|
1155
|
-
const tableConfig = this.tables.get(tableName);
|
|
1156
|
-
if (!tableConfig) throw new Error(`Table ${tableName} not found`);
|
|
1157
|
-
const deserializedRows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
|
|
1158
|
-
const validationSchema = this.validationSchemas.get(tableName);
|
|
1159
|
-
let finalRows = deserializedRows;
|
|
1160
|
-
if (validationSchema && hasBackward(validationSchema)) {
|
|
1161
|
-
const biSchema = validationSchema;
|
|
1162
|
-
finalRows = deserializedRows.map((row) => biSchema.backward(row));
|
|
1163
|
-
}
|
|
1164
|
-
await JsonlWriter.write(tableConfig.jsonlPath, finalRows);
|
|
1165
|
-
}
|
|
1166
|
-
/**
|
|
1167
|
-
* Sync database changes back to JSONL files
|
|
1168
|
-
* Uses backward transformation when available
|
|
1169
|
-
* @param tableName Optional table name to sync. If not provided, syncs all loaded tables
|
|
1170
|
-
*/
|
|
1171
|
-
async sync(tableName) {
|
|
1172
|
-
if (tableName) {
|
|
1173
|
-
if (!this.schemas.has(tableName)) throw new Error(`Table '${tableName}' is not loaded`);
|
|
1174
|
-
await this.syncTable(tableName);
|
|
1175
|
-
} else for (const [name] of this.schemas) await this.syncTable(name);
|
|
1176
|
-
}
|
|
1177
|
-
/**
|
|
1178
|
-
* Execute a function within a transaction
|
|
1179
|
-
* Automatically commits on success or rolls back on error
|
|
1180
|
-
*/
|
|
1181
|
-
async transaction(fn) {
|
|
1182
|
-
if (this.inTransaction) throw new Error("Nested transactions are not supported");
|
|
1183
|
-
this.db.exec("BEGIN TRANSACTION");
|
|
1184
|
-
this.inTransaction = true;
|
|
1185
|
-
try {
|
|
1186
|
-
const result = await fn(this);
|
|
1187
|
-
this.db.exec("COMMIT");
|
|
1188
|
-
this.inTransaction = false;
|
|
1189
|
-
await this.sync();
|
|
1190
|
-
return result;
|
|
1191
|
-
} catch (error) {
|
|
1192
|
-
if (this.inTransaction) this.db.exec("ROLLBACK");
|
|
1193
|
-
this.inTransaction = false;
|
|
1194
|
-
throw error;
|
|
1195
|
-
}
|
|
1196
|
-
}
|
|
1197
|
-
/**
|
|
1198
|
-
* Close the database connection
|
|
1199
|
-
*/
|
|
1200
|
-
async close() {
|
|
1201
|
-
try {
|
|
1202
|
-
this.db.close();
|
|
1203
|
-
} catch (_error) {}
|
|
1204
|
-
}
|
|
1205
|
-
/**
|
|
1206
|
-
* Get the underlying SQLite database instance
|
|
1207
|
-
*/
|
|
1208
|
-
getDb() {
|
|
1209
|
-
return this.db;
|
|
1210
|
-
}
|
|
1211
|
-
};
|
|
1212
|
-
//#endregion
|
|
1213
|
-
//#region src/type-generator.ts
|
|
1214
|
-
var TypeGenerator = class {
|
|
1215
|
-
dataDir;
|
|
1216
|
-
projectRoot;
|
|
1217
|
-
outputFile;
|
|
1218
|
-
dataDirPath;
|
|
1219
|
-
constructor(options) {
|
|
1220
|
-
const envProjectRoot = process.env.LINES_DB_TEST_PROJECT_ROOT;
|
|
1221
|
-
this.projectRoot = envProjectRoot !== void 0 ? envProjectRoot : options.projectRoot || process.cwd();
|
|
1222
|
-
this.dataDir = options.dataDir;
|
|
1223
|
-
this.dataDirPath = (0, node_path.isAbsolute)(this.dataDir) ? this.dataDir : (0, node_path.join)(this.projectRoot, this.dataDir);
|
|
1224
|
-
this.outputFile = options.output ? (0, node_path.isAbsolute)(options.output) ? options.output : (0, node_path.join)(this.projectRoot, options.output) : (0, node_path.join)(this.dataDirPath, "db.ts");
|
|
1225
|
-
}
|
|
1226
|
-
/**
|
|
1227
|
-
* Generate types file from JSONL files and their optional schema files.
|
|
1228
|
-
*/
|
|
1229
|
-
async generate() {
|
|
1230
|
-
const tables = await this.findTables();
|
|
1231
|
-
if (tables.length === 0) throw new Error(`No JSONL files found in ${this.dataDirPath}. Place one or more *.jsonl files in the directory.`);
|
|
1232
|
-
const content = this.generateTypeDeclarations(tables);
|
|
1233
|
-
await (0, node_fs_promises.mkdir)((0, node_path.dirname)(this.outputFile), { recursive: true });
|
|
1234
|
-
await (0, node_fs_promises.writeFile)(this.outputFile, content, "utf-8");
|
|
1235
|
-
console.log(`Generated types at ${this.outputFile}`);
|
|
1236
|
-
return this.outputFile;
|
|
1237
|
-
}
|
|
1238
|
-
/**
|
|
1239
|
-
* Find all *.jsonl files and check if they have corresponding *.schema.ts files
|
|
1240
|
-
*/
|
|
1241
|
-
async findTables() {
|
|
1242
|
-
try {
|
|
1243
|
-
const entries = await (0, node_fs_promises.readdir)(this.dataDirPath, { withFileTypes: true });
|
|
1244
|
-
const tables = [];
|
|
1245
|
-
for (const entry of entries) if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
1246
|
-
const tableName = (0, node_path.basename)(entry.name, ".jsonl");
|
|
1247
|
-
const schemaFilePath = findSchemaFileInEntries(this.dataDirPath, tableName, entries);
|
|
1248
|
-
tables.push({
|
|
1249
|
-
tableName,
|
|
1250
|
-
schemaFile: schemaFilePath
|
|
1251
|
-
});
|
|
1252
|
-
}
|
|
1253
|
-
return tables;
|
|
1254
|
-
} catch (error) {
|
|
1255
|
-
if (error.code === "ENOENT") throw new Error(`Data directory not found: ${this.dataDirPath}. Set lines-db.dataDir to the correct location.`);
|
|
1256
|
-
throw error;
|
|
1257
|
-
}
|
|
1258
|
-
}
|
|
1259
|
-
/**
|
|
1260
|
-
* Generate type declaration content
|
|
1261
|
-
*/
|
|
1262
|
-
generateTypeDeclarations(tables) {
|
|
1263
|
-
const imports = [];
|
|
1264
|
-
const tableEntries = [];
|
|
1265
|
-
const usedAliases = /* @__PURE__ */ new Set();
|
|
1266
|
-
for (const table of tables) {
|
|
1267
|
-
const tableKey = this.formatTableKey(table.tableName);
|
|
1268
|
-
if (table.schemaFile) {
|
|
1269
|
-
const schemaIdentifier = this.createSchemaIdentifier(table.tableName, usedAliases);
|
|
1270
|
-
usedAliases.add(schemaIdentifier);
|
|
1271
|
-
let relativePath = rewriteExtensionForImport((0, node_path.relative)((0, node_path.join)(this.outputFile, ".."), table.schemaFile).replace(/\\/g, "/"));
|
|
1272
|
-
if (!relativePath.startsWith(".")) relativePath = "./" + relativePath;
|
|
1273
|
-
imports.push(`import { schema as ${schemaIdentifier} } from '${relativePath}';`);
|
|
1274
|
-
tableEntries.push(` ${tableKey}: InferOutput<typeof ${schemaIdentifier}>;`);
|
|
1275
|
-
} else tableEntries.push(` ${tableKey}: Record<string, unknown>;`);
|
|
1276
|
-
}
|
|
1277
|
-
return `// Auto-generated by lines-db
|
|
1278
|
-
// Do not edit this file manually
|
|
1279
|
-
|
|
1280
|
-
${imports.length > 0 ? `${imports.join("\n")}\n` : ""}import type { DatabaseConfig${imports.length > 0 ? ", InferOutput" : ""} } from '@toiroakr/lines-db';
|
|
1281
|
-
import { fileURLToPath } from 'node:url';
|
|
1282
|
-
import { dirname } from 'node:path';
|
|
1283
|
-
|
|
1284
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
1285
|
-
const __dirname = dirname(__filename);
|
|
1286
|
-
|
|
1287
|
-
export type Tables = {
|
|
1288
|
-
${tableEntries.join("\n")}
|
|
1289
|
-
};
|
|
1290
|
-
|
|
1291
|
-
export const config: DatabaseConfig<Tables> = {
|
|
1292
|
-
dataDir: __dirname,
|
|
1293
|
-
};
|
|
1294
|
-
`;
|
|
1295
|
-
}
|
|
1296
|
-
createSchemaIdentifier(tableName, usedAliases) {
|
|
1297
|
-
let base = sanitizeIdentifier(toCamelCase(tableName)) || "table";
|
|
1298
|
-
if (!/^[A-Za-z_$]/.test(base)) base = `_${base}`;
|
|
1299
|
-
let candidate = `${base}Schema`;
|
|
1300
|
-
let suffix = 1;
|
|
1301
|
-
while (usedAliases.has(candidate)) candidate = `${base}${++suffix}Schema`;
|
|
1302
|
-
return candidate;
|
|
1303
|
-
}
|
|
1304
|
-
formatTableKey(tableName) {
|
|
1305
|
-
if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(tableName)) return tableName;
|
|
1306
|
-
return `'${tableName.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
|
|
1307
|
-
}
|
|
1308
|
-
};
|
|
1309
|
-
function toCamelCase(value) {
|
|
1310
|
-
const parts = value.split(/[^A-Za-z0-9]+/).filter(Boolean).map((part) => part.toLowerCase());
|
|
1311
|
-
if (parts.length === 0) return value;
|
|
1312
|
-
const [first, ...rest] = parts;
|
|
1313
|
-
return first + rest.map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join("");
|
|
1314
|
-
}
|
|
1315
|
-
function sanitizeIdentifier(value) {
|
|
1316
|
-
return value.replace(/[^A-Za-z0-9_$]/g, "");
|
|
1317
|
-
}
|
|
1318
|
-
//#endregion
|
|
1319
|
-
//#region src/jsonl-migration.ts
|
|
1320
|
-
/**
|
|
1321
|
-
* Validate a table by temporarily supplying in-memory rows while reusing the existing LinesDB validation pipeline.
|
|
1322
|
-
* If validation fails, throws an error with validation details.
|
|
1323
|
-
*/
|
|
1324
|
-
async function ensureTableRowsValid(options) {
|
|
1325
|
-
const tablePath = (0, node_path.join)(options.dataDir, `${options.tableName}.jsonl`);
|
|
1326
|
-
const overrides = new Map([[tablePath, options.rows]]);
|
|
1327
|
-
await JsonlReader.withOverrides(overrides, async () => {
|
|
1328
|
-
const db = LinesDB.create({ dataDir: options.dataDir });
|
|
1329
|
-
try {
|
|
1330
|
-
const result = await db.initialize({ tableName: options.tableName });
|
|
1331
|
-
if (!result.valid) {
|
|
1332
|
-
const errorCount = result.errors.length;
|
|
1333
|
-
const errorDetails = result.errors.map((e) => {
|
|
1334
|
-
const issueMessages = e.issues.map((issue) => issue.message).join(", ");
|
|
1335
|
-
return ` Row ${e.rowIndex}: ${issueMessages}`;
|
|
1336
|
-
}).join("\n");
|
|
1337
|
-
throw new Error(`Validation failed for table '${options.tableName}' (${errorCount} error(s)):\n${errorDetails}`);
|
|
1338
|
-
}
|
|
1339
|
-
} finally {
|
|
1340
|
-
await db.close();
|
|
1341
|
-
}
|
|
1342
|
-
});
|
|
1343
|
-
}
|
|
1344
|
-
//#endregion
|
|
1345
|
-
//#region src/error-formatter.ts
|
|
1346
|
-
var ErrorFormatter = class {
|
|
1347
|
-
verbose;
|
|
1348
|
-
constructor(options = {}) {
|
|
1349
|
-
this.verbose = options.verbose ?? false;
|
|
1350
|
-
}
|
|
1351
|
-
/**
|
|
1352
|
-
* Format validation errors
|
|
1353
|
-
*/
|
|
1354
|
-
formatValidationErrors(errors) {
|
|
1355
|
-
if (this.verbose) return this.formatValidationErrorsVerbose(errors);
|
|
1356
|
-
return this.formatValidationErrorsCompact(errors);
|
|
1357
|
-
}
|
|
1358
|
-
/**
|
|
1359
|
-
* Format foreign key error
|
|
1360
|
-
*/
|
|
1361
|
-
formatForeignKeyError(error) {
|
|
1362
|
-
if (this.verbose) return this.formatForeignKeyErrorVerbose(error);
|
|
1363
|
-
return this.formatForeignKeyErrorCompact(error);
|
|
1364
|
-
}
|
|
1365
|
-
/**
|
|
1366
|
-
* Format compact (default) validation errors
|
|
1367
|
-
*/
|
|
1368
|
-
formatValidationErrorsCompact(errors) {
|
|
1369
|
-
const lines = [];
|
|
1370
|
-
for (const error of errors) for (const issue of error.issues) {
|
|
1371
|
-
const fieldPath = this.getFieldPath(issue);
|
|
1372
|
-
const line = `${error.file}:${error.rowIndex + 1} • ${fieldPath}: ${issue.message}`;
|
|
1373
|
-
lines.push(line);
|
|
1374
|
-
}
|
|
1375
|
-
return lines.join("\n");
|
|
1376
|
-
}
|
|
1377
|
-
/**
|
|
1378
|
-
* Format verbose validation errors
|
|
1379
|
-
*/
|
|
1380
|
-
formatValidationErrorsVerbose(errors) {
|
|
1381
|
-
const blocks = [];
|
|
1382
|
-
for (let i = 0; i < errors.length; i++) {
|
|
1383
|
-
const error = errors[i];
|
|
1384
|
-
const isLast = i === errors.length - 1;
|
|
1385
|
-
const prefix = isLast ? "└─" : "├─";
|
|
1386
|
-
const linePrefix = isLast ? " " : "│ ";
|
|
1387
|
-
const lines = [`${prefix} ${error.file}:${error.rowIndex + 1}`];
|
|
1388
|
-
for (const issue of error.issues) {
|
|
1389
|
-
const fieldPath = this.getFieldPath(issue);
|
|
1390
|
-
lines.push(`${linePrefix}Field: ${fieldPath}`);
|
|
1391
|
-
lines.push(`${linePrefix}Error: ${issue.message}`);
|
|
1392
|
-
}
|
|
1393
|
-
if (error.originalData !== void 0) lines.push(`${linePrefix}Original data: ${JSON.stringify(error.originalData)}`);
|
|
1394
|
-
if (error.data !== void 0) {
|
|
1395
|
-
const label = error.originalData !== void 0 ? "Transformed data" : "Data";
|
|
1396
|
-
lines.push(`${linePrefix}${label}: ${JSON.stringify(error.data)}`);
|
|
1397
|
-
}
|
|
1398
|
-
blocks.push(lines.join("\n"));
|
|
1399
|
-
}
|
|
1400
|
-
return blocks.join("\n│\n");
|
|
1401
|
-
}
|
|
1402
|
-
/**
|
|
1403
|
-
* Format compact foreign key error
|
|
1404
|
-
*/
|
|
1405
|
-
formatForeignKeyErrorCompact(error) {
|
|
1406
|
-
return `${error.file}:${error.rowIndex + 1} • ${error.column}: Foreign key constraint failed - Referenced value ${JSON.stringify(error.value)} does not exist in ${error.referencedTable}(${error.referencedColumn})`;
|
|
1407
|
-
}
|
|
1408
|
-
/**
|
|
1409
|
-
* Format verbose foreign key error
|
|
1410
|
-
*/
|
|
1411
|
-
formatForeignKeyErrorVerbose(error) {
|
|
1412
|
-
const lines = [
|
|
1413
|
-
`└─ ${error.file}:${error.rowIndex + 1}`,
|
|
1414
|
-
` Type: Foreign Key Violation`,
|
|
1415
|
-
` Field: ${error.column}`,
|
|
1416
|
-
` Value: ${JSON.stringify(error.value)}`,
|
|
1417
|
-
` References: ${error.referencedTable}(${error.referencedColumn})`,
|
|
1418
|
-
` Error: Referenced value does not exist in target table`
|
|
1419
|
-
];
|
|
1420
|
-
if (error.data !== void 0) lines.push(` Data: ${JSON.stringify(error.data)}`);
|
|
1421
|
-
return lines.join("\n");
|
|
1422
|
-
}
|
|
1423
|
-
/**
|
|
1424
|
-
* Get field path from issue
|
|
1425
|
-
*/
|
|
1426
|
-
getFieldPath(issue) {
|
|
1427
|
-
if (!issue.path || issue.path.length === 0) return "root";
|
|
1428
|
-
return issue.path.map((segment) => {
|
|
1429
|
-
if (typeof segment === "object" && segment !== null && "key" in segment) return String(segment.key);
|
|
1430
|
-
return String(segment);
|
|
1431
|
-
}).join(".");
|
|
1432
|
-
}
|
|
1433
|
-
/**
|
|
1434
|
-
* Format error header with count
|
|
1435
|
-
*/
|
|
1436
|
-
formatErrorHeader(count, file) {
|
|
1437
|
-
return (0, node_util.styleText)("red", `✗ Found ${count} error(s)${file ? ` in ${file}` : ""}`);
|
|
1438
|
-
}
|
|
1439
|
-
/**
|
|
1440
|
-
* Format migration failure header
|
|
1441
|
-
*/
|
|
1442
|
-
formatMigrationFailureHeader() {
|
|
1443
|
-
return (0, node_util.styleText)("red", "\n✗ Migration failed and was rolled back");
|
|
1444
|
-
}
|
|
1445
|
-
};
|
|
1446
|
-
//#endregion
|
|
1447
|
-
exports.DirectoryScanner = DirectoryScanner;
|
|
1448
|
-
exports.ErrorFormatter = ErrorFormatter;
|
|
1449
|
-
exports.JsonlReader = JsonlReader;
|
|
1450
|
-
exports.JsonlWriter = JsonlWriter;
|
|
1451
|
-
exports.LinesDB = LinesDB;
|
|
1452
|
-
exports.RUNTIME = RUNTIME;
|
|
1453
|
-
exports.SCHEMA_EXTENSIONS = SCHEMA_EXTENSIONS;
|
|
1454
|
-
exports.SchemaLoader = SchemaLoader;
|
|
1455
|
-
exports.TypeGenerator = TypeGenerator;
|
|
1456
|
-
exports.defineSchema = defineSchema;
|
|
1457
|
-
exports.detectRuntime = detectRuntime;
|
|
1458
|
-
exports.ensureTableRowsValid = ensureTableRowsValid;
|
|
1459
|
-
exports.extractTableNameFromSchemaFile = extractTableNameFromSchemaFile;
|
|
1460
|
-
exports.findSchemaFile = findSchemaFile;
|
|
1461
|
-
exports.hasBackward = hasBackward;
|
|
1462
|
-
exports.isSchemaFile = isSchemaFile;
|
|
1463
|
-
exports.rewriteExtensionForImport = rewriteExtensionForImport;
|