@toiroakr/lines-db 0.9.2 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs DELETED
@@ -1,1463 +0,0 @@
1
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
- let node_fs_promises = require("node:fs/promises");
3
- let node_path = require("node:path");
4
- let node_url = require("node:url");
5
- let node_util = require("node:util");
6
- //#region src/runtime.ts
7
- function detectRuntime() {
8
- if (typeof process !== "undefined" && process.versions && process.versions.node) return "node";
9
- return "unknown";
10
- }
11
- const RUNTIME = detectRuntime();
12
- //#endregion
13
- //#region src/sqlite-adapter.ts
14
- /**
15
- * SQLite adapter for Node.js
16
- */
17
- /**
18
- * Create a SQLite database instance for Node.js
19
- */
20
- function createDatabase(path = ":memory:") {
21
- if (RUNTIME === "node") return createNodeDatabase(path);
22
- else throw new Error(`Unsupported runtime: ${RUNTIME}`);
23
- }
24
- /**
25
- * Create a Node.js SQLite database
26
- */
27
- function createNodeDatabase(path) {
28
- const { DatabaseSync } = require("node:sqlite");
29
- const db = new DatabaseSync(path);
30
- db.exec("PRAGMA foreign_keys = ON");
31
- return {
32
- prepare(sql) {
33
- const stmt = db.prepare(sql);
34
- return {
35
- run(...params) {
36
- return stmt.run(...params);
37
- },
38
- get(...params) {
39
- return stmt.get(...params);
40
- },
41
- all(...params) {
42
- return stmt.all(...params);
43
- }
44
- };
45
- },
46
- exec(sql) {
47
- db.exec(sql);
48
- },
49
- close() {
50
- db.close();
51
- }
52
- };
53
- }
54
- //#endregion
55
- //#region src/jsonl-reader.ts
56
- var JsonlReader = class {
57
- static overrides = null;
58
- /**
59
- * Temporarily override the data returned for specific JSONL files.
60
- * Useful for scenarios like migration validation where in-memory data should be used.
61
- */
62
- static async withOverrides(overrides, fn) {
63
- const normalized = /* @__PURE__ */ new Map();
64
- for (const [filePath, rows] of overrides) normalized.set((0, node_path.normalize)(filePath), rows);
65
- const previousOverrides = this.overrides;
66
- this.overrides = normalized;
67
- try {
68
- return await fn();
69
- } finally {
70
- this.overrides = previousOverrides;
71
- }
72
- }
73
- /**
74
- * Read JSONL file and parse each line as JSON
75
- */
76
- static async read(filePath) {
77
- const overrideRows = this.overrides?.get((0, node_path.normalize)(filePath));
78
- if (overrideRows) return overrideRows.map((row) => JSON.parse(JSON.stringify(row)));
79
- return (await (0, node_fs_promises.readFile)(filePath, "utf-8")).trim().split("\n").filter((line) => line.trim().length > 0).map((line) => {
80
- try {
81
- return JSON.parse(line);
82
- } catch (error) {
83
- throw new Error(`Failed to parse JSON line: ${line}`, { cause: error });
84
- }
85
- });
86
- }
87
- /**
88
- * Infer schema from JSONL data
89
- */
90
- static inferSchema(tableName, data) {
91
- if (data.length === 0) throw new Error("Cannot infer schema from empty data");
92
- const columnTypes = /* @__PURE__ */ new Map();
93
- const booleanColumns = /* @__PURE__ */ new Set();
94
- const nonBooleanColumns = /* @__PURE__ */ new Set();
95
- for (const row of data) for (const [key, value] of Object.entries(row)) {
96
- if (!columnTypes.has(key)) columnTypes.set(key, /* @__PURE__ */ new Set());
97
- columnTypes.get(key).add(this.inferType(value));
98
- if (typeof value === "boolean") booleanColumns.add(key);
99
- else if (value !== null) nonBooleanColumns.add(key);
100
- }
101
- const columns = [];
102
- for (const [columnName, types] of columnTypes.entries()) {
103
- const typeArray = Array.from(types);
104
- let sqlType = "TEXT";
105
- if (typeArray.length === 1) sqlType = typeArray[0];
106
- else if (typeArray.every((t) => t === "INTEGER" || t === "REAL")) sqlType = "REAL";
107
- else if (!typeArray.includes("NULL")) sqlType = "TEXT";
108
- else if (typeArray.length === 2 && typeArray.includes("NULL")) sqlType = typeArray.find((t) => t !== "NULL");
109
- const isBooleanColumn = booleanColumns.has(columnName) && !nonBooleanColumns.has(columnName);
110
- columns.push({
111
- name: columnName,
112
- type: sqlType,
113
- notNull: !typeArray.includes("NULL"),
114
- valueType: isBooleanColumn ? "boolean" : void 0
115
- });
116
- }
117
- const idColumn = columns.find((col) => col.name === "id");
118
- if (idColumn) idColumn.primaryKey = true;
119
- return {
120
- name: tableName,
121
- columns
122
- };
123
- }
124
- static inferType(value) {
125
- if (value === null || value === void 0) return "NULL";
126
- if (typeof value === "number") return Number.isInteger(value) ? "INTEGER" : "REAL";
127
- if (typeof value === "string") return "TEXT";
128
- if (typeof value === "boolean") return "INTEGER";
129
- if (typeof value === "object") return "JSON";
130
- return "TEXT";
131
- }
132
- };
133
- //#endregion
134
- //#region src/jsonl-writer.ts
135
- var JsonlWriter = class {
136
- /**
137
- * Write data to JSONL file
138
- */
139
- static async write(filePath, data) {
140
- await (0, node_fs_promises.writeFile)(filePath, data.map((obj) => JSON.stringify(obj)).join("\n") + "\n", "utf-8");
141
- }
142
- /**
143
- * Append data to JSONL file
144
- */
145
- static async append(filePath, data) {
146
- const { readFile, writeFile } = await import("node:fs/promises");
147
- try {
148
- const existing = await readFile(filePath, "utf-8");
149
- const lines = data.map((obj) => JSON.stringify(obj)).join("\n");
150
- await writeFile(filePath, existing.trim() + "\n" + lines + "\n", "utf-8");
151
- } catch (error) {
152
- if (error.code === "ENOENT") await this.write(filePath, data);
153
- else throw error;
154
- }
155
- }
156
- };
157
- //#endregion
158
- //#region src/schema-extensions.ts
159
- /**
160
- * Supported schema file extensions, in priority order.
161
- * The first match wins when discovering schema files.
162
- */
163
- const SCHEMA_EXTENSIONS = [
164
- ".schema.ts",
165
- ".schema.mts",
166
- ".schema.cts"
167
- ];
168
- /**
169
- * Map from schema extensions to their JavaScript import counterparts.
170
- */
171
- const SCHEMA_TO_JS_IMPORT_MAP = {
172
- ".schema.ts": ".schema.js",
173
- ".schema.mts": ".schema.mjs",
174
- ".schema.cts": ".schema.cjs"
175
- };
176
- /**
177
- * Try each supported schema extension and return the full path of the first
178
- * one that exists on disk. Returns undefined if none is found.
179
- */
180
- async function findSchemaFile(dir, tableName) {
181
- for (const ext of SCHEMA_EXTENSIONS) {
182
- const candidate = (0, node_path.join)(dir, `${tableName}${ext}`);
183
- try {
184
- await (0, node_fs_promises.access)(candidate);
185
- return candidate;
186
- } catch {}
187
- }
188
- }
189
- /**
190
- * Synchronously find a schema file among directory entries.
191
- * Returns the full path of the first match, or undefined.
192
- */
193
- function findSchemaFileInEntries(dataDirPath, tableName, entries) {
194
- for (const ext of SCHEMA_EXTENSIONS) {
195
- const candidateName = `${tableName}${ext}`;
196
- if (entries.some((e) => e.isFile() && e.name === candidateName)) return (0, node_path.join)(dataDirPath, candidateName);
197
- }
198
- }
199
- /**
200
- * Check if a filename matches any supported schema file pattern.
201
- */
202
- function isSchemaFile(fileName) {
203
- return SCHEMA_EXTENSIONS.some((ext) => fileName.endsWith(ext));
204
- }
205
- /**
206
- * Extract table name from a schema filename.
207
- * e.g., "users.schema.ts" -> "users", "users.schema.mts" -> "users"
208
- */
209
- function extractTableNameFromSchemaFile(fileName) {
210
- for (const ext of SCHEMA_EXTENSIONS) if (fileName.endsWith(ext)) return fileName.slice(0, -ext.length);
211
- return null;
212
- }
213
- /**
214
- * Rewrite a TypeScript path to its JavaScript counterpart for ESM imports.
215
- * ".schema.ts" -> ".schema.js", ".schema.mts" -> ".schema.mjs", ".schema.cts" -> ".schema.cjs"
216
- */
217
- function rewriteExtensionForImport(filePath) {
218
- for (const [tsExt, jsExt] of Object.entries(SCHEMA_TO_JS_IMPORT_MAP)) if (filePath.endsWith(tsExt)) return filePath.slice(0, -tsExt.length) + jsExt;
219
- return filePath;
220
- }
221
- //#endregion
222
- //#region src/schema-loader.ts
223
- var SchemaLoader = class {
224
- /**
225
- * Check if a schema file exists for a table
226
- */
227
- static async hasSchema(jsonlPath) {
228
- return await findSchemaFile((0, node_path.dirname)(jsonlPath), (0, node_path.basename)(jsonlPath, ".jsonl")) !== void 0;
229
- }
230
- /**
231
- * Load a validation schema file for a table
232
- * Requires ${tableName}.schema.{ts,mts,cts} to exist alongside the JSONL file
233
- */
234
- static async loadSchema(jsonlPath) {
235
- const dir = (0, node_path.dirname)(jsonlPath);
236
- const tableName = (0, node_path.basename)(jsonlPath, ".jsonl");
237
- const schemaPath = await findSchemaFile(dir, tableName);
238
- if (!schemaPath) throw new Error(`Schema file not found for table '${tableName}'. Expected one of: ${SCHEMA_EXTENSIONS.map((ext) => `${tableName}${ext}`).join(", ")}`);
239
- try {
240
- const module = await import(`${(0, node_url.pathToFileURL)(schemaPath).href}?t=${Date.now()}`);
241
- const schema = module.default || module.schema;
242
- if (schema && this.isStandardSchema(schema)) return schema;
243
- throw new Error(`Schema file ${schemaPath} does not export a valid StandardSchema`);
244
- } catch (error) {
245
- throw new Error(`Failed to load schema for table '${tableName}' from ${schemaPath}: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : void 0 });
246
- }
247
- }
248
- /**
249
- * Check if an object implements the StandardSchema interface
250
- */
251
- static isStandardSchema(obj) {
252
- if (!obj || typeof obj !== "object") return false;
253
- const standard = obj["~standard"];
254
- if (!standard || typeof standard !== "object") return false;
255
- const standardObj = standard;
256
- return standardObj.version === 1 && typeof standardObj.vendor === "string" && typeof standardObj.validate === "function";
257
- }
258
- };
259
- //#endregion
260
- //#region src/directory-scanner.ts
261
- var DirectoryScanner = class {
262
- /**
263
- * Scan directory for JSONL files and create table configurations
264
- */
265
- static async scanDirectory(dataDir) {
266
- const tables = /* @__PURE__ */ new Map();
267
- try {
268
- const files = await (0, node_fs_promises.readdir)(dataDir);
269
- for (const file of files) if ((0, node_path.extname)(file) === ".jsonl") {
270
- const tableName = (0, node_path.basename)(file, ".jsonl");
271
- const jsonlPath = (0, node_path.join)(dataDir, file);
272
- tables.set(tableName, {
273
- jsonlPath,
274
- autoInferSchema: true
275
- });
276
- }
277
- if (tables.size === 0) console.warn(`Warning: No JSONL files found in directory: ${dataDir}`);
278
- return tables;
279
- } catch (error) {
280
- throw new Error(`Failed to scan directory ${dataDir}: ${error instanceof Error ? error.message : String(error)}`);
281
- }
282
- }
283
- };
284
- //#endregion
285
- //#region src/schema.ts
286
- /**
287
- * Define a bidirectional schema with optional backward transformation
288
- *
289
- * @param schema - Standard Schema for validation
290
- * @param options - SchemaOptions object. When Input and Output types differ, backward transformation is required
291
- *
292
- * @example
293
- * // No transformation - backward not needed
294
- * const schema = defineSchema(
295
- * v.object({ id: v.number(), name: v.string() })
296
- * );
297
- *
298
- * @example
299
- * // With transformation - backward REQUIRED
300
- * const schema = defineSchema(
301
- * v.pipe(v.string(), v.transform(Number)),
302
- * {
303
- * backward: (num) => String(num) // backward: number → string (REQUIRED)
304
- * }
305
- * );
306
- *
307
- * @example
308
- * // With primary key and foreign key
309
- * const schema = defineSchema(
310
- * v.object({ id: v.number(), customerId: v.number() }),
311
- * {
312
- * primaryKey: 'id',
313
- * foreignKeys: [
314
- * { column: 'customerId', references: { table: 'users', column: 'id' } }
315
- * ]
316
- * }
317
- * );
318
- */
319
- function defineSchema(schema, ...args) {
320
- const options = args[0];
321
- const bidirectionalSchema = Object.create(schema);
322
- if (options) {
323
- if (options.backward) bidirectionalSchema.backward = options.backward;
324
- if (options.primaryKey) bidirectionalSchema.primaryKey = options.primaryKey;
325
- if (options.foreignKeys) bidirectionalSchema.foreignKeys = options.foreignKeys;
326
- if (options.indexes) bidirectionalSchema.indexes = options.indexes;
327
- }
328
- Object.defineProperty(bidirectionalSchema, "~standard", {
329
- value: schema["~standard"],
330
- enumerable: true,
331
- configurable: true
332
- });
333
- return bidirectionalSchema;
334
- }
335
- /**
336
- * Check if a schema has backward transformation
337
- */
338
- function hasBackward(schema) {
339
- return "backward" in schema && typeof schema.backward === "function";
340
- }
341
- //#endregion
342
- //#region src/database.ts
343
- var LinesDB = class LinesDB {
344
- db;
345
- config;
346
- schemas = /* @__PURE__ */ new Map();
347
- validationSchemas = /* @__PURE__ */ new Map();
348
- tables = /* @__PURE__ */ new Map();
349
- inTransaction = false;
350
- constructor(config, dbPath) {
351
- this.config = config;
352
- this.db = createDatabase(dbPath ?? ":memory:");
353
- }
354
- static create(config, dbPath) {
355
- return new LinesDB(config, dbPath);
356
- }
357
- /**
358
- * Initialize database by loading all JSONL files or a specific table
359
- * Uses dependency resolution to ensure foreign key references are loaded in correct order
360
- * @param options Optional configuration for initialization
361
- * @param options.tableName Optional table name to initialize. If not provided, initializes all tables
362
- * @param options.detailedValidate If true, performs detailed validation by inserting rows one by one to catch constraint violations
363
- * @param options.transform Optional transform function to apply to rows before validation (only applied to the specified tableName)
364
- * @returns ValidationResult containing validation status, errors, and warnings
365
- */
366
- async initialize(options) {
367
- const allErrors = [];
368
- const allWarnings = [];
369
- const allRowCounts = /* @__PURE__ */ new Map();
370
- const tableName = options?.tableName;
371
- const detailedValidate = options?.detailedValidate ?? false;
372
- const transform = options?.transform;
373
- this.tables = await DirectoryScanner.scanDirectory(this.config.dataDir);
374
- const tablesToLoad = tableName ? [tableName] : Array.from(this.tables.keys());
375
- for (const tableNameToLoad of tablesToLoad) if (!this.tables.has(tableNameToLoad)) throw new Error(`Table '${tableNameToLoad}' not found in directory '${this.config.dataDir}'`);
376
- const loadedTables = /* @__PURE__ */ new Set();
377
- const loadingTables = /* @__PURE__ */ new Set();
378
- const attemptedTables = /* @__PURE__ */ new Set();
379
- const allDeferredForeignKeys = [];
380
- for (const tableNameToLoad of tablesToLoad) if (!attemptedTables.has(tableNameToLoad)) {
381
- const tableTransform = tableNameToLoad === tableName ? transform : void 0;
382
- const { errors, warnings, rowCounts: tableRowCounts, deferredForeignKeys } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
383
- allErrors.push(...errors);
384
- allWarnings.push(...warnings);
385
- allDeferredForeignKeys.push(...deferredForeignKeys);
386
- for (const [k, v] of tableRowCounts) allRowCounts.set(k, v);
387
- }
388
- if (detailedValidate && allDeferredForeignKeys.length > 0) for (const { tableName: tName, foreignKey: fk, filePath } of allDeferredForeignKeys) {
389
- if (!loadedTables.has(fk.references.table)) continue;
390
- const deferredErrors = this.validateDeferredForeignKey(tName, fk, filePath);
391
- allErrors.push(...deferredErrors);
392
- }
393
- const tableResults = tablesToLoad.map((name) => {
394
- const tableErrors = allErrors.filter((e) => e.tableName === name);
395
- const tableWarnings = allWarnings.filter((w) => w.includes(`'${name}'`));
396
- return {
397
- tableName: name,
398
- valid: tableErrors.length === 0,
399
- rowCount: allRowCounts.get(name) ?? 0,
400
- errors: tableErrors,
401
- warnings: tableWarnings
402
- };
403
- });
404
- return {
405
- valid: allErrors.length === 0,
406
- errors: allErrors,
407
- warnings: allWarnings,
408
- tableResults
409
- };
410
- }
411
- /**
412
- * Load a table and its dependencies recursively
413
- */
414
- async loadTableWithDependencies(tableName, loadedTables, loadingTables, attemptedTables, detailedValidate, transform) {
415
- const errors = [];
416
- const warnings = [];
417
- const rowCounts = /* @__PURE__ */ new Map();
418
- const deferredForeignKeys = [];
419
- if (attemptedTables.has(tableName)) return {
420
- errors,
421
- warnings,
422
- rowCounts,
423
- deferredForeignKeys
424
- };
425
- attemptedTables.add(tableName);
426
- if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
427
- const tableConfig = this.tables.get(tableName);
428
- if (!tableConfig) throw new Error(`Table configuration not found for '${tableName}'`);
429
- loadingTables.add(tableName);
430
- try {
431
- let foreignKeys;
432
- try {
433
- const { pathToFileURL } = await import("node:url");
434
- const schemaPath = await findSchemaFile((0, node_path.dirname)(tableConfig.jsonlPath), (0, node_path.basename)(tableConfig.jsonlPath, ".jsonl"));
435
- if (schemaPath) {
436
- const schemaModule = await import(`${pathToFileURL(schemaPath).href}?t=${Date.now()}`);
437
- foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
438
- }
439
- } catch {}
440
- if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
441
- const referencedTable = fk.references.table;
442
- if (referencedTable === tableName) continue;
443
- if (!attemptedTables.has(referencedTable)) if (this.tables.has(referencedTable)) {
444
- const depResult = await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables, attemptedTables, detailedValidate, void 0);
445
- errors.push(...depResult.errors);
446
- warnings.push(...depResult.warnings);
447
- deferredForeignKeys.push(...depResult.deferredForeignKeys);
448
- for (const [k, v] of depResult.rowCounts) rowCounts.set(k, v);
449
- } else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
450
- }
451
- const failedDependencies = /* @__PURE__ */ new Set();
452
- const circularDependencies = /* @__PURE__ */ new Set();
453
- if (foreignKeys && foreignKeys.length > 0) {
454
- for (const fk of foreignKeys) {
455
- const referencedTable = fk.references.table;
456
- if (referencedTable === tableName) continue;
457
- if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable)) if (loadingTables.has(referencedTable)) circularDependencies.add(referencedTable);
458
- else failedDependencies.add(referencedTable);
459
- }
460
- if (failedDependencies.size > 0) for (const dep of failedDependencies) warnings.push(`Skipping foreign key validation for table '${tableName}': referenced table '${dep}' has validation errors`);
461
- }
462
- const allSkippedDependencies = new Set([...failedDependencies, ...circularDependencies]);
463
- const { loaded, rowCount, errors: loadErrors } = await this.loadTable(tableName, tableConfig, detailedValidate, transform, allSkippedDependencies);
464
- errors.push(...loadErrors);
465
- rowCounts.set(tableName, rowCount);
466
- if (loaded) {
467
- loadedTables.add(tableName);
468
- if (foreignKeys && circularDependencies.size > 0) {
469
- for (const fk of foreignKeys) if (circularDependencies.has(fk.references.table)) deferredForeignKeys.push({
470
- tableName,
471
- foreignKey: fk,
472
- filePath: tableConfig.jsonlPath
473
- });
474
- }
475
- } else {
476
- warnings.push(`Table '${tableName}' was not loaded (no data or skipped)`);
477
- this.tables.delete(tableName);
478
- }
479
- } finally {
480
- loadingTables.delete(tableName);
481
- }
482
- return {
483
- errors,
484
- warnings,
485
- rowCounts,
486
- deferredForeignKeys
487
- };
488
- }
489
- /**
490
- * Load a single table from JSONL file
491
- * @returns Object with loaded status and validation errors
492
- */
493
- async loadTable(tableName, config, detailedValidate, transform, failedDependencies) {
494
- let data = await JsonlReader.read(config.jsonlPath);
495
- if (transform) data = data.map((row) => transform(row));
496
- let validationSchema = config.validationSchema;
497
- const schemaMetadata = {};
498
- if (!validationSchema) try {
499
- validationSchema = await SchemaLoader.loadSchema(config.jsonlPath);
500
- } catch (_error) {}
501
- if (!config.validationSchema) try {
502
- const { pathToFileURL } = await import("node:url");
503
- const schemaPath = await findSchemaFile((0, node_path.dirname)(config.jsonlPath), (0, node_path.basename)(config.jsonlPath, ".jsonl"));
504
- if (!schemaPath) throw new Error("Schema file not found");
505
- const schemaModule = await import(`${pathToFileURL(schemaPath).href}?t=${Date.now()}`);
506
- const schemaExport = schemaModule.schema || schemaModule.default;
507
- if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
508
- else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
509
- if (schemaExport?.foreignKeys) schemaMetadata.foreignKeys = schemaExport.foreignKeys;
510
- else if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
511
- if (schemaExport?.indexes) schemaMetadata.indexes = schemaExport.indexes;
512
- else if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
513
- if (process.env.DEBUG_LINES_DB) {
514
- console.log(`[lines-db] Schema metadata for ${tableName}:`);
515
- console.log(` primaryKey: ${schemaMetadata.primaryKey}`);
516
- console.log(` foreignKeys: ${JSON.stringify(schemaMetadata.foreignKeys)}`);
517
- console.log(` indexes: ${JSON.stringify(schemaMetadata.indexes)}`);
518
- }
519
- } catch (_error) {
520
- if (process.env.DEBUG_LINES_DB) console.warn(`[lines-db] Failed to load schema metadata for ${tableName}:`, _error instanceof Error ? _error.message : String(_error));
521
- }
522
- this.validationSchemas.set(tableName, validationSchema);
523
- const validationErrors = [];
524
- const validatedData = [];
525
- for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
526
- const row = data[rowIndex];
527
- try {
528
- const validatedRow = this.validateAndTransform(tableName, row);
529
- validatedData.push(validatedRow);
530
- } catch (error) {
531
- if (error instanceof Error && error.name === "ValidationError") validationErrors.push({
532
- rowIndex,
533
- rowData: row,
534
- error
535
- });
536
- else throw error;
537
- }
538
- }
539
- const validationErrorDetails = validationErrors.map((ve) => ({
540
- file: config.jsonlPath,
541
- tableName,
542
- rowIndex: ve.rowIndex,
543
- issues: ve.error.issues,
544
- type: "schema"
545
- }));
546
- if (validationErrors.length > 0) return {
547
- loaded: false,
548
- rowCount: data.length,
549
- errors: validationErrorDetails
550
- };
551
- let schema;
552
- let inferredSchema;
553
- if (validatedData.length > 0) inferredSchema = JsonlReader.inferSchema(tableName, validatedData);
554
- if (config.schema) {
555
- schema = config.schema;
556
- if (inferredSchema) for (const inferredCol of inferredSchema.columns) {
557
- const schemaCol = schema.columns.find((c) => c.name === inferredCol.name);
558
- if (schemaCol && inferredCol.valueType && !schemaCol.valueType) schemaCol.valueType = inferredCol.valueType;
559
- }
560
- } else if (config.autoInferSchema !== false) {
561
- if (validatedData.length === 0) return {
562
- loaded: false,
563
- rowCount: 0,
564
- errors: []
565
- };
566
- schema = inferredSchema;
567
- } else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
568
- const biSchema = validationSchema;
569
- const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
570
- const foreignKeys = biSchema?.foreignKeys || schemaMetadata.foreignKeys;
571
- const indexes = biSchema?.indexes || schemaMetadata.indexes;
572
- if (primaryKey && !schema.columns.some((col) => col.primaryKey)) {
573
- const col = schema.columns.find((c) => c.name === primaryKey);
574
- if (col) col.primaryKey = true;
575
- } else if (!primaryKey && !schema.columns.some((col) => col.primaryKey)) {
576
- const idColumn = schema.columns.find((c) => c.name === "id");
577
- if (idColumn) idColumn.primaryKey = true;
578
- }
579
- if (foreignKeys) schema.foreignKeys = failedDependencies && failedDependencies.size > 0 ? foreignKeys.filter((fk) => !failedDependencies.has(fk.references.table)) : foreignKeys;
580
- if (indexes) {
581
- schema.indexes = indexes;
582
- for (const index of indexes) if (index.unique && index.columns.length === 1) {
583
- const col = schema.columns.find((c) => c.name === index.columns[0]);
584
- if (col && !col.unique && !col.primaryKey) col.unique = true;
585
- }
586
- }
587
- this.schemas.set(tableName, schema);
588
- this.createTable(schema);
589
- if (detailedValidate) {
590
- const insertErrors = this.insertDataWithDetailedValidation(tableName, schema, validatedData, config.jsonlPath);
591
- if (insertErrors.length > 0) return {
592
- loaded: false,
593
- rowCount: data.length,
594
- errors: insertErrors
595
- };
596
- } else this.insertData(tableName, schema, validatedData);
597
- return {
598
- loaded: true,
599
- rowCount: data.length,
600
- errors: []
601
- };
602
- }
603
- /**
604
- * Create table in SQLite with constraints and indexes
605
- */
606
- createTable(schema) {
607
- const quotedTableName = this.quoteTableName(schema.name);
608
- const uniqueColumns = /* @__PURE__ */ new Set();
609
- for (const col of schema.columns) if (col.unique) uniqueColumns.add(col.name);
610
- if (schema.indexes) {
611
- for (const index of schema.indexes) if (index.unique && index.columns.length === 1) uniqueColumns.add(index.columns[0]);
612
- }
613
- const columnDefs = schema.columns.map((col) => {
614
- const sqlType = col.type === "JSON" ? "TEXT" : col.type;
615
- const parts = [this.quoteIdentifier(col.name), sqlType];
616
- if (col.primaryKey) parts.push("PRIMARY KEY");
617
- if (col.notNull) parts.push("NOT NULL");
618
- if (uniqueColumns.has(col.name) && !col.primaryKey) parts.push("UNIQUE");
619
- return parts.join(" ");
620
- });
621
- const foreignKeyDefs = [];
622
- if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) {
623
- const fkParts = [`FOREIGN KEY (${this.quoteIdentifier(fk.column)})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`];
624
- if (fk.onDelete) fkParts.push(`ON DELETE ${fk.onDelete}`);
625
- if (fk.onUpdate) fkParts.push(`ON UPDATE ${fk.onUpdate}`);
626
- foreignKeyDefs.push(fkParts.join(" "));
627
- }
628
- const sql = `CREATE TABLE IF NOT EXISTS ${quotedTableName} (${[...columnDefs, ...foreignKeyDefs].join(", ")})`;
629
- this.db.exec(sql);
630
- if (schema.indexes && schema.indexes.length > 0) for (let i = 0; i < schema.indexes.length; i++) {
631
- const index = schema.indexes[i];
632
- const safeTableName = schema.name.replace(/[^a-zA-Z0-9]/g, "_");
633
- const resolvedIndexName = index.name || `idx_${safeTableName}_${index.columns.join("_")}_${i}`;
634
- const indexSql = `CREATE ${index.unique ? "UNIQUE " : ""}INDEX IF NOT EXISTS ${this.quoteIdentifier(resolvedIndexName)} ON ${quotedTableName} (${index.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`;
635
- this.db.exec(indexSql);
636
- }
637
- }
638
- /**
639
- * Quote table name to handle special characters in SQL
640
- */
641
- quoteTableName(tableName) {
642
- return this.quoteIdentifier(tableName);
643
- }
644
- /**
645
- * Quote identifier for SQL statements, escaping embedded quotes
646
- */
647
- quoteIdentifier(identifier) {
648
- return `"${identifier.replace(/"/g, "\"\"")}"`;
649
- }
650
- /**
651
- * Insert data into table using batch insert (multiple rows per SQL)
652
- * SQLite has a parameter limit (default 999), so we batch rows accordingly
653
- * Throws exception if any constraint violation occurs
654
- */
655
- insertData(tableName, schema, data) {
656
- if (data.length === 0) return;
657
- const columnNames = schema.columns.map((col) => col.name);
658
- const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
659
- const columnCount = columnNames.length;
660
- const maxBatchSize = Math.floor(900 / columnCount);
661
- const batchSize = Math.max(1, Math.min(maxBatchSize, 100));
662
- for (let i = 0; i < data.length; i += batchSize) {
663
- const batch = data.slice(i, i + batchSize);
664
- const rowPlaceholders = columnNames.map(() => "?").join(", ");
665
- const valuesPlaceholders = batch.map(() => `(${rowPlaceholders})`).join(", ");
666
- const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES ${valuesPlaceholders}`;
667
- const values = [];
668
- for (const row of batch) for (const col of columnNames) values.push(this.normalizeValue(row[col]));
669
- this.db.prepare(sql).run(...values);
670
- }
671
- }
672
- /**
673
- * Insert data into table one row at a time with detailed error reporting
674
- * This is used for validation to catch constraint violations
675
- */
676
- insertDataWithDetailedValidation(tableName, schema, data, filePath) {
677
- const errors = [];
678
- const columnNames = schema.columns.map((col) => col.name);
679
- const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
680
- const placeholders = columnNames.map(() => "?").join(", ");
681
- const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
682
- const stmt = this.db.prepare(sql);
683
- for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
684
- const row = data[rowIndex];
685
- try {
686
- const values = columnNames.map((col) => this.normalizeValue(row[col]));
687
- stmt.run(...values);
688
- } catch (error) {
689
- const constraintError = this.analyzeConstraintError(error, filePath, tableName, rowIndex, row, schema.foreignKeys || []);
690
- if (constraintError) errors.push(constraintError);
691
- }
692
- }
693
- return errors;
694
- }
695
- /**
696
- * Analyze constraint error and extract detailed information
697
- */
698
- analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys) {
699
- const errorMessage = error instanceof Error ? error.message : String(error);
700
- if (errorMessage.includes("FOREIGN KEY constraint failed")) for (const fk of foreignKeys) {
701
- const fkValue = row[fk.column];
702
- if (fkValue === null || fkValue === void 0) continue;
703
- try {
704
- const result = this.query(`SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`, [this.normalizeValue(fkValue)]);
705
- if (result.length > 0 && result[0].count === 0) return {
706
- file,
707
- tableName,
708
- rowIndex,
709
- issues: [],
710
- type: "foreignKey",
711
- foreignKeyError: {
712
- column: fk.column,
713
- value: fkValue,
714
- referencedTable: fk.references.table,
715
- referencedColumn: fk.references.column
716
- }
717
- };
718
- } catch (_) {}
719
- }
720
- return {
721
- file,
722
- tableName,
723
- rowIndex,
724
- issues: [{
725
- message: errorMessage,
726
- path: []
727
- }],
728
- type: "schema"
729
- };
730
- }
731
- /**
732
- * Validate a deferred foreign key constraint after all tables have been loaded.
733
- * Used for circular dependency FK validation.
734
- */
735
- validateDeferredForeignKey(tableName, fk, filePath) {
736
- const errors = [];
737
- const quotedTable = this.quoteTableName(tableName);
738
- const quotedColumn = this.quoteIdentifier(fk.column);
739
- const quotedRefTable = this.quoteTableName(fk.references.table);
740
- const sql = `SELECT rowid - 1 as idx, ${quotedColumn} as val FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL AND ${quotedColumn} NOT IN (SELECT ${this.quoteIdentifier(fk.references.column)} FROM ${quotedRefTable})`;
741
- try {
742
- const rows = this.query(sql);
743
- for (const row of rows) errors.push({
744
- file: filePath,
745
- tableName,
746
- rowIndex: row.idx,
747
- issues: [],
748
- type: "foreignKey",
749
- foreignKeyError: {
750
- column: fk.column,
751
- value: row.val,
752
- referencedTable: fk.references.table,
753
- referencedColumn: fk.references.column
754
- }
755
- });
756
- } catch (_) {}
757
- return errors;
758
- }
759
- /**
760
- * Execute a raw SQL query
761
- */
762
- query(sql, params = []) {
763
- return this.db.prepare(sql).all(...params);
764
- }
765
- /**
766
- * Execute a SQL query that returns a single row
767
- */
768
- queryOne(sql, params = []) {
769
- const result = this.db.prepare(sql).get(...params);
770
- return result === void 0 ? null : result;
771
- }
772
- /**
773
- * Execute a SQL statement (INSERT, UPDATE, DELETE)
774
- */
775
- execute(sql, params = []) {
776
- return this.db.prepare(sql).run(...params);
777
- }
778
- /**
779
- * Find rows by condition (supports OR/AND with arrays and function filters)
780
- * If where is not provided, returns all rows
781
- */
782
- find(tableName, where) {
783
- if (where === void 0) return this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
784
- if (Array.isArray(where) && where.length === 0) return [];
785
- const { sql, values, functionFilters, hasOrWithFunctionFilters } = this.buildWhereClause(where);
786
- let rows;
787
- if (hasOrWithFunctionFilters) {
788
- rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
789
- return this.applyOrConditionWithFilters(rows, where);
790
- }
791
- if (sql) rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values).map((row) => this.deserializeRow(tableName, row));
792
- else rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
793
- return this.applyFunctionFilters(rows, functionFilters);
794
- }
795
- /**
796
- * Find a single row by condition (supports OR/AND with arrays and function filters)
797
- */
798
- findOne(tableName, where) {
799
- const { sql, values, functionFilters } = this.buildWhereClause(where);
800
- let rows;
801
- if (sql) rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values).map((row) => this.deserializeRow(tableName, row));
802
- else rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
803
- const filtered = this.applyFunctionFilters(rows, functionFilters);
804
- return filtered.length > 0 ? filtered[0] : null;
805
- }
806
- /**
807
- * Deserialize JSON columns in a row
808
- */
809
- deserializeRow(tableName, row) {
810
- const schema = this.schemas.get(tableName);
811
- if (!schema) return row;
812
- const deserializedRow = { ...row };
813
- for (const column of schema.columns) {
814
- const colName = column.name;
815
- if (!(colName in deserializedRow)) continue;
816
- const value = deserializedRow[colName];
817
- if (column.type === "JSON" && typeof value === "string") {
818
- try {
819
- deserializedRow[colName] = JSON.parse(value);
820
- } catch (error) {
821
- console.warn(`Failed to parse JSON column ${colName}:`, error);
822
- }
823
- continue;
824
- }
825
- if (column.valueType === "boolean") {
826
- if (typeof value === "number") deserializedRow[colName] = value === 0 ? false : true;
827
- else if (typeof value === "bigint") deserializedRow[colName] = value === 0n ? false : true;
828
- }
829
- }
830
- return deserializedRow;
831
- }
832
- /**
833
- * Validate data using StandardSchema and return the transformed value
834
- * Note: Only synchronous validation is supported
835
- */
836
- validateAndTransform(tableName, data) {
837
- const schema = this.validationSchemas.get(tableName);
838
- if (!schema) return data;
839
- const result = schema["~standard"].validate(data);
840
- if (result instanceof Promise) throw new Error("Asynchronous validation is not supported. Please use synchronous validation schemas.");
841
- if (result.issues && result.issues.length > 0) {
842
- const errorMessage = `Validation failed for table '${tableName}':\n${result.issues.map((issue) => {
843
- let pathStr = "root";
844
- if (issue.path && issue.path.length > 0) pathStr = issue.path.map((segment) => {
845
- if (typeof segment === "object" && segment !== null && "key" in segment) return String(segment.key);
846
- return String(segment);
847
- }).join(".");
848
- return ` - ${pathStr}: ${issue.message}`;
849
- }).join("\n")}`;
850
- const error = new Error(errorMessage);
851
- error.name = "ValidationError";
852
- error.issues = result.issues;
853
- throw error;
854
- }
855
- const transformedValue = "value" in result ? result.value : data;
856
- const normalizedValue = {};
857
- for (const [key, value] of Object.entries(transformedValue)) normalizedValue[key] = value === void 0 ? null : value;
858
- return normalizedValue;
859
- }
860
- /**
861
- * Validate data using StandardSchema (without returning transformed value)
862
- * Note: Only synchronous validation is supported
863
- */
864
- validateData(tableName, data) {
865
- this.validateAndTransform(tableName, data);
866
- }
867
- /**
868
- * Insert a row into a table with validation
869
- */
870
- insert(tableName, data) {
871
- this.validateData(tableName, data);
872
- if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
873
- const columnNames = Object.keys(data);
874
- const quotedColumns = columnNames.map((col) => this.quoteIdentifier(col));
875
- const placeholders = columnNames.map(() => "?").join(", ");
876
- const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
877
- const values = Object.values(data).map((v) => this.normalizeValue(v));
878
- const result = this.execute(sql, values);
879
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
880
- console.error(`Failed to sync table ${tableName}:`, err);
881
- });
882
- return result;
883
- }
884
- /**
885
- * Batch insert rows with validation per record.
886
- */
887
- batchInsert(tableName, records) {
888
- if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
889
- if (records.length === 0) return {
890
- changes: 0,
891
- lastInsertRowid: 0
892
- };
893
- let totalChanges = 0n;
894
- let lastRowid = 0n;
895
- for (const record of records) {
896
- this.validateData(tableName, record);
897
- const columnNames = Object.keys(record);
898
- const quotedColumns = columnNames.map((col) => this.quoteIdentifier(col));
899
- const placeholders = columnNames.map(() => "?").join(", ");
900
- const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
901
- const values = columnNames.map((col) => this.normalizeValue(record[col]));
902
- const result = this.execute(sql, values);
903
- totalChanges += BigInt(result.changes);
904
- lastRowid = BigInt(result.lastInsertRowid);
905
- }
906
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
907
- console.error(`Failed to sync table ${tableName}:`, err);
908
- });
909
- return {
910
- changes: totalChanges,
911
- lastInsertRowid: lastRowid
912
- };
913
- }
914
- /**
915
- * Update rows in a table with validation (supports OR/AND with arrays)
916
- * Note: Function filters are not supported for update operations
917
- * Note: By default, validation is enabled. For partial updates, existing data is fetched
918
- * and merged before validation. Set options.validate = false to disable validation.
919
- */
920
- update(tableName, data, where, options) {
921
- if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
922
- const shouldValidate = options?.validate !== false;
923
- const hasValidationSchema = this.validationSchemas.has(tableName);
924
- if (shouldValidate && hasValidationSchema) {
925
- const existingRows = this.find(tableName, where);
926
- for (const existingRow of existingRows) {
927
- const mergedData = {
928
- ...existingRow,
929
- ...data
930
- };
931
- this.validateData(tableName, mergedData);
932
- }
933
- }
934
- const { sql: whereSql, values: whereValues, functionFilters } = this.buildWhereClause(where);
935
- if (functionFilters.length > 0) throw new Error("Function filters are not supported in update operations");
936
- const setClauses = Object.keys(data).map((key) => `${this.quoteIdentifier(key)} = ?`).join(", ");
937
- const sql = `UPDATE ${this.quoteTableName(tableName)} SET ${setClauses} WHERE ${whereSql}`;
938
- const values = [...Object.values(data).map((v) => this.normalizeValue(v)), ...whereValues];
939
- const result = this.execute(sql, values);
940
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
941
- console.error(`Failed to sync table ${tableName}:`, err);
942
- });
943
- return result;
944
- }
945
- /**
946
- * Batch update rows with record-specific values and validation.
947
- * Each record must include the primary key to identify the target row.
948
- * Validation runs once per merged record unless explicitly disabled.
949
- */
950
- batchUpdate(tableName, records, options) {
951
- const schema = this.schemas.get(tableName);
952
- if (!schema) throw new Error(`Table ${tableName} does not exist`);
953
- if (records.length === 0) return {
954
- changes: 0,
955
- lastInsertRowid: 0
956
- };
957
- const pkColumn = schema.columns.find((col) => col.primaryKey);
958
- if (!pkColumn) throw new Error(`Table ${tableName} does not have a primary key`);
959
- const pkName = pkColumn.name;
960
- const pkValues = [];
961
- for (const record of records) {
962
- const pkValue = record[pkName];
963
- if (pkValue === void 0) throw new Error(`Record is missing primary key '${String(pkName)}': ${JSON.stringify(record)}`);
964
- pkValues.push(pkValue);
965
- }
966
- const shouldValidate = options?.validate !== false;
967
- const hasValidationSchema = this.validationSchemas.has(tableName);
968
- if (shouldValidate && hasValidationSchema) {
969
- const orCondition = pkValues.map((pkValue) => ({ [pkName]: pkValue }));
970
- const existingRows = this.find(tableName, orCondition);
971
- const existingRowsMap = /* @__PURE__ */ new Map();
972
- for (const row of existingRows) {
973
- const pkValue = row[pkName];
974
- existingRowsMap.set(pkValue, row);
975
- }
976
- const validationErrors = [];
977
- for (let i = 0; i < records.length; i++) {
978
- const record = records[i];
979
- const pkValue = record[pkName];
980
- const existingRow = existingRowsMap.get(pkValue);
981
- if (!existingRow) throw new Error(`No existing row found with ${String(pkName)}=${JSON.stringify(pkValue)}`);
982
- const mergedData = {
983
- ...existingRow,
984
- ...record
985
- };
986
- try {
987
- this.validateData(tableName, mergedData);
988
- } catch (error) {
989
- if (error instanceof Error && error.name === "ValidationError") validationErrors.push({
990
- rowIndex: i,
991
- rowData: mergedData,
992
- pkValue,
993
- error
994
- });
995
- else throw error;
996
- }
997
- }
998
- if (validationErrors.length > 0) {
999
- const enhancedError = /* @__PURE__ */ new Error(`Validation failed for ${validationErrors.length} row(s)`);
1000
- enhancedError.name = "ValidationError";
1001
- enhancedError.validationErrors = validationErrors;
1002
- enhancedError.issues = validationErrors[0].error.issues;
1003
- throw enhancedError;
1004
- }
1005
- }
1006
- let totalChanges = 0n;
1007
- let lastRowid = 0n;
1008
- for (const record of records) {
1009
- const pkValue = record[pkName];
1010
- const where = { [pkName]: pkValue };
1011
- const result = this.update(tableName, record, where, { validate: false });
1012
- totalChanges += BigInt(result.changes);
1013
- lastRowid = BigInt(result.lastInsertRowid);
1014
- }
1015
- return {
1016
- changes: totalChanges,
1017
- lastInsertRowid: lastRowid
1018
- };
1019
- }
1020
- /**
1021
- * Delete rows from a table (supports OR/AND with arrays)
1022
- * Note: Function filters are not supported for delete operations
1023
- */
1024
- delete(tableName, where) {
1025
- if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
1026
- const { sql: whereSql, values, functionFilters } = this.buildWhereClause(where);
1027
- if (functionFilters.length > 0) throw new Error("Function filters are not supported in delete operations");
1028
- const sql = `DELETE FROM ${this.quoteTableName(tableName)} WHERE ${whereSql}`;
1029
- const result = this.execute(sql, values);
1030
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
1031
- console.error(`Failed to sync table ${tableName}:`, err);
1032
- });
1033
- return result;
1034
- }
1035
- /**
1036
- * Batch delete rows by primary key.
1037
- */
1038
- batchDelete(tableName, records) {
1039
- const schema = this.schemas.get(tableName);
1040
- if (!schema) throw new Error(`Table ${tableName} does not exist`);
1041
- if (records.length === 0) return {
1042
- changes: 0,
1043
- lastInsertRowid: 0
1044
- };
1045
- const pkColumn = schema.columns.find((col) => col.primaryKey);
1046
- if (!pkColumn) throw new Error(`Table ${tableName} does not have a primary key`);
1047
- const pkName = pkColumn.name;
1048
- const pkValues = records.map((record, index) => {
1049
- const pkValue = record[pkName];
1050
- if (pkValue === void 0) throw new Error(`Record at index ${index} is missing primary key '${String(pkName)}'`);
1051
- return pkValue;
1052
- });
1053
- const placeholders = pkValues.map(() => "?").join(", ");
1054
- const sql = `DELETE FROM ${this.quoteTableName(tableName)} WHERE ${this.quoteIdentifier(pkName)} IN (${placeholders})`;
1055
- const values = pkValues.map((value) => this.normalizeValue(value));
1056
- const result = this.execute(sql, values);
1057
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
1058
- console.error(`Failed to sync table ${tableName}:`, err);
1059
- });
1060
- return {
1061
- changes: BigInt(result.changes),
1062
- lastInsertRowid: BigInt(result.lastInsertRowid)
1063
- };
1064
- }
1065
- /**
1066
- * Normalize value for SQLite
1067
- */
1068
- normalizeValue(value) {
1069
- if (value === null || value === void 0) return null;
1070
- if (typeof value === "boolean") return value ? 1 : 0;
1071
- if (typeof value === "string" || typeof value === "number" || typeof value === "bigint") return value;
1072
- if (value instanceof Uint8Array) return value;
1073
- return JSON.stringify(value);
1074
- }
1075
- /**
1076
- * Build WHERE clause from condition (supports OR/AND with arrays and functions)
1077
- */
1078
- buildWhereClause(condition) {
1079
- const values = [];
1080
- const functionFilters = [];
1081
- let hasOrWithFunctionFilters = false;
1082
- const buildCondition = (cond, isInOr = false) => {
1083
- if (Array.isArray(cond)) return cond.map((item) => {
1084
- const clause = Array.isArray(item) ? buildCondition(item, true) : buildCondition(item, true);
1085
- return clause ? `(${clause})` : "";
1086
- }).filter((clause) => clause !== "").join(" OR ");
1087
- const conditions = [];
1088
- let hasFunctionFilter = false;
1089
- for (const [key, value] of Object.entries(cond)) if (typeof value === "function") {
1090
- functionFilters.push({
1091
- key,
1092
- fn: value
1093
- });
1094
- hasFunctionFilter = true;
1095
- } else {
1096
- conditions.push(`${this.quoteIdentifier(key)} = ?`);
1097
- values.push(this.normalizeValue(value));
1098
- }
1099
- if (isInOr && hasFunctionFilter) hasOrWithFunctionFilters = true;
1100
- return conditions.join(" AND ");
1101
- };
1102
- return {
1103
- sql: buildCondition(condition),
1104
- values,
1105
- functionFilters,
1106
- hasOrWithFunctionFilters
1107
- };
1108
- }
1109
- /**
1110
- * Apply OR condition with function filters by evaluating each row against the condition
1111
- */
1112
- applyOrConditionWithFilters(rows, condition) {
1113
- return rows.filter((row) => this.matchesOrCondition(row, condition));
1114
- }
1115
- /**
1116
- * Check if a row matches an OR/AND condition (recursively)
1117
- */
1118
- matchesOrCondition(row, condition) {
1119
- if (Array.isArray(condition)) return condition.some((item) => this.matchesOrCondition(row, item));
1120
- return Object.entries(condition).every(([key, value]) => {
1121
- const rowValue = row[key];
1122
- if (typeof value === "function") return value(rowValue);
1123
- return rowValue === value;
1124
- });
1125
- }
1126
- /**
1127
- * Apply function filters to rows
1128
- */
1129
- applyFunctionFilters(rows, functionFilters) {
1130
- if (functionFilters.length === 0) return rows;
1131
- return rows.filter((row) => {
1132
- return functionFilters.every(({ key, fn }) => {
1133
- const value = row[key];
1134
- return fn(value);
1135
- });
1136
- });
1137
- }
1138
- /**
1139
- * Get table schema
1140
- */
1141
- getSchema(tableName) {
1142
- return this.schemas.get(tableName);
1143
- }
1144
- /**
1145
- * Get all table names
1146
- */
1147
- getTableNames() {
1148
- return Array.from(this.schemas.keys());
1149
- }
1150
- /**
1151
- * Sync a specific table back to its JSONL file
1152
- * Uses backward transformation when available
1153
- */
1154
- async syncTable(tableName) {
1155
- const tableConfig = this.tables.get(tableName);
1156
- if (!tableConfig) throw new Error(`Table ${tableName} not found`);
1157
- const deserializedRows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
1158
- const validationSchema = this.validationSchemas.get(tableName);
1159
- let finalRows = deserializedRows;
1160
- if (validationSchema && hasBackward(validationSchema)) {
1161
- const biSchema = validationSchema;
1162
- finalRows = deserializedRows.map((row) => biSchema.backward(row));
1163
- }
1164
- await JsonlWriter.write(tableConfig.jsonlPath, finalRows);
1165
- }
1166
- /**
1167
- * Sync database changes back to JSONL files
1168
- * Uses backward transformation when available
1169
- * @param tableName Optional table name to sync. If not provided, syncs all loaded tables
1170
- */
1171
- async sync(tableName) {
1172
- if (tableName) {
1173
- if (!this.schemas.has(tableName)) throw new Error(`Table '${tableName}' is not loaded`);
1174
- await this.syncTable(tableName);
1175
- } else for (const [name] of this.schemas) await this.syncTable(name);
1176
- }
1177
- /**
1178
- * Execute a function within a transaction
1179
- * Automatically commits on success or rolls back on error
1180
- */
1181
- async transaction(fn) {
1182
- if (this.inTransaction) throw new Error("Nested transactions are not supported");
1183
- this.db.exec("BEGIN TRANSACTION");
1184
- this.inTransaction = true;
1185
- try {
1186
- const result = await fn(this);
1187
- this.db.exec("COMMIT");
1188
- this.inTransaction = false;
1189
- await this.sync();
1190
- return result;
1191
- } catch (error) {
1192
- if (this.inTransaction) this.db.exec("ROLLBACK");
1193
- this.inTransaction = false;
1194
- throw error;
1195
- }
1196
- }
1197
- /**
1198
- * Close the database connection
1199
- */
1200
- async close() {
1201
- try {
1202
- this.db.close();
1203
- } catch (_error) {}
1204
- }
1205
- /**
1206
- * Get the underlying SQLite database instance
1207
- */
1208
- getDb() {
1209
- return this.db;
1210
- }
1211
- };
1212
- //#endregion
1213
- //#region src/type-generator.ts
1214
- var TypeGenerator = class {
1215
- dataDir;
1216
- projectRoot;
1217
- outputFile;
1218
- dataDirPath;
1219
- constructor(options) {
1220
- const envProjectRoot = process.env.LINES_DB_TEST_PROJECT_ROOT;
1221
- this.projectRoot = envProjectRoot !== void 0 ? envProjectRoot : options.projectRoot || process.cwd();
1222
- this.dataDir = options.dataDir;
1223
- this.dataDirPath = (0, node_path.isAbsolute)(this.dataDir) ? this.dataDir : (0, node_path.join)(this.projectRoot, this.dataDir);
1224
- this.outputFile = options.output ? (0, node_path.isAbsolute)(options.output) ? options.output : (0, node_path.join)(this.projectRoot, options.output) : (0, node_path.join)(this.dataDirPath, "db.ts");
1225
- }
1226
- /**
1227
- * Generate types file from JSONL files and their optional schema files.
1228
- */
1229
- async generate() {
1230
- const tables = await this.findTables();
1231
- if (tables.length === 0) throw new Error(`No JSONL files found in ${this.dataDirPath}. Place one or more *.jsonl files in the directory.`);
1232
- const content = this.generateTypeDeclarations(tables);
1233
- await (0, node_fs_promises.mkdir)((0, node_path.dirname)(this.outputFile), { recursive: true });
1234
- await (0, node_fs_promises.writeFile)(this.outputFile, content, "utf-8");
1235
- console.log(`Generated types at ${this.outputFile}`);
1236
- return this.outputFile;
1237
- }
1238
- /**
1239
- * Find all *.jsonl files and check if they have corresponding *.schema.ts files
1240
- */
1241
- async findTables() {
1242
- try {
1243
- const entries = await (0, node_fs_promises.readdir)(this.dataDirPath, { withFileTypes: true });
1244
- const tables = [];
1245
- for (const entry of entries) if (entry.isFile() && entry.name.endsWith(".jsonl")) {
1246
- const tableName = (0, node_path.basename)(entry.name, ".jsonl");
1247
- const schemaFilePath = findSchemaFileInEntries(this.dataDirPath, tableName, entries);
1248
- tables.push({
1249
- tableName,
1250
- schemaFile: schemaFilePath
1251
- });
1252
- }
1253
- return tables;
1254
- } catch (error) {
1255
- if (error.code === "ENOENT") throw new Error(`Data directory not found: ${this.dataDirPath}. Set lines-db.dataDir to the correct location.`);
1256
- throw error;
1257
- }
1258
- }
1259
- /**
1260
- * Generate type declaration content
1261
- */
1262
- generateTypeDeclarations(tables) {
1263
- const imports = [];
1264
- const tableEntries = [];
1265
- const usedAliases = /* @__PURE__ */ new Set();
1266
- for (const table of tables) {
1267
- const tableKey = this.formatTableKey(table.tableName);
1268
- if (table.schemaFile) {
1269
- const schemaIdentifier = this.createSchemaIdentifier(table.tableName, usedAliases);
1270
- usedAliases.add(schemaIdentifier);
1271
- let relativePath = rewriteExtensionForImport((0, node_path.relative)((0, node_path.join)(this.outputFile, ".."), table.schemaFile).replace(/\\/g, "/"));
1272
- if (!relativePath.startsWith(".")) relativePath = "./" + relativePath;
1273
- imports.push(`import { schema as ${schemaIdentifier} } from '${relativePath}';`);
1274
- tableEntries.push(` ${tableKey}: InferOutput<typeof ${schemaIdentifier}>;`);
1275
- } else tableEntries.push(` ${tableKey}: Record<string, unknown>;`);
1276
- }
1277
- return `// Auto-generated by lines-db
1278
- // Do not edit this file manually
1279
-
1280
- ${imports.length > 0 ? `${imports.join("\n")}\n` : ""}import type { DatabaseConfig${imports.length > 0 ? ", InferOutput" : ""} } from '@toiroakr/lines-db';
1281
- import { fileURLToPath } from 'node:url';
1282
- import { dirname } from 'node:path';
1283
-
1284
- const __filename = fileURLToPath(import.meta.url);
1285
- const __dirname = dirname(__filename);
1286
-
1287
- export type Tables = {
1288
- ${tableEntries.join("\n")}
1289
- };
1290
-
1291
- export const config: DatabaseConfig<Tables> = {
1292
- dataDir: __dirname,
1293
- };
1294
- `;
1295
- }
1296
- createSchemaIdentifier(tableName, usedAliases) {
1297
- let base = sanitizeIdentifier(toCamelCase(tableName)) || "table";
1298
- if (!/^[A-Za-z_$]/.test(base)) base = `_${base}`;
1299
- let candidate = `${base}Schema`;
1300
- let suffix = 1;
1301
- while (usedAliases.has(candidate)) candidate = `${base}${++suffix}Schema`;
1302
- return candidate;
1303
- }
1304
- formatTableKey(tableName) {
1305
- if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(tableName)) return tableName;
1306
- return `'${tableName.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
1307
- }
1308
- };
1309
- function toCamelCase(value) {
1310
- const parts = value.split(/[^A-Za-z0-9]+/).filter(Boolean).map((part) => part.toLowerCase());
1311
- if (parts.length === 0) return value;
1312
- const [first, ...rest] = parts;
1313
- return first + rest.map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join("");
1314
- }
1315
- function sanitizeIdentifier(value) {
1316
- return value.replace(/[^A-Za-z0-9_$]/g, "");
1317
- }
1318
- //#endregion
1319
- //#region src/jsonl-migration.ts
1320
- /**
1321
- * Validate a table by temporarily supplying in-memory rows while reusing the existing LinesDB validation pipeline.
1322
- * If validation fails, throws an error with validation details.
1323
- */
1324
- async function ensureTableRowsValid(options) {
1325
- const tablePath = (0, node_path.join)(options.dataDir, `${options.tableName}.jsonl`);
1326
- const overrides = new Map([[tablePath, options.rows]]);
1327
- await JsonlReader.withOverrides(overrides, async () => {
1328
- const db = LinesDB.create({ dataDir: options.dataDir });
1329
- try {
1330
- const result = await db.initialize({ tableName: options.tableName });
1331
- if (!result.valid) {
1332
- const errorCount = result.errors.length;
1333
- const errorDetails = result.errors.map((e) => {
1334
- const issueMessages = e.issues.map((issue) => issue.message).join(", ");
1335
- return ` Row ${e.rowIndex}: ${issueMessages}`;
1336
- }).join("\n");
1337
- throw new Error(`Validation failed for table '${options.tableName}' (${errorCount} error(s)):\n${errorDetails}`);
1338
- }
1339
- } finally {
1340
- await db.close();
1341
- }
1342
- });
1343
- }
1344
- //#endregion
1345
- //#region src/error-formatter.ts
1346
- var ErrorFormatter = class {
1347
- verbose;
1348
- constructor(options = {}) {
1349
- this.verbose = options.verbose ?? false;
1350
- }
1351
- /**
1352
- * Format validation errors
1353
- */
1354
- formatValidationErrors(errors) {
1355
- if (this.verbose) return this.formatValidationErrorsVerbose(errors);
1356
- return this.formatValidationErrorsCompact(errors);
1357
- }
1358
- /**
1359
- * Format foreign key error
1360
- */
1361
- formatForeignKeyError(error) {
1362
- if (this.verbose) return this.formatForeignKeyErrorVerbose(error);
1363
- return this.formatForeignKeyErrorCompact(error);
1364
- }
1365
- /**
1366
- * Format compact (default) validation errors
1367
- */
1368
- formatValidationErrorsCompact(errors) {
1369
- const lines = [];
1370
- for (const error of errors) for (const issue of error.issues) {
1371
- const fieldPath = this.getFieldPath(issue);
1372
- const line = `${error.file}:${error.rowIndex + 1} • ${fieldPath}: ${issue.message}`;
1373
- lines.push(line);
1374
- }
1375
- return lines.join("\n");
1376
- }
1377
- /**
1378
- * Format verbose validation errors
1379
- */
1380
- formatValidationErrorsVerbose(errors) {
1381
- const blocks = [];
1382
- for (let i = 0; i < errors.length; i++) {
1383
- const error = errors[i];
1384
- const isLast = i === errors.length - 1;
1385
- const prefix = isLast ? "└─" : "├─";
1386
- const linePrefix = isLast ? " " : "│ ";
1387
- const lines = [`${prefix} ${error.file}:${error.rowIndex + 1}`];
1388
- for (const issue of error.issues) {
1389
- const fieldPath = this.getFieldPath(issue);
1390
- lines.push(`${linePrefix}Field: ${fieldPath}`);
1391
- lines.push(`${linePrefix}Error: ${issue.message}`);
1392
- }
1393
- if (error.originalData !== void 0) lines.push(`${linePrefix}Original data: ${JSON.stringify(error.originalData)}`);
1394
- if (error.data !== void 0) {
1395
- const label = error.originalData !== void 0 ? "Transformed data" : "Data";
1396
- lines.push(`${linePrefix}${label}: ${JSON.stringify(error.data)}`);
1397
- }
1398
- blocks.push(lines.join("\n"));
1399
- }
1400
- return blocks.join("\n│\n");
1401
- }
1402
- /**
1403
- * Format compact foreign key error
1404
- */
1405
- formatForeignKeyErrorCompact(error) {
1406
- return `${error.file}:${error.rowIndex + 1} • ${error.column}: Foreign key constraint failed - Referenced value ${JSON.stringify(error.value)} does not exist in ${error.referencedTable}(${error.referencedColumn})`;
1407
- }
1408
- /**
1409
- * Format verbose foreign key error
1410
- */
1411
- formatForeignKeyErrorVerbose(error) {
1412
- const lines = [
1413
- `└─ ${error.file}:${error.rowIndex + 1}`,
1414
- ` Type: Foreign Key Violation`,
1415
- ` Field: ${error.column}`,
1416
- ` Value: ${JSON.stringify(error.value)}`,
1417
- ` References: ${error.referencedTable}(${error.referencedColumn})`,
1418
- ` Error: Referenced value does not exist in target table`
1419
- ];
1420
- if (error.data !== void 0) lines.push(` Data: ${JSON.stringify(error.data)}`);
1421
- return lines.join("\n");
1422
- }
1423
- /**
1424
- * Get field path from issue
1425
- */
1426
- getFieldPath(issue) {
1427
- if (!issue.path || issue.path.length === 0) return "root";
1428
- return issue.path.map((segment) => {
1429
- if (typeof segment === "object" && segment !== null && "key" in segment) return String(segment.key);
1430
- return String(segment);
1431
- }).join(".");
1432
- }
1433
- /**
1434
- * Format error header with count
1435
- */
1436
- formatErrorHeader(count, file) {
1437
- return (0, node_util.styleText)("red", `✗ Found ${count} error(s)${file ? ` in ${file}` : ""}`);
1438
- }
1439
- /**
1440
- * Format migration failure header
1441
- */
1442
- formatMigrationFailureHeader() {
1443
- return (0, node_util.styleText)("red", "\n✗ Migration failed and was rolled back");
1444
- }
1445
- };
1446
- //#endregion
1447
- exports.DirectoryScanner = DirectoryScanner;
1448
- exports.ErrorFormatter = ErrorFormatter;
1449
- exports.JsonlReader = JsonlReader;
1450
- exports.JsonlWriter = JsonlWriter;
1451
- exports.LinesDB = LinesDB;
1452
- exports.RUNTIME = RUNTIME;
1453
- exports.SCHEMA_EXTENSIONS = SCHEMA_EXTENSIONS;
1454
- exports.SchemaLoader = SchemaLoader;
1455
- exports.TypeGenerator = TypeGenerator;
1456
- exports.defineSchema = defineSchema;
1457
- exports.detectRuntime = detectRuntime;
1458
- exports.ensureTableRowsValid = ensureTableRowsValid;
1459
- exports.extractTableNameFromSchemaFile = extractTableNameFromSchemaFile;
1460
- exports.findSchemaFile = findSchemaFile;
1461
- exports.hasBackward = hasBackward;
1462
- exports.isSchemaFile = isSchemaFile;
1463
- exports.rewriteExtensionForImport = rewriteExtensionForImport;