@toiroakr/lines-db 0.9.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs DELETED
@@ -1,1499 +0,0 @@
1
- //#region rolldown:runtime
2
- var __create = Object.create;
3
- var __defProp = Object.defineProperty;
4
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
- var __hasOwnProp = Object.prototype.hasOwnProperty;
8
- var __copyProps = (to, from, except, desc) => {
9
- if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
10
- key = keys[i];
11
- if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
12
- get: ((k) => from[k]).bind(null, key),
13
- enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
14
- });
15
- }
16
- return to;
17
- };
18
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
19
- value: mod,
20
- enumerable: true
21
- }) : target, mod));
22
-
23
- //#endregion
24
- let node_fs_promises = require("node:fs/promises");
25
- node_fs_promises = __toESM(node_fs_promises);
26
- let node_path = require("node:path");
27
- node_path = __toESM(node_path);
28
- let node_url = require("node:url");
29
- node_url = __toESM(node_url);
30
- let node_util = require("node:util");
31
- node_util = __toESM(node_util);
32
-
33
- //#region src/runtime.ts
34
- function detectRuntime() {
35
- if (typeof process !== "undefined" && process.versions && process.versions.node) return "node";
36
- return "unknown";
37
- }
38
- const RUNTIME = detectRuntime();
39
-
40
- //#endregion
41
- //#region src/sqlite-adapter.ts
42
- /**
43
- * Create a SQLite database instance for Node.js
44
- */
45
- function createDatabase(path = ":memory:") {
46
- if (RUNTIME === "node") return createNodeDatabase(path);
47
- else throw new Error(`Unsupported runtime: ${RUNTIME}`);
48
- }
49
- /**
50
- * Create a Node.js SQLite database
51
- */
52
- function createNodeDatabase(path) {
53
- const { DatabaseSync } = require("node:sqlite");
54
- const db = new DatabaseSync(path);
55
- db.exec("PRAGMA foreign_keys = ON");
56
- return {
57
- prepare(sql) {
58
- const stmt = db.prepare(sql);
59
- return {
60
- run(...params) {
61
- return stmt.run(...params);
62
- },
63
- get(...params) {
64
- return stmt.get(...params);
65
- },
66
- all(...params) {
67
- return stmt.all(...params);
68
- }
69
- };
70
- },
71
- exec(sql) {
72
- db.exec(sql);
73
- },
74
- close() {
75
- db.close();
76
- }
77
- };
78
- }
79
-
80
- //#endregion
81
- //#region src/jsonl-reader.ts
82
- var JsonlReader = class {
83
- static overrides = null;
84
- /**
85
- * Temporarily override the data returned for specific JSONL files.
86
- * Useful for scenarios like migration validation where in-memory data should be used.
87
- */
88
- static async withOverrides(overrides, fn) {
89
- const normalized = /* @__PURE__ */ new Map();
90
- for (const [filePath, rows] of overrides) normalized.set((0, node_path.normalize)(filePath), rows);
91
- const previousOverrides = this.overrides;
92
- this.overrides = normalized;
93
- try {
94
- return await fn();
95
- } finally {
96
- this.overrides = previousOverrides;
97
- }
98
- }
99
- /**
100
- * Read JSONL file and parse each line as JSON
101
- */
102
- static async read(filePath) {
103
- const overrideRows = this.overrides?.get((0, node_path.normalize)(filePath));
104
- if (overrideRows) return overrideRows.map((row) => JSON.parse(JSON.stringify(row)));
105
- return (await (0, node_fs_promises.readFile)(filePath, "utf-8")).trim().split("\n").filter((line) => line.trim().length > 0).map((line) => {
106
- try {
107
- return JSON.parse(line);
108
- } catch (error) {
109
- throw new Error(`Failed to parse JSON line: ${line}`, { cause: error });
110
- }
111
- });
112
- }
113
- /**
114
- * Infer schema from JSONL data
115
- */
116
- static inferSchema(tableName, data) {
117
- if (data.length === 0) throw new Error("Cannot infer schema from empty data");
118
- const columnTypes = /* @__PURE__ */ new Map();
119
- const booleanColumns = /* @__PURE__ */ new Set();
120
- const nonBooleanColumns = /* @__PURE__ */ new Set();
121
- for (const row of data) for (const [key, value] of Object.entries(row)) {
122
- if (!columnTypes.has(key)) columnTypes.set(key, /* @__PURE__ */ new Set());
123
- columnTypes.get(key).add(this.inferType(value));
124
- if (typeof value === "boolean") booleanColumns.add(key);
125
- else if (value !== null) nonBooleanColumns.add(key);
126
- }
127
- const columns = [];
128
- for (const [columnName, types] of columnTypes.entries()) {
129
- const typeArray = Array.from(types);
130
- let sqlType = "TEXT";
131
- if (typeArray.length === 1) sqlType = typeArray[0];
132
- else if (typeArray.every((t) => t === "INTEGER" || t === "REAL")) sqlType = "REAL";
133
- else if (!typeArray.includes("NULL")) sqlType = "TEXT";
134
- else if (typeArray.length === 2 && typeArray.includes("NULL")) sqlType = typeArray.find((t) => t !== "NULL");
135
- const isBooleanColumn = booleanColumns.has(columnName) && !nonBooleanColumns.has(columnName);
136
- columns.push({
137
- name: columnName,
138
- type: sqlType,
139
- notNull: !typeArray.includes("NULL"),
140
- valueType: isBooleanColumn ? "boolean" : void 0
141
- });
142
- }
143
- const idColumn = columns.find((col) => col.name === "id");
144
- if (idColumn) idColumn.primaryKey = true;
145
- return {
146
- name: tableName,
147
- columns
148
- };
149
- }
150
- static inferType(value) {
151
- if (value === null || value === void 0) return "NULL";
152
- if (typeof value === "number") return Number.isInteger(value) ? "INTEGER" : "REAL";
153
- if (typeof value === "string") return "TEXT";
154
- if (typeof value === "boolean") return "INTEGER";
155
- if (typeof value === "object") return "JSON";
156
- return "TEXT";
157
- }
158
- };
159
-
160
- //#endregion
161
- //#region src/jsonl-writer.ts
162
- var JsonlWriter = class {
163
- /**
164
- * Write data to JSONL file
165
- */
166
- static async write(filePath, data) {
167
- await (0, node_fs_promises.writeFile)(filePath, data.map((obj) => JSON.stringify(obj)).join("\n") + "\n", "utf-8");
168
- }
169
- /**
170
- * Append data to JSONL file
171
- */
172
- static async append(filePath, data) {
173
- const { readFile: readFile$1, writeFile: writeFile$2 } = await import("node:fs/promises");
174
- try {
175
- const existing = await readFile$1(filePath, "utf-8");
176
- const lines = data.map((obj) => JSON.stringify(obj)).join("\n");
177
- await writeFile$2(filePath, existing.trim() + "\n" + lines + "\n", "utf-8");
178
- } catch (error) {
179
- if (error.code === "ENOENT") await this.write(filePath, data);
180
- else throw error;
181
- }
182
- }
183
- };
184
-
185
- //#endregion
186
- //#region src/schema-extensions.ts
187
- /**
188
- * Supported schema file extensions, in priority order.
189
- * The first match wins when discovering schema files.
190
- */
191
- const SCHEMA_EXTENSIONS = [
192
- ".schema.ts",
193
- ".schema.mts",
194
- ".schema.cts"
195
- ];
196
- /**
197
- * Map from schema extensions to their JavaScript import counterparts.
198
- */
199
- const SCHEMA_TO_JS_IMPORT_MAP = {
200
- ".schema.ts": ".schema.js",
201
- ".schema.mts": ".schema.mjs",
202
- ".schema.cts": ".schema.cjs"
203
- };
204
- /**
205
- * Try each supported schema extension and return the full path of the first
206
- * one that exists on disk. Returns undefined if none is found.
207
- */
208
- async function findSchemaFile(dir, tableName) {
209
- for (const ext of SCHEMA_EXTENSIONS) {
210
- const candidate = (0, node_path.join)(dir, `${tableName}${ext}`);
211
- try {
212
- await (0, node_fs_promises.access)(candidate);
213
- return candidate;
214
- } catch {}
215
- }
216
- }
217
- /**
218
- * Synchronously find a schema file among directory entries.
219
- * Returns the full path of the first match, or undefined.
220
- */
221
- function findSchemaFileInEntries(dataDirPath, tableName, entries) {
222
- for (const ext of SCHEMA_EXTENSIONS) {
223
- const candidateName = `${tableName}${ext}`;
224
- if (entries.some((e) => e.isFile() && e.name === candidateName)) return (0, node_path.join)(dataDirPath, candidateName);
225
- }
226
- }
227
- /**
228
- * Check if a filename matches any supported schema file pattern.
229
- */
230
- function isSchemaFile(fileName) {
231
- return SCHEMA_EXTENSIONS.some((ext) => fileName.endsWith(ext));
232
- }
233
- /**
234
- * Extract table name from a schema filename.
235
- * e.g., "users.schema.ts" -> "users", "users.schema.mts" -> "users"
236
- */
237
- function extractTableNameFromSchemaFile(fileName) {
238
- for (const ext of SCHEMA_EXTENSIONS) if (fileName.endsWith(ext)) return fileName.slice(0, -ext.length);
239
- return null;
240
- }
241
- /**
242
- * Rewrite a TypeScript path to its JavaScript counterpart for ESM imports.
243
- * ".schema.ts" -> ".schema.js", ".schema.mts" -> ".schema.mjs", ".schema.cts" -> ".schema.cjs"
244
- */
245
- function rewriteExtensionForImport(filePath) {
246
- for (const [tsExt, jsExt] of Object.entries(SCHEMA_TO_JS_IMPORT_MAP)) if (filePath.endsWith(tsExt)) return filePath.slice(0, -tsExt.length) + jsExt;
247
- return filePath;
248
- }
249
-
250
- //#endregion
251
- //#region src/schema-loader.ts
252
- var SchemaLoader = class {
253
- /**
254
- * Check if a schema file exists for a table
255
- */
256
- static async hasSchema(jsonlPath) {
257
- return await findSchemaFile((0, node_path.dirname)(jsonlPath), (0, node_path.basename)(jsonlPath, ".jsonl")) !== void 0;
258
- }
259
- /**
260
- * Load a validation schema file for a table
261
- * Requires ${tableName}.schema.{ts,mts,cts} to exist alongside the JSONL file
262
- */
263
- static async loadSchema(jsonlPath) {
264
- const dir = (0, node_path.dirname)(jsonlPath);
265
- const tableName = (0, node_path.basename)(jsonlPath, ".jsonl");
266
- const schemaPath = await findSchemaFile(dir, tableName);
267
- if (!schemaPath) throw new Error(`Schema file not found for table '${tableName}'. Expected one of: ${SCHEMA_EXTENSIONS.map((ext) => `${tableName}${ext}`).join(", ")}`);
268
- try {
269
- const module$1 = await import(`${(0, node_url.pathToFileURL)(schemaPath).href}?t=${Date.now()}`);
270
- const schema = module$1.default || module$1.schema;
271
- if (schema && this.isStandardSchema(schema)) return schema;
272
- throw new Error(`Schema file ${schemaPath} does not export a valid StandardSchema`);
273
- } catch (error) {
274
- throw new Error(`Failed to load schema for table '${tableName}' from ${schemaPath}: ${error instanceof Error ? error.message : String(error)}`, { cause: error instanceof Error ? error : void 0 });
275
- }
276
- }
277
- /**
278
- * Check if an object implements the StandardSchema interface
279
- */
280
- static isStandardSchema(obj) {
281
- if (!obj || typeof obj !== "object") return false;
282
- const standard = obj["~standard"];
283
- if (!standard || typeof standard !== "object") return false;
284
- const standardObj = standard;
285
- return standardObj.version === 1 && typeof standardObj.vendor === "string" && typeof standardObj.validate === "function";
286
- }
287
- };
288
-
289
- //#endregion
290
- //#region src/directory-scanner.ts
291
- var DirectoryScanner = class {
292
- /**
293
- * Scan directory for JSONL files and create table configurations
294
- */
295
- static async scanDirectory(dataDir) {
296
- const tables = /* @__PURE__ */ new Map();
297
- try {
298
- const files = await (0, node_fs_promises.readdir)(dataDir);
299
- for (const file of files) if ((0, node_path.extname)(file) === ".jsonl") {
300
- const tableName = (0, node_path.basename)(file, ".jsonl");
301
- const jsonlPath = (0, node_path.join)(dataDir, file);
302
- tables.set(tableName, {
303
- jsonlPath,
304
- autoInferSchema: true
305
- });
306
- }
307
- if (tables.size === 0) console.warn(`Warning: No JSONL files found in directory: ${dataDir}`);
308
- return tables;
309
- } catch (error) {
310
- throw new Error(`Failed to scan directory ${dataDir}: ${error instanceof Error ? error.message : String(error)}`);
311
- }
312
- }
313
- };
314
-
315
- //#endregion
316
- //#region src/schema.ts
317
- /**
318
- * Define a bidirectional schema with optional backward transformation
319
- *
320
- * @param schema - Standard Schema for validation
321
- * @param options - SchemaOptions object. When Input and Output types differ, backward transformation is required
322
- *
323
- * @example
324
- * // No transformation - backward not needed
325
- * const schema = defineSchema(
326
- * v.object({ id: v.number(), name: v.string() })
327
- * );
328
- *
329
- * @example
330
- * // With transformation - backward REQUIRED
331
- * const schema = defineSchema(
332
- * v.pipe(v.string(), v.transform(Number)),
333
- * {
334
- * backward: (num) => String(num) // backward: number → string (REQUIRED)
335
- * }
336
- * );
337
- *
338
- * @example
339
- * // With primary key and foreign key
340
- * const schema = defineSchema(
341
- * v.object({ id: v.number(), customerId: v.number() }),
342
- * {
343
- * primaryKey: 'id',
344
- * foreignKeys: [
345
- * { column: 'customerId', references: { table: 'users', column: 'id' } }
346
- * ]
347
- * }
348
- * );
349
- */
350
- function defineSchema(schema, ...args) {
351
- const options = args[0];
352
- const bidirectionalSchema = Object.create(schema);
353
- if (options) {
354
- if (options.backward) bidirectionalSchema.backward = options.backward;
355
- if (options.primaryKey) bidirectionalSchema.primaryKey = options.primaryKey;
356
- if (options.foreignKeys) bidirectionalSchema.foreignKeys = options.foreignKeys;
357
- if (options.indexes) bidirectionalSchema.indexes = options.indexes;
358
- }
359
- Object.defineProperty(bidirectionalSchema, "~standard", {
360
- value: schema["~standard"],
361
- enumerable: true,
362
- configurable: true
363
- });
364
- return bidirectionalSchema;
365
- }
366
- /**
367
- * Check if a schema has backward transformation
368
- */
369
- function hasBackward(schema) {
370
- return "backward" in schema && typeof schema.backward === "function";
371
- }
372
-
373
- //#endregion
374
- //#region src/database.ts
375
- var LinesDB = class LinesDB {
376
- db;
377
- config;
378
- schemas = /* @__PURE__ */ new Map();
379
- validationSchemas = /* @__PURE__ */ new Map();
380
- tables = /* @__PURE__ */ new Map();
381
- inTransaction = false;
382
- constructor(config, dbPath) {
383
- this.config = config;
384
- this.db = createDatabase(dbPath ?? ":memory:");
385
- }
386
- static create(config, dbPath) {
387
- return new LinesDB(config, dbPath);
388
- }
389
- /**
390
- * Initialize database by loading all JSONL files or a specific table
391
- * Uses dependency resolution to ensure foreign key references are loaded in correct order
392
- * @param options Optional configuration for initialization
393
- * @param options.tableName Optional table name to initialize. If not provided, initializes all tables
394
- * @param options.detailedValidate If true, performs detailed validation by inserting rows one by one to catch constraint violations
395
- * @param options.transform Optional transform function to apply to rows before validation (only applied to the specified tableName)
396
- * @returns ValidationResult containing validation status, errors, and warnings
397
- */
398
- async initialize(options) {
399
- const allErrors = [];
400
- const allWarnings = [];
401
- const allRowCounts = /* @__PURE__ */ new Map();
402
- const tableName = options?.tableName;
403
- const detailedValidate = options?.detailedValidate ?? false;
404
- const transform = options?.transform;
405
- this.tables = await DirectoryScanner.scanDirectory(this.config.dataDir);
406
- const tablesToLoad = tableName ? [tableName] : Array.from(this.tables.keys());
407
- for (const tableNameToLoad of tablesToLoad) if (!this.tables.has(tableNameToLoad)) throw new Error(`Table '${tableNameToLoad}' not found in directory '${this.config.dataDir}'`);
408
- const loadedTables = /* @__PURE__ */ new Set();
409
- const loadingTables = /* @__PURE__ */ new Set();
410
- const attemptedTables = /* @__PURE__ */ new Set();
411
- const allDeferredForeignKeys = [];
412
- for (const tableNameToLoad of tablesToLoad) if (!attemptedTables.has(tableNameToLoad)) {
413
- const tableTransform = tableNameToLoad === tableName ? transform : void 0;
414
- const { errors, warnings, rowCounts: tableRowCounts, deferredForeignKeys } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
415
- allErrors.push(...errors);
416
- allWarnings.push(...warnings);
417
- allDeferredForeignKeys.push(...deferredForeignKeys);
418
- for (const [k, v] of tableRowCounts) allRowCounts.set(k, v);
419
- }
420
- if (detailedValidate && allDeferredForeignKeys.length > 0) for (const { tableName: tName, foreignKey: fk, filePath } of allDeferredForeignKeys) {
421
- if (!loadedTables.has(fk.references.table)) continue;
422
- const deferredErrors = this.validateDeferredForeignKey(tName, fk, filePath);
423
- allErrors.push(...deferredErrors);
424
- }
425
- const tableResults = tablesToLoad.map((name) => {
426
- const tableErrors = allErrors.filter((e) => e.tableName === name);
427
- const tableWarnings = allWarnings.filter((w) => w.includes(`'${name}'`));
428
- return {
429
- tableName: name,
430
- valid: tableErrors.length === 0,
431
- rowCount: allRowCounts.get(name) ?? 0,
432
- errors: tableErrors,
433
- warnings: tableWarnings
434
- };
435
- });
436
- return {
437
- valid: allErrors.length === 0,
438
- errors: allErrors,
439
- warnings: allWarnings,
440
- tableResults
441
- };
442
- }
443
- /**
444
- * Load a table and its dependencies recursively
445
- */
446
- async loadTableWithDependencies(tableName, loadedTables, loadingTables, attemptedTables, detailedValidate, transform) {
447
- const errors = [];
448
- const warnings = [];
449
- const rowCounts = /* @__PURE__ */ new Map();
450
- const deferredForeignKeys = [];
451
- if (attemptedTables.has(tableName)) return {
452
- errors,
453
- warnings,
454
- rowCounts,
455
- deferredForeignKeys
456
- };
457
- attemptedTables.add(tableName);
458
- if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
459
- const tableConfig = this.tables.get(tableName);
460
- if (!tableConfig) throw new Error(`Table configuration not found for '${tableName}'`);
461
- loadingTables.add(tableName);
462
- try {
463
- let foreignKeys;
464
- try {
465
- const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
466
- const schemaPath = await findSchemaFile((0, node_path.dirname)(tableConfig.jsonlPath), (0, node_path.basename)(tableConfig.jsonlPath, ".jsonl"));
467
- if (schemaPath) {
468
- const schemaModule = await import(`${pathToFileURL$1(schemaPath).href}?t=${Date.now()}`);
469
- foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
470
- }
471
- } catch {}
472
- if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
473
- const referencedTable = fk.references.table;
474
- if (referencedTable === tableName) continue;
475
- if (!attemptedTables.has(referencedTable)) if (this.tables.has(referencedTable)) {
476
- const depResult = await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables, attemptedTables, detailedValidate, void 0);
477
- errors.push(...depResult.errors);
478
- warnings.push(...depResult.warnings);
479
- deferredForeignKeys.push(...depResult.deferredForeignKeys);
480
- for (const [k, v] of depResult.rowCounts) rowCounts.set(k, v);
481
- } else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
482
- }
483
- const failedDependencies = /* @__PURE__ */ new Set();
484
- const circularDependencies = /* @__PURE__ */ new Set();
485
- if (foreignKeys && foreignKeys.length > 0) {
486
- for (const fk of foreignKeys) {
487
- const referencedTable = fk.references.table;
488
- if (referencedTable === tableName) continue;
489
- if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable)) if (loadingTables.has(referencedTable)) circularDependencies.add(referencedTable);
490
- else failedDependencies.add(referencedTable);
491
- }
492
- if (failedDependencies.size > 0) for (const dep of failedDependencies) warnings.push(`Skipping foreign key validation for table '${tableName}': referenced table '${dep}' has validation errors`);
493
- }
494
- const allSkippedDependencies = new Set([...failedDependencies, ...circularDependencies]);
495
- const { loaded, rowCount, errors: loadErrors } = await this.loadTable(tableName, tableConfig, detailedValidate, transform, allSkippedDependencies);
496
- errors.push(...loadErrors);
497
- rowCounts.set(tableName, rowCount);
498
- if (loaded) {
499
- loadedTables.add(tableName);
500
- if (foreignKeys && circularDependencies.size > 0) {
501
- for (const fk of foreignKeys) if (circularDependencies.has(fk.references.table)) deferredForeignKeys.push({
502
- tableName,
503
- foreignKey: fk,
504
- filePath: tableConfig.jsonlPath
505
- });
506
- }
507
- } else {
508
- warnings.push(`Table '${tableName}' was not loaded (no data or skipped)`);
509
- this.tables.delete(tableName);
510
- }
511
- } finally {
512
- loadingTables.delete(tableName);
513
- }
514
- return {
515
- errors,
516
- warnings,
517
- rowCounts,
518
- deferredForeignKeys
519
- };
520
- }
521
- /**
522
- * Load a single table from JSONL file
523
- * @returns Object with loaded status and validation errors
524
- */
525
- async loadTable(tableName, config, detailedValidate, transform, failedDependencies) {
526
- let data = await JsonlReader.read(config.jsonlPath);
527
- if (transform) data = data.map((row) => transform(row));
528
- let validationSchema = config.validationSchema;
529
- const schemaMetadata = {};
530
- if (!validationSchema) try {
531
- validationSchema = await SchemaLoader.loadSchema(config.jsonlPath);
532
- } catch (_error) {}
533
- if (!config.validationSchema) try {
534
- const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
535
- const schemaPath = await findSchemaFile((0, node_path.dirname)(config.jsonlPath), (0, node_path.basename)(config.jsonlPath, ".jsonl"));
536
- if (!schemaPath) throw new Error("Schema file not found");
537
- const schemaModule = await import(`${pathToFileURL$1(schemaPath).href}?t=${Date.now()}`);
538
- const schemaExport = schemaModule.schema || schemaModule.default;
539
- if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
540
- else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
541
- if (schemaExport?.foreignKeys) schemaMetadata.foreignKeys = schemaExport.foreignKeys;
542
- else if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
543
- if (schemaExport?.indexes) schemaMetadata.indexes = schemaExport.indexes;
544
- else if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
545
- if (process.env.DEBUG_LINES_DB) {
546
- console.log(`[lines-db] Schema metadata for ${tableName}:`);
547
- console.log(` primaryKey: ${schemaMetadata.primaryKey}`);
548
- console.log(` foreignKeys: ${JSON.stringify(schemaMetadata.foreignKeys)}`);
549
- console.log(` indexes: ${JSON.stringify(schemaMetadata.indexes)}`);
550
- }
551
- } catch (_error) {
552
- if (process.env.DEBUG_LINES_DB) console.warn(`[lines-db] Failed to load schema metadata for ${tableName}:`, _error instanceof Error ? _error.message : String(_error));
553
- }
554
- this.validationSchemas.set(tableName, validationSchema);
555
- const validationErrors = [];
556
- const validatedData = [];
557
- for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
558
- const row = data[rowIndex];
559
- try {
560
- const validatedRow = this.validateAndTransform(tableName, row);
561
- validatedData.push(validatedRow);
562
- } catch (error) {
563
- if (error instanceof Error && error.name === "ValidationError") validationErrors.push({
564
- rowIndex,
565
- rowData: row,
566
- error
567
- });
568
- else throw error;
569
- }
570
- }
571
- const validationErrorDetails = validationErrors.map((ve) => ({
572
- file: config.jsonlPath,
573
- tableName,
574
- rowIndex: ve.rowIndex,
575
- issues: ve.error.issues,
576
- type: "schema"
577
- }));
578
- if (validationErrors.length > 0) return {
579
- loaded: false,
580
- rowCount: data.length,
581
- errors: validationErrorDetails
582
- };
583
- let schema;
584
- let inferredSchema;
585
- if (validatedData.length > 0) inferredSchema = JsonlReader.inferSchema(tableName, validatedData);
586
- if (config.schema) {
587
- schema = config.schema;
588
- if (inferredSchema) for (const inferredCol of inferredSchema.columns) {
589
- const schemaCol = schema.columns.find((c) => c.name === inferredCol.name);
590
- if (schemaCol && inferredCol.valueType && !schemaCol.valueType) schemaCol.valueType = inferredCol.valueType;
591
- }
592
- } else if (config.autoInferSchema !== false) {
593
- if (validatedData.length === 0) return {
594
- loaded: false,
595
- rowCount: 0,
596
- errors: []
597
- };
598
- schema = inferredSchema;
599
- } else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
600
- const biSchema = validationSchema;
601
- const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
602
- const foreignKeys = biSchema?.foreignKeys || schemaMetadata.foreignKeys;
603
- const indexes = biSchema?.indexes || schemaMetadata.indexes;
604
- if (primaryKey && !schema.columns.some((col) => col.primaryKey)) {
605
- const col = schema.columns.find((c) => c.name === primaryKey);
606
- if (col) col.primaryKey = true;
607
- } else if (!primaryKey && !schema.columns.some((col) => col.primaryKey)) {
608
- const idColumn = schema.columns.find((c) => c.name === "id");
609
- if (idColumn) idColumn.primaryKey = true;
610
- }
611
- if (foreignKeys) schema.foreignKeys = failedDependencies && failedDependencies.size > 0 ? foreignKeys.filter((fk) => !failedDependencies.has(fk.references.table)) : foreignKeys;
612
- if (indexes) {
613
- schema.indexes = indexes;
614
- for (const index of indexes) if (index.unique && index.columns.length === 1) {
615
- const col = schema.columns.find((c) => c.name === index.columns[0]);
616
- if (col && !col.unique && !col.primaryKey) col.unique = true;
617
- }
618
- }
619
- this.schemas.set(tableName, schema);
620
- this.createTable(schema);
621
- if (detailedValidate) {
622
- const insertErrors = this.insertDataWithDetailedValidation(tableName, schema, validatedData, config.jsonlPath);
623
- if (insertErrors.length > 0) return {
624
- loaded: false,
625
- rowCount: data.length,
626
- errors: insertErrors
627
- };
628
- } else this.insertData(tableName, schema, validatedData);
629
- return {
630
- loaded: true,
631
- rowCount: data.length,
632
- errors: []
633
- };
634
- }
635
- /**
636
- * Create table in SQLite with constraints and indexes
637
- */
638
- createTable(schema) {
639
- const quotedTableName = this.quoteTableName(schema.name);
640
- const uniqueColumns = /* @__PURE__ */ new Set();
641
- for (const col of schema.columns) if (col.unique) uniqueColumns.add(col.name);
642
- if (schema.indexes) {
643
- for (const index of schema.indexes) if (index.unique && index.columns.length === 1) uniqueColumns.add(index.columns[0]);
644
- }
645
- const columnDefs = schema.columns.map((col) => {
646
- const sqlType = col.type === "JSON" ? "TEXT" : col.type;
647
- const parts = [this.quoteIdentifier(col.name), sqlType];
648
- if (col.primaryKey) parts.push("PRIMARY KEY");
649
- if (col.notNull) parts.push("NOT NULL");
650
- if (uniqueColumns.has(col.name) && !col.primaryKey) parts.push("UNIQUE");
651
- return parts.join(" ");
652
- });
653
- const foreignKeyDefs = [];
654
- if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) {
655
- const fkParts = [`FOREIGN KEY (${this.quoteIdentifier(fk.column)})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`];
656
- if (fk.onDelete) fkParts.push(`ON DELETE ${fk.onDelete}`);
657
- if (fk.onUpdate) fkParts.push(`ON UPDATE ${fk.onUpdate}`);
658
- foreignKeyDefs.push(fkParts.join(" "));
659
- }
660
- const sql = `CREATE TABLE IF NOT EXISTS ${quotedTableName} (${[...columnDefs, ...foreignKeyDefs].join(", ")})`;
661
- this.db.exec(sql);
662
- if (schema.indexes && schema.indexes.length > 0) for (let i = 0; i < schema.indexes.length; i++) {
663
- const index = schema.indexes[i];
664
- const safeTableName = schema.name.replace(/[^a-zA-Z0-9]/g, "_");
665
- const resolvedIndexName = index.name || `idx_${safeTableName}_${index.columns.join("_")}_${i}`;
666
- const indexSql = `CREATE ${index.unique ? "UNIQUE " : ""}INDEX IF NOT EXISTS ${this.quoteIdentifier(resolvedIndexName)} ON ${quotedTableName} (${index.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`;
667
- this.db.exec(indexSql);
668
- }
669
- }
670
- /**
671
- * Quote table name to handle special characters in SQL
672
- */
673
- quoteTableName(tableName) {
674
- return this.quoteIdentifier(tableName);
675
- }
676
- /**
677
- * Quote identifier for SQL statements, escaping embedded quotes
678
- */
679
- quoteIdentifier(identifier) {
680
- return `"${identifier.replace(/"/g, "\"\"")}"`;
681
- }
682
- /**
683
- * Insert data into table using batch insert (multiple rows per SQL)
684
- * SQLite has a parameter limit (default 999), so we batch rows accordingly
685
- * Throws exception if any constraint violation occurs
686
- */
687
- insertData(tableName, schema, data) {
688
- if (data.length === 0) return;
689
- const columnNames = schema.columns.map((col) => col.name);
690
- const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
691
- const columnCount = columnNames.length;
692
- const maxBatchSize = Math.floor(900 / columnCount);
693
- const batchSize = Math.max(1, Math.min(maxBatchSize, 100));
694
- for (let i = 0; i < data.length; i += batchSize) {
695
- const batch = data.slice(i, i + batchSize);
696
- const rowPlaceholders = columnNames.map(() => "?").join(", ");
697
- const valuesPlaceholders = batch.map(() => `(${rowPlaceholders})`).join(", ");
698
- const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES ${valuesPlaceholders}`;
699
- const values = [];
700
- for (const row of batch) for (const col of columnNames) values.push(this.normalizeValue(row[col]));
701
- this.db.prepare(sql).run(...values);
702
- }
703
- }
704
- /**
705
- * Insert data into table one row at a time with detailed error reporting
706
- * This is used for validation to catch constraint violations
707
- */
708
- insertDataWithDetailedValidation(tableName, schema, data, filePath) {
709
- const errors = [];
710
- const columnNames = schema.columns.map((col) => col.name);
711
- const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
712
- const placeholders = columnNames.map(() => "?").join(", ");
713
- const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
714
- const stmt = this.db.prepare(sql);
715
- for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
716
- const row = data[rowIndex];
717
- try {
718
- const values = columnNames.map((col) => this.normalizeValue(row[col]));
719
- stmt.run(...values);
720
- } catch (error) {
721
- const constraintError = this.analyzeConstraintError(error, filePath, tableName, rowIndex, row, schema.foreignKeys || []);
722
- if (constraintError) errors.push(constraintError);
723
- }
724
- }
725
- return errors;
726
- }
727
- /**
728
- * Analyze constraint error and extract detailed information
729
- */
730
- analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys) {
731
- const errorMessage = error instanceof Error ? error.message : String(error);
732
- if (errorMessage.includes("FOREIGN KEY constraint failed")) for (const fk of foreignKeys) {
733
- const fkValue = row[fk.column];
734
- if (fkValue === null || fkValue === void 0) continue;
735
- try {
736
- const result = this.query(`SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`, [this.normalizeValue(fkValue)]);
737
- if (result.length > 0 && result[0].count === 0) return {
738
- file,
739
- tableName,
740
- rowIndex,
741
- issues: [],
742
- type: "foreignKey",
743
- foreignKeyError: {
744
- column: fk.column,
745
- value: fkValue,
746
- referencedTable: fk.references.table,
747
- referencedColumn: fk.references.column
748
- }
749
- };
750
- } catch (_) {}
751
- }
752
- return {
753
- file,
754
- tableName,
755
- rowIndex,
756
- issues: [{
757
- message: errorMessage,
758
- path: []
759
- }],
760
- type: "schema"
761
- };
762
- }
763
- /**
764
- * Validate a deferred foreign key constraint after all tables have been loaded.
765
- * Used for circular dependency FK validation.
766
- */
767
- validateDeferredForeignKey(tableName, fk, filePath) {
768
- const errors = [];
769
- const quotedTable = this.quoteTableName(tableName);
770
- const quotedColumn = this.quoteIdentifier(fk.column);
771
- const quotedRefTable = this.quoteTableName(fk.references.table);
772
- const sql = `SELECT rowid - 1 as idx, ${quotedColumn} as val FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL AND ${quotedColumn} NOT IN (SELECT ${this.quoteIdentifier(fk.references.column)} FROM ${quotedRefTable})`;
773
- try {
774
- const rows = this.query(sql);
775
- for (const row of rows) errors.push({
776
- file: filePath,
777
- tableName,
778
- rowIndex: row.idx,
779
- issues: [],
780
- type: "foreignKey",
781
- foreignKeyError: {
782
- column: fk.column,
783
- value: row.val,
784
- referencedTable: fk.references.table,
785
- referencedColumn: fk.references.column
786
- }
787
- });
788
- } catch (_) {}
789
- return errors;
790
- }
791
- /**
792
- * Execute a raw SQL query
793
- */
794
- query(sql, params = []) {
795
- return this.db.prepare(sql).all(...params);
796
- }
797
- /**
798
- * Execute a SQL query that returns a single row
799
- */
800
- queryOne(sql, params = []) {
801
- const result = this.db.prepare(sql).get(...params);
802
- return result === void 0 ? null : result;
803
- }
804
- /**
805
- * Execute a SQL statement (INSERT, UPDATE, DELETE)
806
- */
807
- execute(sql, params = []) {
808
- return this.db.prepare(sql).run(...params);
809
- }
810
- /**
811
- * Find rows by condition (supports OR/AND with arrays and function filters)
812
- * If where is not provided, returns all rows
813
- */
814
- find(tableName, where) {
815
- if (where === void 0) return this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
816
- if (Array.isArray(where) && where.length === 0) return [];
817
- const { sql, values, functionFilters, hasOrWithFunctionFilters } = this.buildWhereClause(where);
818
- let rows;
819
- if (hasOrWithFunctionFilters) {
820
- rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
821
- return this.applyOrConditionWithFilters(rows, where);
822
- }
823
- if (sql) rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values).map((row) => this.deserializeRow(tableName, row));
824
- else rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
825
- return this.applyFunctionFilters(rows, functionFilters);
826
- }
827
- /**
828
- * Find a single row by condition (supports OR/AND with arrays and function filters)
829
- */
830
- findOne(tableName, where) {
831
- const { sql, values, functionFilters } = this.buildWhereClause(where);
832
- let rows;
833
- if (sql) rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values).map((row) => this.deserializeRow(tableName, row));
834
- else rows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
835
- const filtered = this.applyFunctionFilters(rows, functionFilters);
836
- return filtered.length > 0 ? filtered[0] : null;
837
- }
838
- /**
839
- * Deserialize JSON columns in a row
840
- */
841
- deserializeRow(tableName, row) {
842
- const schema = this.schemas.get(tableName);
843
- if (!schema) return row;
844
- const deserializedRow = { ...row };
845
- for (const column of schema.columns) {
846
- const colName = column.name;
847
- if (!(colName in deserializedRow)) continue;
848
- const value = deserializedRow[colName];
849
- if (column.type === "JSON" && typeof value === "string") {
850
- try {
851
- deserializedRow[colName] = JSON.parse(value);
852
- } catch (error) {
853
- console.warn(`Failed to parse JSON column ${colName}:`, error);
854
- }
855
- continue;
856
- }
857
- if (column.valueType === "boolean") {
858
- if (typeof value === "number") deserializedRow[colName] = value === 0 ? false : true;
859
- else if (typeof value === "bigint") deserializedRow[colName] = value === 0n ? false : true;
860
- }
861
- }
862
- return deserializedRow;
863
- }
864
- /**
865
- * Validate data using StandardSchema and return the transformed value
866
- * Note: Only synchronous validation is supported
867
- */
868
- validateAndTransform(tableName, data) {
869
- const schema = this.validationSchemas.get(tableName);
870
- if (!schema) return data;
871
- const result = schema["~standard"].validate(data);
872
- if (result instanceof Promise) throw new Error("Asynchronous validation is not supported. Please use synchronous validation schemas.");
873
- if (result.issues && result.issues.length > 0) {
874
- const errorMessage = `Validation failed for table '${tableName}':\n${result.issues.map((issue) => {
875
- let pathStr = "root";
876
- if (issue.path && issue.path.length > 0) pathStr = issue.path.map((segment) => {
877
- if (typeof segment === "object" && segment !== null && "key" in segment) return String(segment.key);
878
- return String(segment);
879
- }).join(".");
880
- return ` - ${pathStr}: ${issue.message}`;
881
- }).join("\n")}`;
882
- const error = new Error(errorMessage);
883
- error.name = "ValidationError";
884
- error.issues = result.issues;
885
- throw error;
886
- }
887
- const transformedValue = "value" in result ? result.value : data;
888
- const normalizedValue = {};
889
- for (const [key, value] of Object.entries(transformedValue)) normalizedValue[key] = value === void 0 ? null : value;
890
- return normalizedValue;
891
- }
892
- /**
893
- * Validate data using StandardSchema (without returning transformed value)
894
- * Note: Only synchronous validation is supported
895
- */
896
- validateData(tableName, data) {
897
- this.validateAndTransform(tableName, data);
898
- }
899
- /**
900
- * Insert a row into a table with validation
901
- */
902
- insert(tableName, data) {
903
- this.validateData(tableName, data);
904
- if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
905
- const columnNames = Object.keys(data);
906
- const quotedColumns = columnNames.map((col) => this.quoteIdentifier(col));
907
- const placeholders = columnNames.map(() => "?").join(", ");
908
- const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
909
- const values = Object.values(data).map((v) => this.normalizeValue(v));
910
- const result = this.execute(sql, values);
911
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
912
- console.error(`Failed to sync table ${tableName}:`, err);
913
- });
914
- return result;
915
- }
916
- /**
917
- * Batch insert rows with validation per record.
918
- */
919
- batchInsert(tableName, records) {
920
- if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
921
- if (records.length === 0) return {
922
- changes: 0,
923
- lastInsertRowid: 0
924
- };
925
- let totalChanges = 0n;
926
- let lastRowid = 0n;
927
- for (const record of records) {
928
- this.validateData(tableName, record);
929
- const columnNames = Object.keys(record);
930
- const quotedColumns = columnNames.map((col) => this.quoteIdentifier(col));
931
- const placeholders = columnNames.map(() => "?").join(", ");
932
- const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
933
- const values = columnNames.map((col) => this.normalizeValue(record[col]));
934
- const result = this.execute(sql, values);
935
- totalChanges += BigInt(result.changes);
936
- lastRowid = BigInt(result.lastInsertRowid);
937
- }
938
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
939
- console.error(`Failed to sync table ${tableName}:`, err);
940
- });
941
- return {
942
- changes: totalChanges,
943
- lastInsertRowid: lastRowid
944
- };
945
- }
946
- /**
947
- * Update rows in a table with validation (supports OR/AND with arrays)
948
- * Note: Function filters are not supported for update operations
949
- * Note: By default, validation is enabled. For partial updates, existing data is fetched
950
- * and merged before validation. Set options.validate = false to disable validation.
951
- */
952
- update(tableName, data, where, options) {
953
- if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
954
- const shouldValidate = options?.validate !== false;
955
- const hasValidationSchema = this.validationSchemas.has(tableName);
956
- if (shouldValidate && hasValidationSchema) {
957
- const existingRows = this.find(tableName, where);
958
- for (const existingRow of existingRows) {
959
- const mergedData = {
960
- ...existingRow,
961
- ...data
962
- };
963
- this.validateData(tableName, mergedData);
964
- }
965
- }
966
- const { sql: whereSql, values: whereValues, functionFilters } = this.buildWhereClause(where);
967
- if (functionFilters.length > 0) throw new Error("Function filters are not supported in update operations");
968
- const setClauses = Object.keys(data).map((key) => `${this.quoteIdentifier(key)} = ?`).join(", ");
969
- const sql = `UPDATE ${this.quoteTableName(tableName)} SET ${setClauses} WHERE ${whereSql}`;
970
- const values = [...Object.values(data).map((v) => this.normalizeValue(v)), ...whereValues];
971
- const result = this.execute(sql, values);
972
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
973
- console.error(`Failed to sync table ${tableName}:`, err);
974
- });
975
- return result;
976
- }
977
- /**
978
- * Batch update rows with record-specific values and validation.
979
- * Each record must include the primary key to identify the target row.
980
- * Validation runs once per merged record unless explicitly disabled.
981
- */
982
- batchUpdate(tableName, records, options) {
983
- const schema = this.schemas.get(tableName);
984
- if (!schema) throw new Error(`Table ${tableName} does not exist`);
985
- if (records.length === 0) return {
986
- changes: 0,
987
- lastInsertRowid: 0
988
- };
989
- const pkColumn = schema.columns.find((col) => col.primaryKey);
990
- if (!pkColumn) throw new Error(`Table ${tableName} does not have a primary key`);
991
- const pkName = pkColumn.name;
992
- const pkValues = [];
993
- for (const record of records) {
994
- const pkValue = record[pkName];
995
- if (pkValue === void 0) throw new Error(`Record is missing primary key '${String(pkName)}': ${JSON.stringify(record)}`);
996
- pkValues.push(pkValue);
997
- }
998
- const shouldValidate = options?.validate !== false;
999
- const hasValidationSchema = this.validationSchemas.has(tableName);
1000
- if (shouldValidate && hasValidationSchema) {
1001
- const orCondition = pkValues.map((pkValue) => ({ [pkName]: pkValue }));
1002
- const existingRows = this.find(tableName, orCondition);
1003
- const existingRowsMap = /* @__PURE__ */ new Map();
1004
- for (const row of existingRows) {
1005
- const pkValue = row[pkName];
1006
- existingRowsMap.set(pkValue, row);
1007
- }
1008
- const validationErrors = [];
1009
- for (let i = 0; i < records.length; i++) {
1010
- const record = records[i];
1011
- const pkValue = record[pkName];
1012
- const existingRow = existingRowsMap.get(pkValue);
1013
- if (!existingRow) throw new Error(`No existing row found with ${String(pkName)}=${JSON.stringify(pkValue)}`);
1014
- const mergedData = {
1015
- ...existingRow,
1016
- ...record
1017
- };
1018
- try {
1019
- this.validateData(tableName, mergedData);
1020
- } catch (error) {
1021
- if (error instanceof Error && error.name === "ValidationError") validationErrors.push({
1022
- rowIndex: i,
1023
- rowData: mergedData,
1024
- pkValue,
1025
- error
1026
- });
1027
- else throw error;
1028
- }
1029
- }
1030
- if (validationErrors.length > 0) {
1031
- const enhancedError = /* @__PURE__ */ new Error(`Validation failed for ${validationErrors.length} row(s)`);
1032
- enhancedError.name = "ValidationError";
1033
- enhancedError.validationErrors = validationErrors;
1034
- enhancedError.issues = validationErrors[0].error.issues;
1035
- throw enhancedError;
1036
- }
1037
- }
1038
- let totalChanges = 0n;
1039
- let lastRowid = 0n;
1040
- for (const record of records) {
1041
- const pkValue = record[pkName];
1042
- const where = { [pkName]: pkValue };
1043
- const result = this.update(tableName, record, where, { validate: false });
1044
- totalChanges += BigInt(result.changes);
1045
- lastRowid = BigInt(result.lastInsertRowid);
1046
- }
1047
- return {
1048
- changes: totalChanges,
1049
- lastInsertRowid: lastRowid
1050
- };
1051
- }
1052
- /**
1053
- * Delete rows from a table (supports OR/AND with arrays)
1054
- * Note: Function filters are not supported for delete operations
1055
- */
1056
- delete(tableName, where) {
1057
- if (!this.schemas.get(tableName)) throw new Error(`Table ${tableName} does not exist`);
1058
- const { sql: whereSql, values, functionFilters } = this.buildWhereClause(where);
1059
- if (functionFilters.length > 0) throw new Error("Function filters are not supported in delete operations");
1060
- const sql = `DELETE FROM ${this.quoteTableName(tableName)} WHERE ${whereSql}`;
1061
- const result = this.execute(sql, values);
1062
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
1063
- console.error(`Failed to sync table ${tableName}:`, err);
1064
- });
1065
- return result;
1066
- }
1067
- /**
1068
- * Batch delete rows by primary key.
1069
- */
1070
- batchDelete(tableName, records) {
1071
- const schema = this.schemas.get(tableName);
1072
- if (!schema) throw new Error(`Table ${tableName} does not exist`);
1073
- if (records.length === 0) return {
1074
- changes: 0,
1075
- lastInsertRowid: 0
1076
- };
1077
- const pkColumn = schema.columns.find((col) => col.primaryKey);
1078
- if (!pkColumn) throw new Error(`Table ${tableName} does not have a primary key`);
1079
- const pkName = pkColumn.name;
1080
- const pkValues = records.map((record, index) => {
1081
- const pkValue = record[pkName];
1082
- if (pkValue === void 0) throw new Error(`Record at index ${index} is missing primary key '${String(pkName)}'`);
1083
- return pkValue;
1084
- });
1085
- const placeholders = pkValues.map(() => "?").join(", ");
1086
- const sql = `DELETE FROM ${this.quoteTableName(tableName)} WHERE ${this.quoteIdentifier(pkName)} IN (${placeholders})`;
1087
- const values = pkValues.map((value) => this.normalizeValue(value));
1088
- const result = this.execute(sql, values);
1089
- if (!this.inTransaction) this.syncTable(tableName).catch((err) => {
1090
- console.error(`Failed to sync table ${tableName}:`, err);
1091
- });
1092
- return {
1093
- changes: BigInt(result.changes),
1094
- lastInsertRowid: BigInt(result.lastInsertRowid)
1095
- };
1096
- }
1097
- /**
1098
- * Normalize value for SQLite
1099
- */
1100
- normalizeValue(value) {
1101
- if (value === null || value === void 0) return null;
1102
- if (typeof value === "boolean") return value ? 1 : 0;
1103
- if (typeof value === "string" || typeof value === "number" || typeof value === "bigint") return value;
1104
- if (value instanceof Uint8Array) return value;
1105
- return JSON.stringify(value);
1106
- }
1107
- /**
1108
- * Build WHERE clause from condition (supports OR/AND with arrays and functions)
1109
- */
1110
- buildWhereClause(condition) {
1111
- const values = [];
1112
- const functionFilters = [];
1113
- let hasOrWithFunctionFilters = false;
1114
- const buildCondition = (cond, isInOr = false) => {
1115
- if (Array.isArray(cond)) return cond.map((item) => {
1116
- const clause = Array.isArray(item) ? buildCondition(item, true) : buildCondition(item, true);
1117
- return clause ? `(${clause})` : "";
1118
- }).filter((clause) => clause !== "").join(" OR ");
1119
- const conditions = [];
1120
- let hasFunctionFilter = false;
1121
- for (const [key, value] of Object.entries(cond)) if (typeof value === "function") {
1122
- functionFilters.push({
1123
- key,
1124
- fn: value
1125
- });
1126
- hasFunctionFilter = true;
1127
- } else {
1128
- conditions.push(`${this.quoteIdentifier(key)} = ?`);
1129
- values.push(this.normalizeValue(value));
1130
- }
1131
- if (isInOr && hasFunctionFilter) hasOrWithFunctionFilters = true;
1132
- return conditions.join(" AND ");
1133
- };
1134
- return {
1135
- sql: buildCondition(condition),
1136
- values,
1137
- functionFilters,
1138
- hasOrWithFunctionFilters
1139
- };
1140
- }
1141
- /**
1142
- * Apply OR condition with function filters by evaluating each row against the condition
1143
- */
1144
- applyOrConditionWithFilters(rows, condition) {
1145
- return rows.filter((row) => this.matchesOrCondition(row, condition));
1146
- }
1147
- /**
1148
- * Check if a row matches an OR/AND condition (recursively)
1149
- */
1150
- matchesOrCondition(row, condition) {
1151
- if (Array.isArray(condition)) return condition.some((item) => this.matchesOrCondition(row, item));
1152
- return Object.entries(condition).every(([key, value]) => {
1153
- const rowValue = row[key];
1154
- if (typeof value === "function") return value(rowValue);
1155
- return rowValue === value;
1156
- });
1157
- }
1158
- /**
1159
- * Apply function filters to rows
1160
- */
1161
- applyFunctionFilters(rows, functionFilters) {
1162
- if (functionFilters.length === 0) return rows;
1163
- return rows.filter((row) => {
1164
- return functionFilters.every(({ key, fn }) => {
1165
- const value = row[key];
1166
- return fn(value);
1167
- });
1168
- });
1169
- }
1170
- /**
1171
- * Get table schema
1172
- */
1173
- getSchema(tableName) {
1174
- return this.schemas.get(tableName);
1175
- }
1176
- /**
1177
- * Get all table names
1178
- */
1179
- getTableNames() {
1180
- return Array.from(this.schemas.keys());
1181
- }
1182
- /**
1183
- * Sync a specific table back to its JSONL file
1184
- * Uses backward transformation when available
1185
- */
1186
- async syncTable(tableName) {
1187
- const tableConfig = this.tables.get(tableName);
1188
- if (!tableConfig) throw new Error(`Table ${tableName} not found`);
1189
- const deserializedRows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)}`).map((row) => this.deserializeRow(tableName, row));
1190
- const validationSchema = this.validationSchemas.get(tableName);
1191
- let finalRows = deserializedRows;
1192
- if (validationSchema && hasBackward(validationSchema)) {
1193
- const biSchema = validationSchema;
1194
- finalRows = deserializedRows.map((row) => biSchema.backward(row));
1195
- }
1196
- await JsonlWriter.write(tableConfig.jsonlPath, finalRows);
1197
- }
1198
- /**
1199
- * Sync database changes back to JSONL files
1200
- * Uses backward transformation when available
1201
- * @param tableName Optional table name to sync. If not provided, syncs all loaded tables
1202
- */
1203
- async sync(tableName) {
1204
- if (tableName) {
1205
- if (!this.schemas.has(tableName)) throw new Error(`Table '${tableName}' is not loaded`);
1206
- await this.syncTable(tableName);
1207
- } else for (const [name] of this.schemas) await this.syncTable(name);
1208
- }
1209
- /**
1210
- * Execute a function within a transaction
1211
- * Automatically commits on success or rolls back on error
1212
- */
1213
- async transaction(fn) {
1214
- if (this.inTransaction) throw new Error("Nested transactions are not supported");
1215
- this.db.exec("BEGIN TRANSACTION");
1216
- this.inTransaction = true;
1217
- try {
1218
- const result = await fn(this);
1219
- this.db.exec("COMMIT");
1220
- this.inTransaction = false;
1221
- await this.sync();
1222
- return result;
1223
- } catch (error) {
1224
- if (this.inTransaction) this.db.exec("ROLLBACK");
1225
- this.inTransaction = false;
1226
- throw error;
1227
- }
1228
- }
1229
- /**
1230
- * Close the database connection
1231
- */
1232
- async close() {
1233
- try {
1234
- this.db.close();
1235
- } catch (_error) {}
1236
- }
1237
- /**
1238
- * Get the underlying SQLite database instance
1239
- */
1240
- getDb() {
1241
- return this.db;
1242
- }
1243
- };
1244
-
1245
- //#endregion
1246
- //#region src/type-generator.ts
1247
- var TypeGenerator = class {
1248
- dataDir;
1249
- projectRoot;
1250
- outputFile;
1251
- dataDirPath;
1252
- constructor(options) {
1253
- const envProjectRoot = process.env.LINES_DB_TEST_PROJECT_ROOT;
1254
- this.projectRoot = envProjectRoot !== void 0 ? envProjectRoot : options.projectRoot || process.cwd();
1255
- this.dataDir = options.dataDir;
1256
- this.dataDirPath = (0, node_path.isAbsolute)(this.dataDir) ? this.dataDir : (0, node_path.join)(this.projectRoot, this.dataDir);
1257
- this.outputFile = options.output ? (0, node_path.isAbsolute)(options.output) ? options.output : (0, node_path.join)(this.projectRoot, options.output) : (0, node_path.join)(this.dataDirPath, "db.ts");
1258
- }
1259
- /**
1260
- * Generate types file from JSONL files and their optional schema files.
1261
- */
1262
- async generate() {
1263
- const tables = await this.findTables();
1264
- if (tables.length === 0) throw new Error(`No JSONL files found in ${this.dataDirPath}. Place one or more *.jsonl files in the directory.`);
1265
- const content = this.generateTypeDeclarations(tables);
1266
- await (0, node_fs_promises.mkdir)((0, node_path.dirname)(this.outputFile), { recursive: true });
1267
- await (0, node_fs_promises.writeFile)(this.outputFile, content, "utf-8");
1268
- console.log(`Generated types at ${this.outputFile}`);
1269
- return this.outputFile;
1270
- }
1271
- /**
1272
- * Find all *.jsonl files and check if they have corresponding *.schema.ts files
1273
- */
1274
- async findTables() {
1275
- try {
1276
- const entries = await (0, node_fs_promises.readdir)(this.dataDirPath, { withFileTypes: true });
1277
- const tables = [];
1278
- for (const entry of entries) if (entry.isFile() && entry.name.endsWith(".jsonl")) {
1279
- const tableName = (0, node_path.basename)(entry.name, ".jsonl");
1280
- const schemaFilePath = findSchemaFileInEntries(this.dataDirPath, tableName, entries);
1281
- tables.push({
1282
- tableName,
1283
- schemaFile: schemaFilePath
1284
- });
1285
- }
1286
- return tables;
1287
- } catch (error) {
1288
- if (error.code === "ENOENT") throw new Error(`Data directory not found: ${this.dataDirPath}. Set lines-db.dataDir to the correct location.`);
1289
- throw error;
1290
- }
1291
- }
1292
- /**
1293
- * Generate type declaration content
1294
- */
1295
- generateTypeDeclarations(tables) {
1296
- const imports = [];
1297
- const tableEntries = [];
1298
- const usedAliases = /* @__PURE__ */ new Set();
1299
- for (const table of tables) {
1300
- const tableKey = this.formatTableKey(table.tableName);
1301
- if (table.schemaFile) {
1302
- const schemaIdentifier = this.createSchemaIdentifier(table.tableName, usedAliases);
1303
- usedAliases.add(schemaIdentifier);
1304
- let relativePath = rewriteExtensionForImport((0, node_path.relative)((0, node_path.join)(this.outputFile, ".."), table.schemaFile).replace(/\\/g, "/"));
1305
- if (!relativePath.startsWith(".")) relativePath = "./" + relativePath;
1306
- imports.push(`import { schema as ${schemaIdentifier} } from '${relativePath}';`);
1307
- tableEntries.push(` ${tableKey}: InferOutput<typeof ${schemaIdentifier}>;`);
1308
- } else tableEntries.push(` ${tableKey}: Record<string, unknown>;`);
1309
- }
1310
- return `// Auto-generated by lines-db
1311
- // Do not edit this file manually
1312
-
1313
- ${imports.length > 0 ? `${imports.join("\n")}\n` : ""}import type { DatabaseConfig${imports.length > 0 ? ", InferOutput" : ""} } from '@toiroakr/lines-db';
1314
- import { fileURLToPath } from 'node:url';
1315
- import { dirname } from 'node:path';
1316
-
1317
- const __filename = fileURLToPath(import.meta.url);
1318
- const __dirname = dirname(__filename);
1319
-
1320
- export type Tables = {
1321
- ${tableEntries.join("\n")}
1322
- };
1323
-
1324
- export const config: DatabaseConfig<Tables> = {
1325
- dataDir: __dirname,
1326
- };
1327
- `;
1328
- }
1329
- createSchemaIdentifier(tableName, usedAliases) {
1330
- let base = sanitizeIdentifier(toCamelCase(tableName)) || "table";
1331
- if (!/^[A-Za-z_$]/.test(base)) base = `_${base}`;
1332
- let candidate = `${base}Schema`;
1333
- let suffix = 1;
1334
- while (usedAliases.has(candidate)) candidate = `${base}${++suffix}Schema`;
1335
- return candidate;
1336
- }
1337
- formatTableKey(tableName) {
1338
- if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(tableName)) return tableName;
1339
- return `'${tableName.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
1340
- }
1341
- };
1342
- function toCamelCase(value) {
1343
- const parts = value.split(/[^A-Za-z0-9]+/).filter(Boolean).map((part) => part.toLowerCase());
1344
- if (parts.length === 0) return value;
1345
- const [first, ...rest] = parts;
1346
- return first + rest.map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join("");
1347
- }
1348
- function sanitizeIdentifier(value) {
1349
- return value.replace(/[^A-Za-z0-9_$]/g, "");
1350
- }
1351
-
1352
- //#endregion
1353
- //#region src/jsonl-migration.ts
1354
- /**
1355
- * Validate a table by temporarily supplying in-memory rows while reusing the existing LinesDB validation pipeline.
1356
- * If validation fails, throws an error with validation details.
1357
- */
1358
- async function ensureTableRowsValid(options) {
1359
- const tablePath = (0, node_path.join)(options.dataDir, `${options.tableName}.jsonl`);
1360
- const overrides = new Map([[tablePath, options.rows]]);
1361
- await JsonlReader.withOverrides(overrides, async () => {
1362
- const db = LinesDB.create({ dataDir: options.dataDir });
1363
- try {
1364
- const result = await db.initialize({ tableName: options.tableName });
1365
- if (!result.valid) {
1366
- const errorCount = result.errors.length;
1367
- const errorDetails = result.errors.map((e) => {
1368
- const issueMessages = e.issues.map((issue) => issue.message).join(", ");
1369
- return ` Row ${e.rowIndex}: ${issueMessages}`;
1370
- }).join("\n");
1371
- throw new Error(`Validation failed for table '${options.tableName}' (${errorCount} error(s)):\n${errorDetails}`);
1372
- }
1373
- } finally {
1374
- await db.close();
1375
- }
1376
- });
1377
- }
1378
-
1379
- //#endregion
1380
- //#region src/error-formatter.ts
1381
- var ErrorFormatter = class {
1382
- verbose;
1383
- constructor(options = {}) {
1384
- this.verbose = options.verbose ?? false;
1385
- }
1386
- /**
1387
- * Format validation errors
1388
- */
1389
- formatValidationErrors(errors) {
1390
- if (this.verbose) return this.formatValidationErrorsVerbose(errors);
1391
- return this.formatValidationErrorsCompact(errors);
1392
- }
1393
- /**
1394
- * Format foreign key error
1395
- */
1396
- formatForeignKeyError(error) {
1397
- if (this.verbose) return this.formatForeignKeyErrorVerbose(error);
1398
- return this.formatForeignKeyErrorCompact(error);
1399
- }
1400
- /**
1401
- * Format compact (default) validation errors
1402
- */
1403
- formatValidationErrorsCompact(errors) {
1404
- const lines = [];
1405
- for (const error of errors) for (const issue of error.issues) {
1406
- const fieldPath = this.getFieldPath(issue);
1407
- const line = `${error.file}:${error.rowIndex + 1} • ${fieldPath}: ${issue.message}`;
1408
- lines.push(line);
1409
- }
1410
- return lines.join("\n");
1411
- }
1412
- /**
1413
- * Format verbose validation errors
1414
- */
1415
- formatValidationErrorsVerbose(errors) {
1416
- const blocks = [];
1417
- for (let i = 0; i < errors.length; i++) {
1418
- const error = errors[i];
1419
- const isLast = i === errors.length - 1;
1420
- const prefix = isLast ? "└─" : "├─";
1421
- const linePrefix = isLast ? " " : "│ ";
1422
- const lines = [`${prefix} ${error.file}:${error.rowIndex + 1}`];
1423
- for (const issue of error.issues) {
1424
- const fieldPath = this.getFieldPath(issue);
1425
- lines.push(`${linePrefix}Field: ${fieldPath}`);
1426
- lines.push(`${linePrefix}Error: ${issue.message}`);
1427
- }
1428
- if (error.originalData !== void 0) lines.push(`${linePrefix}Original data: ${JSON.stringify(error.originalData)}`);
1429
- if (error.data !== void 0) {
1430
- const label = error.originalData !== void 0 ? "Transformed data" : "Data";
1431
- lines.push(`${linePrefix}${label}: ${JSON.stringify(error.data)}`);
1432
- }
1433
- blocks.push(lines.join("\n"));
1434
- }
1435
- return blocks.join("\n│\n");
1436
- }
1437
- /**
1438
- * Format compact foreign key error
1439
- */
1440
- formatForeignKeyErrorCompact(error) {
1441
- return `${error.file}:${error.rowIndex + 1} • ${error.column}: Foreign key constraint failed - Referenced value ${JSON.stringify(error.value)} does not exist in ${error.referencedTable}(${error.referencedColumn})`;
1442
- }
1443
- /**
1444
- * Format verbose foreign key error
1445
- */
1446
- formatForeignKeyErrorVerbose(error) {
1447
- const lines = [
1448
- `└─ ${error.file}:${error.rowIndex + 1}`,
1449
- ` Type: Foreign Key Violation`,
1450
- ` Field: ${error.column}`,
1451
- ` Value: ${JSON.stringify(error.value)}`,
1452
- ` References: ${error.referencedTable}(${error.referencedColumn})`,
1453
- ` Error: Referenced value does not exist in target table`
1454
- ];
1455
- if (error.data !== void 0) lines.push(` Data: ${JSON.stringify(error.data)}`);
1456
- return lines.join("\n");
1457
- }
1458
- /**
1459
- * Get field path from issue
1460
- */
1461
- getFieldPath(issue) {
1462
- if (!issue.path || issue.path.length === 0) return "root";
1463
- return issue.path.map((segment) => {
1464
- if (typeof segment === "object" && segment !== null && "key" in segment) return String(segment.key);
1465
- return String(segment);
1466
- }).join(".");
1467
- }
1468
- /**
1469
- * Format error header with count
1470
- */
1471
- formatErrorHeader(count, file) {
1472
- return (0, node_util.styleText)("red", `✗ Found ${count} error(s)${file ? ` in ${file}` : ""}`);
1473
- }
1474
- /**
1475
- * Format migration failure header
1476
- */
1477
- formatMigrationFailureHeader() {
1478
- return (0, node_util.styleText)("red", "\n✗ Migration failed and was rolled back");
1479
- }
1480
- };
1481
-
1482
- //#endregion
1483
- exports.DirectoryScanner = DirectoryScanner;
1484
- exports.ErrorFormatter = ErrorFormatter;
1485
- exports.JsonlReader = JsonlReader;
1486
- exports.JsonlWriter = JsonlWriter;
1487
- exports.LinesDB = LinesDB;
1488
- exports.RUNTIME = RUNTIME;
1489
- exports.SCHEMA_EXTENSIONS = SCHEMA_EXTENSIONS;
1490
- exports.SchemaLoader = SchemaLoader;
1491
- exports.TypeGenerator = TypeGenerator;
1492
- exports.defineSchema = defineSchema;
1493
- exports.detectRuntime = detectRuntime;
1494
- exports.ensureTableRowsValid = ensureTableRowsValid;
1495
- exports.extractTableNameFromSchemaFile = extractTableNameFromSchemaFile;
1496
- exports.findSchemaFile = findSchemaFile;
1497
- exports.hasBackward = hasBackward;
1498
- exports.isSchemaFile = isSchemaFile;
1499
- exports.rewriteExtensionForImport = rewriteExtensionForImport;