@toiroakr/lines-db 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/bin/{cli.js → cli.mjs} +69 -30
- package/dist/index.cjs +71 -57
- package/dist/index.d.cts +24 -20
- package/dist/index.d.cts.map +1 -1
- package/dist/{index.d.ts → index.d.mts} +24 -20
- package/dist/index.d.mts.map +1 -0
- package/dist/{index.js → index.mjs} +70 -30
- package/dist/index.mjs.map +1 -0
- package/package.json +13 -13
- package/src/cli.ts +24 -54
- package/src/database.test.ts +162 -31
- package/src/database.ts +116 -68
- package/src/directory-scanner.test.ts +1 -3
- package/src/directory-scanner.ts +1 -3
- package/src/index.ts +1 -5
- package/src/jsonl-reader.test.ts +1 -3
- package/src/jsonl-reader.ts +1 -4
- package/src/schema-loader.test.ts +4 -12
- package/src/schema-loader.ts +1 -3
- package/src/schema.ts +5 -5
- package/src/type-generator.ts +4 -11
- package/src/types.ts +1 -4
- package/tsconfig.json +1 -0
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# @toiroakr/lines-db
|
|
2
2
|
|
|
3
|
+
## 0.9.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 9982a21: Update runtime dependencies (`commander` to v14.0.3, `@standard-schema/spec` to v1.1.0, `tsx` to v4.21.0) and refresh devDependencies/tooling (migrate from ESLint/Prettier to Oxlint/Oxfmt, TypeScript v6).
|
|
8
|
+
|
|
9
|
+
## 0.9.1
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 4ca2664: fix: validate circular foreign key constraints via deferred validation
|
|
14
|
+
|
|
15
|
+
Previously, when two tables had bidirectional foreign keys (e.g., `_User` → `User` and `User` → `_User`), one direction's FK validation was always skipped due to circular dependency detection. Now, circular dependency FKs are validated in a second pass after all tables have been loaded, using SQL queries instead of SQLite FK constraints.
|
|
16
|
+
|
|
3
17
|
## 0.9.0
|
|
4
18
|
|
|
5
19
|
### Minor Changes
|
|
@@ -13,6 +27,7 @@
|
|
|
13
27
|
- c408b92: feat: display per-table validation results for directory validation
|
|
14
28
|
|
|
15
29
|
The `validate` command now shows individual results per table when validating a directory, including record counts for successful tables (e.g., `✓ users (3 records)`).
|
|
30
|
+
|
|
16
31
|
- Added `TableValidationResult` type and `tableResults` field to `ValidationResult`
|
|
17
32
|
- Each table result includes `tableName`, `valid`, `rowCount`, `errors`, and `warnings`
|
|
18
33
|
|
|
@@ -29,6 +44,7 @@
|
|
|
29
44
|
- 4597383: feat: support .mts and .cts schema file extensions
|
|
30
45
|
|
|
31
46
|
Schema files are now auto-detected with the following priority: `.schema.ts` > `.schema.mts` > `.schema.cts`. Mixed extensions within a single project are supported.
|
|
47
|
+
|
|
32
48
|
- Added `--output` option to `generate` command for specifying the output file path (e.g., `--output ./data/db.mts`)
|
|
33
49
|
- Import paths are correctly rewritten: `.ts`→`.js`, `.mts`→`.mjs`, `.cts`→`.cjs`
|
|
34
50
|
- New exported utilities: `findSchemaFile`, `isSchemaFile`, `extractTableNameFromSchemaFile`, `rewriteExtensionForImport`, `SCHEMA_EXTENSIONS`
|
|
@@ -70,6 +86,7 @@
|
|
|
70
86
|
- b281dc8: Fix constraint validation in validator to properly detect primary key and unique index violations
|
|
71
87
|
|
|
72
88
|
Previously, the validator was not creating indexes from schema metadata and was missing the default primaryKey behavior, causing constraint violations to go undetected. This fix ensures:
|
|
89
|
+
|
|
73
90
|
- Indexes (both unique and non-unique) are now properly created from schema metadata in the validation database
|
|
74
91
|
- Primary key defaults to 'id' column when not explicitly specified, matching database.ts behavior
|
|
75
92
|
- Constraint violations are properly detected by inserting rows into an in-memory database and catching SQLite exceptions
|
|
@@ -110,6 +127,7 @@
|
|
|
110
127
|
- 49089e1: fix: skip validation with warning instead of error when schema file is not found
|
|
111
128
|
|
|
112
129
|
When validating a directory containing JSONL files, if a schema file is missing for some tables, the validator will now:
|
|
130
|
+
|
|
113
131
|
- Skip validation for those files with a warning message instead of throwing an error
|
|
114
132
|
- Display warnings in yellow in the CLI output
|
|
115
133
|
- Continue validation for other files that have schema files
|
package/bin/{cli.js → cli.mjs}
RENAMED
|
@@ -6,10 +6,8 @@ import { pathToFileURL } from "node:url";
|
|
|
6
6
|
import { styleText } from "node:util";
|
|
7
7
|
import { Command } from "commander";
|
|
8
8
|
import { runInNewContext } from "node:vm";
|
|
9
|
-
|
|
10
|
-
//#region rolldown:runtime
|
|
9
|
+
//#region \0rolldown/runtime.js
|
|
11
10
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
12
|
-
|
|
13
11
|
//#endregion
|
|
14
12
|
//#region src/schema-extensions.ts
|
|
15
13
|
/**
|
|
@@ -60,7 +58,6 @@ function rewriteExtensionForImport(filePath) {
|
|
|
60
58
|
for (const [tsExt, jsExt] of Object.entries(SCHEMA_TO_JS_IMPORT_MAP)) if (filePath.endsWith(tsExt)) return filePath.slice(0, -tsExt.length) + jsExt;
|
|
61
59
|
return filePath;
|
|
62
60
|
}
|
|
63
|
-
|
|
64
61
|
//#endregion
|
|
65
62
|
//#region src/type-generator.ts
|
|
66
63
|
var TypeGenerator = class {
|
|
@@ -167,7 +164,6 @@ function toCamelCase(value) {
|
|
|
167
164
|
function sanitizeIdentifier(value) {
|
|
168
165
|
return value.replace(/[^A-Za-z0-9_$]/g, "");
|
|
169
166
|
}
|
|
170
|
-
|
|
171
167
|
//#endregion
|
|
172
168
|
//#region src/runtime.ts
|
|
173
169
|
function detectRuntime() {
|
|
@@ -175,10 +171,12 @@ function detectRuntime() {
|
|
|
175
171
|
return "unknown";
|
|
176
172
|
}
|
|
177
173
|
const RUNTIME = detectRuntime();
|
|
178
|
-
|
|
179
174
|
//#endregion
|
|
180
175
|
//#region src/sqlite-adapter.ts
|
|
181
176
|
/**
|
|
177
|
+
* SQLite adapter for Node.js
|
|
178
|
+
*/
|
|
179
|
+
/**
|
|
182
180
|
* Create a SQLite database instance for Node.js
|
|
183
181
|
*/
|
|
184
182
|
function createDatabase(path = ":memory:") {
|
|
@@ -215,7 +213,6 @@ function createNodeDatabase(path) {
|
|
|
215
213
|
}
|
|
216
214
|
};
|
|
217
215
|
}
|
|
218
|
-
|
|
219
216
|
//#endregion
|
|
220
217
|
//#region src/jsonl-reader.ts
|
|
221
218
|
var JsonlReader = class {
|
|
@@ -295,7 +292,6 @@ var JsonlReader = class {
|
|
|
295
292
|
return "TEXT";
|
|
296
293
|
}
|
|
297
294
|
};
|
|
298
|
-
|
|
299
295
|
//#endregion
|
|
300
296
|
//#region src/jsonl-writer.ts
|
|
301
297
|
var JsonlWriter = class {
|
|
@@ -309,18 +305,17 @@ var JsonlWriter = class {
|
|
|
309
305
|
* Append data to JSONL file
|
|
310
306
|
*/
|
|
311
307
|
static async append(filePath, data) {
|
|
312
|
-
const { readFile
|
|
308
|
+
const { readFile, writeFile } = await import("node:fs/promises");
|
|
313
309
|
try {
|
|
314
|
-
const existing = await readFile
|
|
310
|
+
const existing = await readFile(filePath, "utf-8");
|
|
315
311
|
const lines = data.map((obj) => JSON.stringify(obj)).join("\n");
|
|
316
|
-
await writeFile
|
|
312
|
+
await writeFile(filePath, existing.trim() + "\n" + lines + "\n", "utf-8");
|
|
317
313
|
} catch (error) {
|
|
318
314
|
if (error.code === "ENOENT") await this.write(filePath, data);
|
|
319
315
|
else throw error;
|
|
320
316
|
}
|
|
321
317
|
}
|
|
322
318
|
};
|
|
323
|
-
|
|
324
319
|
//#endregion
|
|
325
320
|
//#region src/schema-loader.ts
|
|
326
321
|
var SchemaLoader = class {
|
|
@@ -359,7 +354,6 @@ var SchemaLoader = class {
|
|
|
359
354
|
return standardObj.version === 1 && typeof standardObj.vendor === "string" && typeof standardObj.validate === "function";
|
|
360
355
|
}
|
|
361
356
|
};
|
|
362
|
-
|
|
363
357
|
//#endregion
|
|
364
358
|
//#region src/directory-scanner.ts
|
|
365
359
|
var DirectoryScanner = class {
|
|
@@ -385,7 +379,6 @@ var DirectoryScanner = class {
|
|
|
385
379
|
}
|
|
386
380
|
}
|
|
387
381
|
};
|
|
388
|
-
|
|
389
382
|
//#endregion
|
|
390
383
|
//#region src/schema.ts
|
|
391
384
|
/**
|
|
@@ -394,7 +387,6 @@ var DirectoryScanner = class {
|
|
|
394
387
|
function hasBackward(schema) {
|
|
395
388
|
return "backward" in schema && typeof schema.backward === "function";
|
|
396
389
|
}
|
|
397
|
-
|
|
398
390
|
//#endregion
|
|
399
391
|
//#region src/database.ts
|
|
400
392
|
var LinesDB = class LinesDB {
|
|
@@ -433,13 +425,20 @@ var LinesDB = class LinesDB {
|
|
|
433
425
|
const loadedTables = /* @__PURE__ */ new Set();
|
|
434
426
|
const loadingTables = /* @__PURE__ */ new Set();
|
|
435
427
|
const attemptedTables = /* @__PURE__ */ new Set();
|
|
428
|
+
const allDeferredForeignKeys = [];
|
|
436
429
|
for (const tableNameToLoad of tablesToLoad) if (!attemptedTables.has(tableNameToLoad)) {
|
|
437
430
|
const tableTransform = tableNameToLoad === tableName ? transform : void 0;
|
|
438
|
-
const { errors, warnings, rowCounts: tableRowCounts } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
|
|
431
|
+
const { errors, warnings, rowCounts: tableRowCounts, deferredForeignKeys } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
|
|
439
432
|
allErrors.push(...errors);
|
|
440
433
|
allWarnings.push(...warnings);
|
|
434
|
+
allDeferredForeignKeys.push(...deferredForeignKeys);
|
|
441
435
|
for (const [k, v] of tableRowCounts) allRowCounts.set(k, v);
|
|
442
436
|
}
|
|
437
|
+
if (detailedValidate && allDeferredForeignKeys.length > 0) for (const { tableName: tName, foreignKey: fk, filePath } of allDeferredForeignKeys) {
|
|
438
|
+
if (!loadedTables.has(fk.references.table)) continue;
|
|
439
|
+
const deferredErrors = this.validateDeferredForeignKey(tName, fk, filePath);
|
|
440
|
+
allErrors.push(...deferredErrors);
|
|
441
|
+
}
|
|
443
442
|
const tableResults = tablesToLoad.map((name) => {
|
|
444
443
|
const tableErrors = allErrors.filter((e) => e.tableName === name);
|
|
445
444
|
const tableWarnings = allWarnings.filter((w) => w.includes(`'${name}'`));
|
|
@@ -465,10 +464,12 @@ var LinesDB = class LinesDB {
|
|
|
465
464
|
const errors = [];
|
|
466
465
|
const warnings = [];
|
|
467
466
|
const rowCounts = /* @__PURE__ */ new Map();
|
|
467
|
+
const deferredForeignKeys = [];
|
|
468
468
|
if (attemptedTables.has(tableName)) return {
|
|
469
469
|
errors,
|
|
470
470
|
warnings,
|
|
471
|
-
rowCounts
|
|
471
|
+
rowCounts,
|
|
472
|
+
deferredForeignKeys
|
|
472
473
|
};
|
|
473
474
|
attemptedTables.add(tableName);
|
|
474
475
|
if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
|
|
@@ -478,10 +479,10 @@ var LinesDB = class LinesDB {
|
|
|
478
479
|
try {
|
|
479
480
|
let foreignKeys;
|
|
480
481
|
try {
|
|
481
|
-
const { pathToFileURL
|
|
482
|
+
const { pathToFileURL } = await import("node:url");
|
|
482
483
|
const schemaPath = await findSchemaFile(dirname(tableConfig.jsonlPath), basename(tableConfig.jsonlPath, ".jsonl"));
|
|
483
484
|
if (schemaPath) {
|
|
484
|
-
const schemaModule = await import(`${pathToFileURL
|
|
485
|
+
const schemaModule = await import(`${pathToFileURL(schemaPath).href}?t=${Date.now()}`);
|
|
485
486
|
foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
|
|
486
487
|
}
|
|
487
488
|
} catch {}
|
|
@@ -492,23 +493,35 @@ var LinesDB = class LinesDB {
|
|
|
492
493
|
const depResult = await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables, attemptedTables, detailedValidate, void 0);
|
|
493
494
|
errors.push(...depResult.errors);
|
|
494
495
|
warnings.push(...depResult.warnings);
|
|
496
|
+
deferredForeignKeys.push(...depResult.deferredForeignKeys);
|
|
495
497
|
for (const [k, v] of depResult.rowCounts) rowCounts.set(k, v);
|
|
496
498
|
} else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
|
|
497
499
|
}
|
|
498
500
|
const failedDependencies = /* @__PURE__ */ new Set();
|
|
501
|
+
const circularDependencies = /* @__PURE__ */ new Set();
|
|
499
502
|
if (foreignKeys && foreignKeys.length > 0) {
|
|
500
503
|
for (const fk of foreignKeys) {
|
|
501
504
|
const referencedTable = fk.references.table;
|
|
502
505
|
if (referencedTable === tableName) continue;
|
|
503
|
-
if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable))
|
|
506
|
+
if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable)) if (loadingTables.has(referencedTable)) circularDependencies.add(referencedTable);
|
|
507
|
+
else failedDependencies.add(referencedTable);
|
|
504
508
|
}
|
|
505
509
|
if (failedDependencies.size > 0) for (const dep of failedDependencies) warnings.push(`Skipping foreign key validation for table '${tableName}': referenced table '${dep}' has validation errors`);
|
|
506
510
|
}
|
|
507
|
-
const
|
|
511
|
+
const allSkippedDependencies = new Set([...failedDependencies, ...circularDependencies]);
|
|
512
|
+
const { loaded, rowCount, errors: loadErrors } = await this.loadTable(tableName, tableConfig, detailedValidate, transform, allSkippedDependencies);
|
|
508
513
|
errors.push(...loadErrors);
|
|
509
514
|
rowCounts.set(tableName, rowCount);
|
|
510
|
-
if (loaded)
|
|
511
|
-
|
|
515
|
+
if (loaded) {
|
|
516
|
+
loadedTables.add(tableName);
|
|
517
|
+
if (foreignKeys && circularDependencies.size > 0) {
|
|
518
|
+
for (const fk of foreignKeys) if (circularDependencies.has(fk.references.table)) deferredForeignKeys.push({
|
|
519
|
+
tableName,
|
|
520
|
+
foreignKey: fk,
|
|
521
|
+
filePath: tableConfig.jsonlPath
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
} else {
|
|
512
525
|
warnings.push(`Table '${tableName}' was not loaded (no data or skipped)`);
|
|
513
526
|
this.tables.delete(tableName);
|
|
514
527
|
}
|
|
@@ -518,7 +531,8 @@ var LinesDB = class LinesDB {
|
|
|
518
531
|
return {
|
|
519
532
|
errors,
|
|
520
533
|
warnings,
|
|
521
|
-
rowCounts
|
|
534
|
+
rowCounts,
|
|
535
|
+
deferredForeignKeys
|
|
522
536
|
};
|
|
523
537
|
}
|
|
524
538
|
/**
|
|
@@ -534,10 +548,10 @@ var LinesDB = class LinesDB {
|
|
|
534
548
|
validationSchema = await SchemaLoader.loadSchema(config.jsonlPath);
|
|
535
549
|
} catch (_error) {}
|
|
536
550
|
if (!config.validationSchema) try {
|
|
537
|
-
const { pathToFileURL
|
|
551
|
+
const { pathToFileURL } = await import("node:url");
|
|
538
552
|
const schemaPath = await findSchemaFile(dirname(config.jsonlPath), basename(config.jsonlPath, ".jsonl"));
|
|
539
553
|
if (!schemaPath) throw new Error("Schema file not found");
|
|
540
|
-
const schemaModule = await import(`${pathToFileURL
|
|
554
|
+
const schemaModule = await import(`${pathToFileURL(schemaPath).href}?t=${Date.now()}`);
|
|
541
555
|
const schemaExport = schemaModule.schema || schemaModule.default;
|
|
542
556
|
if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
|
|
543
557
|
else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
|
|
@@ -764,6 +778,34 @@ var LinesDB = class LinesDB {
|
|
|
764
778
|
};
|
|
765
779
|
}
|
|
766
780
|
/**
|
|
781
|
+
* Validate a deferred foreign key constraint after all tables have been loaded.
|
|
782
|
+
* Used for circular dependency FK validation.
|
|
783
|
+
*/
|
|
784
|
+
validateDeferredForeignKey(tableName, fk, filePath) {
|
|
785
|
+
const errors = [];
|
|
786
|
+
const quotedTable = this.quoteTableName(tableName);
|
|
787
|
+
const quotedColumn = this.quoteIdentifier(fk.column);
|
|
788
|
+
const quotedRefTable = this.quoteTableName(fk.references.table);
|
|
789
|
+
const sql = `SELECT rowid - 1 as idx, ${quotedColumn} as val FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL AND ${quotedColumn} NOT IN (SELECT ${this.quoteIdentifier(fk.references.column)} FROM ${quotedRefTable})`;
|
|
790
|
+
try {
|
|
791
|
+
const rows = this.query(sql);
|
|
792
|
+
for (const row of rows) errors.push({
|
|
793
|
+
file: filePath,
|
|
794
|
+
tableName,
|
|
795
|
+
rowIndex: row.idx,
|
|
796
|
+
issues: [],
|
|
797
|
+
type: "foreignKey",
|
|
798
|
+
foreignKeyError: {
|
|
799
|
+
column: fk.column,
|
|
800
|
+
value: row.val,
|
|
801
|
+
referencedTable: fk.references.table,
|
|
802
|
+
referencedColumn: fk.references.column
|
|
803
|
+
}
|
|
804
|
+
});
|
|
805
|
+
} catch (_) {}
|
|
806
|
+
return errors;
|
|
807
|
+
}
|
|
808
|
+
/**
|
|
767
809
|
* Execute a raw SQL query
|
|
768
810
|
*/
|
|
769
811
|
query(sql, params = []) {
|
|
@@ -1216,7 +1258,6 @@ var LinesDB = class LinesDB {
|
|
|
1216
1258
|
return this.db;
|
|
1217
1259
|
}
|
|
1218
1260
|
};
|
|
1219
|
-
|
|
1220
1261
|
//#endregion
|
|
1221
1262
|
//#region src/error-formatter.ts
|
|
1222
1263
|
var ErrorFormatter = class {
|
|
@@ -1319,7 +1360,6 @@ var ErrorFormatter = class {
|
|
|
1319
1360
|
return styleText("red", "\n✗ Migration failed and was rolled back");
|
|
1320
1361
|
}
|
|
1321
1362
|
};
|
|
1322
|
-
|
|
1323
1363
|
//#endregion
|
|
1324
1364
|
//#region src/cli.ts
|
|
1325
1365
|
register("tsx", import.meta.url, { data: {} });
|
|
@@ -1711,6 +1751,5 @@ async function migrateFile(filePath, transformStr, options) {
|
|
|
1711
1751
|
}
|
|
1712
1752
|
}
|
|
1713
1753
|
program.parse();
|
|
1714
|
-
|
|
1715
1754
|
//#endregion
|
|
1716
|
-
export {
|
|
1755
|
+
export {};
|
package/dist/index.cjs
CHANGED
|
@@ -1,45 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __copyProps = (to, from, except, desc) => {
|
|
9
|
-
if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
|
|
10
|
-
key = keys[i];
|
|
11
|
-
if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
|
|
12
|
-
get: ((k) => from[k]).bind(null, key),
|
|
13
|
-
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
14
|
-
});
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
|
|
19
|
-
value: mod,
|
|
20
|
-
enumerable: true
|
|
21
|
-
}) : target, mod));
|
|
22
|
-
|
|
23
|
-
//#endregion
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
24
2
|
let node_fs_promises = require("node:fs/promises");
|
|
25
|
-
node_fs_promises = __toESM(node_fs_promises);
|
|
26
3
|
let node_path = require("node:path");
|
|
27
|
-
node_path = __toESM(node_path);
|
|
28
4
|
let node_url = require("node:url");
|
|
29
|
-
node_url = __toESM(node_url);
|
|
30
5
|
let node_util = require("node:util");
|
|
31
|
-
node_util = __toESM(node_util);
|
|
32
|
-
|
|
33
6
|
//#region src/runtime.ts
|
|
34
7
|
function detectRuntime() {
|
|
35
8
|
if (typeof process !== "undefined" && process.versions && process.versions.node) return "node";
|
|
36
9
|
return "unknown";
|
|
37
10
|
}
|
|
38
11
|
const RUNTIME = detectRuntime();
|
|
39
|
-
|
|
40
12
|
//#endregion
|
|
41
13
|
//#region src/sqlite-adapter.ts
|
|
42
14
|
/**
|
|
15
|
+
* SQLite adapter for Node.js
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
43
18
|
* Create a SQLite database instance for Node.js
|
|
44
19
|
*/
|
|
45
20
|
function createDatabase(path = ":memory:") {
|
|
@@ -76,7 +51,6 @@ function createNodeDatabase(path) {
|
|
|
76
51
|
}
|
|
77
52
|
};
|
|
78
53
|
}
|
|
79
|
-
|
|
80
54
|
//#endregion
|
|
81
55
|
//#region src/jsonl-reader.ts
|
|
82
56
|
var JsonlReader = class {
|
|
@@ -156,7 +130,6 @@ var JsonlReader = class {
|
|
|
156
130
|
return "TEXT";
|
|
157
131
|
}
|
|
158
132
|
};
|
|
159
|
-
|
|
160
133
|
//#endregion
|
|
161
134
|
//#region src/jsonl-writer.ts
|
|
162
135
|
var JsonlWriter = class {
|
|
@@ -170,18 +143,17 @@ var JsonlWriter = class {
|
|
|
170
143
|
* Append data to JSONL file
|
|
171
144
|
*/
|
|
172
145
|
static async append(filePath, data) {
|
|
173
|
-
const { readFile
|
|
146
|
+
const { readFile, writeFile } = await import("node:fs/promises");
|
|
174
147
|
try {
|
|
175
|
-
const existing = await readFile
|
|
148
|
+
const existing = await readFile(filePath, "utf-8");
|
|
176
149
|
const lines = data.map((obj) => JSON.stringify(obj)).join("\n");
|
|
177
|
-
await writeFile
|
|
150
|
+
await writeFile(filePath, existing.trim() + "\n" + lines + "\n", "utf-8");
|
|
178
151
|
} catch (error) {
|
|
179
152
|
if (error.code === "ENOENT") await this.write(filePath, data);
|
|
180
153
|
else throw error;
|
|
181
154
|
}
|
|
182
155
|
}
|
|
183
156
|
};
|
|
184
|
-
|
|
185
157
|
//#endregion
|
|
186
158
|
//#region src/schema-extensions.ts
|
|
187
159
|
/**
|
|
@@ -246,7 +218,6 @@ function rewriteExtensionForImport(filePath) {
|
|
|
246
218
|
for (const [tsExt, jsExt] of Object.entries(SCHEMA_TO_JS_IMPORT_MAP)) if (filePath.endsWith(tsExt)) return filePath.slice(0, -tsExt.length) + jsExt;
|
|
247
219
|
return filePath;
|
|
248
220
|
}
|
|
249
|
-
|
|
250
221
|
//#endregion
|
|
251
222
|
//#region src/schema-loader.ts
|
|
252
223
|
var SchemaLoader = class {
|
|
@@ -266,8 +237,8 @@ var SchemaLoader = class {
|
|
|
266
237
|
const schemaPath = await findSchemaFile(dir, tableName);
|
|
267
238
|
if (!schemaPath) throw new Error(`Schema file not found for table '${tableName}'. Expected one of: ${SCHEMA_EXTENSIONS.map((ext) => `${tableName}${ext}`).join(", ")}`);
|
|
268
239
|
try {
|
|
269
|
-
const module
|
|
270
|
-
const schema = module
|
|
240
|
+
const module = await import(`${(0, node_url.pathToFileURL)(schemaPath).href}?t=${Date.now()}`);
|
|
241
|
+
const schema = module.default || module.schema;
|
|
271
242
|
if (schema && this.isStandardSchema(schema)) return schema;
|
|
272
243
|
throw new Error(`Schema file ${schemaPath} does not export a valid StandardSchema`);
|
|
273
244
|
} catch (error) {
|
|
@@ -285,7 +256,6 @@ var SchemaLoader = class {
|
|
|
285
256
|
return standardObj.version === 1 && typeof standardObj.vendor === "string" && typeof standardObj.validate === "function";
|
|
286
257
|
}
|
|
287
258
|
};
|
|
288
|
-
|
|
289
259
|
//#endregion
|
|
290
260
|
//#region src/directory-scanner.ts
|
|
291
261
|
var DirectoryScanner = class {
|
|
@@ -311,7 +281,6 @@ var DirectoryScanner = class {
|
|
|
311
281
|
}
|
|
312
282
|
}
|
|
313
283
|
};
|
|
314
|
-
|
|
315
284
|
//#endregion
|
|
316
285
|
//#region src/schema.ts
|
|
317
286
|
/**
|
|
@@ -369,7 +338,6 @@ function defineSchema(schema, ...args) {
|
|
|
369
338
|
function hasBackward(schema) {
|
|
370
339
|
return "backward" in schema && typeof schema.backward === "function";
|
|
371
340
|
}
|
|
372
|
-
|
|
373
341
|
//#endregion
|
|
374
342
|
//#region src/database.ts
|
|
375
343
|
var LinesDB = class LinesDB {
|
|
@@ -408,13 +376,20 @@ var LinesDB = class LinesDB {
|
|
|
408
376
|
const loadedTables = /* @__PURE__ */ new Set();
|
|
409
377
|
const loadingTables = /* @__PURE__ */ new Set();
|
|
410
378
|
const attemptedTables = /* @__PURE__ */ new Set();
|
|
379
|
+
const allDeferredForeignKeys = [];
|
|
411
380
|
for (const tableNameToLoad of tablesToLoad) if (!attemptedTables.has(tableNameToLoad)) {
|
|
412
381
|
const tableTransform = tableNameToLoad === tableName ? transform : void 0;
|
|
413
|
-
const { errors, warnings, rowCounts: tableRowCounts } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
|
|
382
|
+
const { errors, warnings, rowCounts: tableRowCounts, deferredForeignKeys } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
|
|
414
383
|
allErrors.push(...errors);
|
|
415
384
|
allWarnings.push(...warnings);
|
|
385
|
+
allDeferredForeignKeys.push(...deferredForeignKeys);
|
|
416
386
|
for (const [k, v] of tableRowCounts) allRowCounts.set(k, v);
|
|
417
387
|
}
|
|
388
|
+
if (detailedValidate && allDeferredForeignKeys.length > 0) for (const { tableName: tName, foreignKey: fk, filePath } of allDeferredForeignKeys) {
|
|
389
|
+
if (!loadedTables.has(fk.references.table)) continue;
|
|
390
|
+
const deferredErrors = this.validateDeferredForeignKey(tName, fk, filePath);
|
|
391
|
+
allErrors.push(...deferredErrors);
|
|
392
|
+
}
|
|
418
393
|
const tableResults = tablesToLoad.map((name) => {
|
|
419
394
|
const tableErrors = allErrors.filter((e) => e.tableName === name);
|
|
420
395
|
const tableWarnings = allWarnings.filter((w) => w.includes(`'${name}'`));
|
|
@@ -440,10 +415,12 @@ var LinesDB = class LinesDB {
|
|
|
440
415
|
const errors = [];
|
|
441
416
|
const warnings = [];
|
|
442
417
|
const rowCounts = /* @__PURE__ */ new Map();
|
|
418
|
+
const deferredForeignKeys = [];
|
|
443
419
|
if (attemptedTables.has(tableName)) return {
|
|
444
420
|
errors,
|
|
445
421
|
warnings,
|
|
446
|
-
rowCounts
|
|
422
|
+
rowCounts,
|
|
423
|
+
deferredForeignKeys
|
|
447
424
|
};
|
|
448
425
|
attemptedTables.add(tableName);
|
|
449
426
|
if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
|
|
@@ -453,10 +430,10 @@ var LinesDB = class LinesDB {
|
|
|
453
430
|
try {
|
|
454
431
|
let foreignKeys;
|
|
455
432
|
try {
|
|
456
|
-
const { pathToFileURL
|
|
433
|
+
const { pathToFileURL } = await import("node:url");
|
|
457
434
|
const schemaPath = await findSchemaFile((0, node_path.dirname)(tableConfig.jsonlPath), (0, node_path.basename)(tableConfig.jsonlPath, ".jsonl"));
|
|
458
435
|
if (schemaPath) {
|
|
459
|
-
const schemaModule = await import(`${pathToFileURL
|
|
436
|
+
const schemaModule = await import(`${pathToFileURL(schemaPath).href}?t=${Date.now()}`);
|
|
460
437
|
foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
|
|
461
438
|
}
|
|
462
439
|
} catch {}
|
|
@@ -467,23 +444,35 @@ var LinesDB = class LinesDB {
|
|
|
467
444
|
const depResult = await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables, attemptedTables, detailedValidate, void 0);
|
|
468
445
|
errors.push(...depResult.errors);
|
|
469
446
|
warnings.push(...depResult.warnings);
|
|
447
|
+
deferredForeignKeys.push(...depResult.deferredForeignKeys);
|
|
470
448
|
for (const [k, v] of depResult.rowCounts) rowCounts.set(k, v);
|
|
471
449
|
} else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
|
|
472
450
|
}
|
|
473
451
|
const failedDependencies = /* @__PURE__ */ new Set();
|
|
452
|
+
const circularDependencies = /* @__PURE__ */ new Set();
|
|
474
453
|
if (foreignKeys && foreignKeys.length > 0) {
|
|
475
454
|
for (const fk of foreignKeys) {
|
|
476
455
|
const referencedTable = fk.references.table;
|
|
477
456
|
if (referencedTable === tableName) continue;
|
|
478
|
-
if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable))
|
|
457
|
+
if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable)) if (loadingTables.has(referencedTable)) circularDependencies.add(referencedTable);
|
|
458
|
+
else failedDependencies.add(referencedTable);
|
|
479
459
|
}
|
|
480
460
|
if (failedDependencies.size > 0) for (const dep of failedDependencies) warnings.push(`Skipping foreign key validation for table '${tableName}': referenced table '${dep}' has validation errors`);
|
|
481
461
|
}
|
|
482
|
-
const
|
|
462
|
+
const allSkippedDependencies = new Set([...failedDependencies, ...circularDependencies]);
|
|
463
|
+
const { loaded, rowCount, errors: loadErrors } = await this.loadTable(tableName, tableConfig, detailedValidate, transform, allSkippedDependencies);
|
|
483
464
|
errors.push(...loadErrors);
|
|
484
465
|
rowCounts.set(tableName, rowCount);
|
|
485
|
-
if (loaded)
|
|
486
|
-
|
|
466
|
+
if (loaded) {
|
|
467
|
+
loadedTables.add(tableName);
|
|
468
|
+
if (foreignKeys && circularDependencies.size > 0) {
|
|
469
|
+
for (const fk of foreignKeys) if (circularDependencies.has(fk.references.table)) deferredForeignKeys.push({
|
|
470
|
+
tableName,
|
|
471
|
+
foreignKey: fk,
|
|
472
|
+
filePath: tableConfig.jsonlPath
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
} else {
|
|
487
476
|
warnings.push(`Table '${tableName}' was not loaded (no data or skipped)`);
|
|
488
477
|
this.tables.delete(tableName);
|
|
489
478
|
}
|
|
@@ -493,7 +482,8 @@ var LinesDB = class LinesDB {
|
|
|
493
482
|
return {
|
|
494
483
|
errors,
|
|
495
484
|
warnings,
|
|
496
|
-
rowCounts
|
|
485
|
+
rowCounts,
|
|
486
|
+
deferredForeignKeys
|
|
497
487
|
};
|
|
498
488
|
}
|
|
499
489
|
/**
|
|
@@ -509,10 +499,10 @@ var LinesDB = class LinesDB {
|
|
|
509
499
|
validationSchema = await SchemaLoader.loadSchema(config.jsonlPath);
|
|
510
500
|
} catch (_error) {}
|
|
511
501
|
if (!config.validationSchema) try {
|
|
512
|
-
const { pathToFileURL
|
|
502
|
+
const { pathToFileURL } = await import("node:url");
|
|
513
503
|
const schemaPath = await findSchemaFile((0, node_path.dirname)(config.jsonlPath), (0, node_path.basename)(config.jsonlPath, ".jsonl"));
|
|
514
504
|
if (!schemaPath) throw new Error("Schema file not found");
|
|
515
|
-
const schemaModule = await import(`${pathToFileURL
|
|
505
|
+
const schemaModule = await import(`${pathToFileURL(schemaPath).href}?t=${Date.now()}`);
|
|
516
506
|
const schemaExport = schemaModule.schema || schemaModule.default;
|
|
517
507
|
if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
|
|
518
508
|
else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
|
|
@@ -739,6 +729,34 @@ var LinesDB = class LinesDB {
|
|
|
739
729
|
};
|
|
740
730
|
}
|
|
741
731
|
/**
|
|
732
|
+
* Validate a deferred foreign key constraint after all tables have been loaded.
|
|
733
|
+
* Used for circular dependency FK validation.
|
|
734
|
+
*/
|
|
735
|
+
validateDeferredForeignKey(tableName, fk, filePath) {
|
|
736
|
+
const errors = [];
|
|
737
|
+
const quotedTable = this.quoteTableName(tableName);
|
|
738
|
+
const quotedColumn = this.quoteIdentifier(fk.column);
|
|
739
|
+
const quotedRefTable = this.quoteTableName(fk.references.table);
|
|
740
|
+
const sql = `SELECT rowid - 1 as idx, ${quotedColumn} as val FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL AND ${quotedColumn} NOT IN (SELECT ${this.quoteIdentifier(fk.references.column)} FROM ${quotedRefTable})`;
|
|
741
|
+
try {
|
|
742
|
+
const rows = this.query(sql);
|
|
743
|
+
for (const row of rows) errors.push({
|
|
744
|
+
file: filePath,
|
|
745
|
+
tableName,
|
|
746
|
+
rowIndex: row.idx,
|
|
747
|
+
issues: [],
|
|
748
|
+
type: "foreignKey",
|
|
749
|
+
foreignKeyError: {
|
|
750
|
+
column: fk.column,
|
|
751
|
+
value: row.val,
|
|
752
|
+
referencedTable: fk.references.table,
|
|
753
|
+
referencedColumn: fk.references.column
|
|
754
|
+
}
|
|
755
|
+
});
|
|
756
|
+
} catch (_) {}
|
|
757
|
+
return errors;
|
|
758
|
+
}
|
|
759
|
+
/**
|
|
742
760
|
* Execute a raw SQL query
|
|
743
761
|
*/
|
|
744
762
|
query(sql, params = []) {
|
|
@@ -1191,7 +1209,6 @@ var LinesDB = class LinesDB {
|
|
|
1191
1209
|
return this.db;
|
|
1192
1210
|
}
|
|
1193
1211
|
};
|
|
1194
|
-
|
|
1195
1212
|
//#endregion
|
|
1196
1213
|
//#region src/type-generator.ts
|
|
1197
1214
|
var TypeGenerator = class {
|
|
@@ -1298,7 +1315,6 @@ function toCamelCase(value) {
|
|
|
1298
1315
|
function sanitizeIdentifier(value) {
|
|
1299
1316
|
return value.replace(/[^A-Za-z0-9_$]/g, "");
|
|
1300
1317
|
}
|
|
1301
|
-
|
|
1302
1318
|
//#endregion
|
|
1303
1319
|
//#region src/jsonl-migration.ts
|
|
1304
1320
|
/**
|
|
@@ -1325,7 +1341,6 @@ async function ensureTableRowsValid(options) {
|
|
|
1325
1341
|
}
|
|
1326
1342
|
});
|
|
1327
1343
|
}
|
|
1328
|
-
|
|
1329
1344
|
//#endregion
|
|
1330
1345
|
//#region src/error-formatter.ts
|
|
1331
1346
|
var ErrorFormatter = class {
|
|
@@ -1428,7 +1443,6 @@ var ErrorFormatter = class {
|
|
|
1428
1443
|
return (0, node_util.styleText)("red", "\n✗ Migration failed and was rolled back");
|
|
1429
1444
|
}
|
|
1430
1445
|
};
|
|
1431
|
-
|
|
1432
1446
|
//#endregion
|
|
1433
1447
|
exports.DirectoryScanner = DirectoryScanner;
|
|
1434
1448
|
exports.ErrorFormatter = ErrorFormatter;
|
|
@@ -1446,4 +1460,4 @@ exports.extractTableNameFromSchemaFile = extractTableNameFromSchemaFile;
|
|
|
1446
1460
|
exports.findSchemaFile = findSchemaFile;
|
|
1447
1461
|
exports.hasBackward = hasBackward;
|
|
1448
1462
|
exports.isSchemaFile = isSchemaFile;
|
|
1449
|
-
exports.rewriteExtensionForImport = rewriteExtensionForImport;
|
|
1463
|
+
exports.rewriteExtensionForImport = rewriteExtensionForImport;
|