@toiroakr/lines-db 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/bin/{cli.js → cli.mjs} +69 -30
- package/dist/index.cjs +71 -57
- package/dist/index.d.cts +24 -20
- package/dist/index.d.cts.map +1 -1
- package/dist/{index.d.ts → index.d.mts} +24 -20
- package/dist/index.d.mts.map +1 -0
- package/dist/{index.js → index.mjs} +70 -30
- package/dist/index.mjs.map +1 -0
- package/package.json +13 -13
- package/src/cli.ts +24 -54
- package/src/database.test.ts +162 -31
- package/src/database.ts +116 -68
- package/src/directory-scanner.test.ts +1 -3
- package/src/directory-scanner.ts +1 -3
- package/src/index.ts +1 -5
- package/src/jsonl-reader.test.ts +1 -3
- package/src/jsonl-reader.ts +1 -4
- package/src/schema-loader.test.ts +4 -12
- package/src/schema-loader.ts +1 -3
- package/src/schema.ts +5 -5
- package/src/type-generator.ts +4 -11
- package/src/types.ts +1 -4
- package/tsconfig.json +1 -0
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
package/src/database.test.ts
CHANGED
|
@@ -137,10 +137,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
137
137
|
});
|
|
138
138
|
|
|
139
139
|
it('should find rows by condition', async () => {
|
|
140
|
-
await writeTable(
|
|
141
|
-
'users',
|
|
142
|
-
'{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n{"id":3,"name":"Alice"}\n',
|
|
143
|
-
);
|
|
140
|
+
await writeTable('users', '{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n{"id":3,"name":"Alice"}\n');
|
|
144
141
|
|
|
145
142
|
type Tables = {
|
|
146
143
|
users: { id: number; name: string };
|
|
@@ -278,10 +275,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
278
275
|
|
|
279
276
|
describe('update operations', () => {
|
|
280
277
|
it('should update existing rows', async () => {
|
|
281
|
-
await writeTable(
|
|
282
|
-
'users',
|
|
283
|
-
'{"id":1,"name":"Alice","age":30}\n{"id":2,"name":"Bob","age":25}\n',
|
|
284
|
-
);
|
|
278
|
+
await writeTable('users', '{"id":1,"name":"Alice","age":30}\n{"id":2,"name":"Bob","age":25}\n');
|
|
285
279
|
|
|
286
280
|
type Tables = {
|
|
287
281
|
users: { id: number; name: string; age: number };
|
|
@@ -301,10 +295,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
301
295
|
});
|
|
302
296
|
|
|
303
297
|
it('should update multiple rows', async () => {
|
|
304
|
-
await writeTable(
|
|
305
|
-
'users',
|
|
306
|
-
'{"id":1,"name":"Alice","active":true}\n{"id":2,"name":"Bob","active":true}\n',
|
|
307
|
-
);
|
|
298
|
+
await writeTable('users', '{"id":1,"name":"Alice","active":true}\n{"id":2,"name":"Bob","active":true}\n');
|
|
308
299
|
|
|
309
300
|
type Tables = {
|
|
310
301
|
users: { id: number; name: string; active: boolean };
|
|
@@ -321,10 +312,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
321
312
|
});
|
|
322
313
|
|
|
323
314
|
it('should batch update rows with distinct values', async () => {
|
|
324
|
-
await writeTable(
|
|
325
|
-
'users',
|
|
326
|
-
'{"id":1,"name":"Alice","age":30}\n{"id":2,"name":"Bob","age":25}\n',
|
|
327
|
-
);
|
|
315
|
+
await writeTable('users', '{"id":1,"name":"Alice","age":30}\n{"id":2,"name":"Bob","age":25}\n');
|
|
328
316
|
|
|
329
317
|
type Tables = {
|
|
330
318
|
users: { id: number; name: string; age: number };
|
|
@@ -373,10 +361,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
373
361
|
});
|
|
374
362
|
|
|
375
363
|
it('should delete multiple rows', async () => {
|
|
376
|
-
await writeTable(
|
|
377
|
-
'users',
|
|
378
|
-
'{"id":1,"name":"Alice"}\n{"id":2,"name":"Alice"}\n{"id":3,"name":"Bob"}\n',
|
|
379
|
-
);
|
|
364
|
+
await writeTable('users', '{"id":1,"name":"Alice"}\n{"id":2,"name":"Alice"}\n{"id":3,"name":"Bob"}\n');
|
|
380
365
|
|
|
381
366
|
type Tables = {
|
|
382
367
|
users: { id: number; name: string };
|
|
@@ -396,10 +381,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
396
381
|
});
|
|
397
382
|
|
|
398
383
|
it('should batch delete rows by primary key', async () => {
|
|
399
|
-
await writeTable(
|
|
400
|
-
'users',
|
|
401
|
-
'{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n{"id":3,"name":"Carol"}\n',
|
|
402
|
-
);
|
|
384
|
+
await writeTable('users', '{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n{"id":3,"name":"Carol"}\n');
|
|
403
385
|
|
|
404
386
|
type Tables = {
|
|
405
387
|
users: { id: number; name: string };
|
|
@@ -477,10 +459,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
477
459
|
|
|
478
460
|
describe('raw SQL queries', () => {
|
|
479
461
|
it('should execute raw queries', async () => {
|
|
480
|
-
await writeFile(
|
|
481
|
-
join(testDir, 'users.jsonl'),
|
|
482
|
-
'{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n',
|
|
483
|
-
);
|
|
462
|
+
await writeFile(join(testDir, 'users.jsonl'), '{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n');
|
|
484
463
|
|
|
485
464
|
type Tables = {
|
|
486
465
|
users: { id: number; name: string };
|
|
@@ -490,9 +469,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
490
469
|
const db = LinesDB.create(config);
|
|
491
470
|
await db.initialize();
|
|
492
471
|
|
|
493
|
-
const result = db.query<{ id: number; name: string }>('SELECT * FROM users WHERE id > ?', [
|
|
494
|
-
1,
|
|
495
|
-
]);
|
|
472
|
+
const result = db.query<{ id: number; name: string }>('SELECT * FROM users WHERE id > ?', [1]);
|
|
496
473
|
|
|
497
474
|
expect(result).toHaveLength(1);
|
|
498
475
|
expect(result[0].name).toBe('Bob');
|
|
@@ -518,4 +495,158 @@ export const schema = defineSchema(rawSchema);
|
|
|
518
495
|
await db.close();
|
|
519
496
|
});
|
|
520
497
|
});
|
|
498
|
+
|
|
499
|
+
describe('circular foreign key validation', () => {
|
|
500
|
+
const writeSchemaWithFK = async (
|
|
501
|
+
tableName: string,
|
|
502
|
+
primaryKey: string,
|
|
503
|
+
foreignKeys: Array<{ column: string; references: { table: string; column: string } }>,
|
|
504
|
+
uniqueColumns: string[] = [],
|
|
505
|
+
) => {
|
|
506
|
+
const fkJson = JSON.stringify(foreignKeys);
|
|
507
|
+
const indexDefs = uniqueColumns
|
|
508
|
+
.map((col) => `{ name: "${tableName}_${col}_idx", columns: ["${col}"], unique: true }`)
|
|
509
|
+
.join(', ');
|
|
510
|
+
const indexesLine = uniqueColumns.length > 0 ? `indexes: [${indexDefs}],` : '';
|
|
511
|
+
// Use inline StandardSchema without external imports (dynamic import from /tmp can't resolve npm packages)
|
|
512
|
+
const source = `
|
|
513
|
+
const baseSchema = {
|
|
514
|
+
'~standard': {
|
|
515
|
+
version: 1,
|
|
516
|
+
vendor: 'test',
|
|
517
|
+
validate: (value) => ({ value })
|
|
518
|
+
}
|
|
519
|
+
};
|
|
520
|
+
|
|
521
|
+
export const schema = Object.assign(Object.create(baseSchema), {
|
|
522
|
+
'~standard': baseSchema['~standard'],
|
|
523
|
+
primaryKey: "${primaryKey}",
|
|
524
|
+
foreignKeys: ${fkJson},
|
|
525
|
+
${indexesLine}
|
|
526
|
+
});
|
|
527
|
+
`;
|
|
528
|
+
await writeFile(join(testDir, `${tableName}.schema.ts`), source);
|
|
529
|
+
};
|
|
530
|
+
|
|
531
|
+
it('should validate circular foreign keys after all tables are loaded', async () => {
|
|
532
|
+
// authors and profiles reference each other bidirectionally
|
|
533
|
+
await writeFile(
|
|
534
|
+
join(testDir, 'authors.jsonl'),
|
|
535
|
+
'{"id":"1","name":"Alice","profile_code":"a-001"}\n{"id":"2","name":"Bob","profile_code":"b-001"}\n',
|
|
536
|
+
);
|
|
537
|
+
await writeSchemaWithFK(
|
|
538
|
+
'authors',
|
|
539
|
+
'id',
|
|
540
|
+
[{ column: 'profile_code', references: { table: 'profiles', column: 'code' } }],
|
|
541
|
+
['profile_code'],
|
|
542
|
+
);
|
|
543
|
+
|
|
544
|
+
await writeFile(
|
|
545
|
+
join(testDir, 'profiles.jsonl'),
|
|
546
|
+
'{"code":"a-001","author_id":"1","bio":"Author A"}\n{"code":"b-001","author_id":"2","bio":"Author B"}\n',
|
|
547
|
+
);
|
|
548
|
+
await writeSchemaWithFK(
|
|
549
|
+
'profiles',
|
|
550
|
+
'code',
|
|
551
|
+
[{ column: 'author_id', references: { table: 'authors', column: 'id' } }],
|
|
552
|
+
['code'],
|
|
553
|
+
);
|
|
554
|
+
|
|
555
|
+
type Tables = {
|
|
556
|
+
authors: { id: string; name: string; profile_code: string };
|
|
557
|
+
profiles: { code: string; author_id: string; bio: string };
|
|
558
|
+
};
|
|
559
|
+
|
|
560
|
+
const config: DatabaseConfig<Tables> = { dataDir: testDir };
|
|
561
|
+
const db = LinesDB.create(config);
|
|
562
|
+
const result = await db.initialize({ detailedValidate: true });
|
|
563
|
+
|
|
564
|
+
expect(result.valid).toBe(true);
|
|
565
|
+
expect(result.errors).toHaveLength(0);
|
|
566
|
+
|
|
567
|
+
await db.close();
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
it('should detect orphan in authors (no matching profile)', async () => {
|
|
571
|
+
await writeFile(
|
|
572
|
+
join(testDir, 'authors.jsonl'),
|
|
573
|
+
'{"id":"1","name":"Alice","profile_code":"a-001"}\n{"id":"2","name":"Orphan","profile_code":"missing"}\n',
|
|
574
|
+
);
|
|
575
|
+
await writeSchemaWithFK(
|
|
576
|
+
'authors',
|
|
577
|
+
'id',
|
|
578
|
+
[{ column: 'profile_code', references: { table: 'profiles', column: 'code' } }],
|
|
579
|
+
['profile_code'],
|
|
580
|
+
);
|
|
581
|
+
|
|
582
|
+
await writeFile(join(testDir, 'profiles.jsonl'), '{"code":"a-001","author_id":"1","bio":"Author A"}\n');
|
|
583
|
+
await writeSchemaWithFK(
|
|
584
|
+
'profiles',
|
|
585
|
+
'code',
|
|
586
|
+
[{ column: 'author_id', references: { table: 'authors', column: 'id' } }],
|
|
587
|
+
['code'],
|
|
588
|
+
);
|
|
589
|
+
|
|
590
|
+
type Tables = {
|
|
591
|
+
authors: { id: string; name: string; profile_code: string };
|
|
592
|
+
profiles: { code: string; author_id: string; bio: string };
|
|
593
|
+
};
|
|
594
|
+
|
|
595
|
+
const config: DatabaseConfig<Tables> = { dataDir: testDir };
|
|
596
|
+
const db = LinesDB.create(config);
|
|
597
|
+
const result = await db.initialize({ detailedValidate: true });
|
|
598
|
+
|
|
599
|
+
expect(result.valid).toBe(false);
|
|
600
|
+
expect(result.errors).toHaveLength(1);
|
|
601
|
+
expect(result.errors[0].foreignKeyError).toMatchObject({
|
|
602
|
+
column: 'profile_code',
|
|
603
|
+
value: 'missing',
|
|
604
|
+
referencedTable: 'profiles',
|
|
605
|
+
referencedColumn: 'code',
|
|
606
|
+
});
|
|
607
|
+
|
|
608
|
+
await db.close();
|
|
609
|
+
});
|
|
610
|
+
|
|
611
|
+
it('should detect orphan in profiles (no matching author)', async () => {
|
|
612
|
+
await writeFile(join(testDir, 'authors.jsonl'), '{"id":"1","name":"Alice","profile_code":"a-001"}\n');
|
|
613
|
+
await writeSchemaWithFK(
|
|
614
|
+
'authors',
|
|
615
|
+
'id',
|
|
616
|
+
[{ column: 'profile_code', references: { table: 'profiles', column: 'code' } }],
|
|
617
|
+
['profile_code'],
|
|
618
|
+
);
|
|
619
|
+
|
|
620
|
+
await writeFile(
|
|
621
|
+
join(testDir, 'profiles.jsonl'),
|
|
622
|
+
'{"code":"a-001","author_id":"1","bio":"Author A"}\n{"code":"x-999","author_id":"999","bio":"Orphan"}\n',
|
|
623
|
+
);
|
|
624
|
+
await writeSchemaWithFK(
|
|
625
|
+
'profiles',
|
|
626
|
+
'code',
|
|
627
|
+
[{ column: 'author_id', references: { table: 'authors', column: 'id' } }],
|
|
628
|
+
['code'],
|
|
629
|
+
);
|
|
630
|
+
|
|
631
|
+
type Tables = {
|
|
632
|
+
authors: { id: string; name: string; profile_code: string };
|
|
633
|
+
profiles: { code: string; author_id: string; bio: string };
|
|
634
|
+
};
|
|
635
|
+
|
|
636
|
+
const config: DatabaseConfig<Tables> = { dataDir: testDir };
|
|
637
|
+
const db = LinesDB.create(config);
|
|
638
|
+
const result = await db.initialize({ detailedValidate: true });
|
|
639
|
+
|
|
640
|
+
expect(result.valid).toBe(false);
|
|
641
|
+
expect(result.errors).toHaveLength(1);
|
|
642
|
+
expect(result.errors[0].foreignKeyError).toMatchObject({
|
|
643
|
+
column: 'author_id',
|
|
644
|
+
value: '999',
|
|
645
|
+
referencedTable: 'authors',
|
|
646
|
+
referencedColumn: 'id',
|
|
647
|
+
});
|
|
648
|
+
|
|
649
|
+
await db.close();
|
|
650
|
+
});
|
|
651
|
+
});
|
|
521
652
|
});
|
package/src/database.ts
CHANGED
|
@@ -36,10 +36,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
36
36
|
this.db = createDatabase(dbPath ?? ':memory:');
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
static create<Tables extends TableDefs>(
|
|
40
|
-
config: DatabaseConfig<Tables>,
|
|
41
|
-
dbPath?: string,
|
|
42
|
-
): LinesDB<Tables> {
|
|
39
|
+
static create<Tables extends TableDefs>(config: DatabaseConfig<Tables>, dbPath?: string): LinesDB<Tables> {
|
|
43
40
|
return new LinesDB<Tables>(config, dbPath);
|
|
44
41
|
}
|
|
45
42
|
|
|
@@ -73,9 +70,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
73
70
|
// Validate that all requested tables exist BEFORE starting to load
|
|
74
71
|
for (const tableNameToLoad of tablesToLoad) {
|
|
75
72
|
if (!this.tables.has(tableNameToLoad)) {
|
|
76
|
-
throw new Error(
|
|
77
|
-
`Table '${tableNameToLoad}' not found in directory '${this.config.dataDir}'`,
|
|
78
|
-
);
|
|
73
|
+
throw new Error(`Table '${tableNameToLoad}' not found in directory '${this.config.dataDir}'`);
|
|
79
74
|
}
|
|
80
75
|
}
|
|
81
76
|
|
|
@@ -83,6 +78,11 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
83
78
|
const loadedTables = new Set<string>();
|
|
84
79
|
const loadingTables = new Set<string>();
|
|
85
80
|
const attemptedTables = new Set<string>(); // Track all attempted tables (loaded or not)
|
|
81
|
+
const allDeferredForeignKeys: Array<{
|
|
82
|
+
tableName: string;
|
|
83
|
+
foreignKey: ForeignKeyDefinition;
|
|
84
|
+
filePath: string;
|
|
85
|
+
}> = [];
|
|
86
86
|
|
|
87
87
|
// Load tables with dependency resolution
|
|
88
88
|
for (const tableNameToLoad of tablesToLoad) {
|
|
@@ -93,6 +93,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
93
93
|
errors,
|
|
94
94
|
warnings,
|
|
95
95
|
rowCounts: tableRowCounts,
|
|
96
|
+
deferredForeignKeys,
|
|
96
97
|
} = await this.loadTableWithDependencies(
|
|
97
98
|
tableNameToLoad,
|
|
98
99
|
loadedTables,
|
|
@@ -103,12 +104,25 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
103
104
|
);
|
|
104
105
|
allErrors.push(...errors);
|
|
105
106
|
allWarnings.push(...warnings);
|
|
107
|
+
allDeferredForeignKeys.push(...deferredForeignKeys);
|
|
106
108
|
for (const [k, v] of tableRowCounts) {
|
|
107
109
|
allRowCounts.set(k, v);
|
|
108
110
|
}
|
|
109
111
|
}
|
|
110
112
|
}
|
|
111
113
|
|
|
114
|
+
// Validate deferred foreign keys (from circular dependencies) now that all tables are loaded
|
|
115
|
+
if (detailedValidate && allDeferredForeignKeys.length > 0) {
|
|
116
|
+
for (const { tableName: tName, foreignKey: fk, filePath } of allDeferredForeignKeys) {
|
|
117
|
+
// Only validate if the referenced table was actually loaded
|
|
118
|
+
if (!loadedTables.has(fk.references.table)) {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const deferredErrors = this.validateDeferredForeignKey(tName, fk, filePath);
|
|
122
|
+
allErrors.push(...deferredErrors);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
112
126
|
// Build per-table results
|
|
113
127
|
const tableResults: TableValidationResult[] = tablesToLoad.map((name) => {
|
|
114
128
|
const tableErrors = allErrors.filter((e) => e.tableName === name);
|
|
@@ -144,14 +158,24 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
144
158
|
errors: ValidationErrorDetail[];
|
|
145
159
|
warnings: string[];
|
|
146
160
|
rowCounts: Map<string, number>;
|
|
161
|
+
deferredForeignKeys: Array<{
|
|
162
|
+
tableName: string;
|
|
163
|
+
foreignKey: ForeignKeyDefinition;
|
|
164
|
+
filePath: string;
|
|
165
|
+
}>;
|
|
147
166
|
}> {
|
|
148
167
|
const errors: ValidationErrorDetail[] = [];
|
|
149
168
|
const warnings: string[] = [];
|
|
150
169
|
const rowCounts = new Map<string, number>();
|
|
170
|
+
const deferredForeignKeys: Array<{
|
|
171
|
+
tableName: string;
|
|
172
|
+
foreignKey: ForeignKeyDefinition;
|
|
173
|
+
filePath: string;
|
|
174
|
+
}> = [];
|
|
151
175
|
|
|
152
176
|
// Skip if already attempted (loaded or not)
|
|
153
177
|
if (attemptedTables.has(tableName)) {
|
|
154
|
-
return { errors, warnings, rowCounts };
|
|
178
|
+
return { errors, warnings, rowCounts, deferredForeignKeys };
|
|
155
179
|
}
|
|
156
180
|
|
|
157
181
|
// Mark as attempted
|
|
@@ -218,6 +242,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
218
242
|
);
|
|
219
243
|
errors.push(...depResult.errors);
|
|
220
244
|
warnings.push(...depResult.warnings);
|
|
245
|
+
deferredForeignKeys.push(...depResult.deferredForeignKeys);
|
|
221
246
|
for (const [k, v] of depResult.rowCounts) {
|
|
222
247
|
rowCounts.set(k, v);
|
|
223
248
|
}
|
|
@@ -230,14 +255,21 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
230
255
|
}
|
|
231
256
|
}
|
|
232
257
|
|
|
233
|
-
// Determine which FK dependencies failed
|
|
258
|
+
// Determine which FK dependencies failed or are circular (attempted but not loaded)
|
|
234
259
|
const failedDependencies = new Set<string>();
|
|
260
|
+
const circularDependencies = new Set<string>();
|
|
235
261
|
if (foreignKeys && foreignKeys.length > 0) {
|
|
236
262
|
for (const fk of foreignKeys) {
|
|
237
263
|
const referencedTable = fk.references.table;
|
|
238
264
|
if (referencedTable === tableName) continue;
|
|
239
265
|
if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable)) {
|
|
240
|
-
|
|
266
|
+
if (loadingTables.has(referencedTable)) {
|
|
267
|
+
// Circular dependency: table is currently being loaded
|
|
268
|
+
circularDependencies.add(referencedTable);
|
|
269
|
+
} else {
|
|
270
|
+
// Actual failure: table attempted but not loaded
|
|
271
|
+
failedDependencies.add(referencedTable);
|
|
272
|
+
}
|
|
241
273
|
}
|
|
242
274
|
}
|
|
243
275
|
if (failedDependencies.size > 0) {
|
|
@@ -249,23 +281,33 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
249
281
|
}
|
|
250
282
|
}
|
|
251
283
|
|
|
284
|
+
// Combine failed and circular dependencies for table loading (both need FK skipping)
|
|
285
|
+
const allSkippedDependencies = new Set([...failedDependencies, ...circularDependencies]);
|
|
286
|
+
|
|
252
287
|
// Now load this table
|
|
253
288
|
const {
|
|
254
289
|
loaded,
|
|
255
290
|
rowCount,
|
|
256
291
|
errors: loadErrors,
|
|
257
|
-
} = await this.loadTable(
|
|
258
|
-
tableName,
|
|
259
|
-
tableConfig,
|
|
260
|
-
detailedValidate,
|
|
261
|
-
transform,
|
|
262
|
-
failedDependencies,
|
|
263
|
-
);
|
|
292
|
+
} = await this.loadTable(tableName, tableConfig, detailedValidate, transform, allSkippedDependencies);
|
|
264
293
|
errors.push(...loadErrors);
|
|
265
294
|
rowCounts.set(tableName, rowCount);
|
|
266
295
|
|
|
267
296
|
if (loaded) {
|
|
268
297
|
loadedTables.add(tableName);
|
|
298
|
+
|
|
299
|
+
// Track circular dependency FKs for deferred validation
|
|
300
|
+
if (foreignKeys && circularDependencies.size > 0) {
|
|
301
|
+
for (const fk of foreignKeys) {
|
|
302
|
+
if (circularDependencies.has(fk.references.table)) {
|
|
303
|
+
deferredForeignKeys.push({
|
|
304
|
+
tableName,
|
|
305
|
+
foreignKey: fk,
|
|
306
|
+
filePath: tableConfig.jsonlPath,
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
269
311
|
} else {
|
|
270
312
|
// Table was not loaded (e.g., empty data)
|
|
271
313
|
warnings.push(`Table '${tableName}' was not loaded (no data or skipped)`);
|
|
@@ -276,7 +318,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
276
318
|
loadingTables.delete(tableName);
|
|
277
319
|
}
|
|
278
320
|
|
|
279
|
-
return { errors, warnings, rowCounts };
|
|
321
|
+
return { errors, warnings, rowCounts, deferredForeignKeys };
|
|
280
322
|
}
|
|
281
323
|
|
|
282
324
|
/**
|
|
@@ -320,10 +362,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
320
362
|
// Only load if not already provided via config
|
|
321
363
|
try {
|
|
322
364
|
const { pathToFileURL } = await import('node:url');
|
|
323
|
-
const schemaPath = await findSchemaFile(
|
|
324
|
-
dirname(config.jsonlPath),
|
|
325
|
-
basename(config.jsonlPath, '.jsonl'),
|
|
326
|
-
);
|
|
365
|
+
const schemaPath = await findSchemaFile(dirname(config.jsonlPath), basename(config.jsonlPath, '.jsonl'));
|
|
327
366
|
if (!schemaPath) throw new Error('Schema file not found');
|
|
328
367
|
const schemaUrl = pathToFileURL(schemaPath).href;
|
|
329
368
|
const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
|
|
@@ -491,12 +530,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
491
530
|
|
|
492
531
|
// Insert validated data (with detailed validation if requested)
|
|
493
532
|
if (detailedValidate) {
|
|
494
|
-
const insertErrors = this.insertDataWithDetailedValidation(
|
|
495
|
-
tableName,
|
|
496
|
-
schema,
|
|
497
|
-
validatedData,
|
|
498
|
-
config.jsonlPath,
|
|
499
|
-
);
|
|
533
|
+
const insertErrors = this.insertDataWithDetailedValidation(tableName, schema, validatedData, config.jsonlPath);
|
|
500
534
|
if (insertErrors.length > 0) {
|
|
501
535
|
return { loaded: false, rowCount: data.length, errors: insertErrors };
|
|
502
536
|
}
|
|
@@ -573,8 +607,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
573
607
|
const index = schema.indexes[i];
|
|
574
608
|
// Create safe index name by replacing special characters
|
|
575
609
|
const safeTableName = schema.name.replace(/[^a-zA-Z0-9]/g, '_');
|
|
576
|
-
const resolvedIndexName =
|
|
577
|
-
index.name || `idx_${safeTableName}_${index.columns.join('_')}_${i}`;
|
|
610
|
+
const resolvedIndexName = index.name || `idx_${safeTableName}_${index.columns.join('_')}_${i}`;
|
|
578
611
|
const uniqueKeyword = index.unique ? 'UNIQUE ' : '';
|
|
579
612
|
const indexSql = `CREATE ${uniqueKeyword}INDEX IF NOT EXISTS ${this.quoteIdentifier(resolvedIndexName)} ON ${quotedTableName} (${index.columns
|
|
580
613
|
.map((col) => this.quoteIdentifier(col))
|
|
@@ -738,13 +771,52 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
738
771
|
};
|
|
739
772
|
}
|
|
740
773
|
|
|
774
|
+
/**
|
|
775
|
+
* Validate a deferred foreign key constraint after all tables have been loaded.
|
|
776
|
+
* Used for circular dependency FK validation.
|
|
777
|
+
*/
|
|
778
|
+
private validateDeferredForeignKey(
|
|
779
|
+
tableName: string,
|
|
780
|
+
fk: ForeignKeyDefinition,
|
|
781
|
+
filePath: string,
|
|
782
|
+
): ValidationErrorDetail[] {
|
|
783
|
+
const errors: ValidationErrorDetail[] = [];
|
|
784
|
+
const quotedTable = this.quoteTableName(tableName);
|
|
785
|
+
const quotedColumn = this.quoteIdentifier(fk.column);
|
|
786
|
+
const quotedRefTable = this.quoteTableName(fk.references.table);
|
|
787
|
+
const quotedRefColumn = this.quoteIdentifier(fk.references.column);
|
|
788
|
+
|
|
789
|
+
// Find rows where the FK value does not exist in the referenced table
|
|
790
|
+
const sql = `SELECT rowid - 1 as idx, ${quotedColumn} as val FROM ${quotedTable} WHERE ${quotedColumn} IS NOT NULL AND ${quotedColumn} NOT IN (SELECT ${quotedRefColumn} FROM ${quotedRefTable})`;
|
|
791
|
+
|
|
792
|
+
try {
|
|
793
|
+
const rows = this.query<{ idx: number; val: string | number }>(sql);
|
|
794
|
+
for (const row of rows) {
|
|
795
|
+
errors.push({
|
|
796
|
+
file: filePath,
|
|
797
|
+
tableName,
|
|
798
|
+
rowIndex: row.idx,
|
|
799
|
+
issues: [],
|
|
800
|
+
type: 'foreignKey',
|
|
801
|
+
foreignKeyError: {
|
|
802
|
+
column: fk.column,
|
|
803
|
+
value: row.val,
|
|
804
|
+
referencedTable: fk.references.table,
|
|
805
|
+
referencedColumn: fk.references.column,
|
|
806
|
+
},
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
} catch (_) {
|
|
810
|
+
// Table might not exist - skip validation
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
return errors;
|
|
814
|
+
}
|
|
815
|
+
|
|
741
816
|
/**
|
|
742
817
|
* Execute a raw SQL query
|
|
743
818
|
*/
|
|
744
|
-
query<T = unknown>(
|
|
745
|
-
sql: string,
|
|
746
|
-
params: (string | number | bigint | null | Uint8Array)[] = [],
|
|
747
|
-
): T[] {
|
|
819
|
+
query<T = unknown>(sql: string, params: (string | number | bigint | null | Uint8Array)[] = []): T[] {
|
|
748
820
|
const stmt = this.db.prepare(sql);
|
|
749
821
|
return stmt.all(...params) as T[];
|
|
750
822
|
}
|
|
@@ -752,10 +824,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
752
824
|
/**
|
|
753
825
|
* Execute a SQL query that returns a single row
|
|
754
826
|
*/
|
|
755
|
-
queryOne<T = unknown>(
|
|
756
|
-
sql: string,
|
|
757
|
-
params: (string | number | bigint | null | Uint8Array)[] = [],
|
|
758
|
-
): T | null {
|
|
827
|
+
queryOne<T = unknown>(sql: string, params: (string | number | bigint | null | Uint8Array)[] = []): T | null {
|
|
759
828
|
const stmt = this.db.prepare(sql);
|
|
760
829
|
const result = stmt.get(...params);
|
|
761
830
|
return result === undefined ? null : (result as T);
|
|
@@ -801,10 +870,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
801
870
|
|
|
802
871
|
// Normal case: use SQL WHERE clause
|
|
803
872
|
if (sql) {
|
|
804
|
-
const rawRows = this.query(
|
|
805
|
-
`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`,
|
|
806
|
-
values,
|
|
807
|
-
);
|
|
873
|
+
const rawRows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values);
|
|
808
874
|
rows = rawRows.map((row) => this.deserializeRow(tableName, row)) as Tables[K][];
|
|
809
875
|
} else {
|
|
810
876
|
// If only function filters (AND case), get all rows
|
|
@@ -824,10 +890,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
824
890
|
|
|
825
891
|
let rows: Tables[K][];
|
|
826
892
|
if (sql) {
|
|
827
|
-
const rawRows = this.query(
|
|
828
|
-
`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`,
|
|
829
|
-
values,
|
|
830
|
-
);
|
|
893
|
+
const rawRows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values);
|
|
831
894
|
rows = rawRows.map((row) => this.deserializeRow(tableName, row)) as Tables[K][];
|
|
832
895
|
} else {
|
|
833
896
|
// If only function filters, get all rows
|
|
@@ -891,9 +954,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
891
954
|
|
|
892
955
|
// Only synchronous validation is supported
|
|
893
956
|
if (result instanceof Promise) {
|
|
894
|
-
throw new Error(
|
|
895
|
-
'Asynchronous validation is not supported. Please use synchronous validation schemas.',
|
|
896
|
-
);
|
|
957
|
+
throw new Error('Asynchronous validation is not supported. Please use synchronous validation schemas.');
|
|
897
958
|
}
|
|
898
959
|
|
|
899
960
|
if (result.issues && result.issues.length > 0) {
|
|
@@ -1114,9 +1175,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1114
1175
|
for (const record of records) {
|
|
1115
1176
|
const pkValue = record[pkName];
|
|
1116
1177
|
if (pkValue === undefined) {
|
|
1117
|
-
throw new Error(
|
|
1118
|
-
`Record is missing primary key '${String(pkName)}': ${JSON.stringify(record)}`,
|
|
1119
|
-
);
|
|
1178
|
+
throw new Error(`Record is missing primary key '${String(pkName)}': ${JSON.stringify(record)}`);
|
|
1120
1179
|
}
|
|
1121
1180
|
pkValues.push(pkValue);
|
|
1122
1181
|
}
|
|
@@ -1155,9 +1214,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1155
1214
|
const existingRow = existingRowsMap.get(pkValue);
|
|
1156
1215
|
|
|
1157
1216
|
if (!existingRow) {
|
|
1158
|
-
throw new Error(
|
|
1159
|
-
`No existing row found with ${String(pkName)}=${JSON.stringify(pkValue)}`,
|
|
1160
|
-
);
|
|
1217
|
+
throw new Error(`No existing row found with ${String(pkName)}=${JSON.stringify(pkValue)}`);
|
|
1161
1218
|
}
|
|
1162
1219
|
|
|
1163
1220
|
const mergedData = { ...existingRow, ...record };
|
|
@@ -1301,8 +1358,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1301
1358
|
private normalizeValue(value: unknown): string | number | bigint | null | Uint8Array {
|
|
1302
1359
|
if (value === null || value === undefined) return null;
|
|
1303
1360
|
if (typeof value === 'boolean') return value ? 1 : 0;
|
|
1304
|
-
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'bigint')
|
|
1305
|
-
return value;
|
|
1361
|
+
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'bigint') return value;
|
|
1306
1362
|
if (value instanceof Uint8Array) return value;
|
|
1307
1363
|
// For objects, convert to JSON string
|
|
1308
1364
|
return JSON.stringify(value);
|
|
@@ -1331,9 +1387,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1331
1387
|
if (Array.isArray(cond)) {
|
|
1332
1388
|
const clauses = cond
|
|
1333
1389
|
.map((item) => {
|
|
1334
|
-
const clause = Array.isArray(item)
|
|
1335
|
-
? buildCondition(item, true)
|
|
1336
|
-
: buildCondition(item, true);
|
|
1390
|
+
const clause = Array.isArray(item) ? buildCondition(item, true) : buildCondition(item, true);
|
|
1337
1391
|
return clause ? `(${clause})` : '';
|
|
1338
1392
|
})
|
|
1339
1393
|
.filter((clause) => clause !== ''); // Filter out empty clauses
|
|
@@ -1370,20 +1424,14 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1370
1424
|
/**
|
|
1371
1425
|
* Apply OR condition with function filters by evaluating each row against the condition
|
|
1372
1426
|
*/
|
|
1373
|
-
private applyOrConditionWithFilters<T extends Record<string, unknown>>(
|
|
1374
|
-
rows: T[],
|
|
1375
|
-
condition: WhereCondition<T>,
|
|
1376
|
-
): T[] {
|
|
1427
|
+
private applyOrConditionWithFilters<T extends Record<string, unknown>>(rows: T[], condition: WhereCondition<T>): T[] {
|
|
1377
1428
|
return rows.filter((row) => this.matchesOrCondition(row, condition));
|
|
1378
1429
|
}
|
|
1379
1430
|
|
|
1380
1431
|
/**
|
|
1381
1432
|
* Check if a row matches an OR/AND condition (recursively)
|
|
1382
1433
|
*/
|
|
1383
|
-
private matchesOrCondition<T extends Record<string, unknown>>(
|
|
1384
|
-
row: T,
|
|
1385
|
-
condition: WhereCondition<T>,
|
|
1386
|
-
): boolean {
|
|
1434
|
+
private matchesOrCondition<T extends Record<string, unknown>>(row: T, condition: WhereCondition<T>): boolean {
|
|
1387
1435
|
// Handle array (OR conditions)
|
|
1388
1436
|
if (Array.isArray(condition)) {
|
|
1389
1437
|
return condition.some((item) => this.matchesOrCondition(row, item));
|
|
@@ -59,9 +59,7 @@ describe('DirectoryScanner', () => {
|
|
|
59
59
|
it('should throw error for non-existent directory', async () => {
|
|
60
60
|
const nonExistentDir = join(testDir, 'nonexistent');
|
|
61
61
|
|
|
62
|
-
await expect(DirectoryScanner.scanDirectory(nonExistentDir)).rejects.toThrow(
|
|
63
|
-
'Failed to scan directory',
|
|
64
|
-
);
|
|
62
|
+
await expect(DirectoryScanner.scanDirectory(nonExistentDir)).rejects.toThrow('Failed to scan directory');
|
|
65
63
|
});
|
|
66
64
|
|
|
67
65
|
it('should handle multiple JSONL files', async () => {
|
package/src/directory-scanner.ts
CHANGED
|
@@ -30,9 +30,7 @@ export class DirectoryScanner {
|
|
|
30
30
|
|
|
31
31
|
return tables;
|
|
32
32
|
} catch (error) {
|
|
33
|
-
throw new Error(
|
|
34
|
-
`Failed to scan directory ${dataDir}: ${error instanceof Error ? error.message : String(error)}`,
|
|
35
|
-
);
|
|
33
|
+
throw new Error(`Failed to scan directory ${dataDir}: ${error instanceof Error ? error.message : String(error)}`);
|
|
36
34
|
}
|
|
37
35
|
}
|
|
38
36
|
}
|
package/src/index.ts
CHANGED
|
@@ -16,11 +16,7 @@ export {
|
|
|
16
16
|
} from './schema-extensions.js';
|
|
17
17
|
export type { SchemaExtension } from './schema-extensions.js';
|
|
18
18
|
export { ErrorFormatter } from './error-formatter.js';
|
|
19
|
-
export type {
|
|
20
|
-
ErrorFormatterOptions,
|
|
21
|
-
ValidationErrorInfo,
|
|
22
|
-
ForeignKeyErrorInfo,
|
|
23
|
-
} from './error-formatter.js';
|
|
19
|
+
export type { ErrorFormatterOptions, ValidationErrorInfo, ForeignKeyErrorInfo } from './error-formatter.js';
|
|
24
20
|
export { detectRuntime, RUNTIME } from './runtime.js';
|
|
25
21
|
export type { RuntimeEnvironment } from './runtime.js';
|
|
26
22
|
export type { SQLiteDatabase, SQLiteStatement } from './sqlite-adapter.js';
|
package/src/jsonl-reader.test.ts
CHANGED
|
@@ -148,9 +148,7 @@ describe('JsonlReader', () => {
|
|
|
148
148
|
});
|
|
149
149
|
|
|
150
150
|
it('should throw error for empty data', () => {
|
|
151
|
-
expect(() => JsonlReader.inferSchema('empty', [])).toThrow(
|
|
152
|
-
'Cannot infer schema from empty data',
|
|
153
|
-
);
|
|
151
|
+
expect(() => JsonlReader.inferSchema('empty', [])).toThrow('Cannot infer schema from empty data');
|
|
154
152
|
});
|
|
155
153
|
|
|
156
154
|
it('should handle REAL numbers', () => {
|