@toiroakr/lines-db 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/bin/{cli.js → cli.mjs} +12 -23
- package/dist/index.cjs +14 -50
- package/dist/index.d.cts +19 -20
- package/dist/index.d.cts.map +1 -1
- package/dist/{index.d.ts → index.d.mts} +19 -20
- package/dist/index.d.mts.map +1 -0
- package/dist/{index.js → index.mjs} +13 -23
- package/dist/index.mjs.map +1 -0
- package/package.json +13 -13
- package/src/cli.ts +24 -54
- package/src/database.test.ts +10 -39
- package/src/database.ts +17 -64
- package/src/directory-scanner.test.ts +1 -3
- package/src/directory-scanner.ts +1 -3
- package/src/index.ts +1 -5
- package/src/jsonl-reader.test.ts +1 -3
- package/src/jsonl-reader.ts +1 -4
- package/src/schema-loader.test.ts +4 -12
- package/src/schema-loader.ts +1 -3
- package/src/schema.ts +5 -5
- package/src/type-generator.ts +4 -11
- package/src/types.ts +1 -4
- package/tsconfig.json +1 -0
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
package/package.json
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@toiroakr/lines-db",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.2",
|
|
4
4
|
"description": "A database implementation that treats JSONL files as tables using SQLite",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.cjs",
|
|
7
|
-
"module": "./dist/index.
|
|
8
|
-
"types": "./dist/index.d.
|
|
7
|
+
"module": "./dist/index.mjs",
|
|
8
|
+
"types": "./dist/index.d.mts",
|
|
9
9
|
"bin": {
|
|
10
|
-
"lines-db": "./bin/cli.
|
|
10
|
+
"lines-db": "./bin/cli.mjs"
|
|
11
11
|
},
|
|
12
12
|
"exports": {
|
|
13
13
|
".": {
|
|
14
14
|
"import": {
|
|
15
|
-
"types": "./dist/index.d.
|
|
16
|
-
"default": "./dist/index.
|
|
15
|
+
"types": "./dist/index.d.mts",
|
|
16
|
+
"default": "./dist/index.mjs"
|
|
17
17
|
},
|
|
18
18
|
"require": {
|
|
19
19
|
"types": "./dist/index.d.cts",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
}
|
|
23
23
|
},
|
|
24
24
|
"scripts": {
|
|
25
|
-
"build": "tsdown
|
|
25
|
+
"build": "tsdown",
|
|
26
26
|
"typecheck": "tsc --noEmit",
|
|
27
27
|
"test": "vitest run"
|
|
28
28
|
},
|
|
@@ -43,12 +43,12 @@
|
|
|
43
43
|
},
|
|
44
44
|
"homepage": "https://github.com/toiroakr/lines-db#readme",
|
|
45
45
|
"devDependencies": {
|
|
46
|
-
"@types/node": "
|
|
47
|
-
"tsdown": "
|
|
48
|
-
"type-fest": "
|
|
49
|
-
"typescript": "
|
|
50
|
-
"valibot": "
|
|
51
|
-
"vitest": "
|
|
46
|
+
"@types/node": "24.12.2",
|
|
47
|
+
"tsdown": "0.21.10",
|
|
48
|
+
"type-fest": "5.6.0",
|
|
49
|
+
"typescript": "6.0.3",
|
|
50
|
+
"valibot": "1.3.1",
|
|
51
|
+
"vitest": "4.1.5"
|
|
52
52
|
},
|
|
53
53
|
"peerDependencies": {
|
|
54
54
|
"valibot": ">=1.0.0"
|
package/src/cli.ts
CHANGED
|
@@ -16,12 +16,7 @@ import { runInNewContext } from 'node:vm';
|
|
|
16
16
|
|
|
17
17
|
const originalEmitWarning = process.emitWarning;
|
|
18
18
|
process.emitWarning = (warning, ...args) => {
|
|
19
|
-
if (
|
|
20
|
-
typeof warning === 'string' &&
|
|
21
|
-
warning.startsWith('SQLite') &&
|
|
22
|
-
args[0] === 'ExperimentalWarning'
|
|
23
|
-
)
|
|
24
|
-
return;
|
|
19
|
+
if (typeof warning === 'string' && warning.startsWith('SQLite') && args[0] === 'ExperimentalWarning') return;
|
|
25
20
|
originalEmitWarning(warning, ...(args as any[]));
|
|
26
21
|
};
|
|
27
22
|
|
|
@@ -52,10 +47,7 @@ function runInSandbox<T>(expression: string, context: Record<string, unknown> =
|
|
|
52
47
|
|
|
53
48
|
const program = new Command();
|
|
54
49
|
|
|
55
|
-
program
|
|
56
|
-
.name('@toiroakr/lines-db')
|
|
57
|
-
.description('Database utilities for JSONL files')
|
|
58
|
-
.version('1.0.0');
|
|
50
|
+
program.name('@toiroakr/lines-db').description('Database utilities for JSONL files').version('1.0.0');
|
|
59
51
|
|
|
60
52
|
// Generate command
|
|
61
53
|
program
|
|
@@ -113,9 +105,7 @@ program
|
|
|
113
105
|
for (const tableResult of result.tableResults) {
|
|
114
106
|
if (tableResult.valid && tableResult.warnings.length === 0) {
|
|
115
107
|
// Success
|
|
116
|
-
console.log(
|
|
117
|
-
styleText('green', `✓ ${tableResult.tableName} (${tableResult.rowCount} records)`),
|
|
118
|
-
);
|
|
108
|
+
console.log(styleText('green', `✓ ${tableResult.tableName} (${tableResult.rowCount} records)`));
|
|
119
109
|
} else if (tableResult.valid && tableResult.warnings.length > 0) {
|
|
120
110
|
// Warnings
|
|
121
111
|
for (const warning of tableResult.warnings) {
|
|
@@ -127,12 +117,8 @@ program
|
|
|
127
117
|
console.error(formatter.formatErrorHeader(fileErrors.length, fileErrors[0]?.file));
|
|
128
118
|
console.error('');
|
|
129
119
|
|
|
130
|
-
const validationErrors = fileErrors.filter(
|
|
131
|
-
|
|
132
|
-
);
|
|
133
|
-
const foreignKeyErrors = fileErrors.filter(
|
|
134
|
-
(e) => e.type === 'foreignKey' && e.foreignKeyError,
|
|
135
|
-
);
|
|
120
|
+
const validationErrors = fileErrors.filter((e) => e.type !== 'foreignKey' || !e.foreignKeyError);
|
|
121
|
+
const foreignKeyErrors = fileErrors.filter((e) => e.type === 'foreignKey' && e.foreignKeyError);
|
|
136
122
|
|
|
137
123
|
if (validationErrors.length > 0) {
|
|
138
124
|
console.error(
|
|
@@ -196,12 +182,8 @@ program
|
|
|
196
182
|
console.error(formatter.formatErrorHeader(fileErrors.length, fileErrors[0]?.file));
|
|
197
183
|
console.error('');
|
|
198
184
|
|
|
199
|
-
const validationErrors = fileErrors.filter(
|
|
200
|
-
|
|
201
|
-
);
|
|
202
|
-
const foreignKeyErrors = fileErrors.filter(
|
|
203
|
-
(e) => e.type === 'foreignKey' && e.foreignKeyError,
|
|
204
|
-
);
|
|
185
|
+
const validationErrors = fileErrors.filter((e) => e.type !== 'foreignKey' || !e.foreignKeyError);
|
|
186
|
+
const foreignKeyErrors = fileErrors.filter((e) => e.type === 'foreignKey' && e.foreignKeyError);
|
|
205
187
|
|
|
206
188
|
if (validationErrors.length > 0) {
|
|
207
189
|
console.error(
|
|
@@ -271,11 +253,7 @@ program
|
|
|
271
253
|
process.exit(1);
|
|
272
254
|
}
|
|
273
255
|
} catch (error) {
|
|
274
|
-
if (
|
|
275
|
-
error instanceof Error &&
|
|
276
|
-
'code' in error &&
|
|
277
|
-
(error as NodeJS.ErrnoException).code === 'ENOENT'
|
|
278
|
-
) {
|
|
256
|
+
if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
|
|
279
257
|
console.error(`Error: Path not found: ${path}`);
|
|
280
258
|
} else {
|
|
281
259
|
console.error(`Error: ${String(error)}`);
|
|
@@ -374,9 +352,7 @@ async function migrateDirectory(
|
|
|
374
352
|
console.log(`Processing table '${tableName}'...`);
|
|
375
353
|
|
|
376
354
|
// Get rows to migrate
|
|
377
|
-
const rowsToMigrate = filter
|
|
378
|
-
? db.find(tableName, filter as Parameters<typeof db.find>[1])
|
|
379
|
-
: db.find(tableName);
|
|
355
|
+
const rowsToMigrate = filter ? db.find(tableName, filter as Parameters<typeof db.find>[1]) : db.find(tableName);
|
|
380
356
|
|
|
381
357
|
if (rowsToMigrate.length === 0) {
|
|
382
358
|
console.log(` No rows to migrate`);
|
|
@@ -414,23 +390,19 @@ async function migrateDirectory(
|
|
|
414
390
|
};
|
|
415
391
|
|
|
416
392
|
if (validationError.validationErrors) {
|
|
417
|
-
console.error(
|
|
418
|
-
` Found ${validationError.validationErrors.length} validation error(s):\n`,
|
|
419
|
-
);
|
|
393
|
+
console.error(` Found ${validationError.validationErrors.length} validation error(s):\n`);
|
|
420
394
|
|
|
421
395
|
const rowsToMigrate = filter
|
|
422
396
|
? db.find(tableName, filter as Parameters<typeof db.find>[1])
|
|
423
397
|
: db.find(tableName);
|
|
424
398
|
|
|
425
|
-
const errorInfos = validationError.validationErrors.map(
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
}),
|
|
433
|
-
);
|
|
399
|
+
const errorInfos = validationError.validationErrors.map(({ rowIndex, rowData, error: rowError }) => ({
|
|
400
|
+
file: `${dirPath}/${tableName}.jsonl`,
|
|
401
|
+
rowIndex,
|
|
402
|
+
issues: rowError.issues,
|
|
403
|
+
data: rowData,
|
|
404
|
+
originalData: rowsToMigrate[rowIndex],
|
|
405
|
+
}));
|
|
434
406
|
|
|
435
407
|
const formatted = formatter.formatValidationErrors(errorInfos);
|
|
436
408
|
console.error(formatted);
|
|
@@ -622,15 +594,13 @@ async function migrateFile(
|
|
|
622
594
|
`\nFound ${validationError.validationErrors.length} validation error(s) in transformed data:\n`,
|
|
623
595
|
);
|
|
624
596
|
|
|
625
|
-
const errorInfos = validationError.validationErrors.map(
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
}),
|
|
633
|
-
);
|
|
597
|
+
const errorInfos = validationError.validationErrors.map(({ rowIndex, rowData, error: rowError }) => ({
|
|
598
|
+
file: filePath,
|
|
599
|
+
rowIndex,
|
|
600
|
+
issues: rowError.issues,
|
|
601
|
+
data: rowData,
|
|
602
|
+
originalData: rowsToMigrate[rowIndex],
|
|
603
|
+
}));
|
|
634
604
|
|
|
635
605
|
const formatted = formatter.formatValidationErrors(errorInfos);
|
|
636
606
|
console.error(formatted);
|
package/src/database.test.ts
CHANGED
|
@@ -137,10 +137,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
137
137
|
});
|
|
138
138
|
|
|
139
139
|
it('should find rows by condition', async () => {
|
|
140
|
-
await writeTable(
|
|
141
|
-
'users',
|
|
142
|
-
'{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n{"id":3,"name":"Alice"}\n',
|
|
143
|
-
);
|
|
140
|
+
await writeTable('users', '{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n{"id":3,"name":"Alice"}\n');
|
|
144
141
|
|
|
145
142
|
type Tables = {
|
|
146
143
|
users: { id: number; name: string };
|
|
@@ -278,10 +275,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
278
275
|
|
|
279
276
|
describe('update operations', () => {
|
|
280
277
|
it('should update existing rows', async () => {
|
|
281
|
-
await writeTable(
|
|
282
|
-
'users',
|
|
283
|
-
'{"id":1,"name":"Alice","age":30}\n{"id":2,"name":"Bob","age":25}\n',
|
|
284
|
-
);
|
|
278
|
+
await writeTable('users', '{"id":1,"name":"Alice","age":30}\n{"id":2,"name":"Bob","age":25}\n');
|
|
285
279
|
|
|
286
280
|
type Tables = {
|
|
287
281
|
users: { id: number; name: string; age: number };
|
|
@@ -301,10 +295,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
301
295
|
});
|
|
302
296
|
|
|
303
297
|
it('should update multiple rows', async () => {
|
|
304
|
-
await writeTable(
|
|
305
|
-
'users',
|
|
306
|
-
'{"id":1,"name":"Alice","active":true}\n{"id":2,"name":"Bob","active":true}\n',
|
|
307
|
-
);
|
|
298
|
+
await writeTable('users', '{"id":1,"name":"Alice","active":true}\n{"id":2,"name":"Bob","active":true}\n');
|
|
308
299
|
|
|
309
300
|
type Tables = {
|
|
310
301
|
users: { id: number; name: string; active: boolean };
|
|
@@ -321,10 +312,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
321
312
|
});
|
|
322
313
|
|
|
323
314
|
it('should batch update rows with distinct values', async () => {
|
|
324
|
-
await writeTable(
|
|
325
|
-
'users',
|
|
326
|
-
'{"id":1,"name":"Alice","age":30}\n{"id":2,"name":"Bob","age":25}\n',
|
|
327
|
-
);
|
|
315
|
+
await writeTable('users', '{"id":1,"name":"Alice","age":30}\n{"id":2,"name":"Bob","age":25}\n');
|
|
328
316
|
|
|
329
317
|
type Tables = {
|
|
330
318
|
users: { id: number; name: string; age: number };
|
|
@@ -373,10 +361,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
373
361
|
});
|
|
374
362
|
|
|
375
363
|
it('should delete multiple rows', async () => {
|
|
376
|
-
await writeTable(
|
|
377
|
-
'users',
|
|
378
|
-
'{"id":1,"name":"Alice"}\n{"id":2,"name":"Alice"}\n{"id":3,"name":"Bob"}\n',
|
|
379
|
-
);
|
|
364
|
+
await writeTable('users', '{"id":1,"name":"Alice"}\n{"id":2,"name":"Alice"}\n{"id":3,"name":"Bob"}\n');
|
|
380
365
|
|
|
381
366
|
type Tables = {
|
|
382
367
|
users: { id: number; name: string };
|
|
@@ -396,10 +381,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
396
381
|
});
|
|
397
382
|
|
|
398
383
|
it('should batch delete rows by primary key', async () => {
|
|
399
|
-
await writeTable(
|
|
400
|
-
'users',
|
|
401
|
-
'{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n{"id":3,"name":"Carol"}\n',
|
|
402
|
-
);
|
|
384
|
+
await writeTable('users', '{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n{"id":3,"name":"Carol"}\n');
|
|
403
385
|
|
|
404
386
|
type Tables = {
|
|
405
387
|
users: { id: number; name: string };
|
|
@@ -477,10 +459,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
477
459
|
|
|
478
460
|
describe('raw SQL queries', () => {
|
|
479
461
|
it('should execute raw queries', async () => {
|
|
480
|
-
await writeFile(
|
|
481
|
-
join(testDir, 'users.jsonl'),
|
|
482
|
-
'{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n',
|
|
483
|
-
);
|
|
462
|
+
await writeFile(join(testDir, 'users.jsonl'), '{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n');
|
|
484
463
|
|
|
485
464
|
type Tables = {
|
|
486
465
|
users: { id: number; name: string };
|
|
@@ -490,9 +469,7 @@ export const schema = defineSchema(rawSchema);
|
|
|
490
469
|
const db = LinesDB.create(config);
|
|
491
470
|
await db.initialize();
|
|
492
471
|
|
|
493
|
-
const result = db.query<{ id: number; name: string }>('SELECT * FROM users WHERE id > ?', [
|
|
494
|
-
1,
|
|
495
|
-
]);
|
|
472
|
+
const result = db.query<{ id: number; name: string }>('SELECT * FROM users WHERE id > ?', [1]);
|
|
496
473
|
|
|
497
474
|
expect(result).toHaveLength(1);
|
|
498
475
|
expect(result[0].name).toBe('Bob');
|
|
@@ -602,10 +579,7 @@ export const schema = Object.assign(Object.create(baseSchema), {
|
|
|
602
579
|
['profile_code'],
|
|
603
580
|
);
|
|
604
581
|
|
|
605
|
-
await writeFile(
|
|
606
|
-
join(testDir, 'profiles.jsonl'),
|
|
607
|
-
'{"code":"a-001","author_id":"1","bio":"Author A"}\n',
|
|
608
|
-
);
|
|
582
|
+
await writeFile(join(testDir, 'profiles.jsonl'), '{"code":"a-001","author_id":"1","bio":"Author A"}\n');
|
|
609
583
|
await writeSchemaWithFK(
|
|
610
584
|
'profiles',
|
|
611
585
|
'code',
|
|
@@ -635,10 +609,7 @@ export const schema = Object.assign(Object.create(baseSchema), {
|
|
|
635
609
|
});
|
|
636
610
|
|
|
637
611
|
it('should detect orphan in profiles (no matching author)', async () => {
|
|
638
|
-
await writeFile(
|
|
639
|
-
join(testDir, 'authors.jsonl'),
|
|
640
|
-
'{"id":"1","name":"Alice","profile_code":"a-001"}\n',
|
|
641
|
-
);
|
|
612
|
+
await writeFile(join(testDir, 'authors.jsonl'), '{"id":"1","name":"Alice","profile_code":"a-001"}\n');
|
|
642
613
|
await writeSchemaWithFK(
|
|
643
614
|
'authors',
|
|
644
615
|
'id',
|
package/src/database.ts
CHANGED
|
@@ -36,10 +36,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
36
36
|
this.db = createDatabase(dbPath ?? ':memory:');
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
static create<Tables extends TableDefs>(
|
|
40
|
-
config: DatabaseConfig<Tables>,
|
|
41
|
-
dbPath?: string,
|
|
42
|
-
): LinesDB<Tables> {
|
|
39
|
+
static create<Tables extends TableDefs>(config: DatabaseConfig<Tables>, dbPath?: string): LinesDB<Tables> {
|
|
43
40
|
return new LinesDB<Tables>(config, dbPath);
|
|
44
41
|
}
|
|
45
42
|
|
|
@@ -73,9 +70,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
73
70
|
// Validate that all requested tables exist BEFORE starting to load
|
|
74
71
|
for (const tableNameToLoad of tablesToLoad) {
|
|
75
72
|
if (!this.tables.has(tableNameToLoad)) {
|
|
76
|
-
throw new Error(
|
|
77
|
-
`Table '${tableNameToLoad}' not found in directory '${this.config.dataDir}'`,
|
|
78
|
-
);
|
|
73
|
+
throw new Error(`Table '${tableNameToLoad}' not found in directory '${this.config.dataDir}'`);
|
|
79
74
|
}
|
|
80
75
|
}
|
|
81
76
|
|
|
@@ -294,13 +289,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
294
289
|
loaded,
|
|
295
290
|
rowCount,
|
|
296
291
|
errors: loadErrors,
|
|
297
|
-
} = await this.loadTable(
|
|
298
|
-
tableName,
|
|
299
|
-
tableConfig,
|
|
300
|
-
detailedValidate,
|
|
301
|
-
transform,
|
|
302
|
-
allSkippedDependencies,
|
|
303
|
-
);
|
|
292
|
+
} = await this.loadTable(tableName, tableConfig, detailedValidate, transform, allSkippedDependencies);
|
|
304
293
|
errors.push(...loadErrors);
|
|
305
294
|
rowCounts.set(tableName, rowCount);
|
|
306
295
|
|
|
@@ -373,10 +362,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
373
362
|
// Only load if not already provided via config
|
|
374
363
|
try {
|
|
375
364
|
const { pathToFileURL } = await import('node:url');
|
|
376
|
-
const schemaPath = await findSchemaFile(
|
|
377
|
-
dirname(config.jsonlPath),
|
|
378
|
-
basename(config.jsonlPath, '.jsonl'),
|
|
379
|
-
);
|
|
365
|
+
const schemaPath = await findSchemaFile(dirname(config.jsonlPath), basename(config.jsonlPath, '.jsonl'));
|
|
380
366
|
if (!schemaPath) throw new Error('Schema file not found');
|
|
381
367
|
const schemaUrl = pathToFileURL(schemaPath).href;
|
|
382
368
|
const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
|
|
@@ -544,12 +530,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
544
530
|
|
|
545
531
|
// Insert validated data (with detailed validation if requested)
|
|
546
532
|
if (detailedValidate) {
|
|
547
|
-
const insertErrors = this.insertDataWithDetailedValidation(
|
|
548
|
-
tableName,
|
|
549
|
-
schema,
|
|
550
|
-
validatedData,
|
|
551
|
-
config.jsonlPath,
|
|
552
|
-
);
|
|
533
|
+
const insertErrors = this.insertDataWithDetailedValidation(tableName, schema, validatedData, config.jsonlPath);
|
|
553
534
|
if (insertErrors.length > 0) {
|
|
554
535
|
return { loaded: false, rowCount: data.length, errors: insertErrors };
|
|
555
536
|
}
|
|
@@ -626,8 +607,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
626
607
|
const index = schema.indexes[i];
|
|
627
608
|
// Create safe index name by replacing special characters
|
|
628
609
|
const safeTableName = schema.name.replace(/[^a-zA-Z0-9]/g, '_');
|
|
629
|
-
const resolvedIndexName =
|
|
630
|
-
index.name || `idx_${safeTableName}_${index.columns.join('_')}_${i}`;
|
|
610
|
+
const resolvedIndexName = index.name || `idx_${safeTableName}_${index.columns.join('_')}_${i}`;
|
|
631
611
|
const uniqueKeyword = index.unique ? 'UNIQUE ' : '';
|
|
632
612
|
const indexSql = `CREATE ${uniqueKeyword}INDEX IF NOT EXISTS ${this.quoteIdentifier(resolvedIndexName)} ON ${quotedTableName} (${index.columns
|
|
633
613
|
.map((col) => this.quoteIdentifier(col))
|
|
@@ -836,10 +816,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
836
816
|
/**
|
|
837
817
|
* Execute a raw SQL query
|
|
838
818
|
*/
|
|
839
|
-
query<T = unknown>(
|
|
840
|
-
sql: string,
|
|
841
|
-
params: (string | number | bigint | null | Uint8Array)[] = [],
|
|
842
|
-
): T[] {
|
|
819
|
+
query<T = unknown>(sql: string, params: (string | number | bigint | null | Uint8Array)[] = []): T[] {
|
|
843
820
|
const stmt = this.db.prepare(sql);
|
|
844
821
|
return stmt.all(...params) as T[];
|
|
845
822
|
}
|
|
@@ -847,10 +824,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
847
824
|
/**
|
|
848
825
|
* Execute a SQL query that returns a single row
|
|
849
826
|
*/
|
|
850
|
-
queryOne<T = unknown>(
|
|
851
|
-
sql: string,
|
|
852
|
-
params: (string | number | bigint | null | Uint8Array)[] = [],
|
|
853
|
-
): T | null {
|
|
827
|
+
queryOne<T = unknown>(sql: string, params: (string | number | bigint | null | Uint8Array)[] = []): T | null {
|
|
854
828
|
const stmt = this.db.prepare(sql);
|
|
855
829
|
const result = stmt.get(...params);
|
|
856
830
|
return result === undefined ? null : (result as T);
|
|
@@ -896,10 +870,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
896
870
|
|
|
897
871
|
// Normal case: use SQL WHERE clause
|
|
898
872
|
if (sql) {
|
|
899
|
-
const rawRows = this.query(
|
|
900
|
-
`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`,
|
|
901
|
-
values,
|
|
902
|
-
);
|
|
873
|
+
const rawRows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values);
|
|
903
874
|
rows = rawRows.map((row) => this.deserializeRow(tableName, row)) as Tables[K][];
|
|
904
875
|
} else {
|
|
905
876
|
// If only function filters (AND case), get all rows
|
|
@@ -919,10 +890,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
919
890
|
|
|
920
891
|
let rows: Tables[K][];
|
|
921
892
|
if (sql) {
|
|
922
|
-
const rawRows = this.query(
|
|
923
|
-
`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`,
|
|
924
|
-
values,
|
|
925
|
-
);
|
|
893
|
+
const rawRows = this.query(`SELECT * FROM ${this.quoteTableName(tableName)} WHERE ${sql}`, values);
|
|
926
894
|
rows = rawRows.map((row) => this.deserializeRow(tableName, row)) as Tables[K][];
|
|
927
895
|
} else {
|
|
928
896
|
// If only function filters, get all rows
|
|
@@ -986,9 +954,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
986
954
|
|
|
987
955
|
// Only synchronous validation is supported
|
|
988
956
|
if (result instanceof Promise) {
|
|
989
|
-
throw new Error(
|
|
990
|
-
'Asynchronous validation is not supported. Please use synchronous validation schemas.',
|
|
991
|
-
);
|
|
957
|
+
throw new Error('Asynchronous validation is not supported. Please use synchronous validation schemas.');
|
|
992
958
|
}
|
|
993
959
|
|
|
994
960
|
if (result.issues && result.issues.length > 0) {
|
|
@@ -1209,9 +1175,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1209
1175
|
for (const record of records) {
|
|
1210
1176
|
const pkValue = record[pkName];
|
|
1211
1177
|
if (pkValue === undefined) {
|
|
1212
|
-
throw new Error(
|
|
1213
|
-
`Record is missing primary key '${String(pkName)}': ${JSON.stringify(record)}`,
|
|
1214
|
-
);
|
|
1178
|
+
throw new Error(`Record is missing primary key '${String(pkName)}': ${JSON.stringify(record)}`);
|
|
1215
1179
|
}
|
|
1216
1180
|
pkValues.push(pkValue);
|
|
1217
1181
|
}
|
|
@@ -1250,9 +1214,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1250
1214
|
const existingRow = existingRowsMap.get(pkValue);
|
|
1251
1215
|
|
|
1252
1216
|
if (!existingRow) {
|
|
1253
|
-
throw new Error(
|
|
1254
|
-
`No existing row found with ${String(pkName)}=${JSON.stringify(pkValue)}`,
|
|
1255
|
-
);
|
|
1217
|
+
throw new Error(`No existing row found with ${String(pkName)}=${JSON.stringify(pkValue)}`);
|
|
1256
1218
|
}
|
|
1257
1219
|
|
|
1258
1220
|
const mergedData = { ...existingRow, ...record };
|
|
@@ -1396,8 +1358,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1396
1358
|
private normalizeValue(value: unknown): string | number | bigint | null | Uint8Array {
|
|
1397
1359
|
if (value === null || value === undefined) return null;
|
|
1398
1360
|
if (typeof value === 'boolean') return value ? 1 : 0;
|
|
1399
|
-
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'bigint')
|
|
1400
|
-
return value;
|
|
1361
|
+
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'bigint') return value;
|
|
1401
1362
|
if (value instanceof Uint8Array) return value;
|
|
1402
1363
|
// For objects, convert to JSON string
|
|
1403
1364
|
return JSON.stringify(value);
|
|
@@ -1426,9 +1387,7 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1426
1387
|
if (Array.isArray(cond)) {
|
|
1427
1388
|
const clauses = cond
|
|
1428
1389
|
.map((item) => {
|
|
1429
|
-
const clause = Array.isArray(item)
|
|
1430
|
-
? buildCondition(item, true)
|
|
1431
|
-
: buildCondition(item, true);
|
|
1390
|
+
const clause = Array.isArray(item) ? buildCondition(item, true) : buildCondition(item, true);
|
|
1432
1391
|
return clause ? `(${clause})` : '';
|
|
1433
1392
|
})
|
|
1434
1393
|
.filter((clause) => clause !== ''); // Filter out empty clauses
|
|
@@ -1465,20 +1424,14 @@ export class LinesDB<Tables extends TableDefs> {
|
|
|
1465
1424
|
/**
|
|
1466
1425
|
* Apply OR condition with function filters by evaluating each row against the condition
|
|
1467
1426
|
*/
|
|
1468
|
-
private applyOrConditionWithFilters<T extends Record<string, unknown>>(
|
|
1469
|
-
rows: T[],
|
|
1470
|
-
condition: WhereCondition<T>,
|
|
1471
|
-
): T[] {
|
|
1427
|
+
private applyOrConditionWithFilters<T extends Record<string, unknown>>(rows: T[], condition: WhereCondition<T>): T[] {
|
|
1472
1428
|
return rows.filter((row) => this.matchesOrCondition(row, condition));
|
|
1473
1429
|
}
|
|
1474
1430
|
|
|
1475
1431
|
/**
|
|
1476
1432
|
* Check if a row matches an OR/AND condition (recursively)
|
|
1477
1433
|
*/
|
|
1478
|
-
private matchesOrCondition<T extends Record<string, unknown>>(
|
|
1479
|
-
row: T,
|
|
1480
|
-
condition: WhereCondition<T>,
|
|
1481
|
-
): boolean {
|
|
1434
|
+
private matchesOrCondition<T extends Record<string, unknown>>(row: T, condition: WhereCondition<T>): boolean {
|
|
1482
1435
|
// Handle array (OR conditions)
|
|
1483
1436
|
if (Array.isArray(condition)) {
|
|
1484
1437
|
return condition.some((item) => this.matchesOrCondition(row, item));
|
|
@@ -59,9 +59,7 @@ describe('DirectoryScanner', () => {
|
|
|
59
59
|
it('should throw error for non-existent directory', async () => {
|
|
60
60
|
const nonExistentDir = join(testDir, 'nonexistent');
|
|
61
61
|
|
|
62
|
-
await expect(DirectoryScanner.scanDirectory(nonExistentDir)).rejects.toThrow(
|
|
63
|
-
'Failed to scan directory',
|
|
64
|
-
);
|
|
62
|
+
await expect(DirectoryScanner.scanDirectory(nonExistentDir)).rejects.toThrow('Failed to scan directory');
|
|
65
63
|
});
|
|
66
64
|
|
|
67
65
|
it('should handle multiple JSONL files', async () => {
|
package/src/directory-scanner.ts
CHANGED
|
@@ -30,9 +30,7 @@ export class DirectoryScanner {
|
|
|
30
30
|
|
|
31
31
|
return tables;
|
|
32
32
|
} catch (error) {
|
|
33
|
-
throw new Error(
|
|
34
|
-
`Failed to scan directory ${dataDir}: ${error instanceof Error ? error.message : String(error)}`,
|
|
35
|
-
);
|
|
33
|
+
throw new Error(`Failed to scan directory ${dataDir}: ${error instanceof Error ? error.message : String(error)}`);
|
|
36
34
|
}
|
|
37
35
|
}
|
|
38
36
|
}
|
package/src/index.ts
CHANGED
|
@@ -16,11 +16,7 @@ export {
|
|
|
16
16
|
} from './schema-extensions.js';
|
|
17
17
|
export type { SchemaExtension } from './schema-extensions.js';
|
|
18
18
|
export { ErrorFormatter } from './error-formatter.js';
|
|
19
|
-
export type {
|
|
20
|
-
ErrorFormatterOptions,
|
|
21
|
-
ValidationErrorInfo,
|
|
22
|
-
ForeignKeyErrorInfo,
|
|
23
|
-
} from './error-formatter.js';
|
|
19
|
+
export type { ErrorFormatterOptions, ValidationErrorInfo, ForeignKeyErrorInfo } from './error-formatter.js';
|
|
24
20
|
export { detectRuntime, RUNTIME } from './runtime.js';
|
|
25
21
|
export type { RuntimeEnvironment } from './runtime.js';
|
|
26
22
|
export type { SQLiteDatabase, SQLiteStatement } from './sqlite-adapter.js';
|
package/src/jsonl-reader.test.ts
CHANGED
|
@@ -148,9 +148,7 @@ describe('JsonlReader', () => {
|
|
|
148
148
|
});
|
|
149
149
|
|
|
150
150
|
it('should throw error for empty data', () => {
|
|
151
|
-
expect(() => JsonlReader.inferSchema('empty', [])).toThrow(
|
|
152
|
-
'Cannot infer schema from empty data',
|
|
153
|
-
);
|
|
151
|
+
expect(() => JsonlReader.inferSchema('empty', [])).toThrow('Cannot infer schema from empty data');
|
|
154
152
|
});
|
|
155
153
|
|
|
156
154
|
it('should handle REAL numbers', () => {
|
package/src/jsonl-reader.ts
CHANGED
|
@@ -9,10 +9,7 @@ export class JsonlReader {
|
|
|
9
9
|
* Temporarily override the data returned for specific JSONL files.
|
|
10
10
|
* Useful for scenarios like migration validation where in-memory data should be used.
|
|
11
11
|
*/
|
|
12
|
-
static async withOverrides<T>(
|
|
13
|
-
overrides: Map<string, JsonObject[]>,
|
|
14
|
-
fn: () => Promise<T>,
|
|
15
|
-
): Promise<T> {
|
|
12
|
+
static async withOverrides<T>(overrides: Map<string, JsonObject[]>, fn: () => Promise<T>): Promise<T> {
|
|
16
13
|
const normalized = new Map<string, JsonObject[]>();
|
|
17
14
|
for (const [filePath, rows] of overrides) {
|
|
18
15
|
normalized.set(normalize(filePath), rows);
|
|
@@ -157,9 +157,7 @@ describe('SchemaLoader', () => {
|
|
|
157
157
|
const jsonlPath = join(testDir, 'users.jsonl');
|
|
158
158
|
await writeFile(jsonlPath, '{"id":1}\n');
|
|
159
159
|
|
|
160
|
-
await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(
|
|
161
|
-
/\.schema\.ts.*\.schema\.mts.*\.schema\.cts/,
|
|
162
|
-
);
|
|
160
|
+
await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(/\.schema\.ts.*\.schema\.mts.*\.schema\.cts/);
|
|
163
161
|
});
|
|
164
162
|
|
|
165
163
|
it('should throw for invalid schema export', async () => {
|
|
@@ -176,9 +174,7 @@ describe('SchemaLoader', () => {
|
|
|
176
174
|
`,
|
|
177
175
|
);
|
|
178
176
|
|
|
179
|
-
await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(
|
|
180
|
-
/does not export a valid StandardSchema/,
|
|
181
|
-
);
|
|
177
|
+
await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(/does not export a valid StandardSchema/);
|
|
182
178
|
});
|
|
183
179
|
|
|
184
180
|
it('should validate StandardSchema structure', async () => {
|
|
@@ -200,9 +196,7 @@ describe('SchemaLoader', () => {
|
|
|
200
196
|
`,
|
|
201
197
|
);
|
|
202
198
|
|
|
203
|
-
await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(
|
|
204
|
-
/does not export a valid StandardSchema/,
|
|
205
|
-
);
|
|
199
|
+
await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(/does not export a valid StandardSchema/);
|
|
206
200
|
|
|
207
201
|
// Wrong version
|
|
208
202
|
await writeFile(
|
|
@@ -218,9 +212,7 @@ describe('SchemaLoader', () => {
|
|
|
218
212
|
`,
|
|
219
213
|
);
|
|
220
214
|
|
|
221
|
-
await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(
|
|
222
|
-
/does not export a valid StandardSchema/,
|
|
223
|
-
);
|
|
215
|
+
await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(/does not export a valid StandardSchema/);
|
|
224
216
|
});
|
|
225
217
|
});
|
|
226
218
|
});
|
package/src/schema-loader.ts
CHANGED
|
@@ -64,9 +64,7 @@ export class SchemaLoader {
|
|
|
64
64
|
const standardObj = standard as Record<string, unknown>;
|
|
65
65
|
|
|
66
66
|
return (
|
|
67
|
-
standardObj.version === 1 &&
|
|
68
|
-
typeof standardObj.vendor === 'string' &&
|
|
69
|
-
typeof standardObj.validate === 'function'
|
|
67
|
+
standardObj.version === 1 && typeof standardObj.vendor === 'string' && typeof standardObj.validate === 'function'
|
|
70
68
|
);
|
|
71
69
|
}
|
|
72
70
|
}
|
package/src/schema.ts
CHANGED
|
@@ -37,8 +37,10 @@ export type SchemaOptions<Input extends Table, Output extends Table> = {
|
|
|
37
37
|
* BiDirectional Schema interface
|
|
38
38
|
* Extends StandardSchema with optional backward transformation and schema metadata
|
|
39
39
|
*/
|
|
40
|
-
export interface BiDirectionalSchema<Input extends Table = Table, Output extends Table = Input>
|
|
41
|
-
|
|
40
|
+
export interface BiDirectionalSchema<Input extends Table = Table, Output extends Table = Input> extends StandardSchema<
|
|
41
|
+
Input,
|
|
42
|
+
Output
|
|
43
|
+
> {
|
|
42
44
|
/**
|
|
43
45
|
* Backward transformation from Output to Input
|
|
44
46
|
* Required when Input and Output types differ (e.g., with transformations)
|
|
@@ -96,9 +98,7 @@ export interface BiDirectionalSchema<Input extends Table = Table, Output extends
|
|
|
96
98
|
*/
|
|
97
99
|
export function defineSchema<Input extends Table, Output extends Table>(
|
|
98
100
|
schema: StandardSchema<Input, Output>,
|
|
99
|
-
...args: Output extends Input
|
|
100
|
-
? [options?: SchemaOptions<Input, Output>]
|
|
101
|
-
: [options: SchemaOptions<Input, Output>]
|
|
101
|
+
...args: Output extends Input ? [options?: SchemaOptions<Input, Output>] : [options: SchemaOptions<Input, Output>]
|
|
102
102
|
): BiDirectionalSchema<Input, Output> {
|
|
103
103
|
const options = args[0];
|
|
104
104
|
// Create a new object that extends the schema
|