@toiroakr/lines-db 0.6.1 → 0.8.0

package/src/cli.ts

@@ -62,9 +62,10 @@ program
   .command('generate')
   .description('Generate TypeScript type definitions from schema files')
   .argument('<dataDir>', 'Directory containing JSONL and schema files')
-  .action(async (dataDir: string) => {
+  .option('-o, --output <path>', 'Output file path (default: db.ts in dataDir)')
+  .action(async (dataDir: string, options: { output?: string }) => {
     try {
-      const generator = new TypeGenerator({ dataDir });
+      const generator = new TypeGenerator({ dataDir, output: options.output });
       await generator.generate();
       console.log('Type generation completed successfully!');
     } catch (error) {
@@ -105,72 +106,135 @@ program
         await db.close();
       }
 
-      // Display warnings if any
-      if (result.warnings.length > 0) {
-        for (const warning of result.warnings) {
-          console.warn(styleText('yellow', `⚠ ${warning}`));
+      // Directory validation: display per-table results
+      if (!tableName) {
+        const formatter = new ErrorFormatter({ verbose: options.verbose });
+
+        for (const tableResult of result.tableResults) {
+          if (tableResult.valid && tableResult.warnings.length === 0) {
+            // Success
+            console.log(
+              styleText('green', `✓ ${tableResult.tableName} (${tableResult.rowCount} records)`),
+            );
+          } else if (tableResult.valid && tableResult.warnings.length > 0) {
+            // Warnings
+            for (const warning of tableResult.warnings) {
+              console.warn(styleText('yellow', `⚠ ${warning}`));
+            }
+          } else {
+            // Errors
+            const fileErrors = tableResult.errors;
+            console.error(formatter.formatErrorHeader(fileErrors.length, fileErrors[0]?.file));
+            console.error('');
+
+            const validationErrors = fileErrors.filter(
+              (e) => e.type !== 'foreignKey' || !e.foreignKeyError,
+            );
+            const foreignKeyErrors = fileErrors.filter(
+              (e) => e.type === 'foreignKey' && e.foreignKeyError,
+            );
+
+            if (validationErrors.length > 0) {
+              console.error(
+                formatter.formatValidationErrors(
+                  validationErrors.map((e) => ({
+                    file: e.file,
+                    rowIndex: e.rowIndex,
+                    issues: e.issues,
+                  })),
+                ),
+              );
+            }
+
+            for (const error of foreignKeyErrors) {
+              if (error.foreignKeyError) {
+                console.error(
+                  formatter.formatForeignKeyError({
+                    file: error.file,
+                    rowIndex: error.rowIndex,
+                    column: error.foreignKeyError.column,
+                    value: error.foreignKeyError.value,
+                    referencedTable: error.foreignKeyError.referencedTable,
+                    referencedColumn: error.foreignKeyError.referencedColumn,
+                  }),
+                );
+              }
+            }
+
+            console.error('');
+          }
         }
-        console.log('');
-      }
 
-      if (result.valid) {
-        console.log('✓ All records are valid');
-        process.exit(0);
+        if (result.valid) {
+          console.log('');
+          console.log(styleText('green', '✓ All records are valid'));
+          process.exit(0);
+        } else {
+          process.exit(1);
+        }
       } else {
-        const formatter = new ErrorFormatter({ verbose: options.verbose });
-
-        // Group errors by file for header
-        const errorsByFile = new Map<string, typeof result.errors>();
-        for (const error of result.errors) {
-          const fileErrors = errorsByFile.get(error.file) || [];
-          fileErrors.push(error);
-          errorsByFile.set(error.file, fileErrors);
+        // Single file validation: existing behavior
+        if (result.warnings.length > 0) {
+          for (const warning of result.warnings) {
+            console.warn(styleText('yellow', `⚠ ${warning}`));
+          }
+          console.log('');
         }
 
-        // Format and display errors
-        for (const [file, fileErrors] of errorsByFile) {
-          console.error(formatter.formatErrorHeader(fileErrors.length, file));
-          console.error('');
-
-          // Separate validation errors and foreign key errors
-          const validationErrors = fileErrors.filter(
-            (e) => e.type !== 'foreignKey' || !e.foreignKeyError,
-          );
-          const foreignKeyErrors = fileErrors.filter(
-            (e) => e.type === 'foreignKey' && e.foreignKeyError,
-          );
-
-          // Format validation errors
-          if (validationErrors.length > 0) {
-            const formattedValidation = formatter.formatValidationErrors(
-              validationErrors.map((e) => ({
-                file: e.file,
-                rowIndex: e.rowIndex,
-                issues: e.issues,
-              })),
+        if (result.valid) {
+          console.log(styleText('green', '✓ All records are valid'));
+          process.exit(0);
+        } else {
+          const formatter = new ErrorFormatter({ verbose: options.verbose });
+
+          for (const [, fileErrors] of result.errors.reduce((map, error) => {
+            const errors = map.get(error.file) || [];
+            errors.push(error);
+            map.set(error.file, errors);
+            return map;
+          }, new Map<string, typeof result.errors>())) {
+            console.error(formatter.formatErrorHeader(fileErrors.length, fileErrors[0]?.file));
+            console.error('');
+
+            const validationErrors = fileErrors.filter(
+              (e) => e.type !== 'foreignKey' || !e.foreignKeyError,
             );
-            console.error(formattedValidation);
-          }
+            const foreignKeyErrors = fileErrors.filter(
+              (e) => e.type === 'foreignKey' && e.foreignKeyError,
+            );
+
+            if (validationErrors.length > 0) {
+              console.error(
+                formatter.formatValidationErrors(
+                  validationErrors.map((e) => ({
+                    file: e.file,
+                    rowIndex: e.rowIndex,
+                    issues: e.issues,
+                  })),
+                ),
+              );
+            }
 
-          // Format foreign key errors
-          for (const error of foreignKeyErrors) {
-            if (error.foreignKeyError) {
-              const formattedFk = formatter.formatForeignKeyError({
-                file: error.file,
-                rowIndex: error.rowIndex,
-                column: error.foreignKeyError.column,
-                value: error.foreignKeyError.value,
-                referencedTable: error.foreignKeyError.referencedTable,
-                referencedColumn: error.foreignKeyError.referencedColumn,
-              });
-              console.error(formattedFk);
+            for (const error of foreignKeyErrors) {
+              if (error.foreignKeyError) {
+                console.error(
+                  formatter.formatForeignKeyError({
+                    file: error.file,
+                    rowIndex: error.rowIndex,
+                    column: error.foreignKeyError.column,
+                    value: error.foreignKeyError.value,
+                    referencedTable: error.foreignKeyError.referencedTable,
+                    referencedColumn: error.foreignKeyError.referencedColumn,
+                  }),
+                );
+              }
             }
+
+            console.error('');
           }
 
-          console.error('');
+          process.exit(1);
         }
-
-        process.exit(1);
       }
     } catch (error) {
       console.error('Error:', error instanceof Error ? error.message : String(error));

package/src/database.test.ts

@@ -47,6 +47,34 @@ export const schema = defineSchema(rawSchema);
       await db.close();
     });
 
+    it('should load table with .schema.mts schema file', async () => {
+      await writeFile(join(testDir, 'users.jsonl'), '{"id":1,"name":"Alice"}\n');
+      await writeFile(join(testDir, 'users.schema.mts'), GENERIC_SCHEMA_SOURCE);
+
+      const config: DatabaseConfig = { dataDir: testDir };
+      const db = LinesDB.create(config);
+      await db.initialize();
+
+      const tables = db.getTableNames();
+      expect(tables).toContain('users');
+
+      await db.close();
+    });
+
+    it('should load table with .schema.cts schema file', async () => {
+      await writeFile(join(testDir, 'users.jsonl'), '{"id":1,"name":"Alice"}\n');
+      await writeFile(join(testDir, 'users.schema.cts'), GENERIC_SCHEMA_SOURCE);
+
+      const config: DatabaseConfig = { dataDir: testDir };
+      const db = LinesDB.create(config);
+      await db.initialize();
+
+      const tables = db.getTableNames();
+      expect(tables).toContain('users');
+
+      await db.close();
+    });
+
     it('should load multiple tables', async () => {
       await writeTable('users', '{"id":1}\n');
       await writeTable('products', '{"id":1}\n');

package/src/database.ts

@@ -4,6 +4,8 @@ import { JsonlWriter } from './jsonl-writer.js';
 import { SchemaLoader } from './schema-loader.js';
 import { DirectoryScanner } from './directory-scanner.js';
 import { hasBackward } from './schema.js';
+import { findSchemaFile } from './schema-extensions.js';
+import { dirname, basename } from 'node:path';
 import type {
   DatabaseConfig,
   TableSchema,
@@ -16,6 +18,7 @@ import type {
   WhereCondition,
   ValidationResult,
   ValidationErrorDetail,
+  TableValidationResult,
   ForeignKeyDefinition,
 } from './types.js';
 import type { BiDirectionalSchema } from './schema.js';
@@ -56,6 +59,7 @@ export class LinesDB<Tables extends TableDefs> {
   }): Promise<ValidationResult> {
     const allErrors: ValidationErrorDetail[] = [];
     const allWarnings: string[] = [];
+    const allRowCounts = new Map<string, number>();
     const tableName = options?.tableName;
     const detailedValidate = options?.detailedValidate ?? false;
     const transform = options?.transform;
@@ -85,7 +89,11 @@ export class LinesDB<Tables extends TableDefs> {
       if (!attemptedTables.has(tableNameToLoad)) {
         // Only apply transform to the specified table
         const tableTransform = tableNameToLoad === tableName ? transform : undefined;
-        const { errors, warnings } = await this.loadTableWithDependencies(
+        const {
+          errors,
+          warnings,
+          rowCounts: tableRowCounts,
+        } = await this.loadTableWithDependencies(
           tableNameToLoad,
           loadedTables,
           loadingTables,
@@ -95,13 +103,30 @@ export class LinesDB<Tables extends TableDefs> {
         );
         allErrors.push(...errors);
         allWarnings.push(...warnings);
+        for (const [k, v] of tableRowCounts) {
+          allRowCounts.set(k, v);
+        }
       }
     }
 
+    // Build per-table results
+    const tableResults: TableValidationResult[] = tablesToLoad.map((name) => {
+      const tableErrors = allErrors.filter((e) => e.tableName === name);
+      const tableWarnings = allWarnings.filter((w) => w.includes(`'${name}'`));
+      return {
+        tableName: name,
+        valid: tableErrors.length === 0,
+        rowCount: allRowCounts.get(name) ?? 0,
+        errors: tableErrors,
+        warnings: tableWarnings,
+      };
+    });
+
     return {
       valid: allErrors.length === 0,
       errors: allErrors,
       warnings: allWarnings,
+      tableResults,
     };
   }
 
@@ -115,13 +140,18 @@ export class LinesDB<Tables extends TableDefs> {
     attemptedTables: Set<string>,
     detailedValidate: boolean,
     transform?: (row: JsonObject) => JsonObject,
-  ): Promise<{ errors: ValidationErrorDetail[]; warnings: string[] }> {
+  ): Promise<{
+    errors: ValidationErrorDetail[];
+    warnings: string[];
+    rowCounts: Map<string, number>;
+  }> {
     const errors: ValidationErrorDetail[] = [];
     const warnings: string[] = [];
+    const rowCounts = new Map<string, number>();
 
     // Skip if already attempted (loaded or not)
     if (attemptedTables.has(tableName)) {
-      return { errors, warnings };
+      return { errors, warnings, rowCounts };
     }
 
     // Mark as attempted
@@ -148,13 +178,18 @@ export class LinesDB<Tables extends TableDefs> {
 
       try {
         const { pathToFileURL } = await import('node:url');
-        const schemaPath = tableConfig.jsonlPath.replace('.jsonl', '.schema.ts');
-        const schemaUrl = pathToFileURL(schemaPath).href;
-        const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
+        const schemaPath = await findSchemaFile(
+          dirname(tableConfig.jsonlPath),
+          basename(tableConfig.jsonlPath, '.jsonl'),
+        );
+        if (schemaPath) {
+          const schemaUrl = pathToFileURL(schemaPath).href;
+          const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
 
-        // Try to get foreign keys from exported 'schema' or directly from module
-        const schemaExport = schemaModule.schema || schemaModule.default;
-        foreignKeys = schemaExport?.foreignKeys || schemaModule.foreignKeys;
+          // Try to get foreign keys from exported 'schema' or directly from module
+          const schemaExport = schemaModule.schema || schemaModule.default;
+          foreignKeys = schemaExport?.foreignKeys || schemaModule.foreignKeys;
+        }
       } catch {
         // Schema file not found - will continue without validation
       }
@@ -183,6 +218,9 @@ export class LinesDB<Tables extends TableDefs> {
               );
               errors.push(...depResult.errors);
               warnings.push(...depResult.warnings);
+              for (const [k, v] of depResult.rowCounts) {
+                rowCounts.set(k, v);
+              }
             } else {
               throw new Error(
                 `Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`,
@@ -192,14 +230,39 @@ export class LinesDB<Tables extends TableDefs> {
         }
       }
 
+      // Determine which FK dependencies failed validation (attempted but not loaded)
+      const failedDependencies = new Set<string>();
+      if (foreignKeys && foreignKeys.length > 0) {
+        for (const fk of foreignKeys) {
+          const referencedTable = fk.references.table;
+          if (referencedTable === tableName) continue;
+          if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable)) {
+            failedDependencies.add(referencedTable);
+          }
+        }
+        if (failedDependencies.size > 0) {
+          for (const dep of failedDependencies) {
+            warnings.push(
+              `Skipping foreign key validation for table '${tableName}': referenced table '${dep}' has validation errors`,
+            );
+          }
+        }
+      }
+
       // Now load this table
-      const { loaded, errors: loadErrors } = await this.loadTable(
+      const {
+        loaded,
+        rowCount,
+        errors: loadErrors,
+      } = await this.loadTable(
         tableName,
         tableConfig,
         detailedValidate,
         transform,
+        failedDependencies,
       );
       errors.push(...loadErrors);
+      rowCounts.set(tableName, rowCount);
 
       if (loaded) {
         loadedTables.add(tableName);
@@ -213,7 +276,7 @@ export class LinesDB<Tables extends TableDefs> {
       loadingTables.delete(tableName);
     }
 
-    return { errors, warnings };
+    return { errors, warnings, rowCounts };
   }
 
   /**
@@ -225,7 +288,8 @@ export class LinesDB<Tables extends TableDefs> {
     config: TableConfig,
     detailedValidate: boolean,
     transform?: (row: JsonObject) => JsonObject,
-  ): Promise<{ loaded: boolean; errors: ValidationErrorDetail[] }> {
+    failedDependencies?: Set<string>,
+  ): Promise<{ loaded: boolean; rowCount: number; errors: ValidationErrorDetail[] }> {
     // Read JSONL file
     let data = await JsonlReader.read(config.jsonlPath);
 
@@ -256,7 +320,11 @@ export class LinesDB<Tables extends TableDefs> {
       // Only load if not already provided via config
       try {
         const { pathToFileURL } = await import('node:url');
-        const schemaPath = config.jsonlPath.replace('.jsonl', '.schema.ts');
+        const schemaPath = await findSchemaFile(
+          dirname(config.jsonlPath),
+          basename(config.jsonlPath, '.jsonl'),
+        );
+        if (!schemaPath) throw new Error('Schema file not found');
         const schemaUrl = pathToFileURL(schemaPath).href;
         const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
 
@@ -339,7 +407,7 @@ export class LinesDB<Tables extends TableDefs> {
 
     if (validationErrors.length > 0) {
       // Return errors instead of throwing
-      return { loaded: false, errors: validationErrorDetails };
+      return { loaded: false, rowCount: data.length, errors: validationErrorDetails };
     }
 
     // Determine schema - infer from validated data if auto-inference is enabled
@@ -364,7 +432,7 @@ export class LinesDB<Tables extends TableDefs> {
       }
     } else if (config.autoInferSchema !== false) {
       if (validatedData.length === 0) {
-        return { loaded: false, errors: [] };
+        return { loaded: false, rowCount: 0, errors: [] };
       }
       // Use inferred schema
       schema = inferredSchema!;
@@ -395,7 +463,10 @@ export class LinesDB<Tables extends TableDefs> {
       }
     }
     if (foreignKeys) {
-      schema.foreignKeys = foreignKeys;
+      schema.foreignKeys =
+        failedDependencies && failedDependencies.size > 0
+          ? foreignKeys.filter((fk) => !failedDependencies.has(fk.references.table))
+          : foreignKeys;
     }
     if (indexes) {
       schema.indexes = indexes;
@@ -427,13 +498,13 @@ export class LinesDB<Tables extends TableDefs> {
         config.jsonlPath,
       );
       if (insertErrors.length > 0) {
-        return { loaded: false, errors: insertErrors };
+        return { loaded: false, rowCount: data.length, errors: insertErrors };
       }
     } else {
       this.insertData(tableName, schema, validatedData);
     }
 
-    return { loaded: true, errors: [] };
+    return { loaded: true, rowCount: data.length, errors: [] };
   }
 
   /**

package/src/index.ts

@@ -7,6 +7,14 @@ export { defineSchema, hasBackward } from './schema.js';
 export { TypeGenerator } from './type-generator.js';
 export { ensureTableRowsValid } from './jsonl-migration.js';
 export type { TableValidationOptions } from './jsonl-migration.js';
+export {
+  SCHEMA_EXTENSIONS,
+  findSchemaFile,
+  isSchemaFile,
+  extractTableNameFromSchemaFile,
+  rewriteExtensionForImport,
+} from './schema-extensions.js';
+export type { SchemaExtension } from './schema-extensions.js';
 export { detectRuntime, RUNTIME } from './runtime.js';
 export type { RuntimeEnvironment } from './runtime.js';
 export type { SQLiteDatabase, SQLiteStatement } from './sqlite-adapter.js';
@@ -24,6 +32,7 @@ export type {
   StandardSchemaIssue,
   ValidationError,
   ValidationResult,
+  TableValidationResult,
   ValidationErrorDetail,
   InferInput,
   InferOutput,

package/src/schema-extensions.test.ts

@@ -0,0 +1,155 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import {
+  isSchemaFile,
+  extractTableNameFromSchemaFile,
+  rewriteExtensionForImport,
+  findSchemaFile,
+  findSchemaFileInEntries,
+  SCHEMA_EXTENSIONS,
+} from './schema-extensions.js';
+import { writeFile, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+describe('schema-extensions', () => {
+  describe('isSchemaFile', () => {
+    it('should recognize .schema.ts', () => {
+      expect(isSchemaFile('users.schema.ts')).toBe(true);
+    });
+
+    it('should recognize .schema.mts', () => {
+      expect(isSchemaFile('users.schema.mts')).toBe(true);
+    });
+
+    it('should recognize .schema.cts', () => {
+      expect(isSchemaFile('users.schema.cts')).toBe(true);
+    });
+
+    it('should reject non-schema files', () => {
+      expect(isSchemaFile('users.ts')).toBe(false);
+      expect(isSchemaFile('db.ts')).toBe(false);
+      expect(isSchemaFile('schema.ts')).toBe(false);
+    });
+  });
+
+  describe('extractTableNameFromSchemaFile', () => {
+    it('should extract from .schema.ts', () => {
+      expect(extractTableNameFromSchemaFile('users.schema.ts')).toBe('users');
+    });
+
+    it('should extract from .schema.mts', () => {
+      expect(extractTableNameFromSchemaFile('users.schema.mts')).toBe('users');
+    });
+
+    it('should extract from .schema.cts', () => {
+      expect(extractTableNameFromSchemaFile('users.schema.cts')).toBe('users');
+    });
+
+    it('should return null for non-schema files', () => {
+      expect(extractTableNameFromSchemaFile('users.ts')).toBeNull();
+      expect(extractTableNameFromSchemaFile('db.ts')).toBeNull();
+    });
+  });
+
+  describe('rewriteExtensionForImport', () => {
+    it('.schema.ts -> .schema.js', () => {
+      expect(rewriteExtensionForImport('./users.schema.ts')).toBe('./users.schema.js');
+    });
+
+    it('.schema.mts -> .schema.mjs', () => {
+      expect(rewriteExtensionForImport('./users.schema.mts')).toBe('./users.schema.mjs');
+    });
+
+    it('.schema.cts -> .schema.cjs', () => {
+      expect(rewriteExtensionForImport('./users.schema.cts')).toBe('./users.schema.cjs');
+    });
+
+    it('should not modify paths without matching extensions', () => {
+      expect(rewriteExtensionForImport('./users.js')).toBe('./users.js');
+    });
+  });
+
+  describe('findSchemaFile', () => {
+    let testDir: string;
+
+    beforeEach(async () => {
+      testDir = join(tmpdir(), `schema-ext-test-${Date.now()}`);
+      await mkdir(testDir, { recursive: true });
+    });
+
+    afterEach(async () => {
+      await rm(testDir, { recursive: true, force: true });
+    });
+
+    it('should find .schema.ts file', async () => {
+      await writeFile(join(testDir, 'users.schema.ts'), 'export const schema = {};');
+      const result = await findSchemaFile(testDir, 'users');
+      expect(result).toBe(join(testDir, 'users.schema.ts'));
+    });
+
+    it('should find .schema.mts file', async () => {
+      await writeFile(join(testDir, 'users.schema.mts'), 'export const schema = {};');
+      const result = await findSchemaFile(testDir, 'users');
+      expect(result).toBe(join(testDir, 'users.schema.mts'));
+    });
+
+    it('should find .schema.cts file', async () => {
+      await writeFile(join(testDir, 'users.schema.cts'), 'export const schema = {};');
+      const result = await findSchemaFile(testDir, 'users');
+      expect(result).toBe(join(testDir, 'users.schema.cts'));
+    });
+
+    it('should prefer .schema.ts over .schema.mts', async () => {
+      await writeFile(join(testDir, 'users.schema.ts'), 'export const schema = {};');
+      await writeFile(join(testDir, 'users.schema.mts'), 'export const schema = {};');
+      const result = await findSchemaFile(testDir, 'users');
+      expect(result).toBe(join(testDir, 'users.schema.ts'));
+    });
+
+    it('should return undefined when no schema file exists', async () => {
+      const result = await findSchemaFile(testDir, 'users');
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('findSchemaFileInEntries', () => {
+    it('should find .schema.ts in entries', () => {
+      const entries = [
+        { isFile: () => true, name: 'users.jsonl' },
+        { isFile: () => true, name: 'users.schema.ts' },
+      ];
+      const result = findSchemaFileInEntries('/data', 'users', entries);
+      expect(result).toBe(join('/data', 'users.schema.ts'));
+    });
+
+    it('should find .schema.mts in entries', () => {
+      const entries = [
+        { isFile: () => true, name: 'users.jsonl' },
+        { isFile: () => true, name: 'users.schema.mts' },
+      ];
+      const result = findSchemaFileInEntries('/data', 'users', entries);
+      expect(result).toBe(join('/data', 'users.schema.mts'));
+    });
+
+    it('should prefer .schema.ts over .schema.mts in entries', () => {
+      const entries = [
+        { isFile: () => true, name: 'users.schema.ts' },
+        { isFile: () => true, name: 'users.schema.mts' },
+      ];
+      const result = findSchemaFileInEntries('/data', 'users', entries);
+      expect(result).toBe(join('/data', 'users.schema.ts'));
+    });
+
+    it('should return undefined when no schema file in entries', () => {
+      const entries = [{ isFile: () => true, name: 'users.jsonl' }];
+      const result = findSchemaFileInEntries('/data', 'users', entries);
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('SCHEMA_EXTENSIONS', () => {
+    it('should have correct priority order', () => {
+      expect(SCHEMA_EXTENSIONS).toEqual(['.schema.ts', '.schema.mts', '.schema.cts']);
+    });
+  });
+});