@toiroakr/lines-db 0.2.1 → 0.4.0

package/src/database.ts

@@ -40,108 +40,186 @@ export class LinesDB<Tables extends TableDefs> {
 
   /**
    * Initialize database by loading all JSONL files
+   * Uses dependency resolution to ensure foreign key references are loaded in correct order
    */
   async initialize(): Promise<void> {
     // Scan directory for JSONL files
     this.tables = await DirectoryScanner.scanDirectory(this.config.dataDir);
 
-    // Load all tables
-    for (const [tableName, tableConfig] of this.tables) {
+    // Track loaded tables and tables currently being loaded (for circular dependency detection)
+    const loadedTables = new Set<string>();
+    const loadingTables = new Set<string>();
+
+    // Load all tables with dependency resolution
+    for (const [tableName] of this.tables) {
+      if (!loadedTables.has(tableName)) {
+        try {
+          await this.loadTableWithDependencies(tableName, loadedTables, loadingTables);
+        } catch (error) {
+          // Log error but continue loading other tables
+          console.warn(
+            `Warning: Failed to load table '${tableName}':`,
+            error instanceof Error ? error.message : String(error),
+          );
+          // Remove the failed table from the tables map
+          this.tables.delete(tableName);
+          this.schemas.delete(tableName);
+          this.validationSchemas.delete(tableName);
+        }
+      }
+    }
+  }
+
+  /**
+   * Load a table and its dependencies recursively
+   */
+  private async loadTableWithDependencies(
+    tableName: string,
+    loadedTables: Set<string>,
+    loadingTables: Set<string>,
+  ): Promise<void> {
+    // Skip if already loaded
+    if (loadedTables.has(tableName)) {
+      return;
+    }
+
+    // Check for circular dependencies
+    if (loadingTables.has(tableName)) {
+      throw new Error(`Circular dependency detected for table '${tableName}'`);
+    }
+
+    // Get table config
+    const tableConfig = this.tables.get(tableName);
+    if (!tableConfig) {
+      throw new Error(`Table configuration not found for '${tableName}'`);
+    }
+
+    // Mark as currently loading
+    loadingTables.add(tableName);
+
+    try {
+      // Load schema module to check for foreign key dependencies
+      // We need to load the entire module to access foreignKeys export
+      let foreignKeys: BiDirectionalSchema['foreignKeys'];
+
       try {
-        await this.loadTable(tableName, tableConfig);
-      } catch (error) {
-        // Log error but continue loading other tables
-        console.warn(
-          `Warning: Failed to load table '${tableName}':`,
-          error instanceof Error ? error.message : String(error),
-        );
-        // Remove the failed table from the tables map
+        const { pathToFileURL } = await import('node:url');
+        const schemaPath = tableConfig.jsonlPath.replace('.jsonl', '.schema.ts');
+        const schemaUrl = pathToFileURL(schemaPath).href;
+        const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
+
+        // Try to get foreign keys from exported 'schema' or directly from module
+        const schemaExport = schemaModule.schema || schemaModule.default;
+        foreignKeys = schemaExport?.foreignKeys || schemaModule.foreignKeys;
+      } catch {
+        // Schema file not found - will continue without validation
+      }
+
+      // If there are foreign key dependencies, load them first
+      if (foreignKeys && foreignKeys.length > 0) {
+        for (const fk of foreignKeys) {
+          const referencedTable = fk.references.table;
+          if (!loadedTables.has(referencedTable)) {
+            // Check if referenced table exists in our tables map
+            if (this.tables.has(referencedTable)) {
+              await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables);
+            } else {
+              throw new Error(
+                `Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`,
+              );
+            }
+          }
+        }
+      }
+
+      // Now load this table
+      const wasLoaded = await this.loadTable(tableName, tableConfig);
+      if (wasLoaded) {
+        loadedTables.add(tableName);
+      } else {
+        // Table was not loaded (e.g., empty data)
         this.tables.delete(tableName);
       }
+    } finally {
+      // Remove from loading set
+      loadingTables.delete(tableName);
     }
   }
 
   /**
    * Load a single table from JSONL file
+   * @returns true if table was loaded, false if skipped
    */
-  private async loadTable(tableName: string, config: TableConfig): Promise<void> {
+  private async loadTable(tableName: string, config: TableConfig): Promise<boolean> {
     // Read JSONL file
     const data = await JsonlReader.read(config.jsonlPath);
 
-    if (data.length === 0) {
-      console.warn(`Warning: Table ${tableName} has no data`);
-      return;
-    }
-
     // Load validation schema if provided or try to auto-load
     let validationSchema = config.validationSchema;
+    const schemaMetadata: {
+      primaryKey?: string;
+      foreignKeys?: BiDirectionalSchema['foreignKeys'];
+      indexes?: BiDirectionalSchema['indexes'];
+    } = {};
+
     if (!validationSchema) {
       try {
         validationSchema = await SchemaLoader.loadSchema(config.jsonlPath);
-      } catch (error) {
+      } catch (_error) {
         // Schema file not found or failed to load - this is OK, table can still be used without validation
-        console.log(
-          `[LinesDB] No validation schema for table '${tableName}':`,
-          error instanceof Error ? error.message : String(error),
-        );
       }
     }
-    console.log(
-      `[LinesDB] Loaded validation schema for table '${tableName}':`,
-      validationSchema ? 'FOUND' : 'NOT FOUND',
-    );
-    if (validationSchema) {
-      console.log(`[LinesDB] Schema type:`, typeof validationSchema);
-      console.log(`[LinesDB] Schema has '~standard':`, '~standard' in validationSchema);
-    }
-    this.validationSchemas.set(tableName, validationSchema);
 
-    // Determine schema
-    let schema: TableSchema;
-    if (config.schema) {
-      schema = config.schema;
-    } else if (config.autoInferSchema !== false) {
-      schema = JsonlReader.inferSchema(tableName, data);
-    } else {
-      throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
-    }
+    // Load schema metadata (foreignKeys, primaryKey, indexes) from schema module
+    // SchemaLoader.loadSchema() only returns the validation schema object, not metadata
+    if (!config.validationSchema) {
+      // Only load if not already provided via config
+      try {
+        const { pathToFileURL } = await import('node:url');
+        const schemaPath = config.jsonlPath.replace('.jsonl', '.schema.ts');
+        const schemaUrl = pathToFileURL(schemaPath).href;
+        const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
+
+        // Try to get metadata from exported 'schema' or directly from module
+        const schemaExport = schemaModule.schema || schemaModule.default;
+
+        if (schemaExport?.primaryKey) {
+          schemaMetadata.primaryKey = schemaExport.primaryKey;
+        } else if (schemaModule.primaryKey) {
+          schemaMetadata.primaryKey = schemaModule.primaryKey;
+        }
 
-    // Enhance schema with constraints from validation schema (if available)
-    if (validationSchema) {
-      const biSchema = validationSchema as BiDirectionalSchema;
-      if (biSchema.primaryKey && !schema.columns.some((col) => col.primaryKey)) {
-        // Add primary key constraint to columns
-        for (const pkColumn of biSchema.primaryKey) {
-          const col = schema.columns.find((c) => c.name === pkColumn);
-          if (col) {
-            col.primaryKey = true;
-          }
+        if (schemaExport?.foreignKeys) {
+          schemaMetadata.foreignKeys = schemaExport.foreignKeys;
+        } else if (schemaModule.foreignKeys) {
+          schemaMetadata.foreignKeys = schemaModule.foreignKeys;
         }
-      }
-      if (biSchema.foreignKeys) {
-        schema.foreignKeys = biSchema.foreignKeys;
-      }
-      if (biSchema.indexes) {
-        schema.indexes = biSchema.indexes;
+
+        if (schemaExport?.indexes) {
+          schemaMetadata.indexes = schemaExport.indexes;
+        } else if (schemaModule.indexes) {
+          schemaMetadata.indexes = schemaModule.indexes;
+        }
+      } catch (_error) {
+        // Schema file not found - this is OK
       }
     }
 
-    this.schemas.set(tableName, schema);
-
-    // Create table
-    this.createTable(schema);
+    this.validationSchemas.set(tableName, validationSchema);
 
-    // Validate data before inserting
+    // Validate data first and collect validated (transformed) data
     const validationErrors: Array<{
       rowIndex: number;
       rowData: JsonObject;
       error: ValidationError;
     }> = [];
+    const validatedData: JsonObject[] = [];
 
     for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
       const row = data[rowIndex];
       try {
-        this.validateData(tableName, row);
+        const validatedRow = this.validateAndTransform(tableName, row);
+        validatedData.push(validatedRow);
       } catch (error) {
         if (error instanceof Error && error.name === 'ValidationError') {
           validationErrors.push({
@@ -167,15 +245,65 @@ export class LinesDB<Tables extends TableDefs> {
       throw enhancedError;
     }
 
-    this.insertData(tableName, schema, data);
+    // Determine schema - infer from validated data if auto-inference is enabled
+    let schema: TableSchema;
+    if (config.schema) {
+      schema = config.schema;
+    } else if (config.autoInferSchema !== false) {
+      if (validatedData.length === 0) {
+        return false;
+      }
+      // Infer schema from validated data (which may have additional fields added by validation)
+      schema = JsonlReader.inferSchema(tableName, validatedData);
+    } else {
+      throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
+    }
+
+    // Enhance schema with constraints from validation schema and schema metadata
+    // Priority: config.validationSchema (as BiDirectionalSchema) > schemaMetadata
+    const biSchema = validationSchema as BiDirectionalSchema;
+    const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
+    const foreignKeys = biSchema?.foreignKeys || schemaMetadata.foreignKeys;
+    const indexes = biSchema?.indexes || schemaMetadata.indexes;
+
+    if (primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+      // Add primary key constraint to column
+      const col = schema.columns.find((c) => c.name === primaryKey);
+      if (col) {
+        col.primaryKey = true;
+      }
+    } else if (!primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+      // If no primary key is defined, use 'id' column as primary key if it exists
+      // This matches the behavior of JsonlReader.inferSchema()
+      const idColumn = schema.columns.find((c) => c.name === 'id');
+      if (idColumn) {
+        idColumn.primaryKey = true;
+      }
+    }
+    if (foreignKeys) {
+      schema.foreignKeys = foreignKeys;
+    }
+    if (indexes) {
+      schema.indexes = indexes;
+    }
+
+    this.schemas.set(tableName, schema);
+
+    // Create table
+    this.createTable(schema);
+
+    // Insert validated data
+    this.insertData(tableName, schema, validatedData);
+
+    return true;
   }
 
   /**
    * Create table in SQLite with constraints and indexes
    */
   private createTable(schema: TableSchema): void {
-    // Enable foreign key constraints
-    this.db.exec('PRAGMA foreign_keys = ON');
+    // Note: Foreign key constraints are enabled at database connection level (see sqlite-adapter.ts)
+    // No need to enable them here for each table
 
     // Quote table name to handle special characters
     const quotedTableName = this.quoteTableName(schema.name);
@@ -195,10 +323,8 @@ export class LinesDB<Tables extends TableDefs> {
     if (schema.foreignKeys && schema.foreignKeys.length > 0) {
       for (const fk of schema.foreignKeys) {
         const fkParts = [
-          `FOREIGN KEY (${fk.columns.map((col) => this.quoteIdentifier(col)).join(', ')})`,
-          `REFERENCES ${this.quoteTableName(fk.references.table)}(${fk.references.columns
-            .map((col) => this.quoteIdentifier(col))
-            .join(', ')})`,
+          `FOREIGN KEY (${this.quoteIdentifier(fk.column)})`,
+          `REFERENCES ${this.quoteTableName(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`,
         ];
         if (fk.onDelete) {
           fkParts.push(`ON DELETE ${fk.onDelete}`);
@@ -402,19 +528,14 @@ export class LinesDB<Tables extends TableDefs> {
   }
 
   /**
-   * Validate data using StandardSchema
+   * Validate data using StandardSchema and return the transformed value
    * Note: Only synchronous validation is supported
    */
-  private validateData(tableName: string, data: unknown): void {
+  private validateAndTransform(tableName: string, data: unknown): JsonObject {
     const schema = this.validationSchemas.get(tableName);
-    console.log(`[LinesDB] validateData called for table '${tableName}', schema exists:`, !!schema);
     if (!schema) {
-      console.log(
-        `[LinesDB] No validation schema found for table '${tableName}', skipping validation`,
-      );
-      return;
+      return data as JsonObject;
     }
-    console.log(`[LinesDB] Validating data:`, JSON.stringify(data));
 
     const result = schema['~standard'].validate(data);
 
@@ -452,6 +573,27 @@ export class LinesDB<Tables extends TableDefs> {
       error.issues = result.issues;
       throw error;
     }
+
+    // Return the transformed value from validation
+    // When there are no issues, result.value should be present
+    const transformedValue = ('value' in result ? result.value : data) as JsonObject;
+
+    // Convert undefined values to null for JSON compatibility
+    const normalizedValue: JsonObject = {};
+    for (const [key, value] of Object.entries(transformedValue)) {
+      normalizedValue[key] = value === undefined ? null : value;
+    }
+
+    return normalizedValue;
+  }
+
+  /**
+   * Validate data using StandardSchema (without returning transformed value)
+   * Note: Only synchronous validation is supported
+   */
+  private validateData(tableName: string, data: unknown): void {
+    // Use validateAndTransform but discard the result
+    this.validateAndTransform(tableName, data);
   }
 
   /**
@@ -999,7 +1141,9 @@ export class LinesDB<Tables extends TableDefs> {
 
       return result;
     } catch (error) {
-      this.db.exec('ROLLBACK');
+      if (this.inTransaction) {
+        this.db.exec('ROLLBACK');
+      }
       this.inTransaction = false;
       throw error;
     }

package/src/jsonl-reader.ts

@@ -123,7 +123,7 @@ export class JsonlReader {
   }
 
   private static inferType(value: unknown): string {
-    if (value === null) return 'NULL';
+    if (value === null || value === undefined) return 'NULL';
     if (typeof value === 'number') {
       return Number.isInteger(value) ? 'INTEGER' : 'REAL';
     }

package/src/schema.ts

@@ -5,9 +5,9 @@ import type { StandardSchema, Table, ForeignKeyDefinition, IndexDefinition } fro
  */
 export interface SchemaOptions {
   /**
-   * Primary key columns
+   * Primary key column
    */
-  primaryKey?: string[];
+  primaryKey?: string;
 
   /**
    * Foreign key constraints
@@ -39,9 +39,9 @@ export interface BiDirectionalSchema<Input extends Table = Table, Output extends
   backward?: (output: Output) => Input;
 
   /**
-   * Primary key columns
+   * Primary key column
    */
-  primaryKey?: string[];
+  primaryKey?: string;
 
   /**
    * Foreign key constraints
@@ -79,9 +79,9 @@ export interface BiDirectionalSchema<Input extends Table = Table, Output extends
  * const schema = defineSchema(
  *   v.object({ id: v.number(), customerId: v.number() }),
  *   {
- *     primaryKey: ['id'],
+ *     primaryKey: 'id',
  *     foreignKeys: [
- *       { columns: ['customerId'], references: { table: 'users', columns: ['id'] } }
+ *       { column: 'customerId', references: { table: 'users', column: 'id' } }
  *     ]
  *   }
  * );

package/src/sqlite-adapter.ts

@@ -39,6 +39,10 @@ function createNodeDatabase(path: string): SQLiteDatabase {
   const { DatabaseSync } = require('node:sqlite');
   const db = new DatabaseSync(path);
 
+  // CRITICAL: Enable foreign key constraints
+  // SQLite disables foreign keys by default, which is a major data integrity issue
+  db.exec('PRAGMA foreign_keys = ON');
+
   return {
     prepare(sql: string): SQLiteStatement {
       const stmt = db.prepare(sql);

package/src/types.ts

@@ -18,10 +18,10 @@ export type InferInput<T> = T extends StandardSchemaV1<infer I, unknown> ? I : n
 export type InferOutput<T> = T extends StandardSchemaV1<unknown, infer O> ? O : never;
 
 export interface ForeignKeyDefinition {
-  columns: string[];
+  column: string;
   references: {
     table: string;
-    columns: string[];
+    column: string;
   };
   onDelete?: 'CASCADE' | 'SET NULL' | 'RESTRICT' | 'NO ACTION';
   onUpdate?: 'CASCADE' | 'SET NULL' | 'RESTRICT' | 'NO ACTION';

package/src/validator.ts

@@ -1,9 +1,9 @@
 import { readdir, stat } from 'node:fs/promises';
-import { join, basename } from 'node:path';
+import { join, basename, dirname } from 'node:path';
 import { JsonlReader } from './jsonl-reader.js';
 import { SchemaLoader } from './schema-loader.js';
-import type { StandardSchemaIssue, JsonObject } from './types.js';
-import type { BiDirectionalSchema } from './schema.js';
+import { LinesDB } from './database.js';
+import type { StandardSchemaIssue } from './types.js';
 
 export interface ValidationResult {
   valid: boolean;
@@ -91,10 +91,10 @@ export class Validator {
       allWarnings.push(...result.warnings);
     }
 
-    // Then, validate foreign keys across all tables (only for files with schema)
-    if (filesWithSchema.length > 0) {
-      const fkErrors = await this.validateForeignKeys(dirPath, filesWithSchema);
-      allErrors.push(...fkErrors);
+    // Then, validate by actually loading into database
+    if (filesWithSchema.length > 0 && allErrors.length === 0) {
+      const dbErrors = await this.validateWithDatabase(dirPath, filesWithSchema);
+      allErrors.push(...dbErrors);
     }
 
     return {
@@ -105,80 +105,71 @@ export class Validator {
   }
 
   /**
-   * Validate foreign key constraints across all tables
+   * Validate by loading data into an actual database
+   * This catches constraint violations (unique, primary key, foreign key, etc.)
    */
-  private async validateForeignKeys(
+  private async validateWithDatabase(
     dirPath: string,
     jsonlFiles: string[],
   ): Promise<ValidationErrorDetail[]> {
     const errors: ValidationErrorDetail[] = [];
 
-    // Load all table data
-    const tableData = new Map<string, JsonObject[]>();
-    const tableSchemas = new Map<string, BiDirectionalSchema>();
-
-    for (const file of jsonlFiles) {
-      const tableName = basename(file, '.jsonl');
-      const data = await JsonlReader.read(file);
-      const schema = await SchemaLoader.loadSchema(file);
-
-      tableData.set(tableName, data);
-      tableSchemas.set(tableName, schema as BiDirectionalSchema);
-    }
-
-    // Check foreign keys for each table
-    for (const file of jsonlFiles) {
-      const tableName = basename(file, '.jsonl');
-      const schema = tableSchemas.get(tableName);
-      const data = tableData.get(tableName);
-
-      if (!schema || !data || !schema.foreignKeys) {
-        continue;
-      }
-
-      // Check each foreign key constraint
-      for (const fk of schema.foreignKeys) {
-        const referencedTable = fk.references.table;
-        const referencedData = tableData.get(referencedTable);
-
-        if (!referencedData) {
-          // Referenced table not found - skip validation
-          continue;
-        }
-
-        // Build index of referenced values for fast lookup
-        const referencedValues = new Set<string>();
-        for (const refRow of referencedData) {
-          // Build composite key from referenced columns
-          const keyValues = fk.references.columns.map((col) => refRow[col]);
-          const compositeKey = JSON.stringify(keyValues);
-          referencedValues.add(compositeKey);
-        }
+    // Capture console.warn messages
+    const warnMessages: string[] = [];
+    const originalWarn = console.warn;
+    console.warn = (...args: unknown[]) => {
+      const message = args.map((arg) => String(arg)).join(' ');
+      warnMessages.push(message);
+      // Still output to console for debugging
+      originalWarn(...args);
+    };
 
-        // Check each row in current table
-        for (let i = 0; i < data.length; i++) {
-          const row = data[i];
-          const foreignKeyValues = fk.columns.map((col) => row[col]);
-          const compositeKey = JSON.stringify(foreignKeyValues);
-
-          // Check if foreign key value exists in referenced table
-          if (!referencedValues.has(compositeKey)) {
-            errors.push({
-              file,
-              tableName,
-              rowIndex: i, // 0-indexed, will be converted to 1-indexed in formatter
-              issues: [],
-              type: 'foreignKey',
-              foreignKeyError: {
-                column: fk.columns.join(', '),
-                value: foreignKeyValues.length === 1 ? foreignKeyValues[0] : foreignKeyValues,
-                referencedTable: referencedTable,
-                referencedColumn: fk.references.columns.join(', '),
+    try {
+      // Try to initialize database with the data directory
+      const db = LinesDB.create({ dataDir: dirPath });
+      await db.initialize();
+      await db.close();
+
+      // Check if there were any loading errors
+      for (const message of warnMessages) {
+        if (message.includes('Failed to load table')) {
+          // Extract table name from message
+          const tableNameMatch = message.match(/Failed to load table '([^']+)'/);
+          const tableName = tableNameMatch ? tableNameMatch[1] : 'unknown';
+
+          const file = jsonlFiles.find((f) => basename(f, '.jsonl') === tableName);
+
+          errors.push({
+            file: file || `${dirPath}/${tableName}.jsonl`,
+            tableName,
+            rowIndex: 0,
+            issues: [
+              {
+                message: message.replace(/^Warning:\s*/, ''),
+                path: [],
               },
-            });
-          }
+            ],
+            type: 'schema',
+          });
         }
       }
+    } catch (error) {
+      // If initialization itself fails, report it
+      errors.push({
+        file: dirPath,
+        tableName: 'database',
+        rowIndex: 0,
+        issues: [
+          {
+            message: `Database initialization failed: ${error instanceof Error ? error.message : String(error)}`,
+            path: [],
+          },
+        ],
+        type: 'schema',
+      });
+    } finally {
+      // Restore console.warn
+      console.warn = originalWarn;
     }
 
     return errors;
@@ -196,7 +187,7 @@ export class Validator {
 
     const errors: ValidationErrorDetail[] = [];
 
-    // Validate each row
+    // Validate each row with schema
     for (let i = 0; i < data.length; i++) {
       const row = data[i];
       const result = schema['~standard'].validate(row);
@@ -217,6 +208,13 @@ export class Validator {
       }
     }
 
+    // If schema validation passed, validate with database
+    if (errors.length === 0) {
+      const dirPath = dirname(filePath);
+      const dbErrors = await this.validateWithDatabase(dirPath, [filePath]);
+      errors.push(...dbErrors);
+    }
+
     return {
       valid: errors.length === 0,
       errors,