@toiroakr/lines-db 0.3.0 → 0.4.1

package/src/database.ts

@@ -107,7 +107,10 @@ export class LinesDB<Tables extends TableDefs> {
         const schemaPath = tableConfig.jsonlPath.replace('.jsonl', '.schema.ts');
         const schemaUrl = pathToFileURL(schemaPath).href;
         const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
-        foreignKeys = schemaModule.foreignKeys;
+
+        // Try to get foreign keys from exported 'schema' or directly from module
+        const schemaExport = schemaModule.schema || schemaModule.default;
+        foreignKeys = schemaExport?.foreignKeys || schemaModule.foreignKeys;
       } catch {
         // Schema file not found - will continue without validation
       }
@@ -154,7 +157,7 @@ export class LinesDB<Tables extends TableDefs> {
     // Load validation schema if provided or try to auto-load
     let validationSchema = config.validationSchema;
     const schemaMetadata: {
-      primaryKey?: readonly string[];
+      primaryKey?: string;
       foreignKeys?: BiDirectionalSchema['foreignKeys'];
       indexes?: BiDirectionalSchema['indexes'];
     } = {};
@@ -177,13 +180,24 @@ export class LinesDB<Tables extends TableDefs> {
         const schemaUrl = pathToFileURL(schemaPath).href;
         const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
 
-        if (schemaModule.primaryKey) {
+        // Try to get metadata from exported 'schema' or directly from module
+        const schemaExport = schemaModule.schema || schemaModule.default;
+
+        if (schemaExport?.primaryKey) {
+          schemaMetadata.primaryKey = schemaExport.primaryKey;
+        } else if (schemaModule.primaryKey) {
           schemaMetadata.primaryKey = schemaModule.primaryKey;
         }
-        if (schemaModule.foreignKeys) {
+
+        if (schemaExport?.foreignKeys) {
+          schemaMetadata.foreignKeys = schemaExport.foreignKeys;
+        } else if (schemaModule.foreignKeys) {
           schemaMetadata.foreignKeys = schemaModule.foreignKeys;
         }
-        if (schemaModule.indexes) {
+
+        if (schemaExport?.indexes) {
+          schemaMetadata.indexes = schemaExport.indexes;
+        } else if (schemaModule.indexes) {
           schemaMetadata.indexes = schemaModule.indexes;
         }
       } catch (_error) {
@@ -253,12 +267,17 @@ export class LinesDB<Tables extends TableDefs> {
     const indexes = biSchema?.indexes || schemaMetadata.indexes;
 
     if (primaryKey && !schema.columns.some((col) => col.primaryKey)) {
-      // Add primary key constraint to columns
-      for (const pkColumn of primaryKey) {
-        const col = schema.columns.find((c) => c.name === pkColumn);
-        if (col) {
-          col.primaryKey = true;
-        }
+      // Add primary key constraint to column
+      const col = schema.columns.find((c) => c.name === primaryKey);
+      if (col) {
+        col.primaryKey = true;
+      }
+    } else if (!primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+      // If no primary key is defined, use 'id' column as primary key if it exists
+      // This matches the behavior of JsonlReader.inferSchema()
+      const idColumn = schema.columns.find((c) => c.name === 'id');
+      if (idColumn) {
+        idColumn.primaryKey = true;
       }
     }
     if (foreignKeys) {
@@ -304,10 +323,8 @@ export class LinesDB<Tables extends TableDefs> {
     if (schema.foreignKeys && schema.foreignKeys.length > 0) {
       for (const fk of schema.foreignKeys) {
         const fkParts = [
-          `FOREIGN KEY (${fk.columns.map((col) => this.quoteIdentifier(col)).join(', ')})`,
-          `REFERENCES ${this.quoteTableName(fk.references.table)}(${fk.references.columns
-            .map((col) => this.quoteIdentifier(col))
-            .join(', ')})`,
+          `FOREIGN KEY (${this.quoteIdentifier(fk.column)})`,
+          `REFERENCES ${this.quoteTableName(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`,
         ];
         if (fk.onDelete) {
           fkParts.push(`ON DELETE ${fk.onDelete}`);

package/src/schema.ts

@@ -5,9 +5,9 @@ import type { StandardSchema, Table, ForeignKeyDefinition, IndexDefinition } fro
  */
 export interface SchemaOptions {
   /**
-   * Primary key columns
+   * Primary key column
    */
-  primaryKey?: string[];
+  primaryKey?: string;
 
   /**
    * Foreign key constraints
@@ -39,9 +39,9 @@ export interface BiDirectionalSchema<Input extends Table = Table, Output extends
   backward?: (output: Output) => Input;
 
   /**
-   * Primary key columns
+   * Primary key column
    */
-  primaryKey?: string[];
+  primaryKey?: string;
 
   /**
    * Foreign key constraints
@@ -79,9 +79,9 @@ export interface BiDirectionalSchema<Input extends Table = Table, Output extends
  * const schema = defineSchema(
  *   v.object({ id: v.number(), customerId: v.number() }),
  *   {
- *     primaryKey: ['id'],
+ *     primaryKey: 'id',
  *     foreignKeys: [
- *       { columns: ['customerId'], references: { table: 'users', columns: ['id'] } }
+ *       { column: 'customerId', references: { table: 'users', column: 'id' } }
  *     ]
  *   }
  * );

package/src/types.ts

@@ -18,10 +18,10 @@ export type InferInput<T> = T extends StandardSchemaV1<infer I, unknown> ? I : n
 export type InferOutput<T> = T extends StandardSchemaV1<unknown, infer O> ? O : never;
 
 export interface ForeignKeyDefinition {
-  columns: string[];
+  column: string;
   references: {
     table: string;
-    columns: string[];
+    column: string;
   };
   onDelete?: 'CASCADE' | 'SET NULL' | 'RESTRICT' | 'NO ACTION';
   onUpdate?: 'CASCADE' | 'SET NULL' | 'RESTRICT' | 'NO ACTION';

package/src/validator.test.ts

@@ -364,4 +364,144 @@ describe('Validator', () => {
       );
     });
   });
+
+  describe('constraint validation', () => {
+    it('should detect primary key constraint violations', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // Write data with duplicate id
+      await writeFile(
+        jsonlPath,
+        '{"id":"1","name":"Alice","email":"alice@example.com"}\n{"id":"1","name":"Bob","email":"bob@example.com"}\n',
+      );
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          },
+          primaryKey: 'id'
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[0].issues[0].message).toContain('UNIQUE constraint failed');
+      expect(result.errors[0].issues[0].message).toContain('id');
+    });
+
+    it('should detect unique index constraint violations', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // Write data with duplicate email
+      await writeFile(
+        jsonlPath,
+        '{"id":"1","name":"Alice","email":"alice@example.com"}\n{"id":"2","name":"Bob","email":"alice@example.com"}\n',
+      );
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          },
+          primaryKey: 'id',
+          indexes: [
+            { name: 'users_email_unique', columns: ['email'], unique: true }
+          ]
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[0].issues[0].message).toContain('UNIQUE constraint failed');
+      expect(result.errors[0].issues[0].message).toContain('email');
+    });
+
+    it('should use id column as primary key when primaryKey is not specified', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // Write data with duplicate id (no primaryKey specified in schema)
+      await writeFile(jsonlPath, '{"id":"1","name":"Alice"}\n{"id":"1","name":"Bob"}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          }
+          // Note: no primaryKey specified
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[0].issues[0].message).toContain('UNIQUE constraint failed');
+      expect(result.errors[0].issues[0].message).toContain('id');
+    });
+
+    it('should detect multiple constraint violations in single file', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // Write data with both duplicate id and duplicate email
+      await writeFile(
+        jsonlPath,
+        '{"id":"1","name":"Alice","email":"alice@example.com"}\n{"id":"1","name":"Bob","email":"bob@example.com"}\n{"id":"2","name":"Charlie","email":"alice@example.com"}\n',
+      );
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          },
+          primaryKey: 'id',
+          indexes: [
+            { name: 'users_email_unique', columns: ['email'], unique: true }
+          ]
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(2);
+      // First error: duplicate id
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[0].issues[0].message).toContain('id');
+      // Second error: duplicate email
+      expect(result.errors[1].rowIndex).toBe(2);
+      expect(result.errors[1].issues[0].message).toContain('email');
+    });
+  });
 });

package/src/validator.ts

@@ -1,9 +1,9 @@
 import { readdir, stat } from 'node:fs/promises';
-import { join, basename } from 'node:path';
+import { join, basename, dirname } from 'node:path';
 import { JsonlReader } from './jsonl-reader.js';
 import { SchemaLoader } from './schema-loader.js';
-import type { StandardSchemaIssue, JsonObject } from './types.js';
-import type { BiDirectionalSchema } from './schema.js';
+import { LinesDB } from './database.js';
+import type { StandardSchemaIssue } from './types.js';
 
 export interface ValidationResult {
   valid: boolean;
@@ -91,10 +91,10 @@ export class Validator {
       allWarnings.push(...result.warnings);
     }
 
-    // Then, validate foreign keys across all tables (only for files with schema)
-    if (filesWithSchema.length > 0) {
-      const fkErrors = await this.validateForeignKeys(dirPath, filesWithSchema);
-      allErrors.push(...fkErrors);
+    // Validate by loading into database with detailed error tracking
+    if (filesWithSchema.length > 0 && allErrors.length === 0) {
+      const dbErrors = await this.validateWithDatabase(dirPath, filesWithSchema);
+      allErrors.push(...dbErrors);
     }
 
     return {
@@ -105,83 +105,282 @@ export class Validator {
   }
 
   /**
-   * Validate foreign key constraints across all tables
+   * Validate by loading data into database one row at a time
+   * This catches constraint violations and extracts detailed error information
    */
-  private async validateForeignKeys(
+  private async validateWithDatabase(
     dirPath: string,
     jsonlFiles: string[],
   ): Promise<ValidationErrorDetail[]> {
     const errors: ValidationErrorDetail[] = [];
 
-    // Load all table data
-    const tableData = new Map<string, JsonObject[]>();
-    const tableSchemas = new Map<string, BiDirectionalSchema>();
+    try {
+      const db = LinesDB.create({ dataDir: ':memory:' });
 
-    for (const file of jsonlFiles) {
-      const tableName = basename(file, '.jsonl');
-      const data = await JsonlReader.read(file);
-      const schema = await SchemaLoader.loadSchema(file);
+      // Load all tables one by one, checking each row
+      for (const file of jsonlFiles) {
+        const tableName = basename(file, '.jsonl');
+        const data = await JsonlReader.read(file);
+
+        // Load schema and metadata
+        let schema: any;
+        let foreignKeys: any[] = [];
+        let indexes: any[] = [];
+        let primaryKey: string | undefined;
+        try {
+          schema = await SchemaLoader.loadSchema(file);
+          const { pathToFileURL } = await import('node:url');
+          const schemaPath = file.replace('.jsonl', '.schema.ts');
+          const schemaUrl = pathToFileURL(schemaPath).href;
+          const schemaModule = await import(`${schemaUrl}?t=${Date.now()}`);
+          const schemaExport = schemaModule.schema || schemaModule.default;
+          if (schemaExport?.foreignKeys) {
+            foreignKeys = schemaExport.foreignKeys;
+          }
+          if (schemaExport?.indexes) {
+            indexes = schemaExport.indexes;
+          }
+          if (schemaExport?.primaryKey) {
+            primaryKey = schemaExport.primaryKey;
+          }
+        } catch (_error) {
+          // Schema not found or failed to load
+          continue;
+        }
 
-      tableData.set(tableName, data);
-      tableSchemas.set(tableName, schema as BiDirectionalSchema);
+        // Create table schema
+        try {
+          const tableSchema = this.createTableSchema(
+            tableName,
+            data,
+            schema,
+            foreignKeys,
+            indexes,
+            primaryKey,
+          );
+
+          // Create the table in the database
+          this.createTableInDb(db, tableSchema);
+
+          // Insert rows one by one to catch constraint violations
+          for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
+            const row = data[rowIndex];
+            try {
+              this.insertRowIntoDb(db, tableName, tableSchema, row);
+            } catch (error) {
+              // Constraint violation occurred
+              const constraintError = this.analyzeConstraintError(
+                error,
+                file,
+                tableName,
+                rowIndex,
+                row,
+                foreignKeys,
+                db,
+              );
+              if (constraintError) {
+                errors.push(constraintError);
+              }
+            }
+          }
+        } catch (_error) {
+          // Skip this table and continue if table creation fails
+          continue;
+        }
+      }
+
+      await db.close();
+    } catch (error) {
+      // Database initialization failed
+      errors.push({
+        file: dirPath,
+        tableName: 'database',
+        rowIndex: 0,
+        issues: [
+          {
+            message: `Database initialization failed: ${error instanceof Error ? error.message : String(error)}`,
+            path: [],
+          },
+        ],
+        type: 'schema',
+      });
     }
 
-    // Check foreign keys for each table
-    for (const file of jsonlFiles) {
-      const tableName = basename(file, '.jsonl');
-      const schema = tableSchemas.get(tableName);
-      const data = tableData.get(tableName);
+    return errors;
+  }
 
-      if (!schema || !data || !schema.foreignKeys) {
-        continue;
+  /**
+   * Create table schema from data and validation schema
+   */
+  private createTableSchema(
+    tableName: string,
+    data: any[],
+    validationSchema: any,
+    foreignKeys: any[],
+    indexes: any[],
+    primaryKey?: string,
+  ): any {
+    if (data.length === 0) {
+      throw new Error(`No data found in ${tableName}`);
+    }
+
+    // Infer schema from data
+    const schema = JsonlReader.inferSchema(tableName, data);
+
+    // Set primary key if specified
+    if (primaryKey) {
+      const pkColumn = schema.columns.find((col: any) => col.name === primaryKey);
+      if (pkColumn) {
+        pkColumn.primaryKey = true;
+      }
+    } else if (!schema.columns.some((col: any) => col.primaryKey)) {
+      // If no primary key is defined, use 'id' column as primary key if it exists
+      // This matches the behavior of database.ts
+      const idColumn = schema.columns.find((c: any) => c.name === 'id');
+      if (idColumn) {
+        idColumn.primaryKey = true;
       }
+    }
 
-      // Check each foreign key constraint
-      for (const fk of schema.foreignKeys) {
-        const referencedTable = fk.references.table;
-        const referencedData = tableData.get(referencedTable);
+    // Add foreign keys
+    if (foreignKeys && foreignKeys.length > 0) {
+      schema.foreignKeys = foreignKeys;
+    }
 
-        if (!referencedData) {
-          // Referenced table not found - skip validation
-          continue;
-        }
+    // Add indexes
+    if (indexes && indexes.length > 0) {
+      schema.indexes = indexes;
+    }
 
-        // Build index of referenced values for fast lookup
-        const referencedValues = new Set<string>();
-        for (const refRow of referencedData) {
-          // Build composite key from referenced columns
-          const keyValues = fk.references.columns.map((col) => refRow[col]);
-          const compositeKey = JSON.stringify(keyValues);
-          referencedValues.add(compositeKey);
-        }
+    return schema;
+  }
 
-        // Check each row in current table
-        for (let i = 0; i < data.length; i++) {
-          const row = data[i];
-          const foreignKeyValues = fk.columns.map((col) => row[col]);
-          const compositeKey = JSON.stringify(foreignKeyValues);
+  /**
+   * Create table in database
+   */
+  private createTableInDb(db: LinesDB<any>, schema: any): void {
+    const columns = schema.columns.map((col: any) => {
+      let colDef = `${this.quoteIdentifier(col.name)} ${col.type.toUpperCase()}`;
+      if (col.primaryKey) {
+        colDef += ' PRIMARY KEY';
+      }
+      return colDef;
+    });
+
+    // Add foreign key constraints
+    if (schema.foreignKeys && schema.foreignKeys.length > 0) {
+      for (const fk of schema.foreignKeys) {
+        columns.push(
+          `FOREIGN KEY (${this.quoteIdentifier(fk.column)}) REFERENCES ${this.quoteIdentifier(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`,
+        );
+      }
+    }
+
+    const sql = `CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(schema.name)} (${columns.join(', ')})`;
+    db.execute(sql);
+
+    // Create indexes
+    if (schema.indexes && schema.indexes.length > 0) {
+      for (const index of schema.indexes) {
+        const indexName = index.name || `idx_${schema.name}_${index.columns.join('_')}`;
+        const uniqueKeyword = index.unique ? 'UNIQUE' : '';
+        const indexColumns = index.columns
+          .map((col: string) => this.quoteIdentifier(col))
+          .join(', ');
+        const indexSql = `CREATE ${uniqueKeyword} INDEX IF NOT EXISTS ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(schema.name)} (${indexColumns})`;
+        db.execute(indexSql);
+      }
+    }
+  }
+
+  /**
+   * Insert a row into database
+   */
+  private insertRowIntoDb(db: LinesDB<any>, tableName: string, schema: any, row: any): void {
+    const columnNames = schema.columns.map((col: any) => col.name);
+    const quotedColumns = columnNames.map((name: string) => this.quoteIdentifier(name));
+    const placeholders = columnNames.map(() => '?').join(', ');
+    const sql = `INSERT INTO ${this.quoteIdentifier(tableName)} (${quotedColumns.join(', ')}) VALUES (${placeholders})`;
+
+    const values = columnNames.map((col: string) => {
+      const value = row[col];
+      if (value === null || value === undefined) return null;
+      if (typeof value === 'object') return JSON.stringify(value);
+      if (typeof value === 'boolean') return value ? 1 : 0;
+      return value;
+    });
+
+    db.execute(sql, values);
+  }
 
-          // Check if foreign key value exists in referenced table
-          if (!referencedValues.has(compositeKey)) {
-            errors.push({
+  /**
+   * Analyze constraint error and extract detailed information
+   */
+  private analyzeConstraintError(
+    error: any,
+    file: string,
+    tableName: string,
+    rowIndex: number,
+    row: any,
+    foreignKeys: any[],
+    db: LinesDB<any>,
+  ): ValidationErrorDetail | null {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+
+    // Foreign key constraint
+    if (errorMessage.includes('FOREIGN KEY constraint failed')) {
+      // Find which foreign key failed
+      for (const fk of foreignKeys) {
+        const fkValue = row[fk.column];
+        if (fkValue === null || fkValue === undefined) continue;
+
+        // Check if referenced value exists
+        try {
+          const result = db.query(
+            `SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`,
+            [fkValue],
+          );
+          if (result.length > 0 && (result[0] as any).count === 0) {
+            return {
               file,
               tableName,
-              rowIndex: i, // 0-indexed, will be converted to 1-indexed in formatter
+              rowIndex,
               issues: [],
               type: 'foreignKey',
               foreignKeyError: {
-                column: fk.columns.join(', '),
-                value: foreignKeyValues.length === 1 ? foreignKeyValues[0] : foreignKeyValues,
-                referencedTable: referencedTable,
-                referencedColumn: fk.references.columns.join(', '),
+                column: fk.column,
+                value: fkValue,
+                referencedTable: fk.references.table,
+                referencedColumn: fk.references.column,
               },
-            });
+            };
           }
+        } catch (_) {
+          // Referenced table doesn't exist yet
         }
       }
     }
 
-    return errors;
+    // Other constraint errors (primary key, unique, etc.)
+    return {
+      file,
+      tableName,
+      rowIndex,
+      issues: [
+        {
+          message: errorMessage,
+          path: [],
+        },
+      ],
+      type: 'schema',
+    };
+  }
+
+  /**
+   * Quote SQL identifier
+   */
+  private quoteIdentifier(name: string): string {
+    return `"${name.replace(/"/g, '""')}"`;
   }
 
   /**
@@ -196,7 +395,7 @@ export class Validator {
 
     const errors: ValidationErrorDetail[] = [];
 
-    // Validate each row
+    // Validate each row with schema
     for (let i = 0; i < data.length; i++) {
       const row = data[i];
       const result = schema['~standard'].validate(row);
@@ -217,6 +416,22 @@ export class Validator {
       }
     }
 
+    // If schema validation passed, validate database constraints
+    if (errors.length === 0) {
+      const dirPath = dirname(filePath);
+
+      // Get all JSONL files in the directory
+      const entries = await readdir(dirPath, { withFileTypes: true });
+      const allJsonlFiles = entries
+        .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl'))
+        .map((entry) => join(dirPath, entry.name));
+
+      // Validate database constraints (including foreign keys)
+      const dbErrors = await this.validateWithDatabase(dirPath, allJsonlFiles);
+      // Only include errors for the current file
+      errors.push(...dbErrors.filter((e) => e.file === filePath));
+    }
+
     return {
       valid: errors.length === 0,
       errors,