@toiroakr/lines-db 0.3.0 → 0.4.0

package/src/schema.ts

@@ -5,9 +5,9 @@ import type { StandardSchema, Table, ForeignKeyDefinition, IndexDefinition } fro
  */
 export interface SchemaOptions {
   /**
-   * Primary key columns
+   * Primary key column
    */
-  primaryKey?: string[];
+  primaryKey?: string;
 
   /**
    * Foreign key constraints
@@ -39,9 +39,9 @@ export interface BiDirectionalSchema<Input extends Table = Table, Output extends
   backward?: (output: Output) => Input;
 
   /**
-   * Primary key columns
+   * Primary key column
    */
-  primaryKey?: string[];
+  primaryKey?: string;
 
   /**
    * Foreign key constraints
@@ -79,9 +79,9 @@ export interface BiDirectionalSchema<Input extends Table = Table, Output extends
  * const schema = defineSchema(
  *   v.object({ id: v.number(), customerId: v.number() }),
  *   {
- *     primaryKey: ['id'],
+ *     primaryKey: 'id',
  *     foreignKeys: [
- *       { columns: ['customerId'], references: { table: 'users', columns: ['id'] } }
+ *       { column: 'customerId', references: { table: 'users', column: 'id' } }
  *     ]
  *   }
  * );

package/src/types.ts

@@ -18,10 +18,10 @@ export type InferInput<T> = T extends StandardSchemaV1<infer I, unknown> ? I : n
 export type InferOutput<T> = T extends StandardSchemaV1<unknown, infer O> ? O : never;
 
 export interface ForeignKeyDefinition {
-  columns: string[];
+  column: string;
   references: {
     table: string;
-    columns: string[];
+    column: string;
   };
   onDelete?: 'CASCADE' | 'SET NULL' | 'RESTRICT' | 'NO ACTION';
   onUpdate?: 'CASCADE' | 'SET NULL' | 'RESTRICT' | 'NO ACTION';

package/src/validator.ts

@@ -1,9 +1,9 @@
 import { readdir, stat } from 'node:fs/promises';
-import { join, basename } from 'node:path';
+import { join, basename, dirname } from 'node:path';
 import { JsonlReader } from './jsonl-reader.js';
 import { SchemaLoader } from './schema-loader.js';
-import type { StandardSchemaIssue, JsonObject } from './types.js';
-import type { BiDirectionalSchema } from './schema.js';
+import { LinesDB } from './database.js';
+import type { StandardSchemaIssue } from './types.js';
 
 export interface ValidationResult {
   valid: boolean;
@@ -91,10 +91,10 @@ export class Validator {
       allWarnings.push(...result.warnings);
     }
 
-    // Then, validate foreign keys across all tables (only for files with schema)
-    if (filesWithSchema.length > 0) {
-      const fkErrors = await this.validateForeignKeys(dirPath, filesWithSchema);
-      allErrors.push(...fkErrors);
+    // Then, validate by actually loading into database
+    if (filesWithSchema.length > 0 && allErrors.length === 0) {
+      const dbErrors = await this.validateWithDatabase(dirPath, filesWithSchema);
+      allErrors.push(...dbErrors);
     }
 
     return {
@@ -105,80 +105,71 @@ export class Validator {
   }
 
   /**
-   * Validate foreign key constraints across all tables
+   * Validate by loading data into an actual database
+   * This catches constraint violations (unique, primary key, foreign key, etc.)
    */
-  private async validateForeignKeys(
+  private async validateWithDatabase(
     dirPath: string,
     jsonlFiles: string[],
   ): Promise<ValidationErrorDetail[]> {
     const errors: ValidationErrorDetail[] = [];
 
-    // Load all table data
-    const tableData = new Map<string, JsonObject[]>();
-    const tableSchemas = new Map<string, BiDirectionalSchema>();
-
-    for (const file of jsonlFiles) {
-      const tableName = basename(file, '.jsonl');
-      const data = await JsonlReader.read(file);
-      const schema = await SchemaLoader.loadSchema(file);
-
-      tableData.set(tableName, data);
-      tableSchemas.set(tableName, schema as BiDirectionalSchema);
-    }
-
-    // Check foreign keys for each table
-    for (const file of jsonlFiles) {
-      const tableName = basename(file, '.jsonl');
-      const schema = tableSchemas.get(tableName);
-      const data = tableData.get(tableName);
-
-      if (!schema || !data || !schema.foreignKeys) {
-        continue;
-      }
-
-      // Check each foreign key constraint
-      for (const fk of schema.foreignKeys) {
-        const referencedTable = fk.references.table;
-        const referencedData = tableData.get(referencedTable);
-
-        if (!referencedData) {
-          // Referenced table not found - skip validation
-          continue;
-        }
-
-        // Build index of referenced values for fast lookup
-        const referencedValues = new Set<string>();
-        for (const refRow of referencedData) {
-          // Build composite key from referenced columns
-          const keyValues = fk.references.columns.map((col) => refRow[col]);
-          const compositeKey = JSON.stringify(keyValues);
-          referencedValues.add(compositeKey);
-        }
+    // Capture console.warn messages
+    const warnMessages: string[] = [];
+    const originalWarn = console.warn;
+    console.warn = (...args: unknown[]) => {
+      const message = args.map((arg) => String(arg)).join(' ');
+      warnMessages.push(message);
+      // Still output to console for debugging
+      originalWarn(...args);
+    };
 
-        // Check each row in current table
-        for (let i = 0; i < data.length; i++) {
-          const row = data[i];
-          const foreignKeyValues = fk.columns.map((col) => row[col]);
-          const compositeKey = JSON.stringify(foreignKeyValues);
-
-          // Check if foreign key value exists in referenced table
-          if (!referencedValues.has(compositeKey)) {
-            errors.push({
-              file,
-              tableName,
-              rowIndex: i, // 0-indexed, will be converted to 1-indexed in formatter
-              issues: [],
-              type: 'foreignKey',
-              foreignKeyError: {
-                column: fk.columns.join(', '),
-                value: foreignKeyValues.length === 1 ? foreignKeyValues[0] : foreignKeyValues,
-                referencedTable: referencedTable,
-                referencedColumn: fk.references.columns.join(', '),
+    try {
+      // Try to initialize database with the data directory
+      const db = LinesDB.create({ dataDir: dirPath });
+      await db.initialize();
+      await db.close();
+
+      // Check if there were any loading errors
+      for (const message of warnMessages) {
+        if (message.includes('Failed to load table')) {
+          // Extract table name from message
+          const tableNameMatch = message.match(/Failed to load table '([^']+)'/);
+          const tableName = tableNameMatch ? tableNameMatch[1] : 'unknown';
+
+          const file = jsonlFiles.find((f) => basename(f, '.jsonl') === tableName);
+
+          errors.push({
+            file: file || `${dirPath}/${tableName}.jsonl`,
+            tableName,
+            rowIndex: 0,
+            issues: [
+              {
+                message: message.replace(/^Warning:\s*/, ''),
+                path: [],
               },
-            });
-          }
+            ],
+            type: 'schema',
+          });
         }
       }
+    } catch (error) {
+      // If initialization itself fails, report it
+      errors.push({
+        file: dirPath,
+        tableName: 'database',
+        rowIndex: 0,
+        issues: [
+          {
+            message: `Database initialization failed: ${error instanceof Error ? error.message : String(error)}`,
+            path: [],
+          },
+        ],
+        type: 'schema',
+      });
+    } finally {
+      // Restore console.warn
+      console.warn = originalWarn;
     }
 
     return errors;
@@ -196,7 +187,7 @@ export class Validator {
 
     const errors: ValidationErrorDetail[] = [];
 
-    // Validate each row
+    // Validate each row with schema
     for (let i = 0; i < data.length; i++) {
       const row = data[i];
       const result = schema['~standard'].validate(row);
@@ -217,6 +208,13 @@ export class Validator {
       }
     }
 
+    // If schema validation passed, validate with database
+    if (errors.length === 0) {
+      const dirPath = dirname(filePath);
+      const dbErrors = await this.validateWithDatabase(dirPath, [filePath]);
+      errors.push(...dbErrors);
+    }
+
     return {
       valid: errors.length === 0,
       errors,