@toiroakr/lines-db 0.5.0 → 0.6.1

package/src/database.ts

@@ -10,11 +10,13 @@ import type {
   JsonObject,
   TableConfig,
   StandardSchema,
-  StandardSchemaIssue,
   ValidationError,
   Table,
   TableDefs,
   WhereCondition,
+  ValidationResult,
+  ValidationErrorDetail,
+  ForeignKeyDefinition,
 } from './types.js';
 import type { BiDirectionalSchema } from './schema.js';
 
@@ -39,35 +41,68 @@ export class LinesDB<Tables extends TableDefs> {
   }
 
   /**
-   * Initialize database by loading all JSONL files
+   * Initialize database by loading all JSONL files or a specific table
    * Uses dependency resolution to ensure foreign key references are loaded in correct order
+   * @param options Optional configuration for initialization
+   * @param options.tableName Optional table name to initialize. If not provided, initializes all tables
+   * @param options.detailedValidate If true, performs detailed validation by inserting rows one by one to catch constraint violations
+   * @param options.transform Optional transform function to apply to rows before validation (only applied to the specified tableName)
+   * @returns ValidationResult containing validation status, errors, and warnings
    */
-  async initialize(): Promise<void> {
+  async initialize(options?: {
+    tableName?: string;
+    detailedValidate?: boolean;
+    transform?: (row: JsonObject) => JsonObject;
+  }): Promise<ValidationResult> {
+    const allErrors: ValidationErrorDetail[] = [];
+    const allWarnings: string[] = [];
+    const tableName = options?.tableName;
+    const detailedValidate = options?.detailedValidate ?? false;
+    const transform = options?.transform;
+
     // Scan directory for JSONL files
     this.tables = await DirectoryScanner.scanDirectory(this.config.dataDir);
 
+    // Determine which tables to load
+    const tablesToLoad = tableName ? [tableName] : Array.from(this.tables.keys());
+
+    // Validate that all requested tables exist BEFORE starting to load
+    for (const tableNameToLoad of tablesToLoad) {
+      if (!this.tables.has(tableNameToLoad)) {
+        throw new Error(
+          `Table '${tableNameToLoad}' not found in directory '${this.config.dataDir}'`,
+        );
+      }
+    }
+
     // Track loaded tables and tables currently being loaded (for circular dependency detection)
     const loadedTables = new Set<string>();
     const loadingTables = new Set<string>();
-
-    // Load all tables with dependency resolution
-    for (const [tableName] of this.tables) {
-      if (!loadedTables.has(tableName)) {
-        try {
-          await this.loadTableWithDependencies(tableName, loadedTables, loadingTables);
-        } catch (error) {
-          // Log error but continue loading other tables
-          console.warn(
-            `Warning: Failed to load table '${tableName}':`,
-            error instanceof Error ? error.message : String(error),
-          );
-          // Remove the failed table from the tables map
-          this.tables.delete(tableName);
-          this.schemas.delete(tableName);
-          this.validationSchemas.delete(tableName);
-        }
+    const attemptedTables = new Set<string>(); // Track all attempted tables (loaded or not)
+
+    // Load tables with dependency resolution
+    for (const tableNameToLoad of tablesToLoad) {
+      if (!attemptedTables.has(tableNameToLoad)) {
+        // Only apply transform to the specified table
+        const tableTransform = tableNameToLoad === tableName ? transform : undefined;
+        const { errors, warnings } = await this.loadTableWithDependencies(
+          tableNameToLoad,
+          loadedTables,
+          loadingTables,
+          attemptedTables,
+          detailedValidate,
+          tableTransform,
+        );
+        allErrors.push(...errors);
+        allWarnings.push(...warnings);
       }
     }
+
+    return {
+      valid: allErrors.length === 0,
+      errors: allErrors,
+      warnings: allWarnings,
+    };
   }
 
   /**
@@ -77,12 +112,21 @@ export class LinesDB<Tables extends TableDefs> {
     tableName: string,
     loadedTables: Set<string>,
     loadingTables: Set<string>,
-  ): Promise<void> {
-    // Skip if already loaded
-    if (loadedTables.has(tableName)) {
-      return;
+    attemptedTables: Set<string>,
+    detailedValidate: boolean,
+    transform?: (row: JsonObject) => JsonObject,
+  ): Promise<{ errors: ValidationErrorDetail[]; warnings: string[] }> {
+    const errors: ValidationErrorDetail[] = [];
+    const warnings: string[] = [];
+
+    // Skip if already attempted (loaded or not)
+    if (attemptedTables.has(tableName)) {
+      return { errors, warnings };
     }
 
+    // Mark as attempted
+    attemptedTables.add(tableName);
+
     // Check for circular dependencies
     if (loadingTables.has(tableName)) {
       throw new Error(`Circular dependency detected for table '${tableName}'`);
@@ -119,10 +163,26 @@ export class LinesDB<Tables extends TableDefs> {
       if (foreignKeys && foreignKeys.length > 0) {
         for (const fk of foreignKeys) {
           const referencedTable = fk.references.table;
-          if (!loadedTables.has(referencedTable)) {
+
+          // Skip self-referencing foreign keys (e.g., nullable parent_id columns)
+          if (referencedTable === tableName) {
+            continue;
+          }
+
+          if (!attemptedTables.has(referencedTable)) {
             // Check if referenced table exists in our tables map
             if (this.tables.has(referencedTable)) {
-              await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables);
+              // Dependencies should not have transform applied
+              const depResult = await this.loadTableWithDependencies(
+                referencedTable,
+                loadedTables,
+                loadingTables,
+                attemptedTables,
+                detailedValidate,
+                undefined,
+              );
+              errors.push(...depResult.errors);
+              warnings.push(...depResult.warnings);
             } else {
               throw new Error(
                 `Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`,
@@ -133,26 +193,46 @@ export class LinesDB<Tables extends TableDefs> {
       }
 
       // Now load this table
-      const wasLoaded = await this.loadTable(tableName, tableConfig);
-      if (wasLoaded) {
+      const { loaded, errors: loadErrors } = await this.loadTable(
+        tableName,
+        tableConfig,
+        detailedValidate,
+        transform,
+      );
+      errors.push(...loadErrors);
+
+      if (loaded) {
         loadedTables.add(tableName);
       } else {
         // Table was not loaded (e.g., empty data)
+        warnings.push(`Table '${tableName}' was not loaded (no data or skipped)`);
         this.tables.delete(tableName);
       }
     } finally {
       // Remove from loading set
       loadingTables.delete(tableName);
     }
+
+    return { errors, warnings };
   }
 
   /**
    * Load a single table from JSONL file
-   * @returns true if table was loaded, false if skipped
+   * @returns Object with loaded status and validation errors
    */
-  private async loadTable(tableName: string, config: TableConfig): Promise<boolean> {
+  private async loadTable(
+    tableName: string,
+    config: TableConfig,
+    detailedValidate: boolean,
+    transform?: (row: JsonObject) => JsonObject,
+  ): Promise<{ loaded: boolean; errors: ValidationErrorDetail[] }> {
     // Read JSONL file
-    const data = await JsonlReader.read(config.jsonlPath);
+    let data = await JsonlReader.read(config.jsonlPath);
+
+    // Apply transform if provided (before validation)
+    if (transform) {
+      data = data.map((row) => transform(row));
+    }
 
     // Load validation schema if provided or try to auto-load
     let validationSchema = config.validationSchema;
@@ -200,8 +280,23 @@ export class LinesDB<Tables extends TableDefs> {
         } else if (schemaModule.indexes) {
           schemaMetadata.indexes = schemaModule.indexes;
         }
+
+        // Debug: log loaded metadata
+        if (process.env.DEBUG_LINES_DB) {
+          console.log(`[lines-db] Schema metadata for ${tableName}:`);
+          console.log(`  primaryKey: ${schemaMetadata.primaryKey}`);
+          console.log(`  foreignKeys: ${JSON.stringify(schemaMetadata.foreignKeys)}`);
+          console.log(`  indexes: ${JSON.stringify(schemaMetadata.indexes)}`);
+        }
       } catch (_error) {
         // Schema file not found - this is OK
+        // Debug: log error for investigation
+        if (process.env.DEBUG_LINES_DB) {
+          console.warn(
+            `[lines-db] Failed to load schema metadata for ${tableName}:`,
+            _error instanceof Error ? _error.message : String(_error),
+          );
+        }
       }
     }
 
@@ -233,29 +328,48 @@ export class LinesDB<Tables extends TableDefs> {
       }
     }
 
+    // Convert validation errors to ValidationErrorDetail format
+    const validationErrorDetails: ValidationErrorDetail[] = validationErrors.map((ve) => ({
+      file: config.jsonlPath,
+      tableName,
+      rowIndex: ve.rowIndex,
+      issues: ve.error.issues,
+      type: 'schema' as const,
+    }));
+
     if (validationErrors.length > 0) {
-      const enhancedError = new Error(
-        `Validation failed for ${validationErrors.length} row(s) in table ${tableName}`,
-      );
-      enhancedError.name = 'ValidationError';
-      (enhancedError as unknown as { validationErrors: typeof validationErrors }).validationErrors =
-        validationErrors;
-      (enhancedError as unknown as { issues: ReadonlyArray<StandardSchemaIssue> }).issues =
-        validationErrors[0].error.issues;
-      throw enhancedError;
+      // Return errors instead of throwing
+      return { loaded: false, errors: validationErrorDetails };
     }
 
     // Determine schema - infer from validated data if auto-inference is enabled
     let schema: TableSchema;
+    let inferredSchema: TableSchema | undefined;
+
+    // Always infer schema from validated data to capture valueType information (e.g., boolean)
+    if (validatedData.length > 0) {
+      inferredSchema = JsonlReader.inferSchema(tableName, validatedData);
+    }
+
     if (config.schema) {
       schema = config.schema;
+      // Merge valueType information from inferred schema
+      if (inferredSchema) {
+        for (const inferredCol of inferredSchema.columns) {
+          const schemaCol = schema.columns.find((c) => c.name === inferredCol.name);
+          if (schemaCol && inferredCol.valueType && !schemaCol.valueType) {
+            schemaCol.valueType = inferredCol.valueType;
+          }
+        }
+      }
     } else if (config.autoInferSchema !== false) {
       if (validatedData.length === 0) {
-        return false;
+        return { loaded: false, errors: [] };
       }
-      // Infer schema from validated data (which may have additional fields added by validation)
-      schema = JsonlReader.inferSchema(tableName, validatedData);
+      // Use inferred schema
+      schema = inferredSchema!;
     } else {
+      // Critical error - throw exception
       throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
     }
 
@@ -285,6 +399,18 @@ export class LinesDB<Tables extends TableDefs> {
     }
     if (indexes) {
       schema.indexes = indexes;
+
+      // Apply unique constraint from single-column unique indexes to column definitions
+      // This is required for foreign key references, as SQLite requires the referenced column
+      // to have a UNIQUE constraint in the table definition (not just an index)
+      for (const index of indexes) {
+        if (index.unique && index.columns.length === 1) {
+          const col = schema.columns.find((c) => c.name === index.columns[0]);
+          if (col && !col.unique && !col.primaryKey) {
+            col.unique = true;
+          }
+        }
+      }
     }
 
     this.schemas.set(tableName, schema);
@@ -292,10 +418,22 @@ export class LinesDB<Tables extends TableDefs> {
     // Create table
     this.createTable(schema);
 
-    // Insert validated data
-    this.insertData(tableName, schema, validatedData);
+    // Insert validated data (with detailed validation if requested)
+    if (detailedValidate) {
+      const insertErrors = this.insertDataWithDetailedValidation(
+        tableName,
+        schema,
+        validatedData,
+        config.jsonlPath,
+      );
+      if (insertErrors.length > 0) {
+        return { loaded: false, errors: insertErrors };
+      }
+    } else {
+      this.insertData(tableName, schema, validatedData);
+    }
 
-    return true;
+    return { loaded: true, errors: [] };
   }
 
   /**
@@ -308,13 +446,31 @@ export class LinesDB<Tables extends TableDefs> {
     // Quote table name to handle special characters
     const quotedTableName = this.quoteTableName(schema.name);
 
+    // Build a set of columns that should have UNIQUE constraint
+    // This includes columns marked as unique in schema AND single-column unique indexes
+    // The latter is required for foreign key references, as SQLite requires the referenced column
+    // to have a UNIQUE constraint in the table definition (not just a separately created index)
+    const uniqueColumns = new Set<string>();
+    for (const col of schema.columns) {
+      if (col.unique) {
+        uniqueColumns.add(col.name);
+      }
+    }
+    if (schema.indexes) {
+      for (const index of schema.indexes) {
+        if (index.unique && index.columns.length === 1) {
+          uniqueColumns.add(index.columns[0]);
+        }
+      }
+    }
+
     const columnDefs = schema.columns.map((col) => {
       // JSON type is stored as TEXT in SQLite
       const sqlType = col.type === 'JSON' ? 'TEXT' : col.type;
       const parts = [this.quoteIdentifier(col.name), sqlType];
       if (col.primaryKey) parts.push('PRIMARY KEY');
       if (col.notNull) parts.push('NOT NULL');
-      if (col.unique) parts.push('UNIQUE');
+      if (uniqueColumns.has(col.name) && !col.primaryKey) parts.push('UNIQUE');
       return parts.join(' ');
     });
 
@@ -372,9 +528,52 @@ export class LinesDB<Tables extends TableDefs> {
   }
 
   /**
-   * Insert data into table
+   * Insert data into table using batch insert (multiple rows per SQL)
+   * SQLite has a parameter limit (default 999), so we batch rows accordingly
+   * Throws exception if any constraint violation occurs
    */
   private insertData(tableName: string, schema: TableSchema, data: JsonObject[]): void {
+    if (data.length === 0) return;
+
+    const columnNames = schema.columns.map((col) => col.name);
+    const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
+    const columnCount = columnNames.length;
+
+    // Calculate batch size to stay under SQLite's parameter limit (999)
+    // Leave some margin for safety
+    const maxBatchSize = Math.floor(900 / columnCount);
+    const batchSize = Math.max(1, Math.min(maxBatchSize, 100));
+
+    // Process data in batches
+    for (let i = 0; i < data.length; i += batchSize) {
+      const batch = data.slice(i, i + batchSize);
+      const rowPlaceholders = columnNames.map(() => '?').join(', ');
+      const valuesPlaceholders = batch.map(() => `(${rowPlaceholders})`).join(', ');
+      const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(', ')}) VALUES ${valuesPlaceholders}`;
+
+      const values: (string | number | bigint | null | Uint8Array)[] = [];
+      for (const row of batch) {
+        for (const col of columnNames) {
+          values.push(this.normalizeValue(row[col]));
+        }
+      }
+
+      const stmt = this.db.prepare(sql);
+      stmt.run(...values);
+    }
+  }
+
+  /**
+   * Insert data into table one row at a time with detailed error reporting
+   * This is used for validation to catch constraint violations
+   */
+  private insertDataWithDetailedValidation(
+    tableName: string,
+    schema: TableSchema,
+    data: JsonObject[],
+    filePath: string,
+  ): ValidationErrorDetail[] {
+    const errors: ValidationErrorDetail[] = [];
     const columnNames = schema.columns.map((col) => col.name);
     const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
     const placeholders = columnNames.map(() => '?').join(', ');
@@ -382,10 +581,90 @@ export class LinesDB<Tables extends TableDefs> {
 
     const stmt = this.db.prepare(sql);
 
-    for (const row of data) {
-      const values = columnNames.map((col) => this.normalizeValue(row[col]));
-      stmt.run(...values);
+    for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
+      const row = data[rowIndex];
+      try {
+        const values = columnNames.map((col) => this.normalizeValue(row[col]));
+        stmt.run(...values);
+      } catch (error) {
+        // Constraint violation occurred - analyze and record details
+        const constraintError = this.analyzeConstraintError(
+          error,
+          filePath,
+          tableName,
+          rowIndex,
+          row,
+          schema.foreignKeys || [],
+        );
+        if (constraintError) {
+          errors.push(constraintError);
+        }
+      }
     }
+
+    return errors;
+  }
+
+  /**
+   * Analyze constraint error and extract detailed information
+   */
+  private analyzeConstraintError(
+    error: unknown,
+    file: string,
+    tableName: string,
+    rowIndex: number,
+    row: JsonObject,
+    foreignKeys: ForeignKeyDefinition[],
+  ): ValidationErrorDetail | null {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+
+    // Foreign key constraint
+    if (errorMessage.includes('FOREIGN KEY constraint failed')) {
+      // Find which foreign key failed
+      for (const fk of foreignKeys) {
+        const fkValue = row[fk.column];
+        if (fkValue === null || fkValue === undefined) continue;
+
+        // Check if referenced value exists
+        try {
+          const result = this.query(
+            `SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`,
+            [this.normalizeValue(fkValue)],
+          );
+          if (result.length > 0 && (result[0] as { count: number }).count === 0) {
+            return {
+              file,
+              tableName,
+              rowIndex,
+              issues: [],
+              type: 'foreignKey',
+              foreignKeyError: {
+                column: fk.column,
+                value: fkValue,
+                referencedTable: fk.references.table,
+                referencedColumn: fk.references.column,
+              },
+            };
+          }
+        } catch (_) {
+          // Referenced table doesn't exist yet
+        }
+      }
+    }
+
+    // Other constraint errors (primary key, unique, etc.)
+    return {
+      file,
+      tableName,
+      rowIndex,
+      issues: [
+        {
+          message: errorMessage,
+          path: [],
+        },
+      ],
+      type: 'schema',
+    };
   }
 
   /**
@@ -1112,10 +1391,20 @@ export class LinesDB<Tables extends TableDefs> {
   /**
    * Sync database changes back to JSONL files
    * Uses backward transformation when available
+   * @param tableName Optional table name to sync. If not provided, syncs all loaded tables
    */
-  async sync(): Promise<void> {
-    for (const [tableName] of this.tables) {
+  async sync(tableName?: string): Promise<void> {
+    if (tableName) {
+      // Sync only the specified table
+      if (!this.schemas.has(tableName)) {
+        throw new Error(`Table '${tableName}' is not loaded`);
+      }
       await this.syncTable(tableName);
+    } else {
+      // Sync all tables that are loaded (present in schemas map)
+      for (const [name] of this.schemas) {
+        await this.syncTable(name);
+      }
     }
   }
 

package/src/index.ts

@@ -5,14 +5,12 @@ export { SchemaLoader } from './schema-loader.js';
 export { DirectoryScanner } from './directory-scanner.js';
 export { defineSchema, hasBackward } from './schema.js';
 export { TypeGenerator } from './type-generator.js';
-export { Validator } from './validator.js';
 export { ensureTableRowsValid } from './jsonl-migration.js';
 export type { TableValidationOptions } from './jsonl-migration.js';
 export { detectRuntime, RUNTIME } from './runtime.js';
 export type { RuntimeEnvironment } from './runtime.js';
 export type { SQLiteDatabase, SQLiteStatement } from './sqlite-adapter.js';
 export type { TypeGeneratorOptions } from './type-generator.js';
-export type { ValidatorOptions, ValidationResult, ValidationErrorDetail } from './validator.js';
 export type {
   TableSchema,
   ColumnDefinition,
@@ -25,6 +23,8 @@ export type {
   StandardSchemaResult,
   StandardSchemaIssue,
   ValidationError,
+  ValidationResult,
+  ValidationErrorDetail,
   InferInput,
   InferOutput,
   Table,

package/src/jsonl-migration.ts

@@ -11,66 +11,34 @@ export interface TableValidationOptions {
 
 /**
  * Validate a table by temporarily supplying in-memory rows while reusing the existing LinesDB validation pipeline.
- * If validation fails, the underlying LinesDB error is rethrown so callers can inspect validation details.
+ * If validation fails, throws an error with validation details.
  */
 export async function ensureTableRowsValid(options: TableValidationOptions): Promise<void> {
-  console.log('[ensureTableRowsValid] START');
-  console.log('[ensureTableRowsValid] dataDir:', options.dataDir);
-  console.log('[ensureTableRowsValid] tableName:', options.tableName);
-  console.log('[ensureTableRowsValid] rows count:', options.rows.length);
-
   const tablePath = join(options.dataDir, `${options.tableName}.jsonl`);
   const overrides = new Map<string, JsonObject[]>([[tablePath, options.rows]]);
-  console.log('[ensureTableRowsValid] tablePath:', tablePath);
-
-  let capturedError: Error | null = null;
-
-  // Intercept console.warn to capture validation errors
-  const originalWarn = console.warn;
-  const warnMessages: string[] = [];
-  console.warn = (...args: any[]) => {
-    const message = args.join(' ');
-    console.log('[ensureTableRowsValid] Captured warn:', message);
-    warnMessages.push(message);
-    // Check if this is a validation error for our table
-    if (
-      message.includes(`Failed to load table '${options.tableName}'`) &&
-      message.includes('Validation failed')
-    ) {
-      // Extract the original error from the warn message
-      capturedError = new Error(message);
-      console.log('[ensureTableRowsValid] Captured validation error!');
-    }
-  };
 
-  try {
-    console.log('[ensureTableRowsValid] Calling JsonlReader.withOverrides');
-    await JsonlReader.withOverrides(overrides, async () => {
-      console.log('[ensureTableRowsValid] Inside withOverrides callback');
-      const db = LinesDB.create({ dataDir: options.dataDir });
-      console.log('[ensureTableRowsValid] LinesDB created');
-      try {
-        console.log('[ensureTableRowsValid] Calling db.initialize()');
-        await db.initialize();
-        console.log('[ensureTableRowsValid] db.initialize() completed');
-      } finally {
-        console.log('[ensureTableRowsValid] Calling db.close()');
-        await db.close();
+  await JsonlReader.withOverrides(overrides, async () => {
+    const db = LinesDB.create({ dataDir: options.dataDir });
+    try {
+      // Initialize only the target table
+      const result = await db.initialize({ tableName: options.tableName });
+
+      // If validation failed, throw an error with details
+      if (!result.valid) {
+        const errorCount = result.errors.length;
+        const errorDetails = result.errors
+          .map((e) => {
+            const issueMessages = e.issues.map((issue) => issue.message).join(', ');
+            return `  Row ${e.rowIndex}: ${issueMessages}`;
+          })
+          .join('\n');
+
+        throw new Error(
+          `Validation failed for table '${options.tableName}' (${errorCount} error(s)):\n${errorDetails}`,
+        );
       }
-    });
-    console.log('[ensureTableRowsValid] withOverrides completed');
-  } finally {
-    // Restore original console.warn
-    console.warn = originalWarn;
-  }
-
-  console.log('[ensureTableRowsValid] Warnings captured:', warnMessages.length);
-  console.log('[ensureTableRowsValid] capturedError:', capturedError ? 'YES' : 'NO');
-
-  if (capturedError) {
-    console.log('[ensureTableRowsValid] Throwing captured error');
-    throw capturedError;
-  }
-
-  console.log('[ensureTableRowsValid] END (success)');
+    } finally {
+      await db.close();
+    }
+  });
 }