@toiroakr/lines-db 0.4.1 → 0.6.0

package/src/cli.ts

@@ -5,13 +5,13 @@ import { register } from 'node:module';
 register('tsx', import.meta.url, { data: {} });
 
 import { TypeGenerator } from './type-generator.js';
-import { Validator } from './validator.js';
 import { LinesDB } from './database.js';
 import { ErrorFormatter } from './error-formatter.js';
-import type { ValidationError } from './types.js';
+import type { ValidationError, JsonObject } from './types.js';
 import { Command } from 'commander';
 import { styleText } from 'node:util';
-import { writeFile } from 'node:fs/promises';
+import { writeFile, stat, readdir } from 'node:fs/promises';
+import { basename, dirname } from 'node:path';
 import { runInNewContext } from 'node:vm';
 
 const originalEmitWarning = process.emitWarning;
@@ -81,8 +81,29 @@ program
   .option('-v, --verbose', 'Show verbose error output', false)
   .action(async (path: string, options: { verbose: boolean }) => {
     try {
-      const validator = new Validator({ path });
-      const result = await validator.validate();
+      // Determine if path is a file or directory
+      const stats = await stat(path);
+      let dataDir: string;
+      let tableName: string | undefined;
+
+      if (stats.isDirectory()) {
+        dataDir = path;
+        // Validate all tables in directory
+      } else if (stats.isFile() && path.endsWith('.jsonl')) {
+        dataDir = dirname(path);
+        tableName = basename(path, '.jsonl');
+      } else {
+        throw new Error(`Invalid path: ${path}. Must be a directory or .jsonl file.`);
+      }
+
+      // Use LinesDB.initialize() with detailed validation
+      const db = LinesDB.create({ dataDir });
+      let result;
+      try {
+        result = await db.initialize({ tableName, detailedValidate: true });
+      } finally {
+        await db.close();
+      }
 
       // Display warnings if any
       if (result.warnings.length > 0) {
@@ -161,191 +182,454 @@ program
 program
   .command('migrate')
   .description('Migrate data with transformation function')
-  .argument('<file>', 'JSONL file to migrate')
+  .argument('<path>', 'File or directory path to migrate')
   .argument('<transform>', 'Transform function (e.g., "(row) => ({ ...row, age: row.age + 1 })")')
   .option('-f, --filter <expr>', 'Filter expression')
   .option('-e, --errorOutput <path>', 'Output file path for transformed data when migration fails')
   .option('-v, --verbose', 'Show verbose error output', false)
   .action(
     async (
-      filePath: string,
+      path: string,
       transformStr: string,
       options: { filter?: string; errorOutput?: string; verbose: boolean },
     ) => {
-      // Extract table name from file path
-      const fileName = filePath.split('/').pop() || '';
-      const tableName = fileName.replace('.jsonl', '');
-
-      if (!tableName) {
-        console.error('Error: Invalid file path. Must be a .jsonl file');
+      try {
+        const stats = await stat(path);
+
+        if (stats.isDirectory()) {
+          // Migrate all JSONL files in directory
+          await migrateDirectory(path, transformStr, options);
+        } else if (stats.isFile() && path.endsWith('.jsonl')) {
+          // Migrate single file
+          await migrateFile(path, transformStr, options);
+        } else {
+          console.error(`Error: Invalid path: ${path}. Must be a directory or .jsonl file.`);
+          process.exit(1);
+        }
+      } catch (error) {
+        if (
+          error instanceof Error &&
+          'code' in error &&
+          (error as NodeJS.ErrnoException).code === 'ENOENT'
+        ) {
+          console.error(`Error: Path not found: ${path}`);
+        } else {
+          console.error(`Error: ${String(error)}`);
+        }
         process.exit(1);
       }
+    },
+  );
 
-      // Get directory from file path
-      const lastSlashIndex = filePath.lastIndexOf('/');
-      const dataDir = lastSlashIndex > 0 ? filePath.substring(0, lastSlashIndex) : '.';
-
-      // Initialize database
-      const db = LinesDB.create({ dataDir });
-      await db.initialize();
+/**
+ * Migrate all JSONL files in a directory
+ */
+async function migrateDirectory(
+  dirPath: string,
+  transformStr: string,
+  options: { filter?: string; errorOutput?: string; verbose: boolean },
+) {
+  // Find all JSONL files in directory
+  const entries = await readdir(dirPath, { withFileTypes: true });
+  const jsonlFiles = entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl'))
+    .map((entry) => entry.name);
+
+  if (jsonlFiles.length === 0) {
+    console.error(`Error: No JSONL files found in directory: ${dirPath}`);
+    process.exit(1);
+  }
+
+  console.log(`Found ${jsonlFiles.length} JSONL file(s) in directory`);
+
+  // Initialize database
+  const db = LinesDB.create({ dataDir: dirPath });
+  const initResult = await db.initialize({ detailedValidate: true });
+
+  // Display warnings if any
+  if (initResult.warnings.length > 0) {
+    for (const warning of initResult.warnings) {
+      console.warn(styleText('yellow', `⚠ ${warning}`));
+    }
+  }
+
+  // Check for initialization errors
+  if (!initResult.valid) {
+    console.error(`Error: Failed to initialize database due to validation errors:`);
+    const formatter = new ErrorFormatter({ verbose: options.verbose });
+    for (const error of initResult.errors) {
+      console.error(
+        formatter.formatValidationErrors([
+          {
+            file: error.file,
+            rowIndex: error.rowIndex,
+            issues: error.issues,
+          },
+        ]),
+      );
+    }
+    await db.close();
+    process.exit(1);
+  }
+
+  const tableNames = db.getTableNames();
+  if (tableNames.length === 0) {
+    console.error(`Error: No tables could be loaded from directory: ${dirPath}`);
+    await db.close();
+    process.exit(1);
+  }
+
+  console.log(`Loaded ${tableNames.length} table(s): ${tableNames.join(', ')}\n`);
+
+  try {
+    // Parse transform function
+    const transform = runInSandbox<unknown>(`(${transformStr})`);
+
+    if (typeof transform !== 'function') {
+      console.error('Error: Transform must be a function');
+      await db.close();
+      process.exit(1);
+    }
 
+    // Parse filter if provided
+    let filter: unknown = undefined;
+    if (options.filter) {
       try {
-        // Parse transform function
-        const transform = runInSandbox<unknown>(`(${transformStr})`);
+        filter = JSON.parse(options.filter);
+      } catch {
+        filter = runInSandbox(`(${options.filter})`);
+      }
+    }
 
-        if (typeof transform !== 'function') {
-          console.error('Error: Transform must be a function');
-          process.exit(1);
-        }
+    let totalRowsMigrated = 0;
+    let hasErrors = false;
 
-        // Parse filter if provided
-        let filter: unknown = undefined;
-        if (options.filter) {
-          try {
-            // Try JSON parse first
-            filter = JSON.parse(options.filter);
-          } catch {
-            // Fall back to eval for JavaScript expressions
-            filter = runInSandbox(`(${options.filter})`);
-          }
-        }
+    // Process each table
+    for (const tableName of tableNames) {
+      try {
+        console.log(`Processing table '${tableName}'...`);
 
         // Get rows to migrate
-        let rowsToMigrate;
-        try {
-          rowsToMigrate = filter
-            ? db.find(tableName, filter as Parameters<typeof db.find>[1])
-            : db.find(tableName);
-        } catch (error) {
-          console.error(`Error: Failed to access table '${tableName}'`);
-          console.error(`  ${error instanceof Error ? error.message : String(error)}`);
-          console.error(`\nThe table may have failed to load during initialization.`);
-          console.error(`Check the table's data and schema for any constraint violations.`);
-          await db.close();
-          process.exit(1);
-        }
-
-        console.log(`Found ${rowsToMigrate.length} row(s) to migrate in table '${tableName}'`);
+        const rowsToMigrate = filter
+          ? db.find(tableName, filter as Parameters<typeof db.find>[1])
+          : db.find(tableName);
 
         if (rowsToMigrate.length === 0) {
-          console.log('No rows to migrate. Exiting.');
-          await db.close();
-          process.exit(0);
+          console.log(`  No rows to migrate`);
+          continue;
         }
 
+        console.log(`  Found ${rowsToMigrate.length} row(s) to migrate`);
+
         // Apply transformation
-        const transformedRows = rowsToMigrate.map((row) => transform(row));
+        const transformedRows = rowsToMigrate.map((row) => transform(row as JsonObject));
 
         // Perform the migration in a transaction
-        try {
-          await db.transaction(async () => {
-            db.batchUpdate(tableName, transformedRows as Parameters<typeof db.batchUpdate>[1], {
-              validate: true,
-            });
+        await db.transaction(async () => {
+          db.batchUpdate(tableName, transformedRows as Parameters<typeof db.batchUpdate>[1], {
+            validate: true,
           });
+        });
 
-          await db.close();
-
-          console.log(`\nMigration completed successfully:`);
-          console.log(`  ✓ ${rowsToMigrate.length} row(s) updated`);
-          process.exit(0);
-        } catch (error) {
-          await db.close();
-
-          // Write transformed data to error output file if --errorOutput is specified
-          if (options.errorOutput) {
-            try {
-              const jsonlContent = transformedRows.map((row) => JSON.stringify(row)).join('\n');
-              await writeFile(options.errorOutput, jsonlContent, 'utf-8');
-              console.error(
-                styleText(
-                  'yellow',
-                  `\n⚠ Transformed data (${transformedRows.length} rows) written to: ${options.errorOutput}`,
-                ),
-              );
-            } catch (writeError) {
-              console.error(
-                styleText(
-                  'red',
-                  `\n✗ Failed to write error output file: ${writeError instanceof Error ? writeError.message : String(writeError)}`,
-                ),
-              );
-            }
+        console.log(`  ✓ ${rowsToMigrate.length} row(s) updated\n`);
+        totalRowsMigrated += rowsToMigrate.length;
+      } catch (error) {
+        hasErrors = true;
+        console.error(styleText('red', `  ✗ Failed to migrate table '${tableName}'`));
+
+        const formatter = new ErrorFormatter({ verbose: options.verbose });
+
+        if (error instanceof Error && error.name === 'ValidationError') {
+          const validationError = error as ValidationError & {
+            validationErrors?: Array<{
+              rowIndex: number;
+              rowData: unknown;
+              pkValue: unknown;
+              error: ValidationError;
+            }>;
+          };
+
+          if (validationError.validationErrors) {
+            console.error(
+              `  Found ${validationError.validationErrors.length} validation error(s):\n`,
+            );
+
+            const rowsToMigrate = filter
+              ? db.find(tableName, filter as Parameters<typeof db.find>[1])
+              : db.find(tableName);
+
+            const errorInfos = validationError.validationErrors.map(
+              ({ rowIndex, rowData, error: rowError }) => ({
+                file: `${dirPath}/${tableName}.jsonl`,
+                rowIndex,
+                issues: rowError.issues,
+                data: rowData,
+                originalData: rowsToMigrate[rowIndex],
+              }),
+            );
+
+            const formatted = formatter.formatValidationErrors(errorInfos);
+            console.error(formatted);
           }
+        } else if (error instanceof Error) {
+          console.error(`  ${error.message}`);
+        }
 
-          const formatter = new ErrorFormatter({ verbose: options.verbose });
-          console.error(formatter.formatMigrationFailureHeader());
-
-          // Display detailed error information
-          if (error instanceof Error && error.name === 'ValidationError') {
-            const validationError = error as ValidationError & {
-              validationErrors?: Array<{
-                rowIndex: number;
-                rowData: unknown;
-                pkValue: unknown;
-                error: ValidationError;
-              }>;
-            };
+        console.error('');
+      }
+    }
 
-            // Display all validation errors
-            if (validationError.validationErrors) {
-              console.error(
-                `\nFound ${validationError.validationErrors.length} validation error(s) in transformed data:\n`,
-              );
-
-              const errorInfos = validationError.validationErrors.map(
-                ({ rowIndex, rowData, error: rowError }) => ({
-                  file: filePath,
-                  rowIndex,
-                  issues: rowError.issues,
-                  data: rowData,
-                  originalData: rowsToMigrate[rowIndex],
-                }),
-              );
-
-              const formatted = formatter.formatValidationErrors(errorInfos);
-              console.error(formatted);
-            } else {
-              // Fallback for single validation error (backward compatibility)
-              console.error('\nValidation error:\n');
-              const errorInfo = {
-                file: filePath,
-                rowIndex: 0,
-                issues: validationError.issues,
-              };
-              const formatted = formatter.formatValidationErrors([errorInfo]);
-              console.error(formatted);
-            }
-          } else if (error instanceof Error) {
-            console.error(`\n  ${error.message}`);
+    await db.close();
 
-            // Output stack trace for debugging
-            if (options.verbose && error.stack) {
-              console.error(`\nStack trace:\n${error.stack}`);
-            }
+    if (hasErrors) {
+      console.error(styleText('red', `\n✗ Migration completed with errors for some tables`));
+      console.log(`Total rows migrated: ${totalRowsMigrated}`);
+      process.exit(1);
+    } else {
+      console.log(styleText('green', `\n✓ Migration completed successfully for all tables`));
+      console.log(`Total rows migrated: ${totalRowsMigrated}`);
+      process.exit(0);
+    }
+  } catch (error) {
+    await db.close();
+    throw error;
+  }
+}
 
-            // Check if it's a SQLite constraint error
-            if (
-              error.message.includes('UNIQUE constraint failed') ||
-              error.message.includes('FOREIGN KEY constraint failed') ||
-              error.message.includes('NOT NULL constraint failed') ||
-              error.message.includes('CHECK constraint failed')
-            ) {
-              console.error('\n  This is a SQLite constraint violation.');
-              console.error('  Please check your data and schema requirements.');
-            }
+/**
+ * Migrate a single JSONL file
+ */
+async function migrateFile(
+  filePath: string,
+  transformStr: string,
+  options: { filter?: string; errorOutput?: string; verbose: boolean },
+) {
+  // Extract table name from file path
+  const fileName = filePath.split('/').pop() || '';
+  const tableName = fileName.replace('.jsonl', '');
+
+  if (!tableName) {
+    console.error('Error: Invalid file path. Must be a .jsonl file');
+    process.exit(1);
+  }
+
+  // Get directory from file path
+  const lastSlashIndex = filePath.lastIndexOf('/');
+  const dataDir = lastSlashIndex > 0 ? filePath.substring(0, lastSlashIndex) : '.';
+
+  // Parse transform function first (before initialization)
+  let transform: (row: JsonObject) => JsonObject;
+  try {
+    const parsedTransform = runInSandbox<unknown>(`(${transformStr})`);
+    if (typeof parsedTransform !== 'function') {
+      console.error('Error: Transform must be a function');
+      process.exit(1);
+    }
+    transform = parsedTransform as (row: JsonObject) => JsonObject;
+  } catch (error) {
+    console.error('Error: Failed to parse transform function');
+    console.error(`  ${error instanceof Error ? error.message : String(error)}`);
+    process.exit(1);
+  }
+
+  // Initialize database with transform applied
+  const db = LinesDB.create({ dataDir });
+  const initResult = await db.initialize({ tableName, transform, detailedValidate: true });
+
+  // Display warnings if any
+  if (initResult.warnings.length > 0) {
+    for (const warning of initResult.warnings) {
+      console.warn(styleText('yellow', `⚠ ${warning}`));
+    }
+  }
+
+  // Check for initialization errors
+  if (!initResult.valid) {
+    console.error(`Error: Failed to initialize database due to validation errors:`);
+    const formatter = new ErrorFormatter({ verbose: options.verbose });
+    for (const error of initResult.errors) {
+      console.error(
+        formatter.formatValidationErrors([
+          {
+            file: error.file,
+            rowIndex: error.rowIndex,
+            issues: error.issues,
+          },
+        ]),
+      );
+    }
+    await db.close();
+    process.exit(1);
+  }
+
+  try {
+    // Parse filter if provided
+    let filter: unknown = undefined;
+    if (options.filter) {
+      try {
+        // Try JSON parse first
+        filter = JSON.parse(options.filter);
+      } catch {
+        // Fall back to eval for JavaScript expressions
+        filter = runInSandbox(`(${options.filter})`);
+      }
+    }
+
+    // If filter is provided, we need to apply transform only to matching rows
+    if (filter) {
+      // Get rows to migrate
+      let rowsToMigrate;
+      try {
+        rowsToMigrate = db.find(tableName, filter as Parameters<typeof db.find>[1]);
+      } catch (error) {
+        console.error(`Error: Failed to access table '${tableName}'`);
+        console.error(`  ${error instanceof Error ? error.message : String(error)}`);
+        console.error(`\nThe table may have failed to load during initialization.`);
+        console.error(`Check the table's data and schema for any constraint violations.`);
+        await db.close();
+        process.exit(1);
+      }
+
+      console.log(`Found ${rowsToMigrate.length} row(s) to migrate in table '${tableName}'`);
+
+      if (rowsToMigrate.length === 0) {
+        console.log('No rows to migrate. Exiting.');
+        await db.close();
+        process.exit(0);
+      }
+
+      // Apply transformation
+      const transformedRows = rowsToMigrate.map((row) => transform(row as JsonObject));
+
+      // Perform the migration in a transaction
+      try {
+        await db.transaction(async () => {
+          db.batchUpdate(tableName, transformedRows as Parameters<typeof db.batchUpdate>[1], {
+            validate: true,
+          });
+        });
+
+        await db.close();
+
+        console.log(`\nMigration completed successfully:`);
+        console.log(`  ✓ ${rowsToMigrate.length} row(s) updated`);
+        process.exit(0);
+      } catch (error) {
+        await db.close();
+
+        // Write transformed data to error output file if --errorOutput is specified
+        if (options.errorOutput) {
+          try {
+            const jsonlContent = transformedRows.map((row) => JSON.stringify(row)).join('\n');
+            await writeFile(options.errorOutput, jsonlContent, 'utf-8');
+            console.error(
+              styleText(
+                'yellow',
+                `\n⚠ Transformed data (${transformedRows.length} rows) written to: ${options.errorOutput}`,
+              ),
+            );
+          } catch (writeError) {
+            console.error(
+              styleText(
+                'red',
+                `\n✗ Failed to write error output file: ${writeError instanceof Error ? writeError.message : String(writeError)}`,
+              ),
+            );
+          }
+        }
+
+        const formatter = new ErrorFormatter({ verbose: options.verbose });
+        console.error(formatter.formatMigrationFailureHeader());
+
+        // Display detailed error information
+        if (error instanceof Error && error.name === 'ValidationError') {
+          const validationError = error as ValidationError & {
+            validationErrors?: Array<{
+              rowIndex: number;
+              rowData: unknown;
+              pkValue: unknown;
+              error: ValidationError;
+            }>;
+          };
+
+          // Display all validation errors
+          if (validationError.validationErrors) {
+            console.error(
+              `\nFound ${validationError.validationErrors.length} validation error(s) in transformed data:\n`,
+            );
+
+            const errorInfos = validationError.validationErrors.map(
+              ({ rowIndex, rowData, error: rowError }) => ({
+                file: filePath,
+                rowIndex,
+                issues: rowError.issues,
+                data: rowData,
+                originalData: rowsToMigrate[rowIndex],
+              }),
+            );
+
+            const formatted = formatter.formatValidationErrors(errorInfos);
+            console.error(formatted);
           } else {
-            console.error(`\n  ${String(error)}`);
+            // Fallback for single validation error (backward compatibility)
+            console.error('\nValidation error:\n');
+            const errorInfo = {
+              file: filePath,
+              rowIndex: 0,
+              issues: validationError.issues,
+            };
+            const formatted = formatter.formatValidationErrors([errorInfo]);
+            console.error(formatted);
           }
+        } else if (error instanceof Error) {
+          console.error(`\n  ${error.message}`);
 
-          console.error('');
-          process.exit(1);
+          // Output stack trace for debugging
+          if (options.verbose && error.stack) {
+            console.error(`\nStack trace:\n${error.stack}`);
+          }
+
+          // Check if it's a SQLite constraint error
+          if (
+            error.message.includes('UNIQUE constraint failed') ||
+            error.message.includes('FOREIGN KEY constraint failed') ||
+            error.message.includes('NOT NULL constraint failed') ||
+            error.message.includes('CHECK constraint failed')
+          ) {
+            console.error('\n  This is a SQLite constraint violation.');
+            console.error('  Please check your data and schema requirements.');
+          }
+        } else {
+          console.error(`\n  ${String(error)}`);
         }
+
+        console.error('');
+        process.exit(1);
+      }
+    } else {
+      // No filter - all rows have been transformed during initialization
+      // Just sync to write back to JSONL file
+      try {
+        const allRows = db.find(tableName);
+        console.log(`Migrated ${allRows.length} row(s) in table '${tableName}'`);
+
+        await db.sync(tableName);
+        await db.close();
+
+        console.log(`\nMigration completed successfully:`);
+        console.log(`  ✓ ${allRows.length} row(s) updated`);
+        process.exit(0);
       } catch (error) {
         await db.close();
-        throw error;
+        console.error('Error: Failed to sync changes to file');
+        console.error(`  ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
       }
-    },
-  );
+    }
+  } catch (error) {
+    await db.close();
+    throw error;
+  }
+}
 
 program.parse();