@toiroakr/lines-db 0.4.0 → 0.5.0

package/src/cli.ts

@@ -11,7 +11,7 @@ import { ErrorFormatter } from './error-formatter.js';
 import type { ValidationError } from './types.js';
 import { Command } from 'commander';
 import { styleText } from 'node:util';
-import { writeFile } from 'node:fs/promises';
+import { writeFile, stat, readdir } from 'node:fs/promises';
 import { runInNewContext } from 'node:vm';
 
 const originalEmitWarning = process.emitWarning;
@@ -161,191 +161,375 @@ program
 program
   .command('migrate')
   .description('Migrate data with transformation function')
-  .argument('<file>', 'JSONL file to migrate')
+  .argument('<path>', 'File or directory path to migrate')
   .argument('<transform>', 'Transform function (e.g., "(row) => ({ ...row, age: row.age + 1 })")')
   .option('-f, --filter <expr>', 'Filter expression')
   .option('-e, --errorOutput <path>', 'Output file path for transformed data when migration fails')
   .option('-v, --verbose', 'Show verbose error output', false)
   .action(
     async (
-      filePath: string,
+      path: string,
       transformStr: string,
       options: { filter?: string; errorOutput?: string; verbose: boolean },
     ) => {
-      // Extract table name from file path
-      const fileName = filePath.split('/').pop() || '';
-      const tableName = fileName.replace('.jsonl', '');
-
-      if (!tableName) {
-        console.error('Error: Invalid file path. Must be a .jsonl file');
+      try {
+        const stats = await stat(path);
+
+        if (stats.isDirectory()) {
+          // Migrate all JSONL files in directory
+          await migrateDirectory(path, transformStr, options);
+        } else if (stats.isFile() && path.endsWith('.jsonl')) {
+          // Migrate single file
+          await migrateFile(path, transformStr, options);
+        } else {
+          console.error(`Error: Invalid path: ${path}. Must be a directory or .jsonl file.`);
+          process.exit(1);
+        }
+      } catch (error) {
+        if (
+          error instanceof Error &&
+          'code' in error &&
+          (error as NodeJS.ErrnoException).code === 'ENOENT'
+        ) {
+          console.error(`Error: Path not found: ${path}`);
+        } else {
+          console.error(`Error: ${String(error)}`);
+        }
         process.exit(1);
       }
+    },
+  );
 
-      // Get directory from file path
-      const lastSlashIndex = filePath.lastIndexOf('/');
-      const dataDir = lastSlashIndex > 0 ? filePath.substring(0, lastSlashIndex) : '.';
-
-      // Initialize database
-      const db = LinesDB.create({ dataDir });
-      await db.initialize();
+/**
+ * Migrate all JSONL files in a directory
+ */
+async function migrateDirectory(
+  dirPath: string,
+  transformStr: string,
+  options: { filter?: string; errorOutput?: string; verbose: boolean },
+) {
+  // Find all JSONL files in directory
+  const entries = await readdir(dirPath, { withFileTypes: true });
+  const jsonlFiles = entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl'))
+    .map((entry) => entry.name);
+
+  if (jsonlFiles.length === 0) {
+    console.error(`Error: No JSONL files found in directory: ${dirPath}`);
+    process.exit(1);
+  }
+
+  console.log(`Found ${jsonlFiles.length} JSONL file(s) in directory`);
+
+  // Initialize database
+  const db = LinesDB.create({ dataDir: dirPath });
+  await db.initialize();
+
+  const tableNames = db.getTableNames();
+  if (tableNames.length === 0) {
+    console.error(`Error: No tables could be loaded from directory: ${dirPath}`);
+    await db.close();
+    process.exit(1);
+  }
+
+  console.log(`Loaded ${tableNames.length} table(s): ${tableNames.join(', ')}\n`);
+
+  try {
+    // Parse transform function
+    const transform = runInSandbox<unknown>(`(${transformStr})`);
+
+    if (typeof transform !== 'function') {
+      console.error('Error: Transform must be a function');
+      await db.close();
+      process.exit(1);
+    }
 
+    // Parse filter if provided
+    let filter: unknown = undefined;
+    if (options.filter) {
       try {
-        // Parse transform function
-        const transform = runInSandbox<unknown>(`(${transformStr})`);
+        filter = JSON.parse(options.filter);
+      } catch {
+        filter = runInSandbox(`(${options.filter})`);
+      }
+    }
 
-        if (typeof transform !== 'function') {
-          console.error('Error: Transform must be a function');
-          process.exit(1);
-        }
+    let totalRowsMigrated = 0;
+    let hasErrors = false;
 
-        // Parse filter if provided
-        let filter: unknown = undefined;
-        if (options.filter) {
-          try {
-            // Try JSON parse first
-            filter = JSON.parse(options.filter);
-          } catch {
-            // Fall back to eval for JavaScript expressions
-            filter = runInSandbox(`(${options.filter})`);
-          }
-        }
+    // Process each table
+    for (const tableName of tableNames) {
+      try {
+        console.log(`Processing table '${tableName}'...`);
 
         // Get rows to migrate
-        let rowsToMigrate;
-        try {
-          rowsToMigrate = filter
-            ? db.find(tableName, filter as Parameters<typeof db.find>[1])
-            : db.find(tableName);
-        } catch (error) {
-          console.error(`Error: Failed to access table '${tableName}'`);
-          console.error(`  ${error instanceof Error ? error.message : String(error)}`);
-          console.error(`\nThe table may have failed to load during initialization.`);
-          console.error(`Check the table's data and schema for any constraint violations.`);
-          await db.close();
-          process.exit(1);
-        }
-
-        console.log(`Found ${rowsToMigrate.length} row(s) to migrate in table '${tableName}'`);
+        const rowsToMigrate = filter
+          ? db.find(tableName, filter as Parameters<typeof db.find>[1])
+          : db.find(tableName);
 
         if (rowsToMigrate.length === 0) {
-          console.log('No rows to migrate. Exiting.');
-          await db.close();
-          process.exit(0);
+          console.log(`  No rows to migrate`);
+          continue;
         }
 
+        console.log(`  Found ${rowsToMigrate.length} row(s) to migrate`);
+
         // Apply transformation
         const transformedRows = rowsToMigrate.map((row) => transform(row));
 
         // Perform the migration in a transaction
-        try {
-          await db.transaction(async () => {
-            db.batchUpdate(tableName, transformedRows as Parameters<typeof db.batchUpdate>[1], {
-              validate: true,
-            });
+        await db.transaction(async () => {
+          db.batchUpdate(tableName, transformedRows as Parameters<typeof db.batchUpdate>[1], {
+            validate: true,
           });
+        });
 
-          await db.close();
-
-          console.log(`\nMigration completed successfully:`);
-          console.log(`  ✓ ${rowsToMigrate.length} row(s) updated`);
-          process.exit(0);
-        } catch (error) {
-          await db.close();
-
-          // Write transformed data to error output file if --errorOutput is specified
-          if (options.errorOutput) {
-            try {
-              const jsonlContent = transformedRows.map((row) => JSON.stringify(row)).join('\n');
-              await writeFile(options.errorOutput, jsonlContent, 'utf-8');
-              console.error(
-                styleText(
-                  'yellow',
-                  `\n⚠ Transformed data (${transformedRows.length} rows) written to: ${options.errorOutput}`,
-                ),
-              );
-            } catch (writeError) {
-              console.error(
-                styleText(
-                  'red',
-                  `\n✗ Failed to write error output file: ${writeError instanceof Error ? writeError.message : String(writeError)}`,
-                ),
-              );
-            }
-          }
+        console.log(`  ✓ ${rowsToMigrate.length} row(s) updated\n`);
+        totalRowsMigrated += rowsToMigrate.length;
+      } catch (error) {
+        hasErrors = true;
+        console.error(styleText('red', `  ✗ Failed to migrate table '${tableName}'`));
 
-          const formatter = new ErrorFormatter({ verbose: options.verbose });
-          console.error(formatter.formatMigrationFailureHeader());
-
-          // Display detailed error information
-          if (error instanceof Error && error.name === 'ValidationError') {
-            const validationError = error as ValidationError & {
-              validationErrors?: Array<{
-                rowIndex: number;
-                rowData: unknown;
-                pkValue: unknown;
-                error: ValidationError;
-              }>;
-            };
-
-            // Display all validation errors
-            if (validationError.validationErrors) {
-              console.error(
-                `\nFound ${validationError.validationErrors.length} validation error(s) in transformed data:\n`,
-              );
-
-              const errorInfos = validationError.validationErrors.map(
-                ({ rowIndex, rowData, error: rowError }) => ({
-                  file: filePath,
-                  rowIndex,
-                  issues: rowError.issues,
-                  data: rowData,
-                  originalData: rowsToMigrate[rowIndex],
-                }),
-              );
-
-              const formatted = formatter.formatValidationErrors(errorInfos);
-              console.error(formatted);
-            } else {
-              // Fallback for single validation error (backward compatibility)
-              console.error('\nValidation error:\n');
-              const errorInfo = {
-                file: filePath,
-                rowIndex: 0,
-                issues: validationError.issues,
-              };
-              const formatted = formatter.formatValidationErrors([errorInfo]);
-              console.error(formatted);
-            }
-          } else if (error instanceof Error) {
-            console.error(`\n  ${error.message}`);
+        const formatter = new ErrorFormatter({ verbose: options.verbose });
 
-            // Output stack trace for debugging
-            if (options.verbose && error.stack) {
-              console.error(`\nStack trace:\n${error.stack}`);
-            }
+        if (error instanceof Error && error.name === 'ValidationError') {
+          const validationError = error as ValidationError & {
+            validationErrors?: Array<{
+              rowIndex: number;
+              rowData: unknown;
+              pkValue: unknown;
+              error: ValidationError;
+            }>;
+          };
+
+          if (validationError.validationErrors) {
+            console.error(
+              `  Found ${validationError.validationErrors.length} validation error(s):\n`,
+            );
 
-            // Check if it's a SQLite constraint error
-            if (
-              error.message.includes('UNIQUE constraint failed') ||
-              error.message.includes('FOREIGN KEY constraint failed') ||
-              error.message.includes('NOT NULL constraint failed') ||
-              error.message.includes('CHECK constraint failed')
-            ) {
-              console.error('\n  This is a SQLite constraint violation.');
-              console.error('  Please check your data and schema requirements.');
-            }
-          } else {
-            console.error(`\n  ${String(error)}`);
+            const rowsToMigrate = filter
+              ? db.find(tableName, filter as Parameters<typeof db.find>[1])
+              : db.find(tableName);
+
+            const errorInfos = validationError.validationErrors.map(
+              ({ rowIndex, rowData, error: rowError }) => ({
+                file: `${dirPath}/${tableName}.jsonl`,
+                rowIndex,
+                issues: rowError.issues,
+                data: rowData,
+                originalData: rowsToMigrate[rowIndex],
+              }),
+            );
+
+            const formatted = formatter.formatValidationErrors(errorInfos);
+            console.error(formatted);
           }
+        } else if (error instanceof Error) {
+          console.error(`  ${error.message}`);
+        }
 
-          console.error('');
-          process.exit(1);
+        console.error('');
+      }
+    }
+
+    await db.close();
+
+    if (hasErrors) {
+      console.error(styleText('red', `\n✗ Migration completed with errors for some tables`));
+      console.log(`Total rows migrated: ${totalRowsMigrated}`);
+      process.exit(1);
+    } else {
+      console.log(styleText('green', `\n✓ Migration completed successfully for all tables`));
+      console.log(`Total rows migrated: ${totalRowsMigrated}`);
+      process.exit(0);
+    }
+  } catch (error) {
+    await db.close();
+    throw error;
+  }
+}
+
+/**
+ * Migrate a single JSONL file
+ */
+async function migrateFile(
+  filePath: string,
+  transformStr: string,
+  options: { filter?: string; errorOutput?: string; verbose: boolean },
+) {
+  // Extract table name from file path
+  const fileName = filePath.split('/').pop() || '';
+  const tableName = fileName.replace('.jsonl', '');
+
+  if (!tableName) {
+    console.error('Error: Invalid file path. Must be a .jsonl file');
+    process.exit(1);
+  }
+
+  // Get directory from file path
+  const lastSlashIndex = filePath.lastIndexOf('/');
+  const dataDir = lastSlashIndex > 0 ? filePath.substring(0, lastSlashIndex) : '.';
+
+  // Initialize database
+  const db = LinesDB.create({ dataDir });
+  await db.initialize();
+
+  try {
+    // Parse transform function
+    const transform = runInSandbox<unknown>(`(${transformStr})`);
+
+    if (typeof transform !== 'function') {
+      console.error('Error: Transform must be a function');
+      process.exit(1);
+    }
+
+    // Parse filter if provided
+    let filter: unknown = undefined;
+    if (options.filter) {
+      try {
+        // Try JSON parse first
+        filter = JSON.parse(options.filter);
+      } catch {
+        // Fall back to eval for JavaScript expressions
+        filter = runInSandbox(`(${options.filter})`);
+      }
+    }
+
+    // Get rows to migrate
+    let rowsToMigrate;
+    try {
+      rowsToMigrate = filter
+        ? db.find(tableName, filter as Parameters<typeof db.find>[1])
+        : db.find(tableName);
+    } catch (error) {
+      console.error(`Error: Failed to access table '${tableName}'`);
+      console.error(`  ${error instanceof Error ? error.message : String(error)}`);
+      console.error(`\nThe table may have failed to load during initialization.`);
+      console.error(`Check the table's data and schema for any constraint violations.`);
+      await db.close();
+      process.exit(1);
+    }
+
+    console.log(`Found ${rowsToMigrate.length} row(s) to migrate in table '${tableName}'`);
+
+    if (rowsToMigrate.length === 0) {
+      console.log('No rows to migrate. Exiting.');
+      await db.close();
+      process.exit(0);
+    }
+
+    // Apply transformation
+    const transformedRows = rowsToMigrate.map((row) => transform(row));
+
+    // Perform the migration in a transaction
+    try {
+      await db.transaction(async () => {
+        db.batchUpdate(tableName, transformedRows as Parameters<typeof db.batchUpdate>[1], {
+          validate: true,
+        });
+      });
+
+      await db.close();
+
+      console.log(`\nMigration completed successfully:`);
+      console.log(`  ✓ ${rowsToMigrate.length} row(s) updated`);
+      process.exit(0);
+    } catch (error) {
+      await db.close();
+
+      // Write transformed data to error output file if --errorOutput is specified
+      if (options.errorOutput) {
+        try {
+          const jsonlContent = transformedRows.map((row) => JSON.stringify(row)).join('\n');
+          await writeFile(options.errorOutput, jsonlContent, 'utf-8');
+          console.error(
+            styleText(
+              'yellow',
+              `\n⚠ Transformed data (${transformedRows.length} rows) written to: ${options.errorOutput}`,
+            ),
+          );
+        } catch (writeError) {
+          console.error(
+            styleText(
+              'red',
+              `\n✗ Failed to write error output file: ${writeError instanceof Error ? writeError.message : String(writeError)}`,
+            ),
+          );
         }
-      } catch (error) {
-        await db.close();
-        throw error;
       }
-    },
-  );
+
+      const formatter = new ErrorFormatter({ verbose: options.verbose });
+      console.error(formatter.formatMigrationFailureHeader());
+
+      // Display detailed error information
+      if (error instanceof Error && error.name === 'ValidationError') {
+        const validationError = error as ValidationError & {
+          validationErrors?: Array<{
+            rowIndex: number;
+            rowData: unknown;
+            pkValue: unknown;
+            error: ValidationError;
+          }>;
+        };
+
+        // Display all validation errors
+        if (validationError.validationErrors) {
+          console.error(
+            `\nFound ${validationError.validationErrors.length} validation error(s) in transformed data:\n`,
+          );
+
+          const errorInfos = validationError.validationErrors.map(
+            ({ rowIndex, rowData, error: rowError }) => ({
+              file: filePath,
+              rowIndex,
+              issues: rowError.issues,
+              data: rowData,
+              originalData: rowsToMigrate[rowIndex],
+            }),
+          );
+
+          const formatted = formatter.formatValidationErrors(errorInfos);
+          console.error(formatted);
+        } else {
+          // Fallback for single validation error (backward compatibility)
+          console.error('\nValidation error:\n');
+          const errorInfo = {
+            file: filePath,
+            rowIndex: 0,
+            issues: validationError.issues,
+          };
+          const formatted = formatter.formatValidationErrors([errorInfo]);
+          console.error(formatted);
+        }
+      } else if (error instanceof Error) {
+        console.error(`\n  ${error.message}`);
+
+        // Output stack trace for debugging
+        if (options.verbose && error.stack) {
+          console.error(`\nStack trace:\n${error.stack}`);
+        }
+
+        // Check if it's a SQLite constraint error
+        if (
+          error.message.includes('UNIQUE constraint failed') ||
+          error.message.includes('FOREIGN KEY constraint failed') ||
+          error.message.includes('NOT NULL constraint failed') ||
+          error.message.includes('CHECK constraint failed')
+        ) {
+          console.error('\n  This is a SQLite constraint violation.');
+          console.error('  Please check your data and schema requirements.');
+        }
+      } else {
+        console.error(`\n  ${String(error)}`);
+      }
+
+      console.error('');
+      process.exit(1);
+    }
+  } catch (error) {
+    await db.close();
+    throw error;
+  }
+}
 
 program.parse();

package/src/validator.test.ts

@@ -364,4 +364,144 @@ describe('Validator', () => {
       );
     });
   });
+
+  describe('constraint validation', () => {
+    it('should detect primary key constraint violations', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // Write data with duplicate id
+      await writeFile(
+        jsonlPath,
+        '{"id":"1","name":"Alice","email":"alice@example.com"}\n{"id":"1","name":"Bob","email":"bob@example.com"}\n',
+      );
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          },
+          primaryKey: 'id'
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[0].issues[0].message).toContain('UNIQUE constraint failed');
+      expect(result.errors[0].issues[0].message).toContain('id');
+    });
+
+    it('should detect unique index constraint violations', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // Write data with duplicate email
+      await writeFile(
+        jsonlPath,
+        '{"id":"1","name":"Alice","email":"alice@example.com"}\n{"id":"2","name":"Bob","email":"alice@example.com"}\n',
+      );
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          },
+          primaryKey: 'id',
+          indexes: [
+            { name: 'users_email_unique', columns: ['email'], unique: true }
+          ]
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[0].issues[0].message).toContain('UNIQUE constraint failed');
+      expect(result.errors[0].issues[0].message).toContain('email');
+    });
+
+    it('should use id column as primary key when primaryKey is not specified', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // Write data with duplicate id (no primaryKey specified in schema)
+      await writeFile(jsonlPath, '{"id":"1","name":"Alice"}\n{"id":"1","name":"Bob"}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          }
+          // Note: no primaryKey specified
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[0].issues[0].message).toContain('UNIQUE constraint failed');
+      expect(result.errors[0].issues[0].message).toContain('id');
+    });
+
+    it('should detect multiple constraint violations in single file', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // Write data with both duplicate id and duplicate email
+      await writeFile(
+        jsonlPath,
+        '{"id":"1","name":"Alice","email":"alice@example.com"}\n{"id":"1","name":"Bob","email":"bob@example.com"}\n{"id":"2","name":"Charlie","email":"alice@example.com"}\n',
+      );
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          },
+          primaryKey: 'id',
+          indexes: [
+            { name: 'users_email_unique', columns: ['email'], unique: true }
+          ]
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(2);
+      // First error: duplicate id
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[0].issues[0].message).toContain('id');
+      // Second error: duplicate email
+      expect(result.errors[1].rowIndex).toBe(2);
+      expect(result.errors[1].issues[0].message).toContain('email');
+    });
+  });
 });