@javalabs/prisma-client 1.0.0

package/src/scripts/data-migration/migration-tool.ts

@@ -0,0 +1,357 @@
+import { Logger } from "@nestjs/common";
+import * as fs from 'fs';
+import * as path from 'path';
+import { SchemaUtils } from "./schema-utils";
+import { DataTransformer } from "./data-transformer";
+import { DatabaseConnections, ColumnSchema, EntityType } from "./types";
+import { BatchMigrator } from "./batch-migrator";
+import { ForeignKeyManager } from "./foreign-key-manager";
+import { DependencyResolver } from "./dependency-resolver";
+
+interface MigrationOptions {
+  publicOnly?: boolean;
+  multiTenant?: boolean;
+  sourceSchema?: string; // Potentially deprecated with config
+  targetSchema?: string; // Potentially deprecated with config
+  forceSingleTenant?: boolean; // Alias for publicOnly
+  configPath?: string;
+}
+
+interface TableConfig {
+  type: 'public' | 'filteredPublic' | 'tenantInfo' | 'tenant';
+  idField: string;
+  filterColumn?: string;
+  via?: string;
+  providerLink?: string;
+  tenantKey?: string;
+}
+
+interface MigrationConfig {
+  commonSchema: string;
+  tenantInfo: {
+    sourceTable: string;
+    tenantIdColumn: string;
+    providerIdColumn: string;
+  };
+  tables: Record<string, TableConfig>;
+}
+
+export class DataMigrationTool {
+  private readonly logger = new Logger("DataMigrationTool");
+  private readonly schemaUtils: SchemaUtils;
+  private readonly dataTransformer: DataTransformer;
+  private readonly foreignKeyManager: ForeignKeyManager;
+  private readonly batchMigrator: BatchMigrator;
+  private readonly dependencyResolver: DependencyResolver;
+  private migrationConfig: MigrationConfig;
+
+  private schemaCache: Record<string, ColumnSchema[]> = {};
+  private targetSchemaCache: Record<string, ColumnSchema[]> = {};
+
+
+  constructor(private readonly connections: DatabaseConnections) {
+    this.schemaUtils = new SchemaUtils(
+      connections.sourcePool,
+      connections.targetPool
+    );
+    this.dataTransformer = new DataTransformer(this.schemaUtils);
+    this.foreignKeyManager = new ForeignKeyManager(connections);
+    this.dependencyResolver = new DependencyResolver(
+      connections.sourcePool,
+      connections.targetPool
+    );
+    this.batchMigrator = new BatchMigrator(
+      this.dataTransformer,
+      this.schemaUtils,
+      connections,
+      this.dependencyResolver,
+      this.schemaCache,
+      this.targetSchemaCache
+    );
+  }
+
+  private loadConfig(configPath: string = 'migration-config.json'): void {
+    const fullPath = path.resolve(process.cwd(), configPath);
+    if (!fs.existsSync(fullPath)) {
+      throw new Error(`Migration config file not found at: ${fullPath}`);
+    }
+    try {
+      const fileContent = fs.readFileSync(fullPath, 'utf-8');
+      this.migrationConfig = JSON.parse(fileContent);
+      this.logger.log(`Loaded migration config from: ${fullPath}`);
+    } catch (error) {
+      throw new Error(`Error reading or parsing migration config file: ${error.message}`);
+    }
+    if (!this.migrationConfig.commonSchema || !this.migrationConfig.tables || !this.migrationConfig.tenantInfo) {
+        throw new Error('Invalid migration config structure.');
+    }
+  }
+
+  async migrateData(options: MigrationOptions = {}) {
+    try {
+      this.logger.log("Starting data migration process");
+      this.loadConfig(options.configPath);
+
+      // Check the publicOnly/forceSingleTenant flag AFTER loading config
+      if (options.forceSingleTenant || options.publicOnly) {
+        this.logger.log("Public-only migration requested. Migrating only 'public' type tables...");
+        await this.migratePublicTablesOnly(); // Call the dedicated function
+      } else {
+        this.logger.log("Multi-tenant migration requested based on config...");
+        await this.migrateMultiTenantWithConfig();
+      }
+    } catch (error) {
+      this.logger.error(`Migration failed: ${error.message}`);
+      console.error(error);
+      throw error;
+    }
+  }
+
+  // --- Function for Public-Only Migration --- 
+  private async migratePublicTablesOnly() {
+      const { commonSchema, tables: tablesConfig } = this.migrationConfig;
+      const allTablesInOrder = await this.dependencyResolver.analyzeDependencies();
+
+      // Filter only the tables marked as 'public' in the config
+      const publicTablesToMigrate = allTablesInOrder.filter(t => tablesConfig[t]?.type === 'public');
+
+      this.logger.log(`Migrating public tables to '${commonSchema}' schema: ${publicTablesToMigrate.join(", ")}`);
+      if (publicTablesToMigrate.length === 0) {
+          this.logger.warn("No tables configured as 'public'. Public-only migration will not process any tables.");
+          return;
+      }
+
+      await this.connections.targetPool.query(`SET session_replication_role = 'replica';`);
+
+      for (const tableName of publicTablesToMigrate) {
+          const config = tablesConfig[tableName];
+          if (!config) {
+              this.logger.warn(`Config not found for public table ${tableName}, skipping.`); // Should not happen if filter worked
+              continue;
+          }
+          try {
+              await this.batchMigrator.migrateEntityDataInBatches(
+                  this.connections.targetPrisma,
+                  { name: tableName, idField: config.idField },
+                  null, // No provider filter for public-only migration
+                  commonSchema // Target is the common schema
+              );
+          } catch (error) {
+              this.logger.error(`Error migrating public table ${tableName}: ${error.message}`);
+              // Decide whether to stop or continue on error for public-only
+              throw error; 
+          }
+      }
+
+      await this.connections.targetPool.query(`SET session_replication_role = 'origin';`);
+      this.logger.log("Finished migrating public tables.");
+  }
+
+
+  // Keep validateSourceData (potentially enhanced)
+  private async validateSourceData() {
+     this.logger.log("Validating source data...");
+       try {
+          const { sourceTable, providerIdColumn } = this.migrationConfig.tenantInfo;
+          const tablesConfig = this.migrationConfig.tables;
+          
+          // Find the config for the transaction-like table and the provider table
+          let transactionTableKey = 'transactions'; // Default or find dynamically
+          let providerTableKey = 'providers'; // Default or find dynamically
+          let transactionProviderFk = providerIdColumn; // Default
+
+          for (const key in tablesConfig) {
+              if (tablesConfig[key].type === 'filteredPublic' && tablesConfig[key].filterColumn === providerIdColumn && !tablesConfig[key].via) {
+                  transactionTableKey = key;
+                  transactionProviderFk = tablesConfig[key].filterColumn;
+              }
+              if (tablesConfig[key].type === 'public' && tablesConfig[key].idField === providerIdColumn) {
+                  providerTableKey = key;
+              }
+          }
+           this.logger.debug(`Validation check: Using ${transactionTableKey}.${transactionProviderFk} referencing ${providerTableKey}.${providerIdColumn}`);
+
+          // Dynamically build the query
+          // IMPORTANT: Ensure proper SQL identifier quoting if names contain special chars/are reserved words
+          const query = `
+            SELECT t."${tablesConfig[transactionTableKey].idField}" AS transaction_id, t."${transactionProviderFk}"
+            FROM "${transactionTableKey}" t
+            LEFT JOIN "${providerTableKey}" p ON t."${transactionProviderFk}" = p."${providerIdColumn}"
+            WHERE t."${transactionProviderFk}" IS NOT NULL AND p."${providerIdColumn}" IS NULL;
+          `;
+
+          const invalidTransactions = (await this.connections.sourcePrisma.$queryRawUnsafe(query)) as any[];
+
+          if (invalidTransactions.length > 0) {
+            this.logger.error(`${invalidTransactions.length} records found in ${transactionTableKey} with invalid foreign key reference to ${providerTableKey}.${providerIdColumn}`);
+            console.error("Invalid records (limit 10):", invalidTransactions.slice(0, 10)); 
+            throw new Error("Source data validation failed: Invalid foreign key references found. Please check the configuration or fix the source data.");
+          } else {
+            this.logger.log(`Source data validation passed: All ${transactionTableKey}.${transactionProviderFk} references are valid.`);
+          }
+        } catch (error) {
+          this.logger.error("Error during source data validation:");
+          console.error(error);
+          throw error;
+        }
+  }
+
+
+  // Renamed from migrateToTenantSchemas to reflect its purpose
+  private async migrateMultiTenantWithConfig() {
+    await this.validateSourceData(); // Validate first
+
+    const { commonSchema, tables: tablesConfig, tenantInfo } = this.migrationConfig;
+    const allTablesInOrder = await this.dependencyResolver.analyzeDependencies();
+
+    // --- PASADA 0: Migrar tablas públicas --- 
+    this.logger.log(`Migrating public tables to '${commonSchema}' schema...`);
+    await this.connections.targetPool.query(`SET session_replication_role = 'replica';`);
+    const publicTables = allTablesInOrder.filter(t => tablesConfig[t]?.type === 'public');
+
+    for (const tableName of publicTables) {
+        const config = tablesConfig[tableName];
+        if (!config) {
+             this.logger.warn(`Skipping migration for table ${tableName}: Not found in config.`);
+             continue;
+        }
+        try {
+            await this.batchMigrator.migrateEntityDataInBatches(
+                this.connections.targetPrisma,
+                { name: tableName, idField: config.idField }, // Pass config details
+                null, // No provider filter for public tables
+                commonSchema
+            );
+        } catch (error) {
+            this.logger.error(`Error migrating public table ${tableName}: ${error.message}`);
+            throw error; // Stop if essential public table fails
+        }
+    }
+    await this.connections.targetPool.query(`SET session_replication_role = 'origin';`);
+    this.logger.log("Finished migrating public tables.");
+
+    // --- Obtener Inquilinos --- 
+    this.logger.log(`Fetching tenant info from source table '${tenantInfo.sourceTable}'...`);
+    const tenants = await this.connections.sourcePrisma[tenantInfo.sourceTable].findMany({
+        select: {
+            [tenantInfo.tenantIdColumn]: true,
+            [tenantInfo.providerIdColumn]: true,
+        }
+    });
+    this.logger.log(`Found ${tenants.length} potential tenants.`);
+
+    // --- Crear Esquemas y Preparar Migración por Inquilino --- 
+    const validTenants = new Map<string, number | null>(); // Map tenantId -> providerId
+    this.logger.log("Ensuring tenant schemas exist (if strategy requires them)...");
+    let requiresTenantSchemas = Object.values(tablesConfig).some(t => t.type === 'tenant');
+    if(requiresTenantSchemas) {
+        this.logger.log('Tenant-specific schemas are required by config.');
+        for (const tenant of tenants) {
+            const tenantId = tenant[tenantInfo.tenantIdColumn];
+            const providerId = tenant[tenantInfo.providerIdColumn] ?? null;
+            if (tenantId && typeof tenantId === 'string' && tenantId !== commonSchema) {
+                await this.schemaUtils.createSchema(tenantId);
+                validTenants.set(tenantId, providerId);
+                this.logger.log(`Ensured schema exists for tenant: ${tenantId}`);
+            } else {
+                this.logger.warn(`Skipping invalid/public tenant ID: ${tenantId}`);
+            }
+        }
+    } else {
+         this.logger.log('No tables configured with type "tenant". Tenant schema creation skipped.');
+         // Still populate validTenants map for filteredPublic logic
+         for (const tenant of tenants) {
+            const tenantId = tenant[tenantInfo.tenantIdColumn];
+            const providerId = tenant[tenantInfo.providerIdColumn] ?? null;
+            // Use tenantId as key even if schemas aren't created, it represents the logical tenant
+             if (tenantId && typeof tenantId === 'string') { 
+                validTenants.set(tenantId, providerId);
+             }
+         }
+    }
+    this.logger.log(`Prepared ${validTenants.size} valid tenants for migration.`);
+
+    // --- PASADAS POR INQUILINO (para tablas filteredPublic y tenant) --- 
+    const tablesToMigratePerTenant = allTablesInOrder.filter(t =>
+        tablesConfig[t]?.type === 'filteredPublic' || tablesConfig[t]?.type === 'tenant'
+    );
+    this.logger.log(`Migrating filtered/tenant tables: ${tablesToMigratePerTenant.join(", ")}`);
+
+    for (const [tenantId, providerId] of validTenants.entries()) {
+        // Note: tenantId here might just be logical identifier if not creating schemas
+        this.logger.log(`Starting migration pass related to tenant: ${tenantId} (Filtering by Provider ID: ${providerId ?? 'N/A'})`);
+        await this.connections.targetPool.query(`SET session_replication_role = 'replica';`);
+
+        for (const tableName of tablesToMigratePerTenant) {
+            const config = tablesConfig[tableName];
+            if (!config) {
+                this.logger.warn(`Skipping migration for table ${tableName} for tenant ${tenantId}: Not found in config.`);
+                continue;
+            }
+
+            // Determine schema destino basado en config.type
+            const targetSchemaForTable = config.type === 'tenant' ? tenantId : commonSchema;
+
+            try {
+                const entity: EntityType = {
+                    name: tableName,
+                    idField: config.idField,
+                    filterColumn: config.filterColumn,
+                    filterVia: config.via
+                };
+
+                await this.batchMigrator.migrateEntityDataInBatches(
+                    this.connections.targetPrisma,
+                    entity,
+                    providerId, // providerId para FILTRAR origen
+                    targetSchemaForTable // esquema donde INSERTAR
+                );
+            } catch (error) {
+                this.logger.error(`Error migrating table ${tableName} into schema ${targetSchemaForTable} (related to tenant ${tenantId}): ${error.message}`);
+            }
+        }
+        await this.connections.targetPool.query(`SET session_replication_role = 'origin';`);
+        this.logger.log(`Finished migration pass related to tenant: ${tenantId}`);
+    }
+
+    this.logger.log("Data migration process completed.");
+    // await this.verifyMigration(Array.from(validTenants.keys()));
+  }
+
+   private async verifyMigration(tenantIds: string[]) {
+       this.logger.log("Verifying migrated data...");
+        // Verificar commonSchema
+        try {
+            const publicResult = await this.connections.targetPool.query(
+            `SELECT tablename, n_live_tup FROM pg_stat_user_tables WHERE schemaname = $1`,
+            [this.migrationConfig.commonSchema]
+            );
+            this.logger.log(`Verification for schema '${this.migrationConfig.commonSchema}':`);
+            publicResult.rows.forEach(row => this.logger.log(`  Table: ${row.tablename}, Rows: ${row.n_live_tup}`));
+        } catch (err) {
+            this.logger.error(`Error verifying public schema: ${err.message}`);
+        }
+
+        // Verificar schemas de inquilinos (only if they were supposed to be created)
+        let requiresTenantSchemas = Object.values(this.migrationConfig.tables).some(t => t.type === 'tenant');
+        if (requiresTenantSchemas) {
+            for (const tenantId of tenantIds) {
+                try {
+                    const tenantResult = await this.connections.targetPool.query(
+                    `SELECT tablename, n_live_tup FROM pg_stat_user_tables WHERE schemaname = $1`,
+                    [tenantId]
+                    );
+                    this.logger.log(`Verification for tenant schema '${tenantId}':`);
+                    if (tenantResult.rows.length === 0) {
+                        this.logger.warn(`  Schema ${tenantId} has no tables (or pg_stat_user_tables is empty).`);
+                    } else {
+                        tenantResult.rows.forEach(row => this.logger.log(`  Table: ${row.tablename}, Rows: ${row.n_live_tup}`));
+                    }
+                } catch (err) {
+                    this.logger.error(`Error verifying tenant schema ${tenantId}: ${err.message}`);
+                }
+            }
+        }
+        this.logger.log("Verification finished.");
+    }
+}

package/src/scripts/data-migration/schema-utils.ts

@@ -0,0 +1,186 @@
+import { Logger } from "@nestjs/common";
+import { ColumnSchema } from "./types";
+
+export class SchemaUtils {
+  private readonly logger = new Logger("SchemaUtils");
+
+  constructor(private readonly sourcePool: any, private readonly targetPool: any) {}
+
+  // En el método getTableSchema de SchemaUtils
+  async getTableSchema(
+    tableName: string,
+    dbType: "source" | "target",
+    schema: string = "public"
+  ): Promise<ColumnSchema[]> {
+    const pool = dbType === "source" ? this.sourcePool : this.targetPool;
+    
+    const query = `
+      SELECT 
+        c.table_name,
+        c.column_name,
+        c.data_type,
+        c.udt_name,
+        c.character_maximum_length,
+        c.is_nullable,
+        c.column_default,
+        tc.constraint_type
+      FROM 
+        information_schema.columns c
+      LEFT JOIN information_schema.constraint_column_usage ccu 
+        ON c.column_name = ccu.column_name 
+        AND c.table_name = ccu.table_name
+      LEFT JOIN information_schema.table_constraints tc 
+        ON ccu.constraint_name = tc.constraint_name
+      WHERE 
+        c.table_schema = $1 
+        AND c.table_name = $2
+      ORDER BY 
+        c.ordinal_position
+    `;
+    
+    try {
+      const result = await pool.query(query, [schema, tableName]);
+      // No transformar los nombres de las columnas, mantener el case original
+      return result.rows;
+    } catch (error) {
+      this.logger.error(`Error getting schema for ${schema}.${tableName}: ${error.message}`);
+      return [];
+    }
+  }
+
+  async checkSchemaExists(schemaName: string): Promise<boolean> {
+    try {
+      const query = `
+        SELECT schema_name 
+        FROM information_schema.schemata 
+        WHERE schema_name = $1
+      `;
+      const result = await this.targetPool.query(query, [schemaName]);
+      return result.rows.length > 0;
+    } catch (error) {
+      this.logger.error(`Error checking if schema exists: ${error.message}`);
+      return false;
+    }
+  }
+
+  async getTargetTables(schemaName: string): Promise<string[]> {
+    try {
+      const query = `
+        SELECT table_name 
+        FROM information_schema.tables 
+        WHERE table_schema = $1
+        AND table_type = 'BASE TABLE'
+      `;
+      const result = await this.targetPool.query(query, [schemaName]);
+      
+      // Log para depuración
+      this.logger.log(`Found ${result.rows.length} tables in schema ${schemaName}`);
+      result.rows.forEach(row => {
+        this.logger.log(`Table in schema ${schemaName}: ${row.table_name}`);
+      });
+      
+      return result.rows.map((row) => row.table_name);
+    } catch (error) {
+      this.logger.error(`Error getting target tables: ${error.message}`);
+      return [];
+    }
+  }
+
+  async getEnumValues(
+    schemaName: string,
+    enumName: string
+  ): Promise<string[]> {
+    try {
+      // Consulta para obtener los valores posibles de un enum
+      const query = `
+        SELECT e.enumlabel
+        FROM pg_type t 
+        JOIN pg_enum e ON t.oid = e.enumtypid  
+        JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
+        WHERE t.typname = $1
+        AND n.nspname = $2
+      `;
+      const result = await this.targetPool.query(query, [enumName, schemaName]);
+      return result.rows.map((row) => row.enumlabel);
+    } catch (error) {
+      this.logger.error(
+        `Error getting enum values for ${enumName}: ${error.message}`
+      );
+      return [];
+    }
+  }
+
+  async executeQuery(query: string, params: any[] = []): Promise<any> {
+    return this.targetPool.query(query, params);
+  }
+
+  async queryTargetDb(query: string, params: any[]): Promise<{ rows: any[] }> {
+    return this.targetPool.query(query, params);
+  }
+
+  async createMissingColumns(tableName: string, schema: string, columns: ColumnSchema[]) {
+    for (const column of columns) {
+      try {
+        const query = `
+          ALTER TABLE "${schema}"."${tableName}"
+          ADD COLUMN IF NOT EXISTS "${column.column_name}" ${column.data_type}
+          ${column.is_nullable === 'NO' ? 'NOT NULL' : ''}
+          ${column.column_default ? `DEFAULT ${column.column_default}` : ''}
+        `;
+        await this.targetPool.query(query);
+
+        // Add unique constraint if needed
+        if (column.constraint_type === 'UNIQUE') {
+          const constraintName = `${tableName}_${column.column_name}_unique`;
+          const uniqueQuery = `
+            DO $$
+            BEGIN
+              IF NOT EXISTS (
+                SELECT 1 FROM pg_constraint WHERE conname = '${constraintName}'
+              ) THEN
+                ALTER TABLE "${schema}"."${tableName}"
+                ADD CONSTRAINT "${constraintName}" UNIQUE ("${column.column_name}");
+              END IF;
+            END $$;
+          `;
+          await this.targetPool.query(uniqueQuery);
+        }
+      } catch (error) {
+        this.logger.error(`Error creating column ${column.column_name}: ${error.message}`);
+        throw error;
+      }
+    }
+  }
+
+  async createSchema(schemaName: string): Promise<void> {
+    try {
+      // Create schema
+      await this.targetPool.query(`CREATE SCHEMA IF NOT EXISTS "${schemaName}"`);
+
+      // Copy table structure from public schema
+      const tables = await this.getPublicTables();
+      
+      for (const table of tables) {
+        await this.targetPool.query(`
+          CREATE TABLE IF NOT EXISTS "${schemaName}"."${table}" (
+            LIKE public."${table}" INCLUDING ALL
+          );
+        `);
+      }
+
+      this.logger.log(`Created schema ${schemaName} with all tables`);
+    } catch (error) {
+      this.logger.error(`Error creating schema ${schemaName}: ${error.message}`);
+      throw error;
+    }
+  }
+
+  private async getPublicTables(): Promise<string[]> {
+    const result = await this.targetPool.query(`
+      SELECT tablename 
+      FROM pg_tables 
+      WHERE schemaname = 'public'
+    `);
+    return result.rows.map(row => row.tablename);
+  }
+}