@javalabs/prisma-client 1.0.0

package/src/scripts/data-migration/data-transformer.ts

@@ -0,0 +1,377 @@
+import { Logger } from "@nestjs/common";
+import { ColumnSchema, EnumCastValue } from "./types";
+import { SchemaUtils } from "./schema-utils";
+
+export class DataTransformer {
+  private readonly logger = new Logger("DataTransformer");
+  private enumValuesCache: Map<string, any> = new Map();
+  private columnValuesCache: Map<string, Map<string, any>> = new Map();
+
+  constructor(public readonly schemaUtils: SchemaUtils) {}
+
+  async validateEnumValue(
+    schemaName: string,
+    enumType: string,
+    value: string
+  ): Promise<string | null> {
+    try {
+      const cacheKey = `${schemaName}.${enumType}`;
+      if (!this.enumValuesCache.has(cacheKey)) {
+        const query = `
+          SELECT e.enumlabel
+          FROM pg_type t
+          JOIN pg_enum e ON t.oid = e.enumtypid
+          JOIN pg_namespace n ON n.oid = t.typnamespace
+          WHERE t.typname = $1 AND n.nspname = $2
+        `;
+
+        const result = await this.schemaUtils.queryTargetDb(query, [
+          enumType,
+          schemaName,
+        ]);
+        const uniqueValues: Set<string> = new Set(
+          result.rows.map((row): string => String(row.enumlabel))
+        );
+
+        // También almacenar un mapa de valores normalizados a valores originales
+        const normalizedMap = new Map<string, string>();
+        result.rows.forEach((row) => {
+          const original = String(row.enumlabel);
+          const normalized = original.toLowerCase().trim();
+          normalizedMap.set(normalized, original);
+        });
+
+        this.enumValuesCache.set(cacheKey, uniqueValues);
+        this.enumValuesCache.set(`${cacheKey}_normalized`, normalizedMap);
+
+        this.logger.log(
+          `Valid values for ${enumType}: ${[...uniqueValues].join(", ")}`
+        );
+      }
+
+      const validValues = this.enumValuesCache.get(cacheKey);
+      const normalizedMap = this.enumValuesCache.get(
+        `${cacheKey}_normalized`
+      ) as unknown as Map<string, string>;
+
+      if (!validValues || !normalizedMap) {
+        throw new Error(`No valid values found for enum ${enumType}`);
+      }
+
+      // Verificar coincidencia exacta primero
+      if (validValues.has(value)) {
+        return value;
+      }
+
+      // Luego intentar con valor normalizado
+      const normalizedValue = value.toLowerCase().trim();
+      if (normalizedMap.has(normalizedValue)) {
+        // Devolver el valor original correcto, no el normalizado
+        return normalizedMap.get(normalizedValue);
+      }
+
+      this.logger.warn(
+        `Invalid enum value "${value}" for type ${enumType}. Valid values are: ${[
+          ...validValues,
+        ].join(", ")}`
+      );
+      return null;
+    } catch (error) {
+      this.logger.error(
+        `Error validating enum value "${value}" for type ${enumType}: ${error.message}`
+      );
+      return null;
+    }
+  }
+
+  transformToNumeric(value: any): number | null {
+    if (value === null || value === "") {
+      return null;
+    }
+
+    const numericValue = parseFloat(value);
+    if (isNaN(numericValue)) {
+      return null;
+    }
+
+    return numericValue;
+  }
+
+  private async getDefaultEnumValue(
+    enumType: string,
+    tenantId: string = "public"
+  ): Promise<string | null> {
+    try {
+      // Obtener todos los valores posibles del enum desde la base de datos
+      const query = `
+              SELECT e.enumlabel
+              FROM pg_type t
+              JOIN pg_enum e ON t.oid = e.enumtypid
+              JOIN pg_namespace n ON n.oid = t.typnamespace
+              WHERE t.typname = $1 AND n.nspname = $2
+              ORDER BY e.enumsortorder
+              LIMIT 1
+          `;
+
+      const result = await this.schemaUtils.queryTargetDb(query, [
+        enumType,
+        tenantId,
+      ]);
+
+      // Retornar el primer valor del enum como valor por defecto
+      if (result.rows.length > 0) {
+        return result.rows[0].enumlabel;
+      }
+
+      return null;
+    } catch (error) {
+      this.logger.error(
+        `Error getting default enum value for ${enumType}: ${error.message}`
+      );
+      return null;
+    }
+  }
+
+  async prepareEnumValue(
+    tenantId: string,
+    enumType: string,
+    value: string
+  ): Promise<EnumCastValue | null> {
+    if (value === null || value === undefined || value === "") {
+      return null;
+    }
+
+    const validatedValue = await this.validateEnumValue(
+      tenantId,
+      enumType,
+      value
+    );
+
+    if (validatedValue === null) {
+      // Obtener valor por defecto dinámicamente
+      const defaultValue = await this.getDefaultEnumValue(enumType, tenantId);
+      if (defaultValue) {
+        this.logger.warn(
+          `Using first enum value '${defaultValue}' as default for invalid value '${value}' of type ${enumType}`
+        );
+        return {
+          needsEnumCast: true,
+          value: defaultValue,
+          enumType,
+        };
+      }
+      return null;
+    }
+
+    return {
+      needsEnumCast: true,
+      value: validatedValue,
+      enumType,
+    };
+  }
+
+  async transformColumnValue(
+    value: any,
+    columnName: string,
+    targetColumn: ColumnSchema,
+    tenantId: string
+  ): Promise<any> {
+    // Log input values for debugging
+    this.logger.debug(
+      `Transforming column ${columnName}, value: ${JSON.stringify(
+        value
+      )}, type: ${targetColumn.udt_name}`
+    );
+
+    if (value === null && targetColumn.is_nullable === "YES") {
+      return null;
+    }
+
+    try {
+      // Handle enum types
+      if (targetColumn.udt_name.startsWith("enum_")) {
+        // Handle JSONB/object to enum conversion
+        if (typeof value === "object" && value !== null) {
+          const stringValue = this.extractEnumValueFromObject(value);
+          if (stringValue) {
+            return await this.prepareEnumValue(
+              tenantId,
+              targetColumn.udt_name,
+              stringValue
+            );
+          }
+        }
+
+        // Handle direct string values
+        if (typeof value === "string") {
+          return await this.prepareEnumValue(
+            tenantId,
+            targetColumn.udt_name,
+            value
+          );
+        }
+
+        // If no valid value found, get default
+        const defaultValue = await this.getDefaultEnumValue(
+          targetColumn.udt_name,
+          tenantId
+        );
+        if (defaultValue) {
+          this.logger.warn(
+            `Using default enum value '${defaultValue}' for column ${columnName}`
+          );
+          return {
+            needsEnumCast: true,
+            value: defaultValue,
+            enumType: targetColumn.udt_name,
+          };
+        }
+      }
+
+      // Handle numeric types
+      if (
+        targetColumn.udt_name === "numeric" ||
+        targetColumn.data_type === "decimal"
+      ) {
+        const numericValue = this.transformToNumeric(value);
+        if (numericValue !== null || targetColumn.is_nullable === "YES") {
+          return numericValue;
+        }
+        return 0; // Default value for non-nullable numeric fields
+      }
+
+      // Handle boolean types
+      if (
+        targetColumn.udt_name === "bool" ||
+        targetColumn.data_type === "boolean"
+      ) {
+        if (typeof value === "string") {
+          return value.toLowerCase() === "true" || value === "1";
+        }
+        return Boolean(value);
+      }
+
+      // Handle date/timestamp types
+      if (
+        targetColumn.data_type.includes("timestamp") ||
+        targetColumn.data_type.includes("date")
+      ) {
+        if (!value) return null;
+        const date = new Date(value);
+        return isNaN(date.getTime()) ? null : date.toISOString();
+      }
+
+      // Handle JSON/JSONB types
+      if (
+        targetColumn.data_type === "json" ||
+        targetColumn.data_type === "jsonb"
+      ) {
+        if (typeof value === "string") {
+          try {
+            return JSON.parse(value);
+          } catch {
+            return value;
+          }
+        }
+        return value;
+      }
+
+      // Default case: return value as-is
+      return value;
+    } catch (error) {
+      this.logger.error(
+        `Error transforming value for column ${columnName}: ${error.message}`
+      );
+
+      // Return appropriate default values based on column type
+      if (targetColumn.is_nullable === "YES") {
+        return null;
+      }
+
+      return this.getDefaultValueForType(targetColumn.udt_name);
+    }
+  }
+
+  private extractEnumValueFromObject(value: any): string | null {
+    const possibleKeys = ["type", "value", "document_type", "status", "state"];
+
+    for (const key of possibleKeys) {
+      if (value[key] && typeof value[key] === "string") {
+        return value[key];
+      }
+    }
+
+    // Try to find any string value in the object
+    for (const val of Object.values(value)) {
+      if (typeof val === "string") {
+        return val;
+      }
+    }
+
+    return null;
+  }
+
+  private getDefaultValueForType(udtName: string): any {
+    switch (udtName) {
+      case "int4":
+      case "int8":
+      case "numeric":
+        return 0;
+      case "bool":
+        return false;
+      case "text":
+      case "varchar":
+        return "";
+      case "jsonb":
+      case "json":
+        return {};
+      default:
+        if (udtName.startsWith("enum_")) {
+          return null; // Enums should be handled by prepareEnumValue
+        }
+        return null;
+    }
+  }
+
+  async transformRecord(
+    record: any,
+    sourceSchema: ColumnSchema[],
+    targetSchema: ColumnSchema[],
+    tenantId: string
+  ): Promise<any> {
+    const transformedData: any = {};
+
+    for (const [key, value] of Object.entries(record)) {
+      // Primero intentar encontrar una coincidencia exacta
+      let targetColumn = targetSchema.find(
+        (col) => col.column_name === key
+      );
+
+      // Si no se encuentra coincidencia exacta, intentar case-insensitive
+      if (!targetColumn) {
+        targetColumn = targetSchema.find(
+          (col) => col.column_name.toLowerCase() === key.toLowerCase()
+        );
+      }
+
+      if (!targetColumn) {
+        this.logger.warn(`Column ${key} not found in target schema, skipping`);
+        continue;
+      }
+
+      // Usar el nombre exacto de la columna del schema destino
+      const columnName = targetColumn.column_name;
+      
+      const transformedValue = await this.transformColumnValue(
+        value,
+        columnName,
+        targetColumn,
+        tenantId
+      );
+
+      transformedData[columnName] = transformedValue;
+    }
+
+    return transformedData;
+  }
+}

package/src/scripts/data-migration/db-connector.ts

@@ -0,0 +1,67 @@
+import { PrismaClient } from "@prisma/client";
+import * as pg from "pg";
+import * as dotenv from "dotenv";
+import { DatabaseConnections } from "./types";
+
+dotenv.config();
+
+export class DatabaseConnector {
+  static createConnections(): DatabaseConnections {
+    // Source database connection (the one we're migrating from)
+    const sourcePool = new pg.Pool({
+      connectionString: process.env.SOURCE_DATABASE_URL,
+    });
+
+    // Target database connection (our multi-tenant database)
+    const targetPool = new pg.Pool({
+      connectionString: process.env.DATABASE_URL,
+    });
+
+    // Source Prisma client
+    const sourcePrisma = new PrismaClient({
+      datasources: {
+        db: {
+          url: process.env.SOURCE_DATABASE_URL,
+        },
+      },
+    });
+
+    // Target Prisma client
+    const targetPrisma = new PrismaClient({
+      datasources: {
+        db: {
+          url: process.env.DATABASE_URL,
+        },
+      },
+    });
+
+    return {
+      sourcePrisma,
+      targetPrisma,
+      sourcePool,
+      targetPool,
+    };
+  }
+
+  static createTenantPrismaClient(tenantId: string): PrismaClient {
+    return new PrismaClient({
+      datasources: {
+        db: {
+          url: `${process.env.DATABASE_URL}?schema=${tenantId}`,
+        },
+      },
+      // Increase transaction timeout to 2 minutes
+      transactionOptions: {
+        maxWait: 120000, // 2 minutes max wait time
+        timeout: 120000, // 2 minutes timeout
+      },
+    });
+  }
+
+  static async cleanup(connections: DatabaseConnections): Promise<void> {
+    await connections.sourcePrisma.$disconnect();
+    await connections.targetPrisma.$disconnect();
+    await connections.sourcePool.end();
+    await connections.targetPool.end();
+  }
+}