@javalabs/prisma-client 1.0.27 → 1.0.30

package/src/scripts/data-migration/batch-migrator.ts

@@ -1,248 +1,248 @@
-import { Logger } from "@nestjs/common";
-import { SchemaUtils } from "./schema-utils";
-import { DataTransformer } from "./data-transformer";
-import { TypecastManager } from "./typecast-manager";
-import {
-  ColumnSchema,
-  DatabaseConnections,
-  MigrationOptions,
-  TableConfig,
-} from "./types";
-
-const BATCH_SIZE = 100;
-const MAX_RETRIES = 3;
-const RETRY_BASE_DELAY = 1000;
-
-export class BatchMigrator {
-  private readonly logger = new Logger("BatchMigrator");
-  private readonly typecastManager: TypecastManager;
-
-  constructor(
-    private readonly schemaUtils: SchemaUtils,
-    private readonly dataTransformer: DataTransformer,
-    private readonly connections: DatabaseConnections,
-    private readonly options: MigrationOptions,
-    private readonly providerId?: string | null
-  ) {
-    this.typecastManager = new TypecastManager();
-  }
-
-  async validateSchema(
-    sourceColumns: ColumnSchema[],
-    targetColumns: ColumnSchema[]
-  ): Promise<void> {
-    for (const targetColumn of targetColumns) {
-      const sourceColumn = sourceColumns.find(
-        (col) => col.column_name === targetColumn.column_name
-      );
-
-      if (!sourceColumn) {
-        if (targetColumn.is_nullable === "NO") {
-          throw new Error(
-            `Required column ${targetColumn.column_name} not found in source schema`
-          );
-        }
-        this.logger.warn(
-          `Column ${targetColumn.column_name} not found in source schema but is nullable`
-        );
-        continue;
-      }
-
-      if (
-        !this.typecastManager.areTypesCompatible(
-          sourceColumn.data_type,
-          targetColumn.data_type
-        )
-      ) {
-        throw new Error(
-          `Incompatible data types for column ${targetColumn.column_name}: ` +
-            `source ${sourceColumn.data_type} -> target ${targetColumn.data_type}`
-        );
-      }
-    }
-  }
-
-  async migrateEntityDataInBatches(
-    sourceSchema: string,
-    targetSchema: string,
-    tableConfig: TableConfig,
-    tenantId: string
-  ): Promise<void> {
-    const sourceColumns = await this.schemaUtils.getTableColumns(
-      sourceSchema,
-      tableConfig.sourceTable
-    );
-    const targetColumns = await this.schemaUtils.getTableColumns(
-      targetSchema,
-      tableConfig.targetTable
-    );
-
-    await this.validateSchema(sourceColumns, targetColumns);
-
-    let offset = 0;
-    let hasMoreRecords = true;
-    let retryCount = 0;
-
-    while (hasMoreRecords) {
-      try {
-        const records = await this.fetchBatch(
-          sourceSchema,
-          tableConfig,
-          offset,
-          BATCH_SIZE
-        );
-
-        if (records.length === 0) {
-          hasMoreRecords = false;
-          continue;
-        }
-
-        await this.processBatchWithTransaction(
-          records,
-          targetSchema,
-          tableConfig,
-          sourceColumns,
-          targetColumns,
-          tenantId
-        );
-
-        offset += BATCH_SIZE;
-        retryCount = 0; // Reset retry count on success
-      } catch (error) {
-        if (retryCount < MAX_RETRIES) {
-          retryCount++;
-          const delay = RETRY_BASE_DELAY * Math.pow(2, retryCount - 1);
-          this.logger.warn(
-            `Error processing batch, retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES}): ${error.message}`
-          );
-          await new Promise((resolve) => setTimeout(resolve, delay));
-        } else {
-          throw new Error(
-            `Failed to process batch after ${MAX_RETRIES} retries: ${error.message}`
-          );
-        }
-      }
-    }
-  }
-
-  private async fetchBatch(
-    sourceSchema: string,
-    tableConfig: TableConfig,
-    offset: number,
-    limit: number
-  ): Promise<any[]> {
-    const query = `
-      SELECT *
-      FROM "${sourceSchema}"."${tableConfig.sourceTable}"
-      ${this.buildWhereClause(tableConfig)}
-      ORDER BY "${tableConfig.idField}"
-      LIMIT ${limit}
-      OFFSET ${offset}
-    `;
-
-    const result = await this.connections.sourcePool.query(query);
-    return result.rows;
-  }
-
-  private buildWhereClause(tableConfig: TableConfig): string {
-    const conditions = [];
-
-    if (this.providerId) {
-      conditions.push(`"${tableConfig.providerLink}" = '${this.providerId}'`);
-    }
-
-    if (tableConfig.filterColumn) {
-      conditions.push(`"${tableConfig.filterColumn}" IS NOT NULL`);
-    }
-
-    return conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
-  }
-
-  private async processBatchWithTransaction(
-    records: any[],
-    targetSchema: string,
-    tableConfig: TableConfig,
-    sourceColumns: ColumnSchema[],
-    targetColumns: ColumnSchema[],
-    tenantId: string
-  ): Promise<void> {
-    const client = await this.connections.targetPool.connect();
-    try {
-      await client.query("BEGIN");
-
-      for (const record of records) {
-        const transformedRecord = await this.transformRecord(
-          record,
-          sourceColumns,
-          targetColumns,
-          tenantId
-        );
-
-        await this.insertRecord(
-          client,
-          targetSchema,
-          tableConfig.targetTable,
-          transformedRecord
-        );
-      }
-
-      await client.query("COMMIT");
-    } catch (error) {
-      await client.query("ROLLBACK");
-      throw error;
-    } finally {
-      client.release();
-    }
-  }
-
-  private async transformRecord(
-    record: any,
-    sourceColumns: ColumnSchema[],
-    targetColumns: ColumnSchema[],
-    tenantId: string
-  ): Promise<any> {
-    const transformedRecord: any = {};
-
-    for (const targetColumn of targetColumns) {
-      const sourceColumn = sourceColumns.find(
-        (col) => col.column_name === targetColumn.column_name
-      );
-
-      if (!sourceColumn) {
-        transformedRecord[targetColumn.column_name] = null;
-        continue;
-      }
-
-      const value = record[targetColumn.column_name];
-      transformedRecord[targetColumn.column_name] =
-        await this.dataTransformer.transformColumnValue(
-          value,
-          targetColumn.column_name,
-          { ...targetColumn, source_type: sourceColumn.data_type },
-          tenantId
-        );
-    }
-
-    return transformedRecord;
-  }
-
-  private async insertRecord(
-    client: any,
-    schema: string,
-    table: string,
-    record: any
-  ): Promise<void> {
-    const columns = Object.keys(record);
-    const values = Object.values(record);
-    const placeholders = values.map((_, i) => `$${i + 1}`).join(", ");
-
-    const query = `
-      INSERT INTO "${schema}"."${table}"
-      (${columns.map((col) => `"${col}"`).join(", ")})
-      VALUES (${placeholders})
-      ON CONFLICT DO NOTHING
-    `;
-
-    await client.query(query, values);
-  }
-}
+import { Logger } from "@nestjs/common";
+import { SchemaUtils } from "./schema-utils";
+import { DataTransformer } from "./data-transformer";
+import { TypecastManager } from "./typecast-manager";
+import {
+  ColumnSchema,
+  DatabaseConnections,
+  MigrationOptions,
+  TableConfig,
+} from "./types";
+
+const BATCH_SIZE = 100;
+const MAX_RETRIES = 3;
+const RETRY_BASE_DELAY = 1000;
+
+export class BatchMigrator {
+  private readonly logger = new Logger("BatchMigrator");
+  private readonly typecastManager: TypecastManager;
+
+  constructor(
+    private readonly schemaUtils: SchemaUtils,
+    private readonly dataTransformer: DataTransformer,
+    private readonly connections: DatabaseConnections,
+    private readonly options: MigrationOptions,
+    private readonly providerId?: string | null
+  ) {
+    this.typecastManager = new TypecastManager();
+  }
+
+  async validateSchema(
+    sourceColumns: ColumnSchema[],
+    targetColumns: ColumnSchema[]
+  ): Promise<void> {
+    for (const targetColumn of targetColumns) {
+      const sourceColumn = sourceColumns.find(
+        (col) => col.column_name === targetColumn.column_name
+      );
+
+      if (!sourceColumn) {
+        if (targetColumn.is_nullable === "NO") {
+          throw new Error(
+            `Required column ${targetColumn.column_name} not found in source schema`
+          );
+        }
+        this.logger.warn(
+          `Column ${targetColumn.column_name} not found in source schema but is nullable`
+        );
+        continue;
+      }
+
+      if (
+        !this.typecastManager.areTypesCompatible(
+          sourceColumn.data_type,
+          targetColumn.data_type
+        )
+      ) {
+        throw new Error(
+          `Incompatible data types for column ${targetColumn.column_name}: ` +
+            `source ${sourceColumn.data_type} -> target ${targetColumn.data_type}`
+        );
+      }
+    }
+  }
+
+  async migrateEntityDataInBatches(
+    sourceSchema: string,
+    targetSchema: string,
+    tableConfig: TableConfig,
+    tenantId: string
+  ): Promise<void> {
+    const sourceColumns = await this.schemaUtils.getTableColumns(
+      sourceSchema,
+      tableConfig.sourceTable
+    );
+    const targetColumns = await this.schemaUtils.getTableColumns(
+      targetSchema,
+      tableConfig.targetTable
+    );
+
+    await this.validateSchema(sourceColumns, targetColumns);
+
+    let offset = 0;
+    let hasMoreRecords = true;
+    let retryCount = 0;
+
+    while (hasMoreRecords) {
+      try {
+        const records = await this.fetchBatch(
+          sourceSchema,
+          tableConfig,
+          offset,
+          BATCH_SIZE
+        );
+
+        if (records.length === 0) {
+          hasMoreRecords = false;
+          continue;
+        }
+
+        await this.processBatchWithTransaction(
+          records,
+          targetSchema,
+          tableConfig,
+          sourceColumns,
+          targetColumns,
+          tenantId
+        );
+
+        offset += BATCH_SIZE;
+        retryCount = 0; // Reset retry count on success
+      } catch (error) {
+        if (retryCount < MAX_RETRIES) {
+          retryCount++;
+          const delay = RETRY_BASE_DELAY * Math.pow(2, retryCount - 1);
+          this.logger.warn(
+            `Error processing batch, retrying in ${delay}ms (attempt ${retryCount}/${MAX_RETRIES}): ${error.message}`
+          );
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        } else {
+          throw new Error(
+            `Failed to process batch after ${MAX_RETRIES} retries: ${error.message}`
+          );
+        }
+      }
+    }
+  }
+
+  private async fetchBatch(
+    sourceSchema: string,
+    tableConfig: TableConfig,
+    offset: number,
+    limit: number
+  ): Promise<any[]> {
+    const query = `
+      SELECT *
+      FROM "${sourceSchema}"."${tableConfig.sourceTable}"
+      ${this.buildWhereClause(tableConfig)}
+      ORDER BY "${tableConfig.idField}"
+      LIMIT ${limit}
+      OFFSET ${offset}
+    `;
+
+    const result = await this.connections.sourcePool.query(query);
+    return result.rows;
+  }
+
+  private buildWhereClause(tableConfig: TableConfig): string {
+    const conditions = [];
+
+    if (this.providerId) {
+      conditions.push(`"${tableConfig.providerLink}" = '${this.providerId}'`);
+    }
+
+    if (tableConfig.filterColumn) {
+      conditions.push(`"${tableConfig.filterColumn}" IS NOT NULL`);
+    }
+
+    return conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  }
+
+  private async processBatchWithTransaction(
+    records: any[],
+    targetSchema: string,
+    tableConfig: TableConfig,
+    sourceColumns: ColumnSchema[],
+    targetColumns: ColumnSchema[],
+    tenantId: string
+  ): Promise<void> {
+    const client = await this.connections.targetPool.connect();
+    try {
+      await client.query("BEGIN");
+
+      for (const record of records) {
+        const transformedRecord = await this.transformRecord(
+          record,
+          sourceColumns,
+          targetColumns,
+          tenantId
+        );
+
+        await this.insertRecord(
+          client,
+          targetSchema,
+          tableConfig.targetTable,
+          transformedRecord
+        );
+      }
+
+      await client.query("COMMIT");
+    } catch (error) {
+      await client.query("ROLLBACK");
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  private async transformRecord(
+    record: any,
+    sourceColumns: ColumnSchema[],
+    targetColumns: ColumnSchema[],
+    tenantId: string
+  ): Promise<any> {
+    const transformedRecord: any = {};
+
+    for (const targetColumn of targetColumns) {
+      const sourceColumn = sourceColumns.find(
+        (col) => col.column_name === targetColumn.column_name
+      );
+
+      if (!sourceColumn) {
+        transformedRecord[targetColumn.column_name] = null;
+        continue;
+      }
+
+      const value = record[targetColumn.column_name];
+      transformedRecord[targetColumn.column_name] =
+        await this.dataTransformer.transformColumnValue(
+          value,
+          targetColumn.column_name,
+          { ...targetColumn, source_type: sourceColumn.data_type },
+          tenantId
+        );
+    }
+
+    return transformedRecord;
+  }
+
+  private async insertRecord(
+    client: any,
+    schema: string,
+    table: string,
+    record: any
+  ): Promise<void> {
+    const columns = Object.keys(record);
+    const values = Object.values(record);
+    const placeholders = values.map((_, i) => `$${i + 1}`).join(", ");
+
+    const query = `
+      INSERT INTO "${schema}"."${table}"
+      (${columns.map((col) => `"${col}"`).join(", ")})
+      VALUES (${placeholders})
+      ON CONFLICT DO NOTHING
+    `;
+
+    await client.query(query, values);
+  }
+}