@javalabs/prisma-client 1.0.26 → 1.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CODEOWNERS +1 -1
- package/README.md +269 -269
- package/migration-config.json +63 -63
- package/migration-config.json.bk +95 -95
- package/migrations/add_reserved_amount.sql +7 -7
- package/package.json +44 -44
- package/prisma/migrations/add_uuid_to_transactions.sql +13 -13
- package/prisma/schema.prisma +609 -607
- package/src/index.ts +23 -23
- package/src/prisma-factory.service.ts +40 -40
- package/src/prisma.module.ts +9 -9
- package/src/prisma.service.ts +16 -16
- package/src/scripts/add-uuid-to-table.ts +138 -138
- package/src/scripts/create-tenant-schemas.ts +145 -145
- package/src/scripts/data-migration/batch-migrator.ts +248 -248
- package/src/scripts/data-migration/data-transformer.ts +426 -426
- package/src/scripts/data-migration/db-connector.ts +120 -120
- package/src/scripts/data-migration/dependency-resolver.ts +174 -174
- package/src/scripts/data-migration/entity-discovery.ts +196 -196
- package/src/scripts/data-migration/foreign-key-manager.ts +277 -277
- package/src/scripts/data-migration/migration-config.json +63 -63
- package/src/scripts/data-migration/migration-tool.ts +509 -509
- package/src/scripts/data-migration/schema-utils.ts +248 -248
- package/src/scripts/data-migration/tenant-migrator.ts +201 -201
- package/src/scripts/data-migration/typecast-manager.ts +193 -193
- package/src/scripts/data-migration/types.ts +113 -113
- package/src/scripts/database-initializer.ts +49 -49
- package/src/scripts/drop-database.ts +104 -104
- package/src/scripts/dump-source-db.sh +61 -61
- package/src/scripts/encrypt-user-passwords.ts +36 -36
- package/src/scripts/error-handler.ts +117 -117
- package/src/scripts/fix-data-types.ts +241 -241
- package/src/scripts/fix-enum-values.ts +357 -357
- package/src/scripts/fix-schema-discrepancies.ts +317 -317
- package/src/scripts/fix-table-indexes.ts +601 -601
- package/src/scripts/migrate-schema-structure.ts +90 -90
- package/src/scripts/migrate-uuid.ts +76 -76
- package/src/scripts/post-migration-validator.ts +526 -526
- package/src/scripts/pre-migration-validator.ts +610 -610
- package/src/scripts/reset-database.ts +263 -263
- package/src/scripts/retry-failed-migrations.ts +416 -416
- package/src/scripts/run-migration.ts +707 -707
- package/src/scripts/schema-sync.ts +128 -128
- package/src/scripts/sequence-sync-cli.ts +416 -416
- package/src/scripts/sequence-synchronizer.ts +127 -127
- package/src/scripts/sync-enum-types.ts +170 -170
- package/src/scripts/sync-enum-values.ts +563 -563
- package/src/scripts/truncate-database.ts +123 -123
- package/src/scripts/verify-migration-setup.ts +135 -135
- package/tsconfig.json +17 -17
- package/dist/index.d.ts +0 -7
- package/dist/index.js +0 -34
- package/dist/index.js.map +0 -1
- package/dist/prisma-factory.service.d.ts +0 -9
- package/dist/prisma-factory.service.js +0 -47
- package/dist/prisma-factory.service.js.map +0 -1
- package/dist/prisma.module.d.ts +0 -2
- package/dist/prisma.module.js +0 -23
- package/dist/prisma.module.js.map +0 -1
- package/dist/prisma.service.d.ts +0 -6
- package/dist/prisma.service.js +0 -27
- package/dist/prisma.service.js.map +0 -1
- package/dist/scripts/add-uuid-to-table.d.ts +0 -8
- package/dist/scripts/add-uuid-to-table.js +0 -98
- package/dist/scripts/add-uuid-to-table.js.map +0 -1
- package/dist/scripts/create-tenant-schemas.d.ts +0 -1
- package/dist/scripts/create-tenant-schemas.js +0 -117
- package/dist/scripts/create-tenant-schemas.js.map +0 -1
- package/dist/scripts/data-migration/batch-migrator.d.ts +0 -20
- package/dist/scripts/data-migration/batch-migrator.js +0 -134
- package/dist/scripts/data-migration/batch-migrator.js.map +0 -1
- package/dist/scripts/data-migration/data-transformer.d.ts +0 -26
- package/dist/scripts/data-migration/data-transformer.js +0 -278
- package/dist/scripts/data-migration/data-transformer.js.map +0 -1
- package/dist/scripts/data-migration/db-connector.d.ts +0 -12
- package/dist/scripts/data-migration/db-connector.js +0 -94
- package/dist/scripts/data-migration/db-connector.js.map +0 -1
- package/dist/scripts/data-migration/dependency-resolver.d.ts +0 -18
- package/dist/scripts/data-migration/dependency-resolver.js +0 -132
- package/dist/scripts/data-migration/dependency-resolver.js.map +0 -1
- package/dist/scripts/data-migration/entity-discovery.d.ts +0 -11
- package/dist/scripts/data-migration/entity-discovery.js +0 -152
- package/dist/scripts/data-migration/entity-discovery.js.map +0 -1
- package/dist/scripts/data-migration/foreign-key-manager.d.ts +0 -18
- package/dist/scripts/data-migration/foreign-key-manager.js +0 -160
- package/dist/scripts/data-migration/foreign-key-manager.js.map +0 -1
- package/dist/scripts/data-migration/migration-tool.d.ts +0 -48
- package/dist/scripts/data-migration/migration-tool.js +0 -290
- package/dist/scripts/data-migration/migration-tool.js.map +0 -1
- package/dist/scripts/data-migration/schema-utils.d.ts +0 -18
- package/dist/scripts/data-migration/schema-utils.js +0 -207
- package/dist/scripts/data-migration/schema-utils.js.map +0 -1
- package/dist/scripts/data-migration/tenant-migrator.d.ts +0 -15
- package/dist/scripts/data-migration/tenant-migrator.js +0 -117
- package/dist/scripts/data-migration/tenant-migrator.js.map +0 -1
- package/dist/scripts/data-migration/typecast-manager.d.ts +0 -9
- package/dist/scripts/data-migration/typecast-manager.js +0 -179
- package/dist/scripts/data-migration/typecast-manager.js.map +0 -1
- package/dist/scripts/data-migration/types.d.ts +0 -100
- package/dist/scripts/data-migration/types.js +0 -3
- package/dist/scripts/data-migration/types.js.map +0 -1
- package/dist/scripts/database-initializer.d.ts +0 -5
- package/dist/scripts/database-initializer.js +0 -45
- package/dist/scripts/database-initializer.js.map +0 -1
- package/dist/scripts/drop-database.d.ts +0 -10
- package/dist/scripts/drop-database.js +0 -81
- package/dist/scripts/drop-database.js.map +0 -1
- package/dist/scripts/encrypt-user-passwords.d.ts +0 -1
- package/dist/scripts/encrypt-user-passwords.js +0 -33
- package/dist/scripts/encrypt-user-passwords.js.map +0 -1
- package/dist/scripts/error-handler.d.ts +0 -12
- package/dist/scripts/error-handler.js +0 -82
- package/dist/scripts/error-handler.js.map +0 -1
- package/dist/scripts/fix-data-types.d.ts +0 -10
- package/dist/scripts/fix-data-types.js +0 -185
- package/dist/scripts/fix-data-types.js.map +0 -1
- package/dist/scripts/fix-enum-values.d.ts +0 -17
- package/dist/scripts/fix-enum-values.js +0 -234
- package/dist/scripts/fix-enum-values.js.map +0 -1
- package/dist/scripts/fix-schema-discrepancies.d.ts +0 -21
- package/dist/scripts/fix-schema-discrepancies.js +0 -240
- package/dist/scripts/fix-schema-discrepancies.js.map +0 -1
- package/dist/scripts/fix-table-indexes.d.ts +0 -26
- package/dist/scripts/fix-table-indexes.js +0 -460
- package/dist/scripts/fix-table-indexes.js.map +0 -1
- package/dist/scripts/migrate-schema-structure.d.ts +0 -1
- package/dist/scripts/migrate-schema-structure.js +0 -76
- package/dist/scripts/migrate-schema-structure.js.map +0 -1
- package/dist/scripts/migrate-uuid.d.ts +0 -2
- package/dist/scripts/migrate-uuid.js +0 -57
- package/dist/scripts/migrate-uuid.js.map +0 -1
- package/dist/scripts/post-migration-validator.d.ts +0 -34
- package/dist/scripts/post-migration-validator.js +0 -363
- package/dist/scripts/post-migration-validator.js.map +0 -1
- package/dist/scripts/pre-migration-validator.d.ts +0 -25
- package/dist/scripts/pre-migration-validator.js +0 -491
- package/dist/scripts/pre-migration-validator.js.map +0 -1
- package/dist/scripts/reset-database.d.ts +0 -17
- package/dist/scripts/reset-database.js +0 -202
- package/dist/scripts/reset-database.js.map +0 -1
- package/dist/scripts/retry-failed-migrations.d.ts +0 -14
- package/dist/scripts/retry-failed-migrations.js +0 -301
- package/dist/scripts/retry-failed-migrations.js.map +0 -1
- package/dist/scripts/run-migration.d.ts +0 -1
- package/dist/scripts/run-migration.js +0 -512
- package/dist/scripts/run-migration.js.map +0 -1
- package/dist/scripts/schema-sync.d.ts +0 -1
- package/dist/scripts/schema-sync.js +0 -85
- package/dist/scripts/schema-sync.js.map +0 -1
- package/dist/scripts/sequence-sync-cli.d.ts +0 -2
- package/dist/scripts/sequence-sync-cli.js +0 -287
- package/dist/scripts/sequence-sync-cli.js.map +0 -1
- package/dist/scripts/sequence-synchronizer.d.ts +0 -8
- package/dist/scripts/sequence-synchronizer.js +0 -88
- package/dist/scripts/sequence-synchronizer.js.map +0 -1
- package/dist/scripts/sync-enum-types.d.ts +0 -13
- package/dist/scripts/sync-enum-types.js +0 -139
- package/dist/scripts/sync-enum-types.js.map +0 -1
- package/dist/scripts/sync-enum-values.d.ts +0 -20
- package/dist/scripts/sync-enum-values.js +0 -336
- package/dist/scripts/sync-enum-values.js.map +0 -1
- package/dist/scripts/truncate-database.d.ts +0 -10
- package/dist/scripts/truncate-database.js +0 -100
- package/dist/scripts/truncate-database.js.map +0 -1
- package/dist/scripts/verify-migration-setup.d.ts +0 -11
- package/dist/scripts/verify-migration-setup.js +0 -120
- package/dist/scripts/verify-migration-setup.js.map +0 -1
- package/dist/tsconfig.tsbuildinfo +0 -1
- package/prisma/migrations/add_athena_match_fields.sql +0 -18
- package/prisma/migrations/add_bank_receipt_number.sql +0 -9
|
@@ -1,509 +1,509 @@
|
|
|
1
|
-
import { Logger } from "@nestjs/common";
|
|
2
|
-
import * as fs from "fs";
|
|
3
|
-
import * as path from "path";
|
|
4
|
-
import { SchemaUtils } from "./schema-utils";
|
|
5
|
-
import { DataTransformer } from "./data-transformer";
|
|
6
|
-
import { DatabaseConnections, ColumnSchema, EntityType } from "./types";
|
|
7
|
-
import { BatchMigrator } from "./batch-migrator";
|
|
8
|
-
import { ForeignKeyManager } from "./foreign-key-manager";
|
|
9
|
-
import { DependencyResolver } from "./dependency-resolver";
|
|
10
|
-
|
|
11
|
-
interface MigrationOptions {
|
|
12
|
-
publicOnly?: boolean;
|
|
13
|
-
multiTenant?: boolean;
|
|
14
|
-
sourceSchema?: string;
|
|
15
|
-
targetSchema?: string;
|
|
16
|
-
forceSingleTenant?: boolean; // Alias for publicOnly
|
|
17
|
-
configPath?: string;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
interface TableConfig {
|
|
21
|
-
type: "public" | "filteredPublic" | "tenantInfo" | "tenant";
|
|
22
|
-
idField: string;
|
|
23
|
-
filterColumn?: string;
|
|
24
|
-
via?: string;
|
|
25
|
-
providerLink?: string;
|
|
26
|
-
tenantKey?: string;
|
|
27
|
-
dependencies?: string[];
|
|
28
|
-
sourceTable: string;
|
|
29
|
-
targetTable: string;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
interface MigrationConfig {
|
|
33
|
-
commonSchema: string;
|
|
34
|
-
tenantInfo: {
|
|
35
|
-
sourceTable: string;
|
|
36
|
-
tenantIdColumn: string;
|
|
37
|
-
providerIdColumn: string;
|
|
38
|
-
};
|
|
39
|
-
tables: Record<string, TableConfig>;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export class DataMigrationTool {
|
|
43
|
-
private readonly logger = new Logger("DataMigrationTool");
|
|
44
|
-
private readonly schemaUtils: SchemaUtils;
|
|
45
|
-
private readonly dataTransformer: DataTransformer;
|
|
46
|
-
private readonly batchMigrator: BatchMigrator;
|
|
47
|
-
private readonly foreignKeyManager: ForeignKeyManager;
|
|
48
|
-
private readonly dependencyResolver: DependencyResolver;
|
|
49
|
-
|
|
50
|
-
constructor(
|
|
51
|
-
private readonly connections: DatabaseConnections,
|
|
52
|
-
private migrationConfig: MigrationConfig,
|
|
53
|
-
private readonly options: MigrationOptions = {}
|
|
54
|
-
) {
|
|
55
|
-
this.schemaUtils = new SchemaUtils(connections);
|
|
56
|
-
|
|
57
|
-
this.dataTransformer = new DataTransformer(
|
|
58
|
-
this.schemaUtils,
|
|
59
|
-
connections.sourcePool,
|
|
60
|
-
connections
|
|
61
|
-
);
|
|
62
|
-
|
|
63
|
-
this.foreignKeyManager = new ForeignKeyManager(connections);
|
|
64
|
-
|
|
65
|
-
this.batchMigrator = new BatchMigrator(
|
|
66
|
-
this.schemaUtils,
|
|
67
|
-
this.dataTransformer,
|
|
68
|
-
connections,
|
|
69
|
-
this.options
|
|
70
|
-
);
|
|
71
|
-
|
|
72
|
-
this.dependencyResolver = new DependencyResolver(
|
|
73
|
-
connections.sourcePool,
|
|
74
|
-
connections.targetPool
|
|
75
|
-
);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
private loadConfig(configPath: string = "migration-config.json"): void {
|
|
79
|
-
const fullPath = path.resolve(process.cwd(), configPath);
|
|
80
|
-
if (!fs.existsSync(fullPath)) {
|
|
81
|
-
throw new Error(`Migration config file not found at: ${fullPath}`);
|
|
82
|
-
}
|
|
83
|
-
try {
|
|
84
|
-
const fileContent = fs.readFileSync(fullPath, "utf-8");
|
|
85
|
-
this.migrationConfig = JSON.parse(fileContent);
|
|
86
|
-
this.logger.log(`Loaded migration config from: ${fullPath}`);
|
|
87
|
-
|
|
88
|
-
// Pasar la configuración al DependencyResolver
|
|
89
|
-
this.dependencyResolver.setConfig(this.migrationConfig);
|
|
90
|
-
} catch (error) {
|
|
91
|
-
throw new Error(
|
|
92
|
-
`Error reading or parsing migration config file: ${error.message}`
|
|
93
|
-
);
|
|
94
|
-
}
|
|
95
|
-
if (
|
|
96
|
-
!this.migrationConfig.commonSchema ||
|
|
97
|
-
!this.migrationConfig.tables ||
|
|
98
|
-
!this.migrationConfig.tenantInfo
|
|
99
|
-
) {
|
|
100
|
-
throw new Error("Invalid migration config structure.");
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
async migrate(): Promise<void> {
|
|
105
|
-
try {
|
|
106
|
-
this.logger.log("Starting data migration process");
|
|
107
|
-
this.loadConfig(this.options.configPath);
|
|
108
|
-
|
|
109
|
-
// Check the publicOnly/forceSingleTenant flag AFTER loading config
|
|
110
|
-
if (this.options.forceSingleTenant || this.options.publicOnly) {
|
|
111
|
-
this.logger.log(
|
|
112
|
-
"Public-only migration requested. Migrating only 'public' type tables..."
|
|
113
|
-
);
|
|
114
|
-
await this.migratePublicTablesOnly(); // Call the dedicated function
|
|
115
|
-
} else {
|
|
116
|
-
this.logger.log("Multi-tenant migration requested based on config...");
|
|
117
|
-
await this.migrateMultiTenantWithConfig();
|
|
118
|
-
}
|
|
119
|
-
} catch (error) {
|
|
120
|
-
this.logger.error(`Migration failed: ${error.message}`);
|
|
121
|
-
console.error(error);
|
|
122
|
-
throw error;
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// --- Function for Public-Only Migration ---
|
|
127
|
-
private async migratePublicTablesOnly() {
|
|
128
|
-
const { commonSchema, tables: tablesConfig } = this.migrationConfig;
|
|
129
|
-
const allTablesInOrder =
|
|
130
|
-
await this.dependencyResolver.analyzeDependencies();
|
|
131
|
-
|
|
132
|
-
// Filter only the tables marked as 'public' in the config
|
|
133
|
-
const publicTablesToMigrate = allTablesInOrder.filter(
|
|
134
|
-
(t) => tablesConfig[t]?.type === "public"
|
|
135
|
-
);
|
|
136
|
-
|
|
137
|
-
this.logger.log(
|
|
138
|
-
`Migrating public tables to '${commonSchema}' schema: ${publicTablesToMigrate.join(
|
|
139
|
-
", "
|
|
140
|
-
)}`
|
|
141
|
-
);
|
|
142
|
-
if (publicTablesToMigrate.length === 0) {
|
|
143
|
-
this.logger.warn(
|
|
144
|
-
"No tables configured as 'public'. Public-only migration will not process any tables."
|
|
145
|
-
);
|
|
146
|
-
return;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
await this.connections.targetPool.query(
|
|
150
|
-
`SET session_replication_role = 'replica';`
|
|
151
|
-
);
|
|
152
|
-
|
|
153
|
-
for (const tableName of publicTablesToMigrate) {
|
|
154
|
-
const config = tablesConfig[tableName];
|
|
155
|
-
if (!config) {
|
|
156
|
-
this.logger.warn(
|
|
157
|
-
`Config not found for public table ${tableName}, skipping.`
|
|
158
|
-
); // Should not happen if filter worked
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
try {
|
|
162
|
-
await this.batchMigrator.migrateEntityDataInBatches(
|
|
163
|
-
this.migrationConfig.commonSchema,
|
|
164
|
-
commonSchema,
|
|
165
|
-
{
|
|
166
|
-
type: "public",
|
|
167
|
-
idField: config.idField,
|
|
168
|
-
sourceTable: tableName,
|
|
169
|
-
targetTable: tableName,
|
|
170
|
-
},
|
|
171
|
-
null // No provider filter for public-only migration
|
|
172
|
-
);
|
|
173
|
-
} catch (error) {
|
|
174
|
-
this.logger.error(
|
|
175
|
-
`Error migrating public table ${tableName}: ${error.message}`
|
|
176
|
-
);
|
|
177
|
-
// Decide whether to stop or continue on error for public-only
|
|
178
|
-
throw error;
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
await this.connections.targetPool.query(
|
|
183
|
-
`SET session_replication_role = 'origin';`
|
|
184
|
-
);
|
|
185
|
-
this.logger.log("Finished migrating public tables.");
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
// Keep validateSourceData (potentially enhanced)
|
|
189
|
-
private async validateSourceData() {
|
|
190
|
-
this.logger.log("Validating source data...");
|
|
191
|
-
try {
|
|
192
|
-
const { sourceTable, providerIdColumn } = this.migrationConfig.tenantInfo;
|
|
193
|
-
const tablesConfig = this.migrationConfig.tables;
|
|
194
|
-
|
|
195
|
-
// Find the config for the transaction-like table and the provider table
|
|
196
|
-
let transactionTableKey = "transactions"; // Default or find dynamically
|
|
197
|
-
let providerTableKey = "providers"; // Default or find dynamically
|
|
198
|
-
let transactionProviderFk = providerIdColumn; // Default
|
|
199
|
-
|
|
200
|
-
for (const key in tablesConfig) {
|
|
201
|
-
if (
|
|
202
|
-
tablesConfig[key].type === "filteredPublic" &&
|
|
203
|
-
tablesConfig[key].filterColumn === providerIdColumn &&
|
|
204
|
-
!tablesConfig[key].via
|
|
205
|
-
) {
|
|
206
|
-
transactionTableKey = key;
|
|
207
|
-
transactionProviderFk = tablesConfig[key].filterColumn;
|
|
208
|
-
}
|
|
209
|
-
if (
|
|
210
|
-
tablesConfig[key].type === "public" &&
|
|
211
|
-
tablesConfig[key].idField === providerIdColumn
|
|
212
|
-
) {
|
|
213
|
-
providerTableKey = key;
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
this.logger.debug(
|
|
217
|
-
`Validation check: Using ${transactionTableKey}.${transactionProviderFk} referencing ${providerTableKey}.${providerIdColumn}`
|
|
218
|
-
);
|
|
219
|
-
|
|
220
|
-
// Dynamically build the query
|
|
221
|
-
// IMPORTANT: Ensure proper SQL identifier quoting if names contain special chars/are reserved words
|
|
222
|
-
const query = `
|
|
223
|
-
SELECT t."${tablesConfig[transactionTableKey].idField}" AS transaction_id, t."${transactionProviderFk}"
|
|
224
|
-
FROM "${transactionTableKey}" t
|
|
225
|
-
LEFT JOIN "${providerTableKey}" p ON t."${transactionProviderFk}" = p."${providerIdColumn}"
|
|
226
|
-
WHERE t."${transactionProviderFk}" IS NOT NULL AND p."${providerIdColumn}" IS NULL;
|
|
227
|
-
`;
|
|
228
|
-
|
|
229
|
-
const invalidTransactions =
|
|
230
|
-
(await this.connections.sourcePrisma.$queryRawUnsafe(query)) as any[];
|
|
231
|
-
|
|
232
|
-
if (invalidTransactions.length > 0) {
|
|
233
|
-
this.logger.error(
|
|
234
|
-
`${invalidTransactions.length} records found in ${transactionTableKey} with invalid foreign key reference to ${providerTableKey}.${providerIdColumn}`
|
|
235
|
-
);
|
|
236
|
-
console.error(
|
|
237
|
-
"Invalid records (limit 10):",
|
|
238
|
-
invalidTransactions.slice(0, 10)
|
|
239
|
-
);
|
|
240
|
-
throw new Error(
|
|
241
|
-
"Source data validation failed: Invalid foreign key references found. Please check the configuration or fix the source data."
|
|
242
|
-
);
|
|
243
|
-
} else {
|
|
244
|
-
this.logger.log(
|
|
245
|
-
`Source data validation passed: All ${transactionTableKey}.${transactionProviderFk} references are valid.`
|
|
246
|
-
);
|
|
247
|
-
}
|
|
248
|
-
} catch (error) {
|
|
249
|
-
this.logger.error("Error during source data validation:");
|
|
250
|
-
console.error(error);
|
|
251
|
-
throw error;
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
// Renamed from migrateToTenantSchemas to reflect its purpose
|
|
256
|
-
private async migrateMultiTenantWithConfig() {
|
|
257
|
-
await this.validateSourceData();
|
|
258
|
-
|
|
259
|
-
const {
|
|
260
|
-
commonSchema,
|
|
261
|
-
tables: tablesConfig,
|
|
262
|
-
tenantInfo,
|
|
263
|
-
} = this.migrationConfig;
|
|
264
|
-
|
|
265
|
-
// Obtener orden de tablas basado en dependencias
|
|
266
|
-
const allTablesInOrder =
|
|
267
|
-
await this.dependencyResolver.analyzeDependencies();
|
|
268
|
-
this.logger.log(`Orden de migración: ${allTablesInOrder.join(" -> ")}`);
|
|
269
|
-
|
|
270
|
-
// Validar dependencias antes de comenzar
|
|
271
|
-
for (const tableName of allTablesInOrder) {
|
|
272
|
-
const config = tablesConfig[tableName];
|
|
273
|
-
if (config?.dependencies) {
|
|
274
|
-
const valid = await this.dependencyResolver.validateDependencies(
|
|
275
|
-
tableName,
|
|
276
|
-
config.dependencies
|
|
277
|
-
);
|
|
278
|
-
if (!valid) {
|
|
279
|
-
this.logger.warn(
|
|
280
|
-
`Advertencia: Algunas dependencias para ${tableName} no están disponibles`
|
|
281
|
-
);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
// --- PASADA 0: Migrar tablas públicas ---
|
|
287
|
-
this.logger.log(`Migrando tablas públicas al esquema '${commonSchema}'...`);
|
|
288
|
-
await this.connections.targetPool.query(
|
|
289
|
-
`SET session_replication_role = 'replica';`
|
|
290
|
-
);
|
|
291
|
-
|
|
292
|
-
const publicTables = allTablesInOrder.filter(
|
|
293
|
-
(t) => tablesConfig[t]?.type === "public"
|
|
294
|
-
);
|
|
295
|
-
|
|
296
|
-
for (const tableName of publicTables) {
|
|
297
|
-
const config = tablesConfig[tableName];
|
|
298
|
-
if (!config) {
|
|
299
|
-
this.logger.warn(
|
|
300
|
-
`Saltando migración de tabla ${tableName}: No encontrada en configuración`
|
|
301
|
-
);
|
|
302
|
-
continue;
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
try {
|
|
306
|
-
await this.batchMigrator.migrateEntityDataInBatches(
|
|
307
|
-
this.migrationConfig.commonSchema,
|
|
308
|
-
commonSchema,
|
|
309
|
-
{
|
|
310
|
-
type: "public",
|
|
311
|
-
idField: config.idField,
|
|
312
|
-
sourceTable: tableName,
|
|
313
|
-
targetTable: tableName,
|
|
314
|
-
},
|
|
315
|
-
null
|
|
316
|
-
);
|
|
317
|
-
} catch (error) {
|
|
318
|
-
this.logger.error(
|
|
319
|
-
`Error migrando tabla pública ${tableName}: ${error.message}`
|
|
320
|
-
);
|
|
321
|
-
throw error;
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
await this.connections.targetPool.query(
|
|
325
|
-
`SET session_replication_role = 'origin';`
|
|
326
|
-
);
|
|
327
|
-
this.logger.log("Finished migrating public tables.");
|
|
328
|
-
|
|
329
|
-
// --- Obtener Inquilinos ---
|
|
330
|
-
this.logger.log(
|
|
331
|
-
`Fetching tenant info from source table '${tenantInfo.sourceTable}'...`
|
|
332
|
-
);
|
|
333
|
-
const tenants = await this.connections.sourcePrisma[
|
|
334
|
-
tenantInfo.sourceTable
|
|
335
|
-
].findMany({
|
|
336
|
-
select: {
|
|
337
|
-
[tenantInfo.tenantIdColumn]: true,
|
|
338
|
-
[tenantInfo.providerIdColumn]: true,
|
|
339
|
-
},
|
|
340
|
-
});
|
|
341
|
-
this.logger.log(`Found ${tenants.length} potential tenants.`);
|
|
342
|
-
|
|
343
|
-
// --- Crear Esquemas y Preparar Migración por Inquilino ---
|
|
344
|
-
const validTenants = new Map<string, number | null>(); // Map tenantId -> providerId
|
|
345
|
-
this.logger.log(
|
|
346
|
-
"Ensuring tenant schemas exist (if strategy requires them)..."
|
|
347
|
-
);
|
|
348
|
-
let requiresTenantSchemas = Object.values(tablesConfig).some(
|
|
349
|
-
(t) => t.type === "tenant"
|
|
350
|
-
);
|
|
351
|
-
if (requiresTenantSchemas) {
|
|
352
|
-
this.logger.log("Tenant-specific schemas are required by config.");
|
|
353
|
-
for (const tenant of tenants) {
|
|
354
|
-
const tenantId = tenant[tenantInfo.tenantIdColumn];
|
|
355
|
-
const providerId = tenant[tenantInfo.providerIdColumn] ?? null;
|
|
356
|
-
if (
|
|
357
|
-
tenantId &&
|
|
358
|
-
typeof tenantId === "string" &&
|
|
359
|
-
tenantId !== commonSchema
|
|
360
|
-
) {
|
|
361
|
-
await this.schemaUtils.createSchema(tenantId);
|
|
362
|
-
validTenants.set(tenantId, providerId);
|
|
363
|
-
this.logger.log(`Ensured schema exists for tenant: ${tenantId}`);
|
|
364
|
-
} else {
|
|
365
|
-
this.logger.warn(`Skipping invalid/public tenant ID: ${tenantId}`);
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
} else {
|
|
369
|
-
this.logger.log(
|
|
370
|
-
'No tables configured with type "tenant". Tenant schema creation skipped.'
|
|
371
|
-
);
|
|
372
|
-
// Still populate validTenants map for filteredPublic logic
|
|
373
|
-
for (const tenant of tenants) {
|
|
374
|
-
const tenantId = tenant[tenantInfo.tenantIdColumn];
|
|
375
|
-
const providerId = tenant[tenantInfo.providerIdColumn] ?? null;
|
|
376
|
-
// Use tenantId as key even if schemas aren't created, it represents the logical tenant
|
|
377
|
-
if (tenantId && typeof tenantId === "string") {
|
|
378
|
-
validTenants.set(tenantId, providerId);
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
this.logger.log(
|
|
383
|
-
`Prepared ${validTenants.size} valid tenants for migration.`
|
|
384
|
-
);
|
|
385
|
-
|
|
386
|
-
// --- PASADAS POR INQUILINO (para tablas filteredPublic y tenant) ---
|
|
387
|
-
const tablesToMigratePerTenant = allTablesInOrder.filter(
|
|
388
|
-
(t) =>
|
|
389
|
-
tablesConfig[t]?.type === "filteredPublic" ||
|
|
390
|
-
tablesConfig[t]?.type === "tenant"
|
|
391
|
-
);
|
|
392
|
-
this.logger.log(
|
|
393
|
-
`Migrating filtered/tenant tables: ${tablesToMigratePerTenant.join(", ")}`
|
|
394
|
-
);
|
|
395
|
-
|
|
396
|
-
for (const [tenantId, providerId] of validTenants.entries()) {
|
|
397
|
-
// Note: tenantId here might just be logical identifier if not creating schemas
|
|
398
|
-
this.logger.log(
|
|
399
|
-
`Starting migration pass related to tenant: ${tenantId} (Filtering by Provider ID: ${
|
|
400
|
-
providerId ?? "N/A"
|
|
401
|
-
})`
|
|
402
|
-
);
|
|
403
|
-
await this.connections.targetPool.query(
|
|
404
|
-
`SET session_replication_role = 'replica';`
|
|
405
|
-
);
|
|
406
|
-
|
|
407
|
-
for (const tableName of tablesToMigratePerTenant) {
|
|
408
|
-
const config = tablesConfig[tableName];
|
|
409
|
-
if (!config) {
|
|
410
|
-
this.logger.warn(
|
|
411
|
-
`Skipping migration for table ${tableName} for tenant ${tenantId}: Not found in config.`
|
|
412
|
-
);
|
|
413
|
-
continue;
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
// Determine schema destino basado en config.type
|
|
417
|
-
const targetSchemaForTable =
|
|
418
|
-
config.type === "tenant" ? tenantId : commonSchema;
|
|
419
|
-
|
|
420
|
-
try {
|
|
421
|
-
const entity: EntityType = {
|
|
422
|
-
name: tableName,
|
|
423
|
-
idField: config.idField,
|
|
424
|
-
filterColumn: config.filterColumn,
|
|
425
|
-
filterVia: config.via,
|
|
426
|
-
};
|
|
427
|
-
|
|
428
|
-
const tableConfig: TableConfig = {
|
|
429
|
-
type: entity.filterVia ? "filteredPublic" : "tenant",
|
|
430
|
-
idField: entity.idField,
|
|
431
|
-
filterColumn: entity.filterColumn,
|
|
432
|
-
via: entity.filterVia,
|
|
433
|
-
sourceTable: entity.name,
|
|
434
|
-
targetTable: entity.name,
|
|
435
|
-
};
|
|
436
|
-
|
|
437
|
-
await this.batchMigrator.migrateEntityDataInBatches(
|
|
438
|
-
this.migrationConfig.commonSchema,
|
|
439
|
-
targetSchemaForTable,
|
|
440
|
-
tableConfig,
|
|
441
|
-
providerId?.toString() // Convertir providerId a string
|
|
442
|
-
);
|
|
443
|
-
} catch (error) {
|
|
444
|
-
this.logger.error(
|
|
445
|
-
`Error migrating table ${tableName} into schema ${targetSchemaForTable} (related to tenant ${tenantId}): ${error.message}`
|
|
446
|
-
);
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
await this.connections.targetPool.query(
|
|
450
|
-
`SET session_replication_role = 'origin';`
|
|
451
|
-
);
|
|
452
|
-
this.logger.log(`Finished migration pass related to tenant: ${tenantId}`);
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
this.logger.log("Data migration process completed.");
|
|
456
|
-
// await this.verifyMigration(Array.from(validTenants.keys()));
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
private async verifyMigration(tenantIds: string[]) {
|
|
460
|
-
this.logger.log("Verifying migrated data...");
|
|
461
|
-
// Verificar commonSchema
|
|
462
|
-
try {
|
|
463
|
-
const publicResult = await this.connections.targetPool.query(
|
|
464
|
-
`SELECT tablename, n_live_tup FROM pg_stat_user_tables WHERE schemaname = $1`,
|
|
465
|
-
[this.migrationConfig.commonSchema]
|
|
466
|
-
);
|
|
467
|
-
this.logger.log(
|
|
468
|
-
`Verification for schema '${this.migrationConfig.commonSchema}':`
|
|
469
|
-
);
|
|
470
|
-
publicResult.rows.forEach((row) =>
|
|
471
|
-
this.logger.log(` Table: ${row.tablename}, Rows: ${row.n_live_tup}`)
|
|
472
|
-
);
|
|
473
|
-
} catch (err) {
|
|
474
|
-
this.logger.error(`Error verifying public schema: ${err.message}`);
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
// Verificar schemas de inquilinos (only if they were supposed to be created)
|
|
478
|
-
let requiresTenantSchemas = Object.values(this.migrationConfig.tables).some(
|
|
479
|
-
(t) => t.type === "tenant"
|
|
480
|
-
);
|
|
481
|
-
if (requiresTenantSchemas) {
|
|
482
|
-
for (const tenantId of tenantIds) {
|
|
483
|
-
try {
|
|
484
|
-
const tenantResult = await this.connections.targetPool.query(
|
|
485
|
-
`SELECT tablename, n_live_tup FROM pg_stat_user_tables WHERE schemaname = $1`,
|
|
486
|
-
[tenantId]
|
|
487
|
-
);
|
|
488
|
-
this.logger.log(`Verification for tenant schema '${tenantId}':`);
|
|
489
|
-
if (tenantResult.rows.length === 0) {
|
|
490
|
-
this.logger.warn(
|
|
491
|
-
` Schema ${tenantId} has no tables (or pg_stat_user_tables is empty).`
|
|
492
|
-
);
|
|
493
|
-
} else {
|
|
494
|
-
tenantResult.rows.forEach((row) =>
|
|
495
|
-
this.logger.log(
|
|
496
|
-
` Table: ${row.tablename}, Rows: ${row.n_live_tup}`
|
|
497
|
-
)
|
|
498
|
-
);
|
|
499
|
-
}
|
|
500
|
-
} catch (err) {
|
|
501
|
-
this.logger.error(
|
|
502
|
-
`Error verifying tenant schema ${tenantId}: ${err.message}`
|
|
503
|
-
);
|
|
504
|
-
}
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
this.logger.log("Verification finished.");
|
|
508
|
-
}
|
|
509
|
-
}
|
|
1
|
+
import { Logger } from "@nestjs/common";
|
|
2
|
+
import * as fs from "fs";
|
|
3
|
+
import * as path from "path";
|
|
4
|
+
import { SchemaUtils } from "./schema-utils";
|
|
5
|
+
import { DataTransformer } from "./data-transformer";
|
|
6
|
+
import { DatabaseConnections, ColumnSchema, EntityType } from "./types";
|
|
7
|
+
import { BatchMigrator } from "./batch-migrator";
|
|
8
|
+
import { ForeignKeyManager } from "./foreign-key-manager";
|
|
9
|
+
import { DependencyResolver } from "./dependency-resolver";
|
|
10
|
+
|
|
11
|
+
interface MigrationOptions {
|
|
12
|
+
publicOnly?: boolean;
|
|
13
|
+
multiTenant?: boolean;
|
|
14
|
+
sourceSchema?: string;
|
|
15
|
+
targetSchema?: string;
|
|
16
|
+
forceSingleTenant?: boolean; // Alias for publicOnly
|
|
17
|
+
configPath?: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
interface TableConfig {
|
|
21
|
+
type: "public" | "filteredPublic" | "tenantInfo" | "tenant";
|
|
22
|
+
idField: string;
|
|
23
|
+
filterColumn?: string;
|
|
24
|
+
via?: string;
|
|
25
|
+
providerLink?: string;
|
|
26
|
+
tenantKey?: string;
|
|
27
|
+
dependencies?: string[];
|
|
28
|
+
sourceTable: string;
|
|
29
|
+
targetTable: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
interface MigrationConfig {
|
|
33
|
+
commonSchema: string;
|
|
34
|
+
tenantInfo: {
|
|
35
|
+
sourceTable: string;
|
|
36
|
+
tenantIdColumn: string;
|
|
37
|
+
providerIdColumn: string;
|
|
38
|
+
};
|
|
39
|
+
tables: Record<string, TableConfig>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export class DataMigrationTool {
|
|
43
|
+
private readonly logger = new Logger("DataMigrationTool");
|
|
44
|
+
private readonly schemaUtils: SchemaUtils;
|
|
45
|
+
private readonly dataTransformer: DataTransformer;
|
|
46
|
+
private readonly batchMigrator: BatchMigrator;
|
|
47
|
+
private readonly foreignKeyManager: ForeignKeyManager;
|
|
48
|
+
private readonly dependencyResolver: DependencyResolver;
|
|
49
|
+
|
|
50
|
+
constructor(
|
|
51
|
+
private readonly connections: DatabaseConnections,
|
|
52
|
+
private migrationConfig: MigrationConfig,
|
|
53
|
+
private readonly options: MigrationOptions = {}
|
|
54
|
+
) {
|
|
55
|
+
this.schemaUtils = new SchemaUtils(connections);
|
|
56
|
+
|
|
57
|
+
this.dataTransformer = new DataTransformer(
|
|
58
|
+
this.schemaUtils,
|
|
59
|
+
connections.sourcePool,
|
|
60
|
+
connections
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
this.foreignKeyManager = new ForeignKeyManager(connections);
|
|
64
|
+
|
|
65
|
+
this.batchMigrator = new BatchMigrator(
|
|
66
|
+
this.schemaUtils,
|
|
67
|
+
this.dataTransformer,
|
|
68
|
+
connections,
|
|
69
|
+
this.options
|
|
70
|
+
);
|
|
71
|
+
|
|
72
|
+
this.dependencyResolver = new DependencyResolver(
|
|
73
|
+
connections.sourcePool,
|
|
74
|
+
connections.targetPool
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private loadConfig(configPath: string = "migration-config.json"): void {
|
|
79
|
+
const fullPath = path.resolve(process.cwd(), configPath);
|
|
80
|
+
if (!fs.existsSync(fullPath)) {
|
|
81
|
+
throw new Error(`Migration config file not found at: ${fullPath}`);
|
|
82
|
+
}
|
|
83
|
+
try {
|
|
84
|
+
const fileContent = fs.readFileSync(fullPath, "utf-8");
|
|
85
|
+
this.migrationConfig = JSON.parse(fileContent);
|
|
86
|
+
this.logger.log(`Loaded migration config from: ${fullPath}`);
|
|
87
|
+
|
|
88
|
+
// Pasar la configuración al DependencyResolver
|
|
89
|
+
this.dependencyResolver.setConfig(this.migrationConfig);
|
|
90
|
+
} catch (error) {
|
|
91
|
+
throw new Error(
|
|
92
|
+
`Error reading or parsing migration config file: ${error.message}`
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
if (
|
|
96
|
+
!this.migrationConfig.commonSchema ||
|
|
97
|
+
!this.migrationConfig.tables ||
|
|
98
|
+
!this.migrationConfig.tenantInfo
|
|
99
|
+
) {
|
|
100
|
+
throw new Error("Invalid migration config structure.");
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async migrate(): Promise<void> {
|
|
105
|
+
try {
|
|
106
|
+
this.logger.log("Starting data migration process");
|
|
107
|
+
this.loadConfig(this.options.configPath);
|
|
108
|
+
|
|
109
|
+
// Check the publicOnly/forceSingleTenant flag AFTER loading config
|
|
110
|
+
if (this.options.forceSingleTenant || this.options.publicOnly) {
|
|
111
|
+
this.logger.log(
|
|
112
|
+
"Public-only migration requested. Migrating only 'public' type tables..."
|
|
113
|
+
);
|
|
114
|
+
await this.migratePublicTablesOnly(); // Call the dedicated function
|
|
115
|
+
} else {
|
|
116
|
+
this.logger.log("Multi-tenant migration requested based on config...");
|
|
117
|
+
await this.migrateMultiTenantWithConfig();
|
|
118
|
+
}
|
|
119
|
+
} catch (error) {
|
|
120
|
+
this.logger.error(`Migration failed: ${error.message}`);
|
|
121
|
+
console.error(error);
|
|
122
|
+
throw error;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// --- Function for Public-Only Migration ---
|
|
127
|
+
private async migratePublicTablesOnly() {
|
|
128
|
+
const { commonSchema, tables: tablesConfig } = this.migrationConfig;
|
|
129
|
+
const allTablesInOrder =
|
|
130
|
+
await this.dependencyResolver.analyzeDependencies();
|
|
131
|
+
|
|
132
|
+
// Filter only the tables marked as 'public' in the config
|
|
133
|
+
const publicTablesToMigrate = allTablesInOrder.filter(
|
|
134
|
+
(t) => tablesConfig[t]?.type === "public"
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
this.logger.log(
|
|
138
|
+
`Migrating public tables to '${commonSchema}' schema: ${publicTablesToMigrate.join(
|
|
139
|
+
", "
|
|
140
|
+
)}`
|
|
141
|
+
);
|
|
142
|
+
if (publicTablesToMigrate.length === 0) {
|
|
143
|
+
this.logger.warn(
|
|
144
|
+
"No tables configured as 'public'. Public-only migration will not process any tables."
|
|
145
|
+
);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
await this.connections.targetPool.query(
|
|
150
|
+
`SET session_replication_role = 'replica';`
|
|
151
|
+
);
|
|
152
|
+
|
|
153
|
+
for (const tableName of publicTablesToMigrate) {
|
|
154
|
+
const config = tablesConfig[tableName];
|
|
155
|
+
if (!config) {
|
|
156
|
+
this.logger.warn(
|
|
157
|
+
`Config not found for public table ${tableName}, skipping.`
|
|
158
|
+
); // Should not happen if filter worked
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
try {
|
|
162
|
+
await this.batchMigrator.migrateEntityDataInBatches(
|
|
163
|
+
this.migrationConfig.commonSchema,
|
|
164
|
+
commonSchema,
|
|
165
|
+
{
|
|
166
|
+
type: "public",
|
|
167
|
+
idField: config.idField,
|
|
168
|
+
sourceTable: tableName,
|
|
169
|
+
targetTable: tableName,
|
|
170
|
+
},
|
|
171
|
+
null // No provider filter for public-only migration
|
|
172
|
+
);
|
|
173
|
+
} catch (error) {
|
|
174
|
+
this.logger.error(
|
|
175
|
+
`Error migrating public table ${tableName}: ${error.message}`
|
|
176
|
+
);
|
|
177
|
+
// Decide whether to stop or continue on error for public-only
|
|
178
|
+
throw error;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
await this.connections.targetPool.query(
|
|
183
|
+
`SET session_replication_role = 'origin';`
|
|
184
|
+
);
|
|
185
|
+
this.logger.log("Finished migrating public tables.");
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Keep validateSourceData (potentially enhanced)
|
|
189
|
+
private async validateSourceData() {
|
|
190
|
+
this.logger.log("Validating source data...");
|
|
191
|
+
try {
|
|
192
|
+
const { sourceTable, providerIdColumn } = this.migrationConfig.tenantInfo;
|
|
193
|
+
const tablesConfig = this.migrationConfig.tables;
|
|
194
|
+
|
|
195
|
+
// Find the config for the transaction-like table and the provider table
|
|
196
|
+
let transactionTableKey = "transactions"; // Default or find dynamically
|
|
197
|
+
let providerTableKey = "providers"; // Default or find dynamically
|
|
198
|
+
let transactionProviderFk = providerIdColumn; // Default
|
|
199
|
+
|
|
200
|
+
for (const key in tablesConfig) {
|
|
201
|
+
if (
|
|
202
|
+
tablesConfig[key].type === "filteredPublic" &&
|
|
203
|
+
tablesConfig[key].filterColumn === providerIdColumn &&
|
|
204
|
+
!tablesConfig[key].via
|
|
205
|
+
) {
|
|
206
|
+
transactionTableKey = key;
|
|
207
|
+
transactionProviderFk = tablesConfig[key].filterColumn;
|
|
208
|
+
}
|
|
209
|
+
if (
|
|
210
|
+
tablesConfig[key].type === "public" &&
|
|
211
|
+
tablesConfig[key].idField === providerIdColumn
|
|
212
|
+
) {
|
|
213
|
+
providerTableKey = key;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
this.logger.debug(
|
|
217
|
+
`Validation check: Using ${transactionTableKey}.${transactionProviderFk} referencing ${providerTableKey}.${providerIdColumn}`
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
// Dynamically build the query
|
|
221
|
+
// IMPORTANT: Ensure proper SQL identifier quoting if names contain special chars/are reserved words
|
|
222
|
+
const query = `
|
|
223
|
+
SELECT t."${tablesConfig[transactionTableKey].idField}" AS transaction_id, t."${transactionProviderFk}"
|
|
224
|
+
FROM "${transactionTableKey}" t
|
|
225
|
+
LEFT JOIN "${providerTableKey}" p ON t."${transactionProviderFk}" = p."${providerIdColumn}"
|
|
226
|
+
WHERE t."${transactionProviderFk}" IS NOT NULL AND p."${providerIdColumn}" IS NULL;
|
|
227
|
+
`;
|
|
228
|
+
|
|
229
|
+
const invalidTransactions =
|
|
230
|
+
(await this.connections.sourcePrisma.$queryRawUnsafe(query)) as any[];
|
|
231
|
+
|
|
232
|
+
if (invalidTransactions.length > 0) {
|
|
233
|
+
this.logger.error(
|
|
234
|
+
`${invalidTransactions.length} records found in ${transactionTableKey} with invalid foreign key reference to ${providerTableKey}.${providerIdColumn}`
|
|
235
|
+
);
|
|
236
|
+
console.error(
|
|
237
|
+
"Invalid records (limit 10):",
|
|
238
|
+
invalidTransactions.slice(0, 10)
|
|
239
|
+
);
|
|
240
|
+
throw new Error(
|
|
241
|
+
"Source data validation failed: Invalid foreign key references found. Please check the configuration or fix the source data."
|
|
242
|
+
);
|
|
243
|
+
} else {
|
|
244
|
+
this.logger.log(
|
|
245
|
+
`Source data validation passed: All ${transactionTableKey}.${transactionProviderFk} references are valid.`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
} catch (error) {
|
|
249
|
+
this.logger.error("Error during source data validation:");
|
|
250
|
+
console.error(error);
|
|
251
|
+
throw error;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Renamed from migrateToTenantSchemas to reflect its purpose
|
|
256
|
+
private async migrateMultiTenantWithConfig() {
|
|
257
|
+
await this.validateSourceData();
|
|
258
|
+
|
|
259
|
+
const {
|
|
260
|
+
commonSchema,
|
|
261
|
+
tables: tablesConfig,
|
|
262
|
+
tenantInfo,
|
|
263
|
+
} = this.migrationConfig;
|
|
264
|
+
|
|
265
|
+
// Obtener orden de tablas basado en dependencias
|
|
266
|
+
const allTablesInOrder =
|
|
267
|
+
await this.dependencyResolver.analyzeDependencies();
|
|
268
|
+
this.logger.log(`Orden de migración: ${allTablesInOrder.join(" -> ")}`);
|
|
269
|
+
|
|
270
|
+
// Validar dependencias antes de comenzar
|
|
271
|
+
for (const tableName of allTablesInOrder) {
|
|
272
|
+
const config = tablesConfig[tableName];
|
|
273
|
+
if (config?.dependencies) {
|
|
274
|
+
const valid = await this.dependencyResolver.validateDependencies(
|
|
275
|
+
tableName,
|
|
276
|
+
config.dependencies
|
|
277
|
+
);
|
|
278
|
+
if (!valid) {
|
|
279
|
+
this.logger.warn(
|
|
280
|
+
`Advertencia: Algunas dependencias para ${tableName} no están disponibles`
|
|
281
|
+
);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// --- PASADA 0: Migrar tablas públicas ---
|
|
287
|
+
this.logger.log(`Migrando tablas públicas al esquema '${commonSchema}'...`);
|
|
288
|
+
await this.connections.targetPool.query(
|
|
289
|
+
`SET session_replication_role = 'replica';`
|
|
290
|
+
);
|
|
291
|
+
|
|
292
|
+
const publicTables = allTablesInOrder.filter(
|
|
293
|
+
(t) => tablesConfig[t]?.type === "public"
|
|
294
|
+
);
|
|
295
|
+
|
|
296
|
+
for (const tableName of publicTables) {
|
|
297
|
+
const config = tablesConfig[tableName];
|
|
298
|
+
if (!config) {
|
|
299
|
+
this.logger.warn(
|
|
300
|
+
`Saltando migración de tabla ${tableName}: No encontrada en configuración`
|
|
301
|
+
);
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
try {
|
|
306
|
+
await this.batchMigrator.migrateEntityDataInBatches(
|
|
307
|
+
this.migrationConfig.commonSchema,
|
|
308
|
+
commonSchema,
|
|
309
|
+
{
|
|
310
|
+
type: "public",
|
|
311
|
+
idField: config.idField,
|
|
312
|
+
sourceTable: tableName,
|
|
313
|
+
targetTable: tableName,
|
|
314
|
+
},
|
|
315
|
+
null
|
|
316
|
+
);
|
|
317
|
+
} catch (error) {
|
|
318
|
+
this.logger.error(
|
|
319
|
+
`Error migrando tabla pública ${tableName}: ${error.message}`
|
|
320
|
+
);
|
|
321
|
+
throw error;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
await this.connections.targetPool.query(
|
|
325
|
+
`SET session_replication_role = 'origin';`
|
|
326
|
+
);
|
|
327
|
+
this.logger.log("Finished migrating public tables.");
|
|
328
|
+
|
|
329
|
+
// --- Obtener Inquilinos ---
|
|
330
|
+
this.logger.log(
|
|
331
|
+
`Fetching tenant info from source table '${tenantInfo.sourceTable}'...`
|
|
332
|
+
);
|
|
333
|
+
const tenants = await this.connections.sourcePrisma[
|
|
334
|
+
tenantInfo.sourceTable
|
|
335
|
+
].findMany({
|
|
336
|
+
select: {
|
|
337
|
+
[tenantInfo.tenantIdColumn]: true,
|
|
338
|
+
[tenantInfo.providerIdColumn]: true,
|
|
339
|
+
},
|
|
340
|
+
});
|
|
341
|
+
this.logger.log(`Found ${tenants.length} potential tenants.`);
|
|
342
|
+
|
|
343
|
+
// --- Crear Esquemas y Preparar Migración por Inquilino ---
|
|
344
|
+
const validTenants = new Map<string, number | null>(); // Map tenantId -> providerId
|
|
345
|
+
this.logger.log(
|
|
346
|
+
"Ensuring tenant schemas exist (if strategy requires them)..."
|
|
347
|
+
);
|
|
348
|
+
let requiresTenantSchemas = Object.values(tablesConfig).some(
|
|
349
|
+
(t) => t.type === "tenant"
|
|
350
|
+
);
|
|
351
|
+
if (requiresTenantSchemas) {
|
|
352
|
+
this.logger.log("Tenant-specific schemas are required by config.");
|
|
353
|
+
for (const tenant of tenants) {
|
|
354
|
+
const tenantId = tenant[tenantInfo.tenantIdColumn];
|
|
355
|
+
const providerId = tenant[tenantInfo.providerIdColumn] ?? null;
|
|
356
|
+
if (
|
|
357
|
+
tenantId &&
|
|
358
|
+
typeof tenantId === "string" &&
|
|
359
|
+
tenantId !== commonSchema
|
|
360
|
+
) {
|
|
361
|
+
await this.schemaUtils.createSchema(tenantId);
|
|
362
|
+
validTenants.set(tenantId, providerId);
|
|
363
|
+
this.logger.log(`Ensured schema exists for tenant: ${tenantId}`);
|
|
364
|
+
} else {
|
|
365
|
+
this.logger.warn(`Skipping invalid/public tenant ID: ${tenantId}`);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
} else {
|
|
369
|
+
this.logger.log(
|
|
370
|
+
'No tables configured with type "tenant". Tenant schema creation skipped.'
|
|
371
|
+
);
|
|
372
|
+
// Still populate validTenants map for filteredPublic logic
|
|
373
|
+
for (const tenant of tenants) {
|
|
374
|
+
const tenantId = tenant[tenantInfo.tenantIdColumn];
|
|
375
|
+
const providerId = tenant[tenantInfo.providerIdColumn] ?? null;
|
|
376
|
+
// Use tenantId as key even if schemas aren't created, it represents the logical tenant
|
|
377
|
+
if (tenantId && typeof tenantId === "string") {
|
|
378
|
+
validTenants.set(tenantId, providerId);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
this.logger.log(
|
|
383
|
+
`Prepared ${validTenants.size} valid tenants for migration.`
|
|
384
|
+
);
|
|
385
|
+
|
|
386
|
+
// --- PASADAS POR INQUILINO (para tablas filteredPublic y tenant) ---
|
|
387
|
+
const tablesToMigratePerTenant = allTablesInOrder.filter(
|
|
388
|
+
(t) =>
|
|
389
|
+
tablesConfig[t]?.type === "filteredPublic" ||
|
|
390
|
+
tablesConfig[t]?.type === "tenant"
|
|
391
|
+
);
|
|
392
|
+
this.logger.log(
|
|
393
|
+
`Migrating filtered/tenant tables: ${tablesToMigratePerTenant.join(", ")}`
|
|
394
|
+
);
|
|
395
|
+
|
|
396
|
+
for (const [tenantId, providerId] of validTenants.entries()) {
|
|
397
|
+
// Note: tenantId here might just be logical identifier if not creating schemas
|
|
398
|
+
this.logger.log(
|
|
399
|
+
`Starting migration pass related to tenant: ${tenantId} (Filtering by Provider ID: ${
|
|
400
|
+
providerId ?? "N/A"
|
|
401
|
+
})`
|
|
402
|
+
);
|
|
403
|
+
await this.connections.targetPool.query(
|
|
404
|
+
`SET session_replication_role = 'replica';`
|
|
405
|
+
);
|
|
406
|
+
|
|
407
|
+
for (const tableName of tablesToMigratePerTenant) {
|
|
408
|
+
const config = tablesConfig[tableName];
|
|
409
|
+
if (!config) {
|
|
410
|
+
this.logger.warn(
|
|
411
|
+
`Skipping migration for table ${tableName} for tenant ${tenantId}: Not found in config.`
|
|
412
|
+
);
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// Determine schema destino basado en config.type
|
|
417
|
+
const targetSchemaForTable =
|
|
418
|
+
config.type === "tenant" ? tenantId : commonSchema;
|
|
419
|
+
|
|
420
|
+
try {
|
|
421
|
+
const entity: EntityType = {
|
|
422
|
+
name: tableName,
|
|
423
|
+
idField: config.idField,
|
|
424
|
+
filterColumn: config.filterColumn,
|
|
425
|
+
filterVia: config.via,
|
|
426
|
+
};
|
|
427
|
+
|
|
428
|
+
const tableConfig: TableConfig = {
|
|
429
|
+
type: entity.filterVia ? "filteredPublic" : "tenant",
|
|
430
|
+
idField: entity.idField,
|
|
431
|
+
filterColumn: entity.filterColumn,
|
|
432
|
+
via: entity.filterVia,
|
|
433
|
+
sourceTable: entity.name,
|
|
434
|
+
targetTable: entity.name,
|
|
435
|
+
};
|
|
436
|
+
|
|
437
|
+
await this.batchMigrator.migrateEntityDataInBatches(
|
|
438
|
+
this.migrationConfig.commonSchema,
|
|
439
|
+
targetSchemaForTable,
|
|
440
|
+
tableConfig,
|
|
441
|
+
providerId?.toString() // Convertir providerId a string
|
|
442
|
+
);
|
|
443
|
+
} catch (error) {
|
|
444
|
+
this.logger.error(
|
|
445
|
+
`Error migrating table ${tableName} into schema ${targetSchemaForTable} (related to tenant ${tenantId}): ${error.message}`
|
|
446
|
+
);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
await this.connections.targetPool.query(
|
|
450
|
+
`SET session_replication_role = 'origin';`
|
|
451
|
+
);
|
|
452
|
+
this.logger.log(`Finished migration pass related to tenant: ${tenantId}`);
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
this.logger.log("Data migration process completed.");
|
|
456
|
+
// await this.verifyMigration(Array.from(validTenants.keys()));
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
private async verifyMigration(tenantIds: string[]) {
|
|
460
|
+
this.logger.log("Verifying migrated data...");
|
|
461
|
+
// Verificar commonSchema
|
|
462
|
+
try {
|
|
463
|
+
const publicResult = await this.connections.targetPool.query(
|
|
464
|
+
`SELECT tablename, n_live_tup FROM pg_stat_user_tables WHERE schemaname = $1`,
|
|
465
|
+
[this.migrationConfig.commonSchema]
|
|
466
|
+
);
|
|
467
|
+
this.logger.log(
|
|
468
|
+
`Verification for schema '${this.migrationConfig.commonSchema}':`
|
|
469
|
+
);
|
|
470
|
+
publicResult.rows.forEach((row) =>
|
|
471
|
+
this.logger.log(` Table: ${row.tablename}, Rows: ${row.n_live_tup}`)
|
|
472
|
+
);
|
|
473
|
+
} catch (err) {
|
|
474
|
+
this.logger.error(`Error verifying public schema: ${err.message}`);
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// Verificar schemas de inquilinos (only if they were supposed to be created)
|
|
478
|
+
let requiresTenantSchemas = Object.values(this.migrationConfig.tables).some(
|
|
479
|
+
(t) => t.type === "tenant"
|
|
480
|
+
);
|
|
481
|
+
if (requiresTenantSchemas) {
|
|
482
|
+
for (const tenantId of tenantIds) {
|
|
483
|
+
try {
|
|
484
|
+
const tenantResult = await this.connections.targetPool.query(
|
|
485
|
+
`SELECT tablename, n_live_tup FROM pg_stat_user_tables WHERE schemaname = $1`,
|
|
486
|
+
[tenantId]
|
|
487
|
+
);
|
|
488
|
+
this.logger.log(`Verification for tenant schema '${tenantId}':`);
|
|
489
|
+
if (tenantResult.rows.length === 0) {
|
|
490
|
+
this.logger.warn(
|
|
491
|
+
` Schema ${tenantId} has no tables (or pg_stat_user_tables is empty).`
|
|
492
|
+
);
|
|
493
|
+
} else {
|
|
494
|
+
tenantResult.rows.forEach((row) =>
|
|
495
|
+
this.logger.log(
|
|
496
|
+
` Table: ${row.tablename}, Rows: ${row.n_live_tup}`
|
|
497
|
+
)
|
|
498
|
+
);
|
|
499
|
+
}
|
|
500
|
+
} catch (err) {
|
|
501
|
+
this.logger.error(
|
|
502
|
+
`Error verifying tenant schema ${tenantId}: ${err.message}`
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
this.logger.log("Verification finished.");
|
|
508
|
+
}
|
|
509
|
+
}
|