appwrite-utils-cli 1.2.5 → 1.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,11 @@
|
|
1
|
-
import { converterFunctions, tryAwaitWithRetry, parseAttribute } from "appwrite-utils";
|
1
|
+
import { converterFunctions, tryAwaitWithRetry, parseAttribute, objectNeedsUpdate } from "appwrite-utils";
|
2
2
|
import {
|
3
3
|
Client,
|
4
4
|
Databases,
|
5
5
|
Storage,
|
6
6
|
Users,
|
7
7
|
Functions,
|
8
|
+
Teams,
|
8
9
|
type Models,
|
9
10
|
Query,
|
10
11
|
} from "node-appwrite";
|
@@ -36,6 +37,7 @@ export interface ComprehensiveTransferOptions {
|
|
36
37
|
targetProject: string;
|
37
38
|
targetKey: string;
|
38
39
|
transferUsers?: boolean;
|
40
|
+
transferTeams?: boolean;
|
39
41
|
transferDatabases?: boolean;
|
40
42
|
transferBuckets?: boolean;
|
41
43
|
transferFunctions?: boolean;
|
@@ -45,6 +47,7 @@ export interface ComprehensiveTransferOptions {
|
|
45
47
|
|
46
48
|
export interface TransferResults {
|
47
49
|
users: { transferred: number; skipped: number; failed: number };
|
50
|
+
teams: { transferred: number; skipped: number; failed: number };
|
48
51
|
databases: { transferred: number; skipped: number; failed: number };
|
49
52
|
buckets: { transferred: number; skipped: number; failed: number };
|
50
53
|
functions: { transferred: number; skipped: number; failed: number };
|
@@ -56,6 +59,8 @@ export class ComprehensiveTransfer {
|
|
56
59
|
private targetClient: Client;
|
57
60
|
private sourceUsers: Users;
|
58
61
|
private targetUsers: Users;
|
62
|
+
private sourceTeams: Teams;
|
63
|
+
private targetTeams: Teams;
|
59
64
|
private sourceDatabases: Databases;
|
60
65
|
private targetDatabases: Databases;
|
61
66
|
private sourceStorage: Storage;
|
@@ -84,6 +89,8 @@ export class ComprehensiveTransfer {
|
|
84
89
|
|
85
90
|
this.sourceUsers = new Users(this.sourceClient);
|
86
91
|
this.targetUsers = new Users(this.targetClient);
|
92
|
+
this.sourceTeams = new Teams(this.sourceClient);
|
93
|
+
this.targetTeams = new Teams(this.targetClient);
|
87
94
|
this.sourceDatabases = new Databases(this.sourceClient);
|
88
95
|
this.targetDatabases = new Databases(this.targetClient);
|
89
96
|
this.sourceStorage = new Storage(this.sourceClient);
|
@@ -101,6 +108,7 @@ export class ComprehensiveTransfer {
|
|
101
108
|
this.fileLimit = pLimit(Math.max(1, Math.floor(baseLimit / 4)));
|
102
109
|
this.results = {
|
103
110
|
users: { transferred: 0, skipped: 0, failed: 0 },
|
111
|
+
teams: { transferred: 0, skipped: 0, failed: 0 },
|
104
112
|
databases: { transferred: 0, skipped: 0, failed: 0 },
|
105
113
|
buckets: { transferred: 0, skipped: 0, failed: 0 },
|
106
114
|
functions: { transferred: 0, skipped: 0, failed: 0 },
|
@@ -135,6 +143,10 @@ export class ComprehensiveTransfer {
|
|
135
143
|
await this.transferAllUsers();
|
136
144
|
}
|
137
145
|
|
146
|
+
if (this.options.transferTeams !== false) {
|
147
|
+
await this.transferAllTeams();
|
148
|
+
}
|
149
|
+
|
138
150
|
if (this.options.transferDatabases !== false) {
|
139
151
|
await this.transferAllDatabases();
|
140
152
|
}
|
@@ -193,6 +205,80 @@ export class ComprehensiveTransfer {
|
|
193
205
|
}
|
194
206
|
}
|
195
207
|
|
208
|
+
private async transferAllTeams(): Promise<void> {
|
209
|
+
MessageFormatter.info("Starting team transfer phase", { prefix: "Transfer" });
|
210
|
+
|
211
|
+
try {
|
212
|
+
// Fetch all teams from source with pagination
|
213
|
+
const allSourceTeams = await this.fetchAllTeams(this.sourceTeams);
|
214
|
+
const allTargetTeams = await this.fetchAllTeams(this.targetTeams);
|
215
|
+
|
216
|
+
if (this.options.dryRun) {
|
217
|
+
let totalMemberships = 0;
|
218
|
+
for (const team of allSourceTeams) {
|
219
|
+
const memberships = await this.sourceTeams.listMemberships(team.$id, [Query.limit(1)]);
|
220
|
+
totalMemberships += memberships.total;
|
221
|
+
}
|
222
|
+
MessageFormatter.info(`DRY RUN: Would transfer ${allSourceTeams.length} teams with ${totalMemberships} memberships`, { prefix: "Transfer" });
|
223
|
+
return;
|
224
|
+
}
|
225
|
+
|
226
|
+
const transferTasks = allSourceTeams.map(team =>
|
227
|
+
this.limit(async () => {
|
228
|
+
try {
|
229
|
+
// Check if team exists in target
|
230
|
+
const existingTeam = allTargetTeams.find(tt => tt.$id === team.$id);
|
231
|
+
|
232
|
+
if (!existingTeam) {
|
233
|
+
// Fetch all memberships to extract unique roles before creating team
|
234
|
+
MessageFormatter.info(`Fetching memberships for team ${team.name} to extract roles`, { prefix: "Transfer" });
|
235
|
+
const memberships = await this.fetchAllMemberships(team.$id);
|
236
|
+
|
237
|
+
// Extract unique roles from all memberships
|
238
|
+
const allRoles = new Set<string>();
|
239
|
+
memberships.forEach(membership => {
|
240
|
+
membership.roles.forEach(role => allRoles.add(role));
|
241
|
+
});
|
242
|
+
const uniqueRoles = Array.from(allRoles);
|
243
|
+
|
244
|
+
MessageFormatter.info(`Found ${uniqueRoles.length} unique roles for team ${team.name}: ${uniqueRoles.join(', ')}`, { prefix: "Transfer" });
|
245
|
+
|
246
|
+
// Create team in target with the collected roles
|
247
|
+
await this.targetTeams.create(
|
248
|
+
team.$id,
|
249
|
+
team.name,
|
250
|
+
uniqueRoles
|
251
|
+
);
|
252
|
+
MessageFormatter.success(`Created team: ${team.name} with roles: ${uniqueRoles.join(', ')}`, { prefix: "Transfer" });
|
253
|
+
} else {
|
254
|
+
MessageFormatter.info(`Team ${team.name} already exists, updating if needed`, { prefix: "Transfer" });
|
255
|
+
|
256
|
+
// Update team if needed
|
257
|
+
if (existingTeam.name !== team.name) {
|
258
|
+
await this.targetTeams.updateName(team.$id, team.name);
|
259
|
+
MessageFormatter.success(`Updated team name: ${team.name}`, { prefix: "Transfer" });
|
260
|
+
}
|
261
|
+
}
|
262
|
+
|
263
|
+
// Transfer team memberships
|
264
|
+
await this.transferTeamMemberships(team.$id);
|
265
|
+
|
266
|
+
this.results.teams.transferred++;
|
267
|
+
MessageFormatter.success(`Team ${team.name} transferred successfully`, { prefix: "Transfer" });
|
268
|
+
} catch (error) {
|
269
|
+
MessageFormatter.error(`Team ${team.name} transfer failed`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
|
270
|
+
this.results.teams.failed++;
|
271
|
+
}
|
272
|
+
})
|
273
|
+
);
|
274
|
+
|
275
|
+
await Promise.all(transferTasks);
|
276
|
+
MessageFormatter.success("Team transfer phase completed", { prefix: "Transfer" });
|
277
|
+
} catch (error) {
|
278
|
+
MessageFormatter.error("Team transfer phase failed", error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
|
279
|
+
}
|
280
|
+
}
|
281
|
+
|
196
282
|
private async transferAllDatabases(): Promise<void> {
|
197
283
|
MessageFormatter.info("Starting database transfer phase", { prefix: "Transfer" });
|
198
284
|
|
@@ -433,6 +519,45 @@ export class ComprehensiveTransfer {
|
|
433
519
|
// Create bucket with fallback strategy for maximumFileSize
|
434
520
|
await this.createBucketWithFallback(bucket);
|
435
521
|
MessageFormatter.success(`Created bucket: ${bucket.name}`, { prefix: "Transfer" });
|
522
|
+
} else {
|
523
|
+
// Compare bucket permissions and update if needed
|
524
|
+
const sourcePermissions = JSON.stringify(bucket.$permissions?.sort() || []);
|
525
|
+
const targetPermissions = JSON.stringify(existingBucket.$permissions?.sort() || []);
|
526
|
+
|
527
|
+
if (sourcePermissions !== targetPermissions ||
|
528
|
+
existingBucket.name !== bucket.name ||
|
529
|
+
existingBucket.fileSecurity !== bucket.fileSecurity ||
|
530
|
+
existingBucket.enabled !== bucket.enabled) {
|
531
|
+
|
532
|
+
MessageFormatter.warning(
|
533
|
+
`Bucket ${bucket.name} exists but has different settings. Updating to match source.`,
|
534
|
+
{ prefix: "Transfer" }
|
535
|
+
);
|
536
|
+
|
537
|
+
try {
|
538
|
+
await this.targetStorage.updateBucket(
|
539
|
+
bucket.$id,
|
540
|
+
bucket.name,
|
541
|
+
bucket.$permissions,
|
542
|
+
bucket.fileSecurity,
|
543
|
+
bucket.enabled,
|
544
|
+
bucket.maximumFileSize,
|
545
|
+
bucket.allowedFileExtensions,
|
546
|
+
bucket.compression as any,
|
547
|
+
bucket.encryption,
|
548
|
+
bucket.antivirus
|
549
|
+
);
|
550
|
+
MessageFormatter.success(`Updated bucket ${bucket.name} to match source`, { prefix: "Transfer" });
|
551
|
+
} catch (updateError) {
|
552
|
+
MessageFormatter.error(
|
553
|
+
`Failed to update bucket ${bucket.name}`,
|
554
|
+
updateError instanceof Error ? updateError : new Error(String(updateError)),
|
555
|
+
{ prefix: "Transfer" }
|
556
|
+
);
|
557
|
+
}
|
558
|
+
} else {
|
559
|
+
MessageFormatter.info(`Bucket ${bucket.name} already exists with matching settings`, { prefix: "Transfer" });
|
560
|
+
}
|
436
561
|
}
|
437
562
|
|
438
563
|
// Transfer bucket files with enhanced validation
|
@@ -617,10 +742,40 @@ export class ComprehensiveTransfer {
|
|
617
742
|
const fileTasks = files.files.map(file =>
|
618
743
|
this.fileLimit(async () => {
|
619
744
|
try {
|
620
|
-
// Check if file already exists
|
745
|
+
// Check if file already exists and compare permissions
|
746
|
+
let existingFile: Models.File | null = null;
|
621
747
|
try {
|
622
|
-
await this.targetStorage.getFile(targetBucketId, file.$id);
|
623
|
-
|
748
|
+
existingFile = await this.targetStorage.getFile(targetBucketId, file.$id);
|
749
|
+
|
750
|
+
// Compare permissions between source and target file
|
751
|
+
const sourcePermissions = JSON.stringify(file.$permissions?.sort() || []);
|
752
|
+
const targetPermissions = JSON.stringify(existingFile.$permissions?.sort() || []);
|
753
|
+
|
754
|
+
if (sourcePermissions !== targetPermissions) {
|
755
|
+
MessageFormatter.warning(
|
756
|
+
`File ${file.name} (${file.$id}) exists but has different permissions. Source: ${sourcePermissions}, Target: ${targetPermissions}`,
|
757
|
+
{ prefix: "Transfer" }
|
758
|
+
);
|
759
|
+
|
760
|
+
// Update file permissions to match source
|
761
|
+
try {
|
762
|
+
await this.targetStorage.updateFile(
|
763
|
+
targetBucketId,
|
764
|
+
file.$id,
|
765
|
+
file.name,
|
766
|
+
file.$permissions
|
767
|
+
);
|
768
|
+
MessageFormatter.success(`Updated file ${file.name} permissions to match source`, { prefix: "Transfer" });
|
769
|
+
} catch (updateError) {
|
770
|
+
MessageFormatter.error(
|
771
|
+
`Failed to update permissions for file ${file.name}`,
|
772
|
+
updateError instanceof Error ? updateError : new Error(String(updateError)),
|
773
|
+
{ prefix: "Transfer" }
|
774
|
+
);
|
775
|
+
}
|
776
|
+
} else {
|
777
|
+
MessageFormatter.info(`File ${file.name} already exists with matching permissions, skipping`, { prefix: "Transfer" });
|
778
|
+
}
|
624
779
|
return;
|
625
780
|
} catch (error) {
|
626
781
|
// File doesn't exist, proceed with transfer
|
@@ -912,7 +1067,7 @@ export class ComprehensiveTransfer {
|
|
912
1067
|
}
|
913
1068
|
|
914
1069
|
/**
|
915
|
-
* Helper method to transfer documents between databases
|
1070
|
+
* Helper method to transfer documents between databases using bulk operations with content and permission-based filtering
|
916
1071
|
*/
|
917
1072
|
private async transferDocumentsBetweenDatabases(
|
918
1073
|
sourceDb: Databases,
|
@@ -922,69 +1077,568 @@ export class ComprehensiveTransfer {
|
|
922
1077
|
sourceCollectionId: string,
|
923
1078
|
targetCollectionId: string
|
924
1079
|
): Promise<void> {
|
925
|
-
MessageFormatter.info(`Transferring documents from ${sourceCollectionId} to ${targetCollectionId}`, { prefix: "Transfer" });
|
1080
|
+
MessageFormatter.info(`Transferring documents from ${sourceCollectionId} to ${targetCollectionId} with bulk operations, content comparison, and permission filtering`, { prefix: "Transfer" });
|
926
1081
|
|
927
1082
|
let lastId: string | undefined;
|
928
1083
|
let totalTransferred = 0;
|
1084
|
+
let totalSkipped = 0;
|
1085
|
+
let totalUpdated = 0;
|
1086
|
+
|
1087
|
+
// Check if bulk operations are supported
|
1088
|
+
const supportsBulk = this.options.sourceEndpoint.includes('cloud.appwrite.io') ||
|
1089
|
+
this.options.targetEndpoint.includes('cloud.appwrite.io');
|
1090
|
+
|
1091
|
+
if (supportsBulk) {
|
1092
|
+
MessageFormatter.info(`Using bulk operations for enhanced performance`, { prefix: "Transfer" });
|
1093
|
+
}
|
929
1094
|
|
930
1095
|
while (true) {
|
931
|
-
|
1096
|
+
// Fetch source documents in larger batches (1000 instead of 50)
|
1097
|
+
const queries = [Query.limit(1000)];
|
932
1098
|
if (lastId) {
|
933
1099
|
queries.push(Query.cursorAfter(lastId));
|
934
1100
|
}
|
935
1101
|
|
936
|
-
const
|
1102
|
+
const sourceDocuments = await tryAwaitWithRetry(async () =>
|
937
1103
|
sourceDb.listDocuments(sourceDbId, sourceCollectionId, queries)
|
938
1104
|
);
|
939
1105
|
|
940
|
-
if (
|
1106
|
+
if (sourceDocuments.documents.length === 0) {
|
941
1107
|
break;
|
942
1108
|
}
|
943
1109
|
|
944
|
-
|
945
|
-
|
946
|
-
|
1110
|
+
MessageFormatter.info(`Processing batch of ${sourceDocuments.documents.length} source documents`, { prefix: "Transfer" });
|
1111
|
+
|
1112
|
+
// Extract document IDs from the current batch
|
1113
|
+
const sourceDocIds = sourceDocuments.documents.map(doc => doc.$id);
|
1114
|
+
|
1115
|
+
// Fetch existing documents from target in a single query
|
1116
|
+
const existingTargetDocs = await this.fetchTargetDocumentsBatch(
|
1117
|
+
targetDb,
|
1118
|
+
targetDbId,
|
1119
|
+
targetCollectionId,
|
1120
|
+
sourceDocIds
|
1121
|
+
);
|
1122
|
+
|
1123
|
+
// Create a map for quick lookup of existing documents
|
1124
|
+
const existingDocsMap = new Map<string, Models.Document>();
|
1125
|
+
existingTargetDocs.forEach(doc => {
|
1126
|
+
existingDocsMap.set(doc.$id, doc);
|
1127
|
+
});
|
1128
|
+
|
1129
|
+
// Filter documents based on existence, content comparison, and permission comparison
|
1130
|
+
const documentsToTransfer: Models.Document[] = [];
|
1131
|
+
const documentsToUpdate: { doc: Models.Document; targetDoc: Models.Document; reason: string }[] = [];
|
1132
|
+
|
1133
|
+
for (const sourceDoc of sourceDocuments.documents) {
|
1134
|
+
const existingTargetDoc = existingDocsMap.get(sourceDoc.$id);
|
1135
|
+
|
1136
|
+
if (!existingTargetDoc) {
|
1137
|
+
// Document doesn't exist in target, needs to be transferred
|
1138
|
+
documentsToTransfer.push(sourceDoc);
|
1139
|
+
} else {
|
1140
|
+
// Document exists, compare both content and permissions
|
1141
|
+
const sourcePermissions = JSON.stringify((sourceDoc.$permissions || []).sort());
|
1142
|
+
const targetPermissions = JSON.stringify((existingTargetDoc.$permissions || []).sort());
|
1143
|
+
const permissionsDiffer = sourcePermissions !== targetPermissions;
|
1144
|
+
|
1145
|
+
// Use objectNeedsUpdate to compare document content (excluding system fields)
|
1146
|
+
const contentDiffers = objectNeedsUpdate(existingTargetDoc, sourceDoc);
|
1147
|
+
|
1148
|
+
if (contentDiffers && permissionsDiffer) {
|
1149
|
+
// Both content and permissions differ
|
1150
|
+
documentsToUpdate.push({
|
1151
|
+
doc: sourceDoc,
|
1152
|
+
targetDoc: existingTargetDoc,
|
1153
|
+
reason: "content and permissions differ"
|
1154
|
+
});
|
1155
|
+
MessageFormatter.info(
|
1156
|
+
`Document ${sourceDoc.$id} exists but content and permissions differ - will update`,
|
1157
|
+
{ prefix: "Transfer" }
|
1158
|
+
);
|
1159
|
+
} else if (contentDiffers) {
|
1160
|
+
// Only content differs
|
1161
|
+
documentsToUpdate.push({
|
1162
|
+
doc: sourceDoc,
|
1163
|
+
targetDoc: existingTargetDoc,
|
1164
|
+
reason: "content differs"
|
1165
|
+
});
|
1166
|
+
MessageFormatter.info(
|
1167
|
+
`Document ${sourceDoc.$id} exists but content differs - will update`,
|
1168
|
+
{ prefix: "Transfer" }
|
1169
|
+
);
|
1170
|
+
} else if (permissionsDiffer) {
|
1171
|
+
// Only permissions differ
|
1172
|
+
documentsToUpdate.push({
|
1173
|
+
doc: sourceDoc,
|
1174
|
+
targetDoc: existingTargetDoc,
|
1175
|
+
reason: "permissions differ"
|
1176
|
+
});
|
1177
|
+
MessageFormatter.info(
|
1178
|
+
`Document ${sourceDoc.$id} exists but permissions differ - will update`,
|
1179
|
+
{ prefix: "Transfer" }
|
1180
|
+
);
|
1181
|
+
} else {
|
1182
|
+
// Document exists with identical content AND permissions, skip
|
1183
|
+
totalSkipped++;
|
1184
|
+
MessageFormatter.info(`Document ${sourceDoc.$id} exists with matching content and permissions - skipping`, { prefix: "Transfer" });
|
1185
|
+
}
|
1186
|
+
}
|
1187
|
+
}
|
1188
|
+
|
1189
|
+
MessageFormatter.info(
|
1190
|
+
`Batch analysis: ${documentsToTransfer.length} to create, ${documentsToUpdate.length} to update, ${totalSkipped} skipped so far`,
|
1191
|
+
{ prefix: "Transfer" }
|
1192
|
+
);
|
1193
|
+
|
1194
|
+
// Process new documents with bulk operations if supported and available
|
1195
|
+
if (documentsToTransfer.length > 0) {
|
1196
|
+
if (supportsBulk && documentsToTransfer.length >= 10) {
|
1197
|
+
// Use bulk operations for large batches
|
1198
|
+
await this.transferDocumentsBulk(
|
1199
|
+
targetDb,
|
1200
|
+
targetDbId,
|
1201
|
+
targetCollectionId,
|
1202
|
+
documentsToTransfer
|
1203
|
+
);
|
1204
|
+
totalTransferred += documentsToTransfer.length;
|
1205
|
+
MessageFormatter.success(`Bulk transferred ${documentsToTransfer.length} new documents`, { prefix: "Transfer" });
|
1206
|
+
} else {
|
1207
|
+
// Use individual transfers for smaller batches or non-bulk endpoints
|
1208
|
+
const transferCount = await this.transferDocumentsIndividual(
|
1209
|
+
targetDb,
|
1210
|
+
targetDbId,
|
1211
|
+
targetCollectionId,
|
1212
|
+
documentsToTransfer
|
1213
|
+
);
|
1214
|
+
totalTransferred += transferCount;
|
1215
|
+
}
|
1216
|
+
}
|
1217
|
+
|
1218
|
+
// Process document updates (always individual since bulk update with permissions needs special handling)
|
1219
|
+
if (documentsToUpdate.length > 0) {
|
1220
|
+
const updateCount = await this.updateDocumentsIndividual(
|
1221
|
+
targetDb,
|
1222
|
+
targetDbId,
|
1223
|
+
targetCollectionId,
|
1224
|
+
documentsToUpdate
|
1225
|
+
);
|
1226
|
+
totalUpdated += updateCount;
|
1227
|
+
}
|
1228
|
+
|
1229
|
+
if (sourceDocuments.documents.length < 1000) {
|
1230
|
+
break;
|
1231
|
+
}
|
1232
|
+
|
1233
|
+
lastId = sourceDocuments.documents[sourceDocuments.documents.length - 1].$id;
|
1234
|
+
}
|
1235
|
+
|
1236
|
+
MessageFormatter.info(
|
1237
|
+
`Transfer complete: ${totalTransferred} new, ${totalUpdated} updated, ${totalSkipped} skipped from ${sourceCollectionId} to ${targetCollectionId}`,
|
1238
|
+
{ prefix: "Transfer" }
|
1239
|
+
);
|
1240
|
+
}
|
1241
|
+
|
1242
|
+
/**
|
1243
|
+
* Fetch target documents by IDs in batches to check existence and permissions
|
1244
|
+
*/
|
1245
|
+
private async fetchTargetDocumentsBatch(
|
1246
|
+
targetDb: Databases,
|
1247
|
+
targetDbId: string,
|
1248
|
+
targetCollectionId: string,
|
1249
|
+
docIds: string[]
|
1250
|
+
): Promise<Models.Document[]> {
|
1251
|
+
const documents: Models.Document[] = [];
|
1252
|
+
|
1253
|
+
// Split IDs into chunks of 100 for Query.equal limitations
|
1254
|
+
const idChunks = this.chunkArray(docIds, 100);
|
1255
|
+
|
1256
|
+
for (const chunk of idChunks) {
|
1257
|
+
try {
|
1258
|
+
const result = await tryAwaitWithRetry(async () =>
|
1259
|
+
targetDb.listDocuments(targetDbId, targetCollectionId, [
|
1260
|
+
Query.equal('$id', chunk),
|
1261
|
+
Query.limit(100)
|
1262
|
+
])
|
1263
|
+
);
|
1264
|
+
documents.push(...result.documents);
|
1265
|
+
} catch (error) {
|
1266
|
+
// If query fails, fall back to individual gets (less efficient but more reliable)
|
1267
|
+
MessageFormatter.warning(
|
1268
|
+
`Batch query failed for ${chunk.length} documents, falling back to individual checks`,
|
1269
|
+
{ prefix: "Transfer" }
|
1270
|
+
);
|
1271
|
+
|
1272
|
+
for (const docId of chunk) {
|
947
1273
|
try {
|
948
|
-
|
1274
|
+
const doc = await targetDb.getDocument(targetDbId, targetCollectionId, docId);
|
1275
|
+
documents.push(doc);
|
1276
|
+
} catch (getError) {
|
1277
|
+
// Document doesn't exist, which is fine
|
1278
|
+
}
|
1279
|
+
}
|
1280
|
+
}
|
1281
|
+
}
|
1282
|
+
|
1283
|
+
return documents;
|
1284
|
+
}
|
1285
|
+
|
1286
|
+
/**
|
1287
|
+
* Transfer documents using bulk operations with proper batch size handling
|
1288
|
+
*/
|
1289
|
+
private async transferDocumentsBulk(
|
1290
|
+
targetDb: Databases,
|
1291
|
+
targetDbId: string,
|
1292
|
+
targetCollectionId: string,
|
1293
|
+
documents: Models.Document[]
|
1294
|
+
): Promise<void> {
|
1295
|
+
// Prepare documents for bulk upsert
|
1296
|
+
const preparedDocs = documents.map(doc => {
|
1297
|
+
const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
|
1298
|
+
return {
|
1299
|
+
$id,
|
1300
|
+
$permissions,
|
1301
|
+
...docData
|
1302
|
+
};
|
1303
|
+
});
|
1304
|
+
|
1305
|
+
// Process in smaller chunks for bulk operations (1000 for Pro, 100 for Free tier)
|
1306
|
+
const batchSizes = [1000, 100]; // Start with Pro plan, fallback to Free
|
1307
|
+
let processed = false;
|
1308
|
+
|
1309
|
+
for (const maxBatchSize of batchSizes) {
|
1310
|
+
const documentBatches = this.chunkArray(preparedDocs, maxBatchSize);
|
1311
|
+
|
1312
|
+
try {
|
1313
|
+
for (const batch of documentBatches) {
|
1314
|
+
MessageFormatter.info(`Bulk upserting ${batch.length} documents...`, { prefix: "Transfer" });
|
1315
|
+
|
1316
|
+
await this.bulkUpsertDocuments(
|
1317
|
+
this.targetClient,
|
1318
|
+
targetDbId,
|
1319
|
+
targetCollectionId,
|
1320
|
+
batch
|
1321
|
+
);
|
1322
|
+
|
1323
|
+
MessageFormatter.success(`✅ Bulk upserted ${batch.length} documents`, { prefix: "Transfer" });
|
1324
|
+
|
1325
|
+
// Add delay between batches to respect rate limits
|
1326
|
+
if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
|
1327
|
+
await new Promise(resolve => setTimeout(resolve, 200));
|
1328
|
+
}
|
1329
|
+
}
|
1330
|
+
|
1331
|
+
processed = true;
|
1332
|
+
break; // Success, exit batch size loop
|
1333
|
+
} catch (error) {
|
1334
|
+
MessageFormatter.warning(
|
1335
|
+
`Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`,
|
1336
|
+
{ prefix: "Transfer" }
|
1337
|
+
);
|
1338
|
+
continue; // Try next smaller batch size
|
1339
|
+
}
|
1340
|
+
}
|
1341
|
+
|
1342
|
+
if (!processed) {
|
1343
|
+
MessageFormatter.warning(
|
1344
|
+
`All bulk operations failed, falling back to individual transfers`,
|
1345
|
+
{ prefix: "Transfer" }
|
1346
|
+
);
|
1347
|
+
|
1348
|
+
// Fall back to individual transfers
|
1349
|
+
await this.transferDocumentsIndividual(targetDb, targetDbId, targetCollectionId, documents);
|
1350
|
+
}
|
1351
|
+
}
|
1352
|
+
|
1353
|
+
/**
|
1354
|
+
* Direct HTTP implementation of bulk upsert API
|
1355
|
+
*/
|
1356
|
+
private async bulkUpsertDocuments(
|
1357
|
+
client: any,
|
1358
|
+
dbId: string,
|
1359
|
+
collectionId: string,
|
1360
|
+
documents: any[]
|
1361
|
+
): Promise<any> {
|
1362
|
+
const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
|
1363
|
+
const url = new URL(client.config.endpoint + apiPath);
|
1364
|
+
|
1365
|
+
const headers = {
|
1366
|
+
'Content-Type': 'application/json',
|
1367
|
+
'X-Appwrite-Project': client.config.project,
|
1368
|
+
'X-Appwrite-Key': client.config.key
|
1369
|
+
};
|
1370
|
+
|
1371
|
+
const response = await fetch(url.toString(), {
|
1372
|
+
method: 'PUT',
|
1373
|
+
headers,
|
1374
|
+
body: JSON.stringify({ documents })
|
1375
|
+
});
|
1376
|
+
|
1377
|
+
if (!response.ok) {
|
1378
|
+
const errorData: any = await response.json().catch(() => ({ message: 'Unknown error' }));
|
1379
|
+
throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
|
1380
|
+
}
|
1381
|
+
|
1382
|
+
return await response.json();
|
1383
|
+
}
|
1384
|
+
|
1385
|
+
/**
|
1386
|
+
* Transfer documents individually with rate limiting
|
1387
|
+
*/
|
1388
|
+
private async transferDocumentsIndividual(
|
1389
|
+
targetDb: Databases,
|
1390
|
+
targetDbId: string,
|
1391
|
+
targetCollectionId: string,
|
1392
|
+
documents: Models.Document[]
|
1393
|
+
): Promise<number> {
|
1394
|
+
let successCount = 0;
|
1395
|
+
|
1396
|
+
const transferTasks = documents.map(doc =>
|
1397
|
+
this.limit(async () => {
|
1398
|
+
try {
|
1399
|
+
const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
|
1400
|
+
|
1401
|
+
await tryAwaitWithRetry(async () =>
|
1402
|
+
targetDb.createDocument(
|
1403
|
+
targetDbId,
|
1404
|
+
targetCollectionId,
|
1405
|
+
doc.$id,
|
1406
|
+
docData,
|
1407
|
+
doc.$permissions
|
1408
|
+
)
|
1409
|
+
);
|
1410
|
+
|
1411
|
+
successCount++;
|
1412
|
+
MessageFormatter.success(`Transferred document ${doc.$id}`, { prefix: "Transfer" });
|
1413
|
+
} catch (error) {
|
1414
|
+
MessageFormatter.error(
|
1415
|
+
`Failed to transfer document ${doc.$id}`,
|
1416
|
+
error instanceof Error ? error : new Error(String(error)),
|
1417
|
+
{ prefix: "Transfer" }
|
1418
|
+
);
|
1419
|
+
}
|
1420
|
+
})
|
1421
|
+
);
|
1422
|
+
|
1423
|
+
await Promise.all(transferTasks);
|
1424
|
+
return successCount;
|
1425
|
+
}
|
1426
|
+
|
1427
|
+
/**
|
1428
|
+
* Update documents individually with content and/or permission changes
|
1429
|
+
*/
|
1430
|
+
private async updateDocumentsIndividual(
|
1431
|
+
targetDb: Databases,
|
1432
|
+
targetDbId: string,
|
1433
|
+
targetCollectionId: string,
|
1434
|
+
documentPairs: { doc: Models.Document; targetDoc: Models.Document; reason: string }[]
|
1435
|
+
): Promise<number> {
|
1436
|
+
let successCount = 0;
|
1437
|
+
|
1438
|
+
const updateTasks = documentPairs.map(({ doc, targetDoc, reason }) =>
|
1439
|
+
this.limit(async () => {
|
1440
|
+
try {
|
1441
|
+
const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
|
1442
|
+
|
1443
|
+
await tryAwaitWithRetry(async () =>
|
1444
|
+
targetDb.updateDocument(
|
1445
|
+
targetDbId,
|
1446
|
+
targetCollectionId,
|
1447
|
+
doc.$id,
|
1448
|
+
docData,
|
1449
|
+
doc.$permissions
|
1450
|
+
)
|
1451
|
+
);
|
1452
|
+
|
1453
|
+
successCount++;
|
1454
|
+
MessageFormatter.success(
|
1455
|
+
`Updated document ${doc.$id} (${reason}) - permissions: [${targetDoc.$permissions?.join(', ')}] → [${doc.$permissions?.join(', ')}]`,
|
1456
|
+
{ prefix: "Transfer" }
|
1457
|
+
);
|
1458
|
+
} catch (error) {
|
1459
|
+
MessageFormatter.error(
|
1460
|
+
`Failed to update document ${doc.$id} (${reason})`,
|
1461
|
+
error instanceof Error ? error : new Error(String(error)),
|
1462
|
+
{ prefix: "Transfer" }
|
1463
|
+
);
|
1464
|
+
}
|
1465
|
+
})
|
1466
|
+
);
|
1467
|
+
|
1468
|
+
await Promise.all(updateTasks);
|
1469
|
+
return successCount;
|
1470
|
+
}
|
1471
|
+
|
1472
|
+
/**
|
1473
|
+
* Utility method to chunk arrays
|
1474
|
+
*/
|
1475
|
+
private chunkArray<T>(array: T[], size: number): T[][] {
|
1476
|
+
const chunks: T[][] = [];
|
1477
|
+
for (let i = 0; i < array.length; i += size) {
|
1478
|
+
chunks.push(array.slice(i, i + size));
|
1479
|
+
}
|
1480
|
+
return chunks;
|
1481
|
+
}
|
1482
|
+
|
1483
|
+
/**
|
1484
|
+
* Helper method to fetch all teams with pagination
|
1485
|
+
*/
|
1486
|
+
private async fetchAllTeams(teams: Teams): Promise<Models.Team<Models.Preferences>[]> {
|
1487
|
+
const teamsList: Models.Team<Models.Preferences>[] = [];
|
1488
|
+
let lastId: string | undefined;
|
1489
|
+
|
1490
|
+
while (true) {
|
1491
|
+
const queries = [Query.limit(100)];
|
1492
|
+
if (lastId) {
|
1493
|
+
queries.push(Query.cursorAfter(lastId));
|
1494
|
+
}
|
1495
|
+
|
1496
|
+
const result = await tryAwaitWithRetry(async () => teams.list(queries));
|
1497
|
+
|
1498
|
+
if (result.teams.length === 0) {
|
1499
|
+
break;
|
1500
|
+
}
|
1501
|
+
|
1502
|
+
teamsList.push(...result.teams);
|
1503
|
+
|
1504
|
+
if (result.teams.length < 100) {
|
1505
|
+
break;
|
1506
|
+
}
|
1507
|
+
|
1508
|
+
lastId = result.teams[result.teams.length - 1].$id;
|
1509
|
+
}
|
1510
|
+
|
1511
|
+
return teamsList;
|
1512
|
+
}
|
1513
|
+
|
1514
|
+
/**
|
1515
|
+
* Helper method to fetch all memberships for a team with pagination
|
1516
|
+
*/
|
1517
|
+
private async fetchAllMemberships(teamId: string): Promise<Models.Membership[]> {
|
1518
|
+
const membershipsList: Models.Membership[] = [];
|
1519
|
+
let lastId: string | undefined;
|
1520
|
+
|
1521
|
+
while (true) {
|
1522
|
+
const queries = [Query.limit(100)];
|
1523
|
+
if (lastId) {
|
1524
|
+
queries.push(Query.cursorAfter(lastId));
|
1525
|
+
}
|
1526
|
+
|
1527
|
+
const result = await tryAwaitWithRetry(async () =>
|
1528
|
+
this.sourceTeams.listMemberships(teamId, queries)
|
1529
|
+
);
|
1530
|
+
|
1531
|
+
if (result.memberships.length === 0) {
|
1532
|
+
break;
|
1533
|
+
}
|
1534
|
+
|
1535
|
+
membershipsList.push(...result.memberships);
|
1536
|
+
|
1537
|
+
if (result.memberships.length < 100) {
|
1538
|
+
break;
|
1539
|
+
}
|
1540
|
+
|
1541
|
+
lastId = result.memberships[result.memberships.length - 1].$id;
|
1542
|
+
}
|
1543
|
+
|
1544
|
+
return membershipsList;
|
1545
|
+
}
|
1546
|
+
|
1547
|
+
/**
|
1548
|
+
* Helper method to transfer team memberships
|
1549
|
+
*/
|
1550
|
+
private async transferTeamMemberships(teamId: string): Promise<void> {
|
1551
|
+
MessageFormatter.info(`Transferring memberships for team ${teamId}`, { prefix: "Transfer" });
|
1552
|
+
|
1553
|
+
try {
|
1554
|
+
// Fetch all memberships for this team
|
1555
|
+
const memberships = await this.fetchAllMemberships(teamId);
|
1556
|
+
|
1557
|
+
if (memberships.length === 0) {
|
1558
|
+
MessageFormatter.info(`No memberships found for team ${teamId}`, { prefix: "Transfer" });
|
1559
|
+
return;
|
1560
|
+
}
|
1561
|
+
|
1562
|
+
MessageFormatter.info(`Found ${memberships.length} memberships for team ${teamId}`, { prefix: "Transfer" });
|
1563
|
+
|
1564
|
+
let totalTransferred = 0;
|
1565
|
+
|
1566
|
+
// Transfer memberships with rate limiting
|
1567
|
+
const transferTasks = memberships.map(membership =>
|
1568
|
+
this.userLimit(async () => { // Use userLimit for team operations (more sensitive)
|
1569
|
+
try {
|
1570
|
+
// Check if membership already exists and compare roles
|
1571
|
+
let existingMembership: Models.Membership | null = null;
|
949
1572
|
try {
|
950
|
-
await
|
951
|
-
|
1573
|
+
existingMembership = await this.targetTeams.getMembership(teamId, membership.$id);
|
1574
|
+
|
1575
|
+
// Compare roles between source and target membership
|
1576
|
+
const sourceRoles = JSON.stringify(membership.roles?.sort() || []);
|
1577
|
+
const targetRoles = JSON.stringify(existingMembership.roles?.sort() || []);
|
1578
|
+
|
1579
|
+
if (sourceRoles !== targetRoles) {
|
1580
|
+
MessageFormatter.warning(
|
1581
|
+
`Membership ${membership.$id} exists but has different roles. Source: ${sourceRoles}, Target: ${targetRoles}`,
|
1582
|
+
{ prefix: "Transfer" }
|
1583
|
+
);
|
1584
|
+
|
1585
|
+
// Update membership roles to match source
|
1586
|
+
try {
|
1587
|
+
await this.targetTeams.updateMembership(
|
1588
|
+
teamId,
|
1589
|
+
membership.$id,
|
1590
|
+
membership.roles
|
1591
|
+
);
|
1592
|
+
MessageFormatter.success(`Updated membership ${membership.$id} roles to match source`, { prefix: "Transfer" });
|
1593
|
+
} catch (updateError) {
|
1594
|
+
MessageFormatter.error(
|
1595
|
+
`Failed to update roles for membership ${membership.$id}`,
|
1596
|
+
updateError instanceof Error ? updateError : new Error(String(updateError)),
|
1597
|
+
{ prefix: "Transfer" }
|
1598
|
+
);
|
1599
|
+
}
|
1600
|
+
} else {
|
1601
|
+
MessageFormatter.info(`Membership ${membership.$id} already exists with matching roles, skipping`, { prefix: "Transfer" });
|
1602
|
+
}
|
952
1603
|
return;
|
953
1604
|
} catch (error) {
|
954
|
-
//
|
1605
|
+
// Membership doesn't exist, proceed with creation
|
955
1606
|
}
|
956
1607
|
|
957
|
-
//
|
958
|
-
|
959
|
-
|
1608
|
+
// Get user data from target (users should already be transferred)
|
1609
|
+
let userData: Models.User<Record<string, any>> | null = null;
|
1610
|
+
try {
|
1611
|
+
userData = await this.targetUsers.get(membership.userId);
|
1612
|
+
} catch (error) {
|
1613
|
+
MessageFormatter.warning(`User ${membership.userId} not found in target, membership ${membership.$id} may fail`, { prefix: "Transfer" });
|
1614
|
+
}
|
1615
|
+
|
1616
|
+
// Create membership using the comprehensive user data
|
960
1617
|
await tryAwaitWithRetry(async () =>
|
961
|
-
|
962
|
-
|
963
|
-
|
964
|
-
|
965
|
-
|
966
|
-
|
1618
|
+
this.targetTeams.createMembership(
|
1619
|
+
teamId,
|
1620
|
+
membership.roles,
|
1621
|
+
userData?.email || membership.userEmail, // Use target user email if available, fallback to membership email
|
1622
|
+
membership.userId, // User ID
|
1623
|
+
userData?.phone || undefined, // Use target user phone if available
|
1624
|
+
undefined, // Invitation URL placeholder
|
1625
|
+
userData?.name || membership.userName // Use target user name if available, fallback to membership name
|
967
1626
|
)
|
968
1627
|
);
|
969
1628
|
|
970
1629
|
totalTransferred++;
|
971
|
-
MessageFormatter.success(`Transferred
|
1630
|
+
MessageFormatter.success(`Transferred membership ${membership.$id} for user ${userData?.name || membership.userName}`, { prefix: "Transfer" });
|
972
1631
|
} catch (error) {
|
973
|
-
MessageFormatter.error(`Failed to transfer
|
1632
|
+
MessageFormatter.error(`Failed to transfer membership ${membership.$id}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
|
974
1633
|
}
|
975
1634
|
})
|
976
1635
|
);
|
977
1636
|
|
978
1637
|
await Promise.all(transferTasks);
|
979
|
-
|
980
|
-
|
981
|
-
|
982
|
-
}
|
983
|
-
|
984
|
-
lastId = documents.documents[documents.documents.length - 1].$id;
|
1638
|
+
MessageFormatter.info(`Transferred ${totalTransferred} memberships for team ${teamId}`, { prefix: "Transfer" });
|
1639
|
+
} catch (error) {
|
1640
|
+
MessageFormatter.error(`Failed to transfer memberships for team ${teamId}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
|
985
1641
|
}
|
986
|
-
|
987
|
-
MessageFormatter.info(`Transferred ${totalTransferred} documents from ${sourceCollectionId} to ${targetCollectionId}`, { prefix: "Transfer" });
|
988
1642
|
}
|
989
1643
|
|
990
1644
|
private printSummary(): void {
|
@@ -993,12 +1647,13 @@ export class ComprehensiveTransfer {
|
|
993
1647
|
MessageFormatter.info("=== COMPREHENSIVE TRANSFER SUMMARY ===", { prefix: "Transfer" });
|
994
1648
|
MessageFormatter.info(`Total Time: ${duration}s`, { prefix: "Transfer" });
|
995
1649
|
MessageFormatter.info(`Users: ${this.results.users.transferred} transferred, ${this.results.users.skipped} skipped, ${this.results.users.failed} failed`, { prefix: "Transfer" });
|
1650
|
+
MessageFormatter.info(`Teams: ${this.results.teams.transferred} transferred, ${this.results.teams.skipped} skipped, ${this.results.teams.failed} failed`, { prefix: "Transfer" });
|
996
1651
|
MessageFormatter.info(`Databases: ${this.results.databases.transferred} transferred, ${this.results.databases.skipped} skipped, ${this.results.databases.failed} failed`, { prefix: "Transfer" });
|
997
1652
|
MessageFormatter.info(`Buckets: ${this.results.buckets.transferred} transferred, ${this.results.buckets.skipped} skipped, ${this.results.buckets.failed} failed`, { prefix: "Transfer" });
|
998
1653
|
MessageFormatter.info(`Functions: ${this.results.functions.transferred} transferred, ${this.results.functions.skipped} skipped, ${this.results.functions.failed} failed`, { prefix: "Transfer" });
|
999
1654
|
|
1000
|
-
const totalTransferred = this.results.users.transferred + this.results.databases.transferred + this.results.buckets.transferred + this.results.functions.transferred;
|
1001
|
-
const totalFailed = this.results.users.failed + this.results.databases.failed + this.results.buckets.failed + this.results.functions.failed;
|
1655
|
+
const totalTransferred = this.results.users.transferred + this.results.teams.transferred + this.results.databases.transferred + this.results.buckets.transferred + this.results.functions.transferred;
|
1656
|
+
const totalFailed = this.results.users.failed + this.results.teams.failed + this.results.databases.failed + this.results.buckets.failed + this.results.functions.failed;
|
1002
1657
|
|
1003
1658
|
if (totalFailed === 0) {
|
1004
1659
|
MessageFormatter.success(`All ${totalTransferred} items transferred successfully!`, { prefix: "Transfer" });
|