appwrite-utils-cli 0.0.286 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +162 -96
- package/dist/collections/attributes.d.ts +4 -0
- package/dist/collections/attributes.js +224 -0
- package/dist/collections/indexes.d.ts +4 -0
- package/dist/collections/indexes.js +27 -0
- package/dist/collections/methods.d.ts +16 -0
- package/dist/collections/methods.js +216 -0
- package/dist/databases/methods.d.ts +6 -0
- package/dist/databases/methods.js +33 -0
- package/dist/interactiveCLI.d.ts +19 -0
- package/dist/interactiveCLI.js +555 -0
- package/dist/main.js +224 -62
- package/dist/migrations/afterImportActions.js +37 -40
- package/dist/migrations/appwriteToX.d.ts +26 -25
- package/dist/migrations/appwriteToX.js +42 -6
- package/dist/migrations/attributes.js +21 -20
- package/dist/migrations/backup.d.ts +93 -87
- package/dist/migrations/collections.d.ts +6 -0
- package/dist/migrations/collections.js +149 -20
- package/dist/migrations/converters.d.ts +2 -18
- package/dist/migrations/converters.js +13 -2
- package/dist/migrations/dataLoader.d.ts +276 -161
- package/dist/migrations/dataLoader.js +535 -292
- package/dist/migrations/databases.js +8 -2
- package/dist/migrations/helper.d.ts +3 -0
- package/dist/migrations/helper.js +21 -0
- package/dist/migrations/importController.d.ts +5 -2
- package/dist/migrations/importController.js +125 -88
- package/dist/migrations/importDataActions.d.ts +9 -1
- package/dist/migrations/importDataActions.js +15 -3
- package/dist/migrations/indexes.js +3 -2
- package/dist/migrations/logging.js +20 -8
- package/dist/migrations/migrationHelper.d.ts +9 -4
- package/dist/migrations/migrationHelper.js +6 -5
- package/dist/migrations/openapi.d.ts +1 -1
- package/dist/migrations/openapi.js +33 -18
- package/dist/migrations/queue.js +3 -2
- package/dist/migrations/relationships.d.ts +2 -2
- package/dist/migrations/schemaStrings.js +53 -41
- package/dist/migrations/setupDatabase.d.ts +2 -4
- package/dist/migrations/setupDatabase.js +24 -105
- package/dist/migrations/storage.d.ts +3 -1
- package/dist/migrations/storage.js +110 -16
- package/dist/migrations/transfer.d.ts +30 -0
- package/dist/migrations/transfer.js +337 -0
- package/dist/migrations/users.d.ts +2 -1
- package/dist/migrations/users.js +78 -43
- package/dist/schemas/authUser.d.ts +2 -2
- package/dist/storage/methods.d.ts +15 -0
- package/dist/storage/methods.js +207 -0
- package/dist/storage/schemas.d.ts +687 -0
- package/dist/storage/schemas.js +175 -0
- package/dist/utils/getClientFromConfig.d.ts +4 -0
- package/dist/utils/getClientFromConfig.js +16 -0
- package/dist/utils/helperFunctions.d.ts +11 -1
- package/dist/utils/helperFunctions.js +38 -0
- package/dist/utils/retryFailedPromises.d.ts +2 -0
- package/dist/utils/retryFailedPromises.js +21 -0
- package/dist/utils/schemaStrings.d.ts +13 -0
- package/dist/utils/schemaStrings.js +403 -0
- package/dist/utils/setupFiles.js +110 -61
- package/dist/utilsController.d.ts +40 -22
- package/dist/utilsController.js +164 -84
- package/package.json +13 -15
- package/src/collections/attributes.ts +483 -0
- package/src/collections/indexes.ts +53 -0
- package/src/collections/methods.ts +331 -0
- package/src/databases/methods.ts +47 -0
- package/src/init.ts +64 -64
- package/src/interactiveCLI.ts +767 -0
- package/src/main.ts +289 -83
- package/src/migrations/afterImportActions.ts +553 -490
- package/src/migrations/appwriteToX.ts +237 -174
- package/src/migrations/attributes.ts +483 -422
- package/src/migrations/backup.ts +205 -205
- package/src/migrations/collections.ts +545 -300
- package/src/migrations/converters.ts +161 -150
- package/src/migrations/dataLoader.ts +1615 -1304
- package/src/migrations/databases.ts +44 -25
- package/src/migrations/dbHelpers.ts +92 -92
- package/src/migrations/helper.ts +40 -0
- package/src/migrations/importController.ts +448 -384
- package/src/migrations/importDataActions.ts +315 -307
- package/src/migrations/indexes.ts +40 -37
- package/src/migrations/logging.ts +29 -16
- package/src/migrations/migrationHelper.ts +207 -201
- package/src/migrations/openapi.ts +83 -70
- package/src/migrations/queue.ts +118 -119
- package/src/migrations/relationships.ts +324 -324
- package/src/migrations/schemaStrings.ts +472 -460
- package/src/migrations/setupDatabase.ts +118 -219
- package/src/migrations/storage.ts +538 -358
- package/src/migrations/transfer.ts +608 -0
- package/src/migrations/users.ts +362 -285
- package/src/migrations/validationRules.ts +63 -63
- package/src/schemas/authUser.ts +23 -23
- package/src/setup.ts +8 -8
- package/src/storage/methods.ts +371 -0
- package/src/storage/schemas.ts +205 -0
- package/src/types.ts +9 -9
- package/src/utils/getClientFromConfig.ts +17 -0
- package/src/utils/helperFunctions.ts +181 -127
- package/src/utils/index.ts +2 -2
- package/src/utils/loadConfigs.ts +59 -59
- package/src/utils/retryFailedPromises.ts +27 -0
- package/src/utils/schemaStrings.ts +473 -0
- package/src/utils/setupFiles.ts +228 -182
- package/src/utilsController.ts +325 -194
- package/tsconfig.json +37 -37
@@ -0,0 +1,30 @@
|
|
1
|
+
import { Databases, Storage, type Models } from "node-appwrite";
|
2
|
+
export interface TransferOptions {
|
3
|
+
fromDb: Models.Database;
|
4
|
+
targetDb: Models.Database;
|
5
|
+
isRemote: boolean;
|
6
|
+
collections?: string[];
|
7
|
+
transferEndpoint?: string;
|
8
|
+
transferProject?: string;
|
9
|
+
transferKey?: string;
|
10
|
+
sourceBucket?: Models.Bucket;
|
11
|
+
targetBucket?: Models.Bucket;
|
12
|
+
}
|
13
|
+
export declare const transferStorageLocalToLocal: (storage: Storage, fromBucketId: string, toBucketId: string) => Promise<void>;
|
14
|
+
export declare const transferStorageLocalToRemote: (localStorage: Storage, endpoint: string, projectId: string, apiKey: string, fromBucketId: string, toBucketId: string) => Promise<void>;
|
15
|
+
/**
|
16
|
+
* Transfers all documents from one collection to another in a different database
|
17
|
+
* within the same Appwrite Project
|
18
|
+
*/
|
19
|
+
export declare const transferDocumentsBetweenDbsLocalToLocal: (db: Databases, fromDbId: string, toDbId: string, fromCollId: string, toCollId: string) => Promise<void>;
|
20
|
+
export declare const transferDocumentsBetweenDbsLocalToRemote: (localDb: Databases, endpoint: string, projectId: string, apiKey: string, fromDbId: string, toDbId: string, fromCollId: string, toCollId: string) => Promise<void>;
|
21
|
+
/**
|
22
|
+
* Transfers all collections and documents from one local database to another local database.
|
23
|
+
*
|
24
|
+
* @param {Databases} localDb - The local database instance.
|
25
|
+
* @param {string} fromDbId - The ID of the source database.
|
26
|
+
* @param {string} targetDbId - The ID of the target database.
|
27
|
+
* @return {Promise<void>} A promise that resolves when the transfer is complete.
|
28
|
+
*/
|
29
|
+
export declare const transferDatabaseLocalToLocal: (localDb: Databases, fromDbId: string, targetDbId: string) => Promise<void>;
|
30
|
+
export declare const transferDatabaseLocalToRemote: (localDb: Databases, endpoint: string, projectId: string, apiKey: string, fromDbId: string, toDbId: string) => Promise<void>;
|
@@ -0,0 +1,337 @@
|
|
1
|
+
import { tryAwaitWithRetry } from "appwrite-utils";
|
2
|
+
import { Client, Databases, IndexType, Query, Storage, } from "node-appwrite";
|
3
|
+
import { InputFile } from "node-appwrite/file";
|
4
|
+
import { getAppwriteClient } from "../utils/helperFunctions.js";
|
5
|
+
import { createOrUpdateAttribute } from "./attributes.js";
|
6
|
+
import { parseAttribute } from "appwrite-utils";
|
7
|
+
export const transferStorageLocalToLocal = async (storage, fromBucketId, toBucketId) => {
|
8
|
+
console.log(`Transferring files from ${fromBucketId} to ${toBucketId}`);
|
9
|
+
let lastFileId;
|
10
|
+
let fromFiles = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [Query.limit(100)]));
|
11
|
+
const allFromFiles = fromFiles.files;
|
12
|
+
let numberOfFiles = 0;
|
13
|
+
const downloadFileWithRetry = async (bucketId, fileId) => {
|
14
|
+
let attempts = 3;
|
15
|
+
while (attempts > 0) {
|
16
|
+
try {
|
17
|
+
return await storage.getFileDownload(bucketId, fileId);
|
18
|
+
}
|
19
|
+
catch (error) {
|
20
|
+
console.error(`Error downloading file ${fileId}: ${error}`);
|
21
|
+
attempts--;
|
22
|
+
if (attempts === 0)
|
23
|
+
throw error;
|
24
|
+
}
|
25
|
+
}
|
26
|
+
};
|
27
|
+
if (fromFiles.files.length < 100) {
|
28
|
+
for (const file of allFromFiles) {
|
29
|
+
const fileData = await tryAwaitWithRetry(async () => await downloadFileWithRetry(file.bucketId, file.$id));
|
30
|
+
if (!fileData) {
|
31
|
+
console.error(`Error downloading file ${file.$id}`);
|
32
|
+
continue;
|
33
|
+
}
|
34
|
+
const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
|
35
|
+
console.log(`Creating file: ${file.name}`);
|
36
|
+
tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
|
37
|
+
numberOfFiles++;
|
38
|
+
}
|
39
|
+
}
|
40
|
+
else {
|
41
|
+
lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
|
42
|
+
while (lastFileId) {
|
43
|
+
const files = await tryAwaitWithRetry(async () => await storage.listFiles(fromBucketId, [
|
44
|
+
Query.limit(100),
|
45
|
+
Query.cursorAfter(lastFileId),
|
46
|
+
]));
|
47
|
+
allFromFiles.push(...files.files);
|
48
|
+
if (files.files.length < 100) {
|
49
|
+
lastFileId = undefined;
|
50
|
+
}
|
51
|
+
else {
|
52
|
+
lastFileId = files.files[files.files.length - 1].$id;
|
53
|
+
}
|
54
|
+
}
|
55
|
+
for (const file of allFromFiles) {
|
56
|
+
const fileData = await tryAwaitWithRetry(async () => await downloadFileWithRetry(file.bucketId, file.$id));
|
57
|
+
if (!fileData) {
|
58
|
+
console.error(`Error downloading file ${file.$id}`);
|
59
|
+
continue;
|
60
|
+
}
|
61
|
+
const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
|
62
|
+
await tryAwaitWithRetry(async () => await storage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
|
63
|
+
numberOfFiles++;
|
64
|
+
}
|
65
|
+
}
|
66
|
+
console.log(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`);
|
67
|
+
};
|
68
|
+
export const transferStorageLocalToRemote = async (localStorage, endpoint, projectId, apiKey, fromBucketId, toBucketId) => {
|
69
|
+
console.log(`Transferring files from current storage ${fromBucketId} to ${endpoint} bucket ${toBucketId}`);
|
70
|
+
const client = getAppwriteClient(endpoint, apiKey, projectId);
|
71
|
+
const remoteStorage = new Storage(client);
|
72
|
+
let numberOfFiles = 0;
|
73
|
+
let lastFileId;
|
74
|
+
let fromFiles = await tryAwaitWithRetry(async () => await localStorage.listFiles(fromBucketId, [Query.limit(100)]));
|
75
|
+
const allFromFiles = fromFiles.files;
|
76
|
+
if (fromFiles.files.length === 100) {
|
77
|
+
lastFileId = fromFiles.files[fromFiles.files.length - 1].$id;
|
78
|
+
while (lastFileId) {
|
79
|
+
const files = await tryAwaitWithRetry(async () => await localStorage.listFiles(fromBucketId, [
|
80
|
+
Query.limit(100),
|
81
|
+
Query.cursorAfter(lastFileId),
|
82
|
+
]));
|
83
|
+
allFromFiles.push(...files.files);
|
84
|
+
if (files.files.length < 100) {
|
85
|
+
break;
|
86
|
+
}
|
87
|
+
lastFileId = files.files[files.files.length - 1].$id;
|
88
|
+
}
|
89
|
+
}
|
90
|
+
for (const file of allFromFiles) {
|
91
|
+
const fileData = await tryAwaitWithRetry(async () => await localStorage.getFileDownload(file.bucketId, file.$id));
|
92
|
+
const fileToCreate = InputFile.fromBuffer(new Uint8Array(fileData), file.name);
|
93
|
+
await tryAwaitWithRetry(async () => await remoteStorage.createFile(toBucketId, file.$id, fileToCreate, file.$permissions));
|
94
|
+
numberOfFiles++;
|
95
|
+
}
|
96
|
+
console.log(`Transferred ${numberOfFiles} files from ${fromBucketId} to ${toBucketId}`);
|
97
|
+
};
|
98
|
+
/**
|
99
|
+
* Transfers all documents from one collection to another in a different database
|
100
|
+
* within the same Appwrite Project
|
101
|
+
*/
|
102
|
+
export const transferDocumentsBetweenDbsLocalToLocal = async (db, fromDbId, toDbId, fromCollId, toCollId) => {
|
103
|
+
let fromCollDocs = await tryAwaitWithRetry(async () => db.listDocuments(fromDbId, fromCollId, [Query.limit(50)]));
|
104
|
+
let totalDocumentsTransferred = 0;
|
105
|
+
if (fromCollDocs.documents.length === 0) {
|
106
|
+
console.log(`No documents found in collection ${fromCollId}`);
|
107
|
+
return;
|
108
|
+
}
|
109
|
+
else if (fromCollDocs.documents.length < 50) {
|
110
|
+
const batchedPromises = fromCollDocs.documents.map((doc) => {
|
111
|
+
const toCreateObject = {
|
112
|
+
...doc,
|
113
|
+
};
|
114
|
+
delete toCreateObject.$databaseId;
|
115
|
+
delete toCreateObject.$collectionId;
|
116
|
+
delete toCreateObject.$createdAt;
|
117
|
+
delete toCreateObject.$updatedAt;
|
118
|
+
delete toCreateObject.$id;
|
119
|
+
delete toCreateObject.$permissions;
|
120
|
+
return tryAwaitWithRetry(async () => await db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
|
121
|
+
});
|
122
|
+
await Promise.all(batchedPromises);
|
123
|
+
totalDocumentsTransferred += fromCollDocs.documents.length;
|
124
|
+
}
|
125
|
+
else {
|
126
|
+
const batchedPromises = fromCollDocs.documents.map((doc) => {
|
127
|
+
const toCreateObject = {
|
128
|
+
...doc,
|
129
|
+
};
|
130
|
+
delete toCreateObject.$databaseId;
|
131
|
+
delete toCreateObject.$collectionId;
|
132
|
+
delete toCreateObject.$createdAt;
|
133
|
+
delete toCreateObject.$updatedAt;
|
134
|
+
delete toCreateObject.$id;
|
135
|
+
delete toCreateObject.$permissions;
|
136
|
+
return tryAwaitWithRetry(async () => db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
|
137
|
+
});
|
138
|
+
await Promise.all(batchedPromises);
|
139
|
+
totalDocumentsTransferred += fromCollDocs.documents.length;
|
140
|
+
while (fromCollDocs.documents.length === 50) {
|
141
|
+
fromCollDocs = await tryAwaitWithRetry(async () => await db.listDocuments(fromDbId, fromCollId, [
|
142
|
+
Query.limit(50),
|
143
|
+
Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
|
144
|
+
]));
|
145
|
+
const batchedPromises = fromCollDocs.documents.map((doc) => {
|
146
|
+
const toCreateObject = {
|
147
|
+
...doc,
|
148
|
+
};
|
149
|
+
delete toCreateObject.$databaseId;
|
150
|
+
delete toCreateObject.$collectionId;
|
151
|
+
delete toCreateObject.$createdAt;
|
152
|
+
delete toCreateObject.$updatedAt;
|
153
|
+
delete toCreateObject.$id;
|
154
|
+
delete toCreateObject.$permissions;
|
155
|
+
return tryAwaitWithRetry(async () => await db.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
|
156
|
+
});
|
157
|
+
await Promise.all(batchedPromises);
|
158
|
+
totalDocumentsTransferred += fromCollDocs.documents.length;
|
159
|
+
}
|
160
|
+
}
|
161
|
+
console.log(`Transferred ${totalDocumentsTransferred} documents from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}`);
|
162
|
+
};
|
163
|
+
export const transferDocumentsBetweenDbsLocalToRemote = async (localDb, endpoint, projectId, apiKey, fromDbId, toDbId, fromCollId, toCollId) => {
|
164
|
+
const client = new Client()
|
165
|
+
.setEndpoint(endpoint)
|
166
|
+
.setProject(projectId)
|
167
|
+
.setKey(apiKey);
|
168
|
+
let totalDocumentsTransferred = 0;
|
169
|
+
const remoteDb = new Databases(client);
|
170
|
+
let fromCollDocs = await tryAwaitWithRetry(async () => localDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)]));
|
171
|
+
if (fromCollDocs.documents.length === 0) {
|
172
|
+
console.log(`No documents found in collection ${fromCollId}`);
|
173
|
+
return;
|
174
|
+
}
|
175
|
+
else if (fromCollDocs.documents.length < 50) {
|
176
|
+
const batchedPromises = fromCollDocs.documents.map((doc) => {
|
177
|
+
const toCreateObject = {
|
178
|
+
...doc,
|
179
|
+
};
|
180
|
+
delete toCreateObject.$databaseId;
|
181
|
+
delete toCreateObject.$collectionId;
|
182
|
+
delete toCreateObject.$createdAt;
|
183
|
+
delete toCreateObject.$updatedAt;
|
184
|
+
delete toCreateObject.$id;
|
185
|
+
delete toCreateObject.$permissions;
|
186
|
+
return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
|
187
|
+
});
|
188
|
+
await Promise.all(batchedPromises);
|
189
|
+
totalDocumentsTransferred += fromCollDocs.documents.length;
|
190
|
+
}
|
191
|
+
else {
|
192
|
+
const batchedPromises = fromCollDocs.documents.map((doc) => {
|
193
|
+
const toCreateObject = {
|
194
|
+
...doc,
|
195
|
+
};
|
196
|
+
delete toCreateObject.$databaseId;
|
197
|
+
delete toCreateObject.$collectionId;
|
198
|
+
delete toCreateObject.$createdAt;
|
199
|
+
delete toCreateObject.$updatedAt;
|
200
|
+
delete toCreateObject.$id;
|
201
|
+
delete toCreateObject.$permissions;
|
202
|
+
return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
|
203
|
+
});
|
204
|
+
await Promise.all(batchedPromises);
|
205
|
+
totalDocumentsTransferred += fromCollDocs.documents.length;
|
206
|
+
while (fromCollDocs.documents.length === 50) {
|
207
|
+
fromCollDocs = await tryAwaitWithRetry(async () => localDb.listDocuments(fromDbId, fromCollId, [
|
208
|
+
Query.limit(50),
|
209
|
+
Query.cursorAfter(fromCollDocs.documents[fromCollDocs.documents.length - 1].$id),
|
210
|
+
]));
|
211
|
+
const batchedPromises = fromCollDocs.documents.map((doc) => {
|
212
|
+
const toCreateObject = {
|
213
|
+
...doc,
|
214
|
+
};
|
215
|
+
delete toCreateObject.$databaseId;
|
216
|
+
delete toCreateObject.$collectionId;
|
217
|
+
delete toCreateObject.$createdAt;
|
218
|
+
delete toCreateObject.$updatedAt;
|
219
|
+
delete toCreateObject.$id;
|
220
|
+
delete toCreateObject.$permissions;
|
221
|
+
return tryAwaitWithRetry(async () => remoteDb.createDocument(toDbId, toCollId, doc.$id, toCreateObject, doc.$permissions));
|
222
|
+
});
|
223
|
+
await Promise.all(batchedPromises);
|
224
|
+
totalDocumentsTransferred += fromCollDocs.documents.length;
|
225
|
+
}
|
226
|
+
}
|
227
|
+
console.log(`Total documents transferred from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}: ${totalDocumentsTransferred}`);
|
228
|
+
};
|
229
|
+
/**
|
230
|
+
* Transfers all collections and documents from one local database to another local database.
|
231
|
+
*
|
232
|
+
* @param {Databases} localDb - The local database instance.
|
233
|
+
* @param {string} fromDbId - The ID of the source database.
|
234
|
+
* @param {string} targetDbId - The ID of the target database.
|
235
|
+
* @return {Promise<void>} A promise that resolves when the transfer is complete.
|
236
|
+
*/
|
237
|
+
export const transferDatabaseLocalToLocal = async (localDb, fromDbId, targetDbId) => {
|
238
|
+
let lastCollectionId;
|
239
|
+
let fromCollections = await tryAwaitWithRetry(async () => await localDb.listCollections(fromDbId, [Query.limit(50)]));
|
240
|
+
const allFromCollections = fromCollections.collections;
|
241
|
+
if (fromCollections.collections.length < 50) {
|
242
|
+
lastCollectionId = undefined;
|
243
|
+
}
|
244
|
+
else {
|
245
|
+
lastCollectionId =
|
246
|
+
fromCollections.collections[fromCollections.collections.length - 1].$id;
|
247
|
+
while (lastCollectionId) {
|
248
|
+
const collections = await localDb.listCollections(fromDbId, [
|
249
|
+
Query.limit(50),
|
250
|
+
Query.cursorAfter(lastCollectionId),
|
251
|
+
]);
|
252
|
+
allFromCollections.push(...collections.collections);
|
253
|
+
if (collections.collections.length < 50) {
|
254
|
+
break;
|
255
|
+
}
|
256
|
+
lastCollectionId =
|
257
|
+
collections.collections[collections.collections.length - 1].$id;
|
258
|
+
}
|
259
|
+
}
|
260
|
+
lastCollectionId = undefined;
|
261
|
+
let toCollections = await tryAwaitWithRetry(async () => await localDb.listCollections(targetDbId, [Query.limit(50)]));
|
262
|
+
const allToCollections = toCollections.collections;
|
263
|
+
if (toCollections.collections.length < 50) {
|
264
|
+
}
|
265
|
+
else {
|
266
|
+
lastCollectionId =
|
267
|
+
toCollections.collections[toCollections.collections.length - 1].$id;
|
268
|
+
while (lastCollectionId) {
|
269
|
+
const collections = await localDb.listCollections(targetDbId, [
|
270
|
+
Query.limit(50),
|
271
|
+
Query.cursorAfter(lastCollectionId),
|
272
|
+
]);
|
273
|
+
allToCollections.push(...collections.collections);
|
274
|
+
if (collections.collections.length < 50) {
|
275
|
+
lastCollectionId = undefined;
|
276
|
+
}
|
277
|
+
else {
|
278
|
+
lastCollectionId =
|
279
|
+
collections.collections[collections.collections.length - 1].$id;
|
280
|
+
}
|
281
|
+
}
|
282
|
+
}
|
283
|
+
for (const collection of allFromCollections) {
|
284
|
+
const toCollection = allToCollections.find((c) => c.$id === collection.$id);
|
285
|
+
if (toCollection) {
|
286
|
+
await transferDocumentsBetweenDbsLocalToLocal(localDb, fromDbId, targetDbId, collection.$id, toCollection.$id);
|
287
|
+
}
|
288
|
+
else {
|
289
|
+
console.log(`Collection ${collection.name} not found in destination database, creating...`);
|
290
|
+
const newCollection = await tryAwaitWithRetry(async () => await localDb.createCollection(targetDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
|
291
|
+
console.log(`Collection ${newCollection.name} created`);
|
292
|
+
for (const attribute of collection.attributes) {
|
293
|
+
await tryAwaitWithRetry(async () => await createOrUpdateAttribute(localDb, targetDbId, newCollection, parseAttribute(attribute)));
|
294
|
+
}
|
295
|
+
for (const index of collection.indexes) {
|
296
|
+
await tryAwaitWithRetry(async () => await localDb.createIndex(targetDbId, newCollection.$id, index.key, index.type, index.attributes, index.orders));
|
297
|
+
}
|
298
|
+
await transferDocumentsBetweenDbsLocalToLocal(localDb, fromDbId, targetDbId, collection.$id, newCollection.$id);
|
299
|
+
}
|
300
|
+
}
|
301
|
+
};
|
302
|
+
export const transferDatabaseLocalToRemote = async (localDb, endpoint, projectId, apiKey, fromDbId, toDbId) => {
|
303
|
+
const client = getAppwriteClient(endpoint, projectId, apiKey);
|
304
|
+
const remoteDb = new Databases(client);
|
305
|
+
let lastCollectionId;
|
306
|
+
let fromCollections = await tryAwaitWithRetry(async () => await localDb.listCollections(fromDbId, [Query.limit(50)]));
|
307
|
+
const allFromCollections = fromCollections.collections;
|
308
|
+
if (fromCollections.collections.length < 50) {
|
309
|
+
}
|
310
|
+
else {
|
311
|
+
lastCollectionId =
|
312
|
+
fromCollections.collections[fromCollections.collections.length - 1].$id;
|
313
|
+
while (lastCollectionId) {
|
314
|
+
const collections = await tryAwaitWithRetry(async () => await localDb.listCollections(fromDbId, [
|
315
|
+
Query.limit(50),
|
316
|
+
Query.cursorAfter(lastCollectionId),
|
317
|
+
]));
|
318
|
+
allFromCollections.push(...collections.collections);
|
319
|
+
if (collections.collections.length < 50) {
|
320
|
+
break;
|
321
|
+
}
|
322
|
+
lastCollectionId =
|
323
|
+
collections.collections[collections.collections.length - 1].$id;
|
324
|
+
}
|
325
|
+
}
|
326
|
+
for (const collection of allFromCollections) {
|
327
|
+
const toCollection = await tryAwaitWithRetry(async () => await remoteDb.createCollection(toDbId, collection.$id, collection.name, collection.$permissions, collection.documentSecurity, collection.enabled));
|
328
|
+
console.log(`Collection ${toCollection.name} created`);
|
329
|
+
for (const attribute of collection.attributes) {
|
330
|
+
await tryAwaitWithRetry(async () => await createOrUpdateAttribute(remoteDb, toDbId, toCollection, parseAttribute(attribute)));
|
331
|
+
}
|
332
|
+
for (const index of collection.indexes) {
|
333
|
+
await tryAwaitWithRetry(async () => await remoteDb.createIndex(toDbId, toCollection.$id, index.key, index.type, index.attributes, index.orders));
|
334
|
+
}
|
335
|
+
await transferDocumentsBetweenDbsLocalToRemote(localDb, endpoint, projectId, apiKey, fromDbId, toDbId, collection.$id, toCollection.$id);
|
336
|
+
}
|
337
|
+
};
|
@@ -9,7 +9,8 @@ export declare class UsersController {
|
|
9
9
|
wipeUsers(): Promise<void>;
|
10
10
|
getAllUsers(): Promise<Models.User<Models.Preferences>[]>;
|
11
11
|
createUsersAndReturn(items: AuthUserCreate[]): Promise<Models.User<Models.Preferences>[]>;
|
12
|
-
createUserAndReturn(item: AuthUserCreate): Promise<Models.User<Models.Preferences>>;
|
12
|
+
createUserAndReturn(item: AuthUserCreate, numAttempts?: number): Promise<Models.User<Models.Preferences>>;
|
13
13
|
createAndCheckForUserAndReturn(item: AuthUserCreate): Promise<Models.User<Models.Preferences> | undefined>;
|
14
14
|
getUserIdByEmailOrPhone(email?: string, phone?: string): Promise<string | undefined>;
|
15
|
+
transferUsersBetweenDbsLocalToRemote: (endpoint: string, projectId: string, apiKey: string) => Promise<void>;
|
15
16
|
}
|
package/dist/migrations/users.js
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
-
import { Databases, ID, Query, Users } from "node-appwrite";
|
1
|
+
import { AppwriteException, Databases, ID, Query, Users, } from "node-appwrite";
|
2
2
|
import { AuthUserSchema, } from "../schemas/authUser.js";
|
3
3
|
import _ from "lodash";
|
4
4
|
import { logger } from "./logging.js";
|
5
5
|
import { splitIntoBatches } from "./migrationHelper.js";
|
6
|
+
import { getAppwriteClient, tryAwaitWithRetry, } from "../utils/helperFunctions.js";
|
6
7
|
export class UsersController {
|
7
8
|
config;
|
8
9
|
users;
|
@@ -22,66 +23,45 @@ export class UsersController {
|
|
22
23
|
this.users = new Users(this.config.appwriteClient);
|
23
24
|
}
|
24
25
|
async wipeUsers() {
|
25
|
-
const
|
26
|
-
const allUsers = users.users;
|
27
|
-
let lastDocumentId;
|
28
|
-
if (users.users.length >= 200) {
|
29
|
-
lastDocumentId = users.users[users.users.length - 1].$id;
|
30
|
-
}
|
31
|
-
while (lastDocumentId) {
|
32
|
-
const moreUsers = await this.users.list([
|
33
|
-
Query.limit(200),
|
34
|
-
Query.cursorAfter(lastDocumentId),
|
35
|
-
]);
|
36
|
-
allUsers.push(...moreUsers.users);
|
37
|
-
if (moreUsers.users.length < 200) {
|
38
|
-
break;
|
39
|
-
}
|
40
|
-
lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
|
41
|
-
}
|
26
|
+
const allUsers = await this.getAllUsers();
|
42
27
|
console.log("Deleting all users...");
|
43
|
-
const createBatches = (finalData) => {
|
44
|
-
let maxBatchLength = 5;
|
28
|
+
const createBatches = (finalData, batchSize) => {
|
45
29
|
const finalBatches = [];
|
46
|
-
for (let i = 0; i < finalData.length; i
|
47
|
-
|
48
|
-
finalBatches.push([]);
|
49
|
-
}
|
50
|
-
finalBatches[finalBatches.length - 1].push(finalData[i]);
|
30
|
+
for (let i = 0; i < finalData.length; i += batchSize) {
|
31
|
+
finalBatches.push(finalData.slice(i, i + batchSize));
|
51
32
|
}
|
52
33
|
return finalBatches;
|
53
34
|
};
|
54
|
-
// const userPromises: Promise<string>[] = [];
|
55
35
|
let usersDeleted = 0;
|
56
|
-
|
57
|
-
|
58
|
-
|
36
|
+
const batchedUserPromises = createBatches(allUsers, 50); // Batch size of 10
|
37
|
+
for (const batch of batchedUserPromises) {
|
38
|
+
console.log(`Deleting ${batch.length} users...`);
|
39
|
+
await Promise.all(batch.map((user) => tryAwaitWithRetry(async () => await this.users.delete(user.$id))));
|
40
|
+
usersDeleted += batch.length;
|
59
41
|
if (usersDeleted % 100 === 0) {
|
60
42
|
console.log(`Deleted ${usersDeleted} users...`);
|
61
43
|
}
|
62
44
|
}
|
63
|
-
// const batchedUserPromises = createBatches(userPromises);
|
64
|
-
// for (const batch of batchedUserPromises) {
|
65
|
-
// console.log(`Deleting ${batch.length} users...`);
|
66
|
-
// await Promise.all(batch);
|
67
|
-
// }
|
68
45
|
}
|
69
46
|
async getAllUsers() {
|
70
47
|
const allUsers = [];
|
71
|
-
const users = await this.users.list([Query.limit(200)]);
|
48
|
+
const users = await tryAwaitWithRetry(async () => await this.users.list([Query.limit(200)]));
|
49
|
+
if (users.users.length === 0) {
|
50
|
+
return [];
|
51
|
+
}
|
72
52
|
if (users.users.length === 200) {
|
73
53
|
let lastDocumentId = users.users[users.users.length - 1].$id;
|
74
54
|
allUsers.push(...users.users);
|
75
55
|
while (lastDocumentId) {
|
76
|
-
const moreUsers = await this.users.list([
|
56
|
+
const moreUsers = await tryAwaitWithRetry(async () => await this.users.list([
|
77
57
|
Query.limit(200),
|
78
58
|
Query.cursorAfter(lastDocumentId),
|
79
|
-
]);
|
59
|
+
]));
|
80
60
|
allUsers.push(...moreUsers.users);
|
81
|
-
lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
|
82
61
|
if (moreUsers.users.length < 200) {
|
83
62
|
break;
|
84
63
|
}
|
64
|
+
lastDocumentId = moreUsers.users[moreUsers.users.length - 1].$id;
|
85
65
|
}
|
86
66
|
}
|
87
67
|
else {
|
@@ -93,13 +73,11 @@ export class UsersController {
|
|
93
73
|
const users = await Promise.all(items.map((item) => this.createUserAndReturn(item)));
|
94
74
|
return users;
|
95
75
|
}
|
96
|
-
async createUserAndReturn(item) {
|
76
|
+
async createUserAndReturn(item, numAttempts) {
|
97
77
|
try {
|
98
78
|
const user = await this.users.create(item.userId || ID.unique(), item.email || undefined, item.phone && item.phone.length < 15 && item.phone.startsWith("+")
|
99
79
|
? item.phone
|
100
|
-
: undefined, item.
|
101
|
-
`changeMe${item.email?.toLowerCase()}` ||
|
102
|
-
`changeMePlease`, item.name || undefined);
|
80
|
+
: undefined, `changeMe${item.email?.toLowerCase()}` || `changeMePlease`, item.name || undefined);
|
103
81
|
if (item.labels) {
|
104
82
|
await this.users.updateLabels(user.$id, item.labels);
|
105
83
|
}
|
@@ -109,9 +87,21 @@ export class UsersController {
|
|
109
87
|
return user;
|
110
88
|
}
|
111
89
|
catch (e) {
|
90
|
+
if (e instanceof AppwriteException) {
|
91
|
+
if (e.message.toLowerCase().includes("fetch failed") ||
|
92
|
+
e.message.toLowerCase().includes("server error")) {
|
93
|
+
const numberOfAttempts = numAttempts || 0;
|
94
|
+
if (numberOfAttempts > 5) {
|
95
|
+
throw e;
|
96
|
+
}
|
97
|
+
const user = await this.createUserAndReturn(item, numberOfAttempts + 1);
|
98
|
+
return user;
|
99
|
+
}
|
100
|
+
}
|
112
101
|
if (e instanceof Error) {
|
113
|
-
logger.error("FAILED CREATING USER: ", e.message);
|
102
|
+
logger.error("FAILED CREATING USER: ", e.message, item);
|
114
103
|
}
|
104
|
+
console.log("FAILED CREATING USER: ", e, item);
|
115
105
|
throw e;
|
116
106
|
}
|
117
107
|
}
|
@@ -242,4 +232,49 @@ export class UsersController {
|
|
242
232
|
}
|
243
233
|
}
|
244
234
|
}
|
235
|
+
transferUsersBetweenDbsLocalToRemote = async (endpoint, projectId, apiKey) => {
|
236
|
+
const localUsers = this.users;
|
237
|
+
const client = getAppwriteClient(endpoint, projectId, apiKey);
|
238
|
+
const remoteUsers = new Users(client);
|
239
|
+
let fromUsers = await localUsers.list([Query.limit(50)]);
|
240
|
+
if (fromUsers.users.length === 0) {
|
241
|
+
console.log(`No users found`);
|
242
|
+
return;
|
243
|
+
}
|
244
|
+
else if (fromUsers.users.length < 50) {
|
245
|
+
console.log(`Transferring ${fromUsers.users.length} users to remote`);
|
246
|
+
const batchedPromises = fromUsers.users.map((user) => {
|
247
|
+
return tryAwaitWithRetry(async () => {
|
248
|
+
const toCreateObject = {
|
249
|
+
...user,
|
250
|
+
};
|
251
|
+
delete toCreateObject.$id;
|
252
|
+
delete toCreateObject.$createdAt;
|
253
|
+
delete toCreateObject.$updatedAt;
|
254
|
+
await remoteUsers.create(user.$id, user.email, user.phone, user.password, user.name);
|
255
|
+
});
|
256
|
+
});
|
257
|
+
await Promise.all(batchedPromises);
|
258
|
+
}
|
259
|
+
else {
|
260
|
+
while (fromUsers.users.length === 50) {
|
261
|
+
fromUsers = await localUsers.list([
|
262
|
+
Query.limit(50),
|
263
|
+
Query.cursorAfter(fromUsers.users[fromUsers.users.length - 1].$id),
|
264
|
+
]);
|
265
|
+
const batchedPromises = fromUsers.users.map((user) => {
|
266
|
+
return tryAwaitWithRetry(async () => {
|
267
|
+
const toCreateObject = {
|
268
|
+
...user,
|
269
|
+
};
|
270
|
+
delete toCreateObject.$id;
|
271
|
+
delete toCreateObject.$createdAt;
|
272
|
+
delete toCreateObject.$updatedAt;
|
273
|
+
await remoteUsers.create(user.$id, user.email, user.phone, user.password, user.name);
|
274
|
+
});
|
275
|
+
});
|
276
|
+
await Promise.all(batchedPromises);
|
277
|
+
}
|
278
|
+
}
|
279
|
+
};
|
245
280
|
}
|
@@ -44,16 +44,16 @@ export declare const AuthUserCreateSchema: z.ZodObject<z.objectUtil.extendShape<
|
|
44
44
|
prefs: Record<string, string>;
|
45
45
|
labels: string[];
|
46
46
|
email?: string | null | undefined;
|
47
|
-
password?: string | undefined;
|
48
47
|
name?: string | null | undefined;
|
48
|
+
password?: string | undefined;
|
49
49
|
$createdAt?: string | undefined;
|
50
50
|
$updatedAt?: string | undefined;
|
51
51
|
phone?: string | null | undefined;
|
52
52
|
userId?: string | undefined;
|
53
53
|
}, {
|
54
54
|
email?: string | null | undefined;
|
55
|
-
password?: string | undefined;
|
56
55
|
name?: string | null | undefined;
|
56
|
+
password?: string | undefined;
|
57
57
|
$createdAt?: string | undefined;
|
58
58
|
$updatedAt?: string | undefined;
|
59
59
|
phone?: string | null | undefined;
|
@@ -0,0 +1,15 @@
|
|
1
|
+
import { Databases, Storage, type Models } from "node-appwrite";
|
2
|
+
import { type AppwriteConfig } from "appwrite-utils";
|
3
|
+
export declare const getStorage: (config: AppwriteConfig) => Storage;
|
4
|
+
export declare const listBuckets: (storage: Storage, queries?: string[], search?: string) => Promise<Models.BucketList>;
|
5
|
+
export declare const getBucket: (storage: Storage, bucketId: string) => Promise<Models.Bucket>;
|
6
|
+
export declare const createBucket: (storage: Storage, bucket: Omit<Models.Bucket, "$id" | "$createdAt" | "$updatedAt">, bucketId?: string) => Promise<Models.Bucket>;
|
7
|
+
export declare const updateBucket: (storage: Storage, bucket: Models.Bucket, bucketId: string) => Promise<Models.Bucket>;
|
8
|
+
export declare const deleteBucket: (storage: Storage, bucketId: string) => Promise<{}>;
|
9
|
+
export declare const getFile: (storage: Storage, bucketId: string, fileId: string) => Promise<Models.File>;
|
10
|
+
export declare const listFiles: (storage: Storage, bucketId: string, queries?: string[], search?: string) => Promise<Models.FileList>;
|
11
|
+
export declare const deleteFile: (storage: Storage, bucketId: string, fileId: string) => Promise<{}>;
|
12
|
+
export declare const wipeDocumentStorage: (storage: Storage, bucketId: string) => Promise<void>;
|
13
|
+
export declare const initOrGetDocumentStorage: (storage: Storage, config: AppwriteConfig, dbId: string, bucketName?: string) => Promise<Models.Bucket>;
|
14
|
+
export declare const initOrGetBackupStorage: (config: AppwriteConfig, storage: Storage) => Promise<Models.Bucket>;
|
15
|
+
export declare const backupDatabase: (config: AppwriteConfig, database: Databases, databaseId: string, storage: Storage) => Promise<void>;
|