appwrite-utils-cli 0.10.80 → 0.10.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/collections/methods.js +13 -6
- package/dist/interactiveCLI.js +6 -1
- package/dist/migrations/backup.d.ts +2 -2
- package/dist/migrations/dataLoader.d.ts +5 -15
- package/dist/migrations/importController.js +21 -13
- package/dist/storage/schemas.d.ts +2 -2
- package/package.json +2 -2
- package/src/collections/methods.ts +16 -6
- package/src/interactiveCLI.ts +9 -1
- package/src/migrations/dataLoader.ts +2 -2
- package/src/migrations/importController.ts +72 -23
package/README.md
CHANGED
@@ -150,6 +150,7 @@ This updated CLI ensures that developers have robust tools at their fingertips t
|
|
150
150
|
|
151
151
|
## Changelog
|
152
152
|
|
153
|
+
- 0.10.81: Fixed `wipeCollection` -- it wasn't properly deleting all files in a loop
|
153
154
|
- 0.10.80: Updated `appwrite-utils` req
|
154
155
|
- 0.10.78: Fixed `attributesSame` so it will properly update attributes that have changed
|
155
156
|
- 0.10.77: Added disclaimer to update function specifications for bug
|
@@ -2,7 +2,7 @@ import { Client, Databases, ID, Permission, Query, } from "node-appwrite";
|
|
2
2
|
import { nameToIdMapping, processQueue } from "../migrations/queue.js";
|
3
3
|
import { createUpdateCollectionAttributes } from "./attributes.js";
|
4
4
|
import { createOrUpdateIndexes } from "./indexes.js";
|
5
|
-
import _ from "lodash";
|
5
|
+
import _, { initial } from "lodash";
|
6
6
|
import { SchemaGenerator } from "../migrations/schemaStrings.js";
|
7
7
|
import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
|
8
8
|
export const documentExists = async (db, dbId, targetCollectionId, toCreateObject) => {
|
@@ -91,18 +91,25 @@ async function wipeDocumentsFromCollection(database, databaseId, collectionId) {
|
|
91
91
|
const initialDocuments = await database.listDocuments(databaseId, collectionId, [Query.limit(1000)]);
|
92
92
|
let documents = initialDocuments.documents;
|
93
93
|
let totalDocuments = documents.length;
|
94
|
-
|
94
|
+
let cursor = initialDocuments.documents.length >= 1000
|
95
|
+
? initialDocuments.documents[initialDocuments.documents.length - 1].$id
|
96
|
+
: undefined;
|
97
|
+
while (cursor) {
|
95
98
|
const docsResponse = await database.listDocuments(databaseId, collectionId, [Query.limit(1000)]);
|
96
|
-
documents
|
99
|
+
documents.push(...docsResponse.documents);
|
97
100
|
totalDocuments = documents.length;
|
101
|
+
cursor =
|
102
|
+
docsResponse.documents.length >= 1000
|
103
|
+
? docsResponse.documents[docsResponse.documents.length - 1].$id
|
104
|
+
: undefined;
|
98
105
|
}
|
99
106
|
console.log(`Found ${totalDocuments} documents to delete`);
|
100
|
-
const maxStackSize =
|
107
|
+
const maxStackSize = 50; // Reduced batch size
|
101
108
|
for (let i = 0; i < documents.length; i += maxStackSize) {
|
102
109
|
const batch = documents.slice(i, i + maxStackSize);
|
103
110
|
const deletePromises = batch.map(async (doc) => {
|
104
111
|
try {
|
105
|
-
await database.deleteDocument(databaseId, collectionId, doc.$id);
|
112
|
+
await tryAwaitWithRetry(async () => database.deleteDocument(databaseId, collectionId, doc.$id));
|
106
113
|
}
|
107
114
|
catch (error) {
|
108
115
|
// Skip if document doesn't exist or other non-critical errors
|
@@ -112,7 +119,7 @@ async function wipeDocumentsFromCollection(database, databaseId, collectionId) {
|
|
112
119
|
}
|
113
120
|
});
|
114
121
|
await Promise.all(deletePromises);
|
115
|
-
await delay(
|
122
|
+
await delay(100); // Increased delay between batches
|
116
123
|
console.log(`Deleted batch of ${batch.length} documents (${i + batch.length}/${totalDocuments})`);
|
117
124
|
}
|
118
125
|
console.log(`Completed deletion of ${totalDocuments} documents from collection ${collectionId}`);
|
package/dist/interactiveCLI.js
CHANGED
@@ -176,16 +176,18 @@ export class InteractiveCLI {
|
|
176
176
|
await this.initControllerIfNeeded();
|
177
177
|
const configCollections = this.getLocalCollections();
|
178
178
|
let remoteCollections = [];
|
179
|
+
let shouldFilterByDatabase = true;
|
179
180
|
const dbExists = await databasesClient.list([
|
180
181
|
Query.equal("name", database.name),
|
181
182
|
]);
|
182
183
|
if (dbExists.total === 0) {
|
183
184
|
console.log(chalk.red(`Database "${database.name}" does not exist, using only local collection options`));
|
185
|
+
shouldFilterByDatabase = false;
|
184
186
|
}
|
185
187
|
else {
|
186
188
|
remoteCollections = await fetchAllCollections(database.$id, databasesClient);
|
187
189
|
}
|
188
|
-
|
190
|
+
let allCollections = preferLocal
|
189
191
|
? remoteCollections.reduce((acc, remoteCollection) => {
|
190
192
|
if (!acc.some((c) => c.name === remoteCollection.name)) {
|
191
193
|
acc.push(remoteCollection);
|
@@ -196,6 +198,9 @@ export class InteractiveCLI {
|
|
196
198
|
...remoteCollections,
|
197
199
|
...configCollections.filter((c) => !remoteCollections.some((rc) => rc.name === c.name)),
|
198
200
|
];
|
201
|
+
if (shouldFilterByDatabase) {
|
202
|
+
allCollections = allCollections.filter((c) => c.databaseId === database.$id);
|
203
|
+
}
|
199
204
|
const hasLocalAndRemote = allCollections.some((coll) => configCollections.some((c) => c.name === coll.name)) &&
|
200
205
|
allCollections.some((coll) => !configCollections.some((c) => c.name === coll.name));
|
201
206
|
const choices = allCollections
|
@@ -275,7 +275,6 @@ export declare const getMigrationCollectionSchemas: () => {
|
|
275
275
|
targetField?: string | undefined;
|
276
276
|
} | undefined;
|
277
277
|
})[];
|
278
|
-
$id: string;
|
279
278
|
$permissions: {
|
280
279
|
permission: string;
|
281
280
|
target: string;
|
@@ -326,6 +325,7 @@ export declare const getMigrationCollectionSchemas: () => {
|
|
326
325
|
} | undefined;
|
327
326
|
}[];
|
328
327
|
description?: string | undefined;
|
328
|
+
$id?: string | undefined;
|
329
329
|
enabled?: boolean | undefined;
|
330
330
|
documentSecurity?: boolean | undefined;
|
331
331
|
databaseId?: string | undefined;
|
@@ -530,7 +530,6 @@ export declare const getMigrationCollectionSchemas: () => {
|
|
530
530
|
targetField?: string | undefined;
|
531
531
|
} | undefined;
|
532
532
|
})[];
|
533
|
-
$id: string;
|
534
533
|
$permissions: {
|
535
534
|
permission: string;
|
536
535
|
target: string;
|
@@ -581,6 +580,7 @@ export declare const getMigrationCollectionSchemas: () => {
|
|
581
580
|
} | undefined;
|
582
581
|
}[];
|
583
582
|
description?: string | undefined;
|
583
|
+
$id?: string | undefined;
|
584
584
|
enabled?: boolean | undefined;
|
585
585
|
documentSecurity?: boolean | undefined;
|
586
586
|
databaseId?: string | undefined;
|
@@ -6,7 +6,7 @@ import { AuthUserCreateSchema } from "../schemas/authUser.js";
|
|
6
6
|
export declare const CollectionImportDataSchema: z.ZodObject<{
|
7
7
|
collection: z.ZodOptional<z.ZodObject<Omit<{
|
8
8
|
name: z.ZodString;
|
9
|
-
$id: z.
|
9
|
+
$id: z.ZodOptional<z.ZodString>;
|
10
10
|
enabled: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
|
11
11
|
documentSecurity: z.ZodOptional<z.ZodDefault<z.ZodBoolean>>;
|
12
12
|
description: z.ZodOptional<z.ZodString>;
|
@@ -79,17 +79,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
79
79
|
key: string;
|
80
80
|
type?: "integer" | undefined;
|
81
81
|
description?: string | Record<string, string> | null | undefined;
|
82
|
-
required
|
83
|
-
/**
|
84
|
-
* Prepares the data for creating user collection documents.
|
85
|
-
* This involves loading the data, transforming it according to the import definition,
|
86
|
-
* and handling the creation of new unique IDs for each item.
|
87
|
-
*
|
88
|
-
* @param db - The database configuration.
|
89
|
-
* @param collection - The collection configuration.
|
90
|
-
* @param importDef - The import definition containing the attribute mappings and other relevant info.
|
91
|
-
*/
|
92
|
-
?: boolean | undefined;
|
82
|
+
required?: boolean | undefined;
|
93
83
|
array?: boolean | undefined;
|
94
84
|
error?: string | undefined;
|
95
85
|
xdefault?: number | null | undefined;
|
@@ -648,7 +638,6 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
648
638
|
targetField?: string | undefined;
|
649
639
|
} | undefined;
|
650
640
|
})[];
|
651
|
-
$id: string;
|
652
641
|
$permissions: {
|
653
642
|
permission: string;
|
654
643
|
target: string;
|
@@ -699,6 +688,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
699
688
|
} | undefined;
|
700
689
|
}[];
|
701
690
|
description?: string | undefined;
|
691
|
+
$id?: string | undefined;
|
702
692
|
enabled?: boolean | undefined;
|
703
693
|
documentSecurity?: boolean | undefined;
|
704
694
|
databaseId?: string | undefined;
|
@@ -1268,7 +1258,6 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
1268
1258
|
targetField?: string | undefined;
|
1269
1259
|
} | undefined;
|
1270
1260
|
})[];
|
1271
|
-
$id: string;
|
1272
1261
|
$permissions: {
|
1273
1262
|
permission: string;
|
1274
1263
|
target: string;
|
@@ -1319,6 +1308,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
1319
1308
|
} | undefined;
|
1320
1309
|
}[];
|
1321
1310
|
description?: string | undefined;
|
1311
|
+
$id?: string | undefined;
|
1322
1312
|
enabled?: boolean | undefined;
|
1323
1313
|
documentSecurity?: boolean | undefined;
|
1324
1314
|
databaseId?: string | undefined;
|
@@ -1672,7 +1662,6 @@ export declare class DataLoader {
|
|
1672
1662
|
targetField?: string | undefined;
|
1673
1663
|
} | undefined;
|
1674
1664
|
})[];
|
1675
|
-
$id: string;
|
1676
1665
|
$permissions: {
|
1677
1666
|
permission: string;
|
1678
1667
|
target: string;
|
@@ -1723,6 +1712,7 @@ export declare class DataLoader {
|
|
1723
1712
|
} | undefined;
|
1724
1713
|
}[];
|
1725
1714
|
description?: string | undefined;
|
1715
|
+
$id?: string | undefined;
|
1726
1716
|
enabled?: boolean | undefined;
|
1727
1717
|
documentSecurity?: boolean | undefined;
|
1728
1718
|
databaseId?: string | undefined;
|
@@ -76,24 +76,24 @@ export class ImportController {
|
|
76
76
|
const updatedDbConfig = this.config.databases.find((db) => db.$id === updatedDb.$id);
|
77
77
|
const targetDbConfig = this.config.databases.find((db) => db.$id === targetDb.$id);
|
78
78
|
const allBuckets = await this.storage.listBuckets([Query.limit(1000)]);
|
79
|
-
const bucketsWithDbIdInThem = allBuckets.buckets.filter(bucket => bucket.name.toLowerCase().includes(updatedDb.$id.toLowerCase()));
|
79
|
+
const bucketsWithDbIdInThem = allBuckets.buckets.filter((bucket) => bucket.name.toLowerCase().includes(updatedDb.$id.toLowerCase()));
|
80
80
|
const configuredUpdatedBucketId = `${this.config.documentBucketId}_${updatedDb.$id.toLowerCase().trim().replace(" ", "")}`;
|
81
81
|
const configuredTargetBucketId = `${this.config.documentBucketId}_${targetDb.$id.toLowerCase().trim().replace(" ", "")}`;
|
82
82
|
let sourceBucketId;
|
83
83
|
let targetBucketId;
|
84
|
-
if (bucketsWithDbIdInThem.find(bucket => bucket.$id === configuredUpdatedBucketId)) {
|
84
|
+
if (bucketsWithDbIdInThem.find((bucket) => bucket.$id === configuredUpdatedBucketId)) {
|
85
85
|
sourceBucketId = configuredUpdatedBucketId;
|
86
86
|
}
|
87
|
-
else if (bucketsWithDbIdInThem.find(bucket => bucket.$id === configuredTargetBucketId)) {
|
87
|
+
else if (bucketsWithDbIdInThem.find((bucket) => bucket.$id === configuredTargetBucketId)) {
|
88
88
|
targetBucketId = configuredTargetBucketId;
|
89
89
|
}
|
90
90
|
if (!sourceBucketId) {
|
91
|
-
sourceBucketId =
|
92
|
-
bucketsWithDbIdInThem[0]?.$id;
|
91
|
+
sourceBucketId =
|
92
|
+
updatedDbConfig?.bucket?.$id || bucketsWithDbIdInThem[0]?.$id;
|
93
93
|
}
|
94
94
|
if (!targetBucketId) {
|
95
|
-
targetBucketId =
|
96
|
-
bucketsWithDbIdInThem[0]?.$id;
|
95
|
+
targetBucketId =
|
96
|
+
targetDbConfig?.bucket?.$id || bucketsWithDbIdInThem[0]?.$id;
|
97
97
|
}
|
98
98
|
if (sourceBucketId && targetBucketId) {
|
99
99
|
await transferStorageLocalToLocal(this.storage, sourceBucketId, targetBucketId);
|
@@ -101,11 +101,15 @@ export class ImportController {
|
|
101
101
|
}
|
102
102
|
}
|
103
103
|
async importCollections(db, dataLoader, specificCollections) {
|
104
|
-
const collectionsToImport = specificCollections ||
|
104
|
+
const collectionsToImport = specificCollections ||
|
105
|
+
(this.config.collections
|
106
|
+
? this.config.collections.map((c) => c.name)
|
107
|
+
: []);
|
105
108
|
for (const collection of this.config.collections || []) {
|
106
109
|
if (collectionsToImport.includes(collection.name)) {
|
107
|
-
let isUsersCollection = this.config.usersCollectionName &&
|
108
|
-
dataLoader.getCollectionKey(
|
110
|
+
let isUsersCollection = this.config.usersCollectionName &&
|
111
|
+
dataLoader.getCollectionKey(this.config.usersCollectionName) ===
|
112
|
+
dataLoader.getCollectionKey(collection.name);
|
109
113
|
const importOperationId = dataLoader.collectionImportOperations.get(dataLoader.getCollectionKey(collection.name));
|
110
114
|
const createBatches = (finalData) => {
|
111
115
|
let maxBatchLength = 50;
|
@@ -222,11 +226,15 @@ export class ImportController {
|
|
222
226
|
}
|
223
227
|
async executePostImportActions(dbId, dataLoader, specificCollections) {
|
224
228
|
console.log("Executing post-import actions...");
|
225
|
-
const collectionsToProcess = specificCollections && specificCollections.length > 0
|
229
|
+
const collectionsToProcess = specificCollections && specificCollections.length > 0
|
230
|
+
? specificCollections
|
231
|
+
: this.config.collections
|
232
|
+
? this.config.collections.map((c) => c.name)
|
233
|
+
: Array.from(dataLoader.importMap.keys());
|
226
234
|
console.log("Collections to process:", collectionsToProcess);
|
227
235
|
// Iterate over each collection in the importMap
|
228
|
-
for (const [collectionKey, collectionData] of dataLoader.importMap.entries()) {
|
229
|
-
const allCollectionKeys = collectionsToProcess.map(c => dataLoader.getCollectionKey(c));
|
236
|
+
for (const [collectionKey, collectionData,] of dataLoader.importMap.entries()) {
|
237
|
+
const allCollectionKeys = collectionsToProcess.map((c) => dataLoader.getCollectionKey(c));
|
230
238
|
if (allCollectionKeys.includes(collectionKey)) {
|
231
239
|
console.log(`Processing post-import actions for collection: ${collectionKey}`);
|
232
240
|
// Iterate over each item in the collectionData.data
|
@@ -275,7 +275,6 @@ export declare const getMigrationCollectionSchemas: () => {
|
|
275
275
|
targetField?: string | undefined;
|
276
276
|
} | undefined;
|
277
277
|
})[];
|
278
|
-
$id: string;
|
279
278
|
$permissions: {
|
280
279
|
permission: string;
|
281
280
|
target: string;
|
@@ -326,6 +325,7 @@ export declare const getMigrationCollectionSchemas: () => {
|
|
326
325
|
} | undefined;
|
327
326
|
}[];
|
328
327
|
description?: string | undefined;
|
328
|
+
$id?: string | undefined;
|
329
329
|
enabled?: boolean | undefined;
|
330
330
|
documentSecurity?: boolean | undefined;
|
331
331
|
databaseId?: string | undefined;
|
@@ -530,7 +530,6 @@ export declare const getMigrationCollectionSchemas: () => {
|
|
530
530
|
targetField?: string | undefined;
|
531
531
|
} | undefined;
|
532
532
|
})[];
|
533
|
-
$id: string;
|
534
533
|
$permissions: {
|
535
534
|
permission: string;
|
536
535
|
target: string;
|
@@ -581,6 +580,7 @@ export declare const getMigrationCollectionSchemas: () => {
|
|
581
580
|
} | undefined;
|
582
581
|
}[];
|
583
582
|
description?: string | undefined;
|
583
|
+
$id?: string | undefined;
|
584
584
|
enabled?: boolean | undefined;
|
585
585
|
documentSecurity?: boolean | undefined;
|
586
586
|
databaseId?: string | undefined;
|
package/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
{
|
2
2
|
"name": "appwrite-utils-cli",
|
3
3
|
"description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
|
4
|
-
"version": "0.10.
|
4
|
+
"version": "0.10.81",
|
5
5
|
"main": "src/main.ts",
|
6
6
|
"type": "module",
|
7
7
|
"repository": {
|
@@ -31,7 +31,7 @@
|
|
31
31
|
},
|
32
32
|
"dependencies": {
|
33
33
|
"@types/inquirer": "^9.0.7",
|
34
|
-
"appwrite-utils": "^0.4.
|
34
|
+
"appwrite-utils": "^0.4.1",
|
35
35
|
"chalk": "^5.3.0",
|
36
36
|
"cli-progress": "^3.12.0",
|
37
37
|
"commander": "^12.1.0",
|
@@ -10,7 +10,7 @@ import type { AppwriteConfig, CollectionCreate, Indexes } from "appwrite-utils";
|
|
10
10
|
import { nameToIdMapping, processQueue } from "../migrations/queue.js";
|
11
11
|
import { createUpdateCollectionAttributes } from "./attributes.js";
|
12
12
|
import { createOrUpdateIndexes } from "./indexes.js";
|
13
|
-
import _ from "lodash";
|
13
|
+
import _, { initial } from "lodash";
|
14
14
|
import { SchemaGenerator } from "../migrations/schemaStrings.js";
|
15
15
|
import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
|
16
16
|
|
@@ -137,25 +137,35 @@ async function wipeDocumentsFromCollection(
|
|
137
137
|
);
|
138
138
|
let documents = initialDocuments.documents;
|
139
139
|
let totalDocuments = documents.length;
|
140
|
+
let cursor =
|
141
|
+
initialDocuments.documents.length >= 1000
|
142
|
+
? initialDocuments.documents[initialDocuments.documents.length - 1].$id
|
143
|
+
: undefined;
|
140
144
|
|
141
|
-
while (
|
145
|
+
while (cursor) {
|
142
146
|
const docsResponse = await database.listDocuments(
|
143
147
|
databaseId,
|
144
148
|
collectionId,
|
145
149
|
[Query.limit(1000)]
|
146
150
|
);
|
147
|
-
documents
|
151
|
+
documents.push(...docsResponse.documents);
|
148
152
|
totalDocuments = documents.length;
|
153
|
+
cursor =
|
154
|
+
docsResponse.documents.length >= 1000
|
155
|
+
? docsResponse.documents[docsResponse.documents.length - 1].$id
|
156
|
+
: undefined;
|
149
157
|
}
|
150
158
|
|
151
159
|
console.log(`Found ${totalDocuments} documents to delete`);
|
152
160
|
|
153
|
-
const maxStackSize =
|
161
|
+
const maxStackSize = 50; // Reduced batch size
|
154
162
|
for (let i = 0; i < documents.length; i += maxStackSize) {
|
155
163
|
const batch = documents.slice(i, i + maxStackSize);
|
156
164
|
const deletePromises = batch.map(async (doc) => {
|
157
165
|
try {
|
158
|
-
await
|
166
|
+
await tryAwaitWithRetry(async () =>
|
167
|
+
database.deleteDocument(databaseId, collectionId, doc.$id)
|
168
|
+
);
|
159
169
|
} catch (error: any) {
|
160
170
|
// Skip if document doesn't exist or other non-critical errors
|
161
171
|
if (
|
@@ -172,7 +182,7 @@ async function wipeDocumentsFromCollection(
|
|
172
182
|
});
|
173
183
|
|
174
184
|
await Promise.all(deletePromises);
|
175
|
-
await delay(
|
185
|
+
await delay(100); // Increased delay between batches
|
176
186
|
|
177
187
|
console.log(
|
178
188
|
`Deleted batch of ${batch.length} documents (${
|
package/src/interactiveCLI.ts
CHANGED
@@ -239,6 +239,7 @@ export class InteractiveCLI {
|
|
239
239
|
|
240
240
|
const configCollections = this.getLocalCollections();
|
241
241
|
let remoteCollections: Models.Collection[] = [];
|
242
|
+
let shouldFilterByDatabase = true;
|
242
243
|
|
243
244
|
const dbExists = await databasesClient.list([
|
244
245
|
Query.equal("name", database.name),
|
@@ -249,6 +250,7 @@ export class InteractiveCLI {
|
|
249
250
|
`Database "${database.name}" does not exist, using only local collection options`
|
250
251
|
)
|
251
252
|
);
|
253
|
+
shouldFilterByDatabase = false;
|
252
254
|
} else {
|
253
255
|
remoteCollections = await fetchAllCollections(
|
254
256
|
database.$id,
|
@@ -256,7 +258,7 @@ export class InteractiveCLI {
|
|
256
258
|
);
|
257
259
|
}
|
258
260
|
|
259
|
-
|
261
|
+
let allCollections = preferLocal
|
260
262
|
? remoteCollections.reduce(
|
261
263
|
(acc, remoteCollection) => {
|
262
264
|
if (!acc.some((c) => c.name === remoteCollection.name)) {
|
@@ -273,6 +275,12 @@ export class InteractiveCLI {
|
|
273
275
|
),
|
274
276
|
];
|
275
277
|
|
278
|
+
if (shouldFilterByDatabase) {
|
279
|
+
allCollections = allCollections.filter(
|
280
|
+
(c) => c.databaseId === database.$id
|
281
|
+
);
|
282
|
+
}
|
283
|
+
|
276
284
|
const hasLocalAndRemote =
|
277
285
|
allCollections.some((coll) =>
|
278
286
|
configCollections.some((c) => c.name === coll.name)
|
@@ -953,7 +953,7 @@ export class DataLoader {
|
|
953
953
|
if (!operationId) {
|
954
954
|
const collectionImportOperation = await findOrCreateOperation(
|
955
955
|
this.database,
|
956
|
-
collection.$id
|
956
|
+
collection.$id!,
|
957
957
|
"importData"
|
958
958
|
);
|
959
959
|
// Store the operation ID in the map
|
@@ -1180,7 +1180,7 @@ export class DataLoader {
|
|
1180
1180
|
if (!operationId) {
|
1181
1181
|
const collectionImportOperation = await findOrCreateOperation(
|
1182
1182
|
this.database,
|
1183
|
-
collection.$id
|
1183
|
+
collection.$id!,
|
1184
1184
|
"importData"
|
1185
1185
|
);
|
1186
1186
|
// Store the operation ID in the map
|
@@ -104,7 +104,11 @@ export class ImportController {
|
|
104
104
|
await dataLoader.start(db.$id);
|
105
105
|
await this.importCollections(db, dataLoader, specificCollections);
|
106
106
|
await resolveAndUpdateRelationships(db.$id, this.database, this.config);
|
107
|
-
await this.executePostImportActions(
|
107
|
+
await this.executePostImportActions(
|
108
|
+
db.$id,
|
109
|
+
dataLoader,
|
110
|
+
specificCollections
|
111
|
+
);
|
108
112
|
} else if (databaseRan.$id !== db.$id) {
|
109
113
|
await this.updateOthersToFinalData(databaseRan, db);
|
110
114
|
}
|
@@ -137,27 +141,41 @@ export class ImportController {
|
|
137
141
|
);
|
138
142
|
|
139
143
|
const allBuckets = await this.storage.listBuckets([Query.limit(1000)]);
|
140
|
-
const bucketsWithDbIdInThem = allBuckets.buckets.filter(bucket =>
|
141
|
-
|
142
|
-
|
144
|
+
const bucketsWithDbIdInThem = allBuckets.buckets.filter((bucket) =>
|
145
|
+
bucket.name.toLowerCase().includes(updatedDb.$id.toLowerCase())
|
146
|
+
);
|
147
|
+
const configuredUpdatedBucketId = `${
|
148
|
+
this.config.documentBucketId
|
149
|
+
}_${updatedDb.$id.toLowerCase().trim().replace(" ", "")}`;
|
150
|
+
const configuredTargetBucketId = `${
|
151
|
+
this.config.documentBucketId
|
152
|
+
}_${targetDb.$id.toLowerCase().trim().replace(" ", "")}`;
|
143
153
|
|
144
154
|
let sourceBucketId: string | undefined;
|
145
155
|
let targetBucketId: string | undefined;
|
146
156
|
|
147
|
-
if (
|
157
|
+
if (
|
158
|
+
bucketsWithDbIdInThem.find(
|
159
|
+
(bucket) => bucket.$id === configuredUpdatedBucketId
|
160
|
+
)
|
161
|
+
) {
|
148
162
|
sourceBucketId = configuredUpdatedBucketId;
|
149
|
-
} else if (
|
163
|
+
} else if (
|
164
|
+
bucketsWithDbIdInThem.find(
|
165
|
+
(bucket) => bucket.$id === configuredTargetBucketId
|
166
|
+
)
|
167
|
+
) {
|
150
168
|
targetBucketId = configuredTargetBucketId;
|
151
169
|
}
|
152
170
|
|
153
171
|
if (!sourceBucketId) {
|
154
|
-
sourceBucketId =
|
155
|
-
bucketsWithDbIdInThem[0]?.$id;
|
172
|
+
sourceBucketId =
|
173
|
+
updatedDbConfig?.bucket?.$id || bucketsWithDbIdInThem[0]?.$id;
|
156
174
|
}
|
157
175
|
|
158
176
|
if (!targetBucketId) {
|
159
|
-
targetBucketId =
|
160
|
-
bucketsWithDbIdInThem[0]?.$id;
|
177
|
+
targetBucketId =
|
178
|
+
targetDbConfig?.bucket?.$id || bucketsWithDbIdInThem[0]?.$id;
|
161
179
|
}
|
162
180
|
|
163
181
|
if (sourceBucketId && targetBucketId) {
|
@@ -170,13 +188,23 @@ export class ImportController {
|
|
170
188
|
}
|
171
189
|
}
|
172
190
|
|
173
|
-
async importCollections(
|
174
|
-
|
191
|
+
async importCollections(
|
192
|
+
db: ConfigDatabase,
|
193
|
+
dataLoader: DataLoader,
|
194
|
+
specificCollections?: string[]
|
195
|
+
) {
|
196
|
+
const collectionsToImport =
|
197
|
+
specificCollections ||
|
198
|
+
(this.config.collections
|
199
|
+
? this.config.collections.map((c) => c.name)
|
200
|
+
: []);
|
175
201
|
|
176
202
|
for (const collection of this.config.collections || []) {
|
177
203
|
if (collectionsToImport.includes(collection.name)) {
|
178
|
-
let isUsersCollection =
|
179
|
-
|
204
|
+
let isUsersCollection =
|
205
|
+
this.config.usersCollectionName &&
|
206
|
+
dataLoader.getCollectionKey(this.config.usersCollectionName) ===
|
207
|
+
dataLoader.getCollectionKey(collection.name);
|
180
208
|
const importOperationId = dataLoader.collectionImportOperations.get(
|
181
209
|
dataLoader.getCollectionKey(collection.name)
|
182
210
|
);
|
@@ -197,7 +225,10 @@ export class ImportController {
|
|
197
225
|
dataLoader.getCollectionKey("users")
|
198
226
|
);
|
199
227
|
const usersData = usersDataMap?.data;
|
200
|
-
const usersController = new UsersController(
|
228
|
+
const usersController = new UsersController(
|
229
|
+
this.config,
|
230
|
+
this.database
|
231
|
+
);
|
201
232
|
if (usersData) {
|
202
233
|
console.log("Found users data", usersData.length);
|
203
234
|
const userDataBatches = createBatches(usersData);
|
@@ -230,7 +261,9 @@ export class ImportController {
|
|
230
261
|
);
|
231
262
|
return usersController.createUserAndReturn(item.finalData);
|
232
263
|
});
|
233
|
-
const promiseResults = await Promise.allSettled(
|
264
|
+
const promiseResults = await Promise.allSettled(
|
265
|
+
userBatchPromises
|
266
|
+
);
|
234
267
|
for (const item of batch) {
|
235
268
|
if (item && item.finalData) {
|
236
269
|
dataLoader.userExistsMap.set(
|
@@ -298,7 +331,7 @@ export class ImportController {
|
|
298
331
|
async () =>
|
299
332
|
await this.database.createDocument(
|
300
333
|
db.$id,
|
301
|
-
collection.$id
|
334
|
+
collection.$id!,
|
302
335
|
id,
|
303
336
|
item.finalData
|
304
337
|
)
|
@@ -324,13 +357,27 @@ export class ImportController {
|
|
324
357
|
}
|
325
358
|
}
|
326
359
|
|
327
|
-
async executePostImportActions(
|
360
|
+
async executePostImportActions(
|
361
|
+
dbId: string,
|
362
|
+
dataLoader: DataLoader,
|
363
|
+
specificCollections?: string[]
|
364
|
+
) {
|
328
365
|
console.log("Executing post-import actions...");
|
329
|
-
const collectionsToProcess =
|
366
|
+
const collectionsToProcess =
|
367
|
+
specificCollections && specificCollections.length > 0
|
368
|
+
? specificCollections
|
369
|
+
: this.config.collections
|
370
|
+
? this.config.collections.map((c) => c.name)
|
371
|
+
: Array.from(dataLoader.importMap.keys());
|
330
372
|
console.log("Collections to process:", collectionsToProcess);
|
331
373
|
// Iterate over each collection in the importMap
|
332
|
-
for (const [
|
333
|
-
|
374
|
+
for (const [
|
375
|
+
collectionKey,
|
376
|
+
collectionData,
|
377
|
+
] of dataLoader.importMap.entries()) {
|
378
|
+
const allCollectionKeys = collectionsToProcess.map((c) =>
|
379
|
+
dataLoader.getCollectionKey(c)
|
380
|
+
);
|
334
381
|
if (allCollectionKeys.includes(collectionKey)) {
|
335
382
|
console.log(
|
336
383
|
`Processing post-import actions for collection: ${collectionKey}`
|
@@ -360,8 +407,10 @@ export class ImportController {
|
|
360
407
|
}
|
361
408
|
}
|
362
409
|
} else {
|
363
|
-
console.log(
|
410
|
+
console.log(
|
411
|
+
`Skipping collection: ${collectionKey} because it's not valid for post-import actions`
|
412
|
+
);
|
364
413
|
}
|
365
414
|
}
|
366
415
|
}
|
367
|
-
}
|
416
|
+
}
|