appwrite-utils-cli 0.0.18 → 0.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -59,6 +59,7 @@ export class ImportController {
|
|
|
59
59
|
await this.importCollections(db);
|
|
60
60
|
await resolveAndUpdateRelationships(db.$id, this.database, this.config);
|
|
61
61
|
await this.executePostImportActions(db.$id);
|
|
62
|
+
await this.executePostImportActions(db.$id);
|
|
62
63
|
console.log(`---------------------------------`);
|
|
63
64
|
console.log(`Finished import data for database: ${db.name}`);
|
|
64
65
|
console.log(`---------------------------------`);
|
|
@@ -207,23 +208,19 @@ export class ImportController {
|
|
|
207
208
|
const attributeMappingsWithActions = this.getAttributeMappingsWithActions(importDef.attributeMappings, afterImportActionContext, finalItem);
|
|
208
209
|
if (attributeMappingsWithActions.some((m) => m.postImportActions)) {
|
|
209
210
|
logger.info(`Pushing to post-import actions queue for ${context.docId}`);
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
// finalItem: finalItem,
|
|
214
|
-
// attributeMappings: attributeMappingsWithActions,
|
|
215
|
-
// context: afterImportActionContext,
|
|
216
|
-
// });
|
|
217
|
-
// await createOrFindAfterImportOperation(
|
|
218
|
-
// this.database,
|
|
219
|
-
// context.collId,
|
|
220
|
-
// afterImportOperationContext
|
|
221
|
-
// );
|
|
222
|
-
this.postImportActionsQueue.push({
|
|
223
|
-
context: afterImportActionContext,
|
|
211
|
+
const afterImportOperationContext = ContextObject.parse({
|
|
212
|
+
dbId: db.$id,
|
|
213
|
+
collectionId: collection.$id,
|
|
224
214
|
finalItem: finalItem,
|
|
225
215
|
attributeMappings: attributeMappingsWithActions,
|
|
216
|
+
context: afterImportActionContext,
|
|
226
217
|
});
|
|
218
|
+
await createOrFindAfterImportOperation(this.database, context.collId, afterImportOperationContext);
|
|
219
|
+
// this.postImportActionsQueue.push({
|
|
220
|
+
// context: afterImportActionContext,
|
|
221
|
+
// finalItem: finalItem,
|
|
222
|
+
// attributeMappings: attributeMappingsWithActions,
|
|
223
|
+
// });
|
|
227
224
|
}
|
|
228
225
|
}));
|
|
229
226
|
results.forEach((result) => {
|
|
@@ -318,29 +315,41 @@ export class ImportController {
|
|
|
318
315
|
});
|
|
319
316
|
}
|
|
320
317
|
async executePostImportActions(dbId) {
|
|
321
|
-
let actionQueue = [];
|
|
322
|
-
for (const action of this.postImportActionsQueue) {
|
|
323
|
-
|
|
318
|
+
// let actionQueue: Promise<any>[] = [];
|
|
319
|
+
// for (const action of this.postImportActionsQueue) {
|
|
320
|
+
// actionQueue.push(
|
|
321
|
+
// this.importDataActions.executeAfterImportActions(
|
|
322
|
+
// action.finalItem,
|
|
323
|
+
// action.attributeMappings,
|
|
324
|
+
// action.context
|
|
325
|
+
// )
|
|
326
|
+
// );
|
|
327
|
+
// }
|
|
328
|
+
// const BATCH_LIMIT = 5;
|
|
329
|
+
// const splitQueue = _.chunk(actionQueue, BATCH_LIMIT);
|
|
330
|
+
// for (const queue of splitQueue) {
|
|
331
|
+
// const results = await Promise.allSettled(queue);
|
|
332
|
+
// results.forEach((result) => {
|
|
333
|
+
// if (result.status === "rejected") {
|
|
334
|
+
// console.error("An action promise was rejected:", result.reason);
|
|
335
|
+
// logger.error(
|
|
336
|
+
// `An action promise was rejected: ${
|
|
337
|
+
// result.reason
|
|
338
|
+
// } -- ${JSON.stringify(result)}`
|
|
339
|
+
// );
|
|
340
|
+
// }
|
|
341
|
+
// });
|
|
342
|
+
// }
|
|
343
|
+
const collectionActionsPromises = [];
|
|
344
|
+
for (const collection of this.config.collections) {
|
|
345
|
+
collectionActionsPromises.push(this.executeActionsInParallel(dbId, collection));
|
|
324
346
|
}
|
|
325
|
-
const results = await Promise.allSettled(
|
|
347
|
+
const results = await Promise.allSettled(collectionActionsPromises);
|
|
326
348
|
results.forEach((result) => {
|
|
327
349
|
if (result.status === "rejected") {
|
|
328
|
-
console.error("
|
|
329
|
-
logger.error(`An action promise was rejected: ${result.reason} -- ${JSON.stringify(result)}`);
|
|
350
|
+
console.error("A process batch promise was rejected:", result.reason);
|
|
330
351
|
}
|
|
331
352
|
});
|
|
332
|
-
// const collectionActionsPromises = [];
|
|
333
|
-
// for (const collection of this.config.collections) {
|
|
334
|
-
// collectionActionsPromises.push(
|
|
335
|
-
// this.executeActionsInParallel(dbId, collection)
|
|
336
|
-
// );
|
|
337
|
-
// }
|
|
338
|
-
// const results = await Promise.allSettled(collectionActionsPromises);
|
|
339
|
-
// results.forEach((result) => {
|
|
340
|
-
// if (result.status === "rejected") {
|
|
341
|
-
// console.error("A process batch promise was rejected:", result.reason);
|
|
342
|
-
// }
|
|
343
|
-
// });
|
|
344
353
|
}
|
|
345
354
|
async executeActionsInParallel(dbId, collection) {
|
|
346
355
|
const collectionExists = await checkForCollection(this.database, dbId, collection);
|
|
@@ -69,11 +69,7 @@ export const getAfterImportOperations = async (database, collectionId) => {
|
|
|
69
69
|
export const setAllPendingAfterImportActionsToReady = async (database, dbId, collectionId) => {
|
|
70
70
|
let lastDocumentId;
|
|
71
71
|
do {
|
|
72
|
-
const query = [
|
|
73
|
-
Query.equal("collectionId", collectionId),
|
|
74
|
-
Query.equal("status", "pending"),
|
|
75
|
-
Query.limit(100),
|
|
76
|
-
];
|
|
72
|
+
const query = [Query.equal("collectionId", collectionId), Query.limit(100)];
|
|
77
73
|
if (lastDocumentId) {
|
|
78
74
|
query.push(Query.cursorAfter(lastDocumentId));
|
|
79
75
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "appwrite-utils-cli",
|
|
3
3
|
"description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.20",
|
|
5
5
|
"main": "src/main.ts",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"repository": {
|
|
@@ -104,6 +104,7 @@ export class ImportController {
|
|
|
104
104
|
await this.importCollections(db);
|
|
105
105
|
await resolveAndUpdateRelationships(db.$id, this.database!, this.config!);
|
|
106
106
|
await this.executePostImportActions(db.$id);
|
|
107
|
+
await this.executePostImportActions(db.$id);
|
|
107
108
|
console.log(`---------------------------------`);
|
|
108
109
|
console.log(`Finished import data for database: ${db.name}`);
|
|
109
110
|
console.log(`---------------------------------`);
|
|
@@ -357,23 +358,23 @@ export class ImportController {
|
|
|
357
358
|
logger.info(
|
|
358
359
|
`Pushing to post-import actions queue for ${context.docId}`
|
|
359
360
|
);
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
// finalItem: finalItem,
|
|
364
|
-
// attributeMappings: attributeMappingsWithActions,
|
|
365
|
-
// context: afterImportActionContext,
|
|
366
|
-
// });
|
|
367
|
-
// await createOrFindAfterImportOperation(
|
|
368
|
-
// this.database,
|
|
369
|
-
// context.collId,
|
|
370
|
-
// afterImportOperationContext
|
|
371
|
-
// );
|
|
372
|
-
this.postImportActionsQueue.push({
|
|
373
|
-
context: afterImportActionContext,
|
|
361
|
+
const afterImportOperationContext = ContextObject.parse({
|
|
362
|
+
dbId: db.$id,
|
|
363
|
+
collectionId: collection.$id,
|
|
374
364
|
finalItem: finalItem,
|
|
375
365
|
attributeMappings: attributeMappingsWithActions,
|
|
366
|
+
context: afterImportActionContext,
|
|
376
367
|
});
|
|
368
|
+
await createOrFindAfterImportOperation(
|
|
369
|
+
this.database,
|
|
370
|
+
context.collId,
|
|
371
|
+
afterImportOperationContext
|
|
372
|
+
);
|
|
373
|
+
// this.postImportActionsQueue.push({
|
|
374
|
+
// context: afterImportActionContext,
|
|
375
|
+
// finalItem: finalItem,
|
|
376
|
+
// attributeMappings: attributeMappingsWithActions,
|
|
377
|
+
// });
|
|
377
378
|
}
|
|
378
379
|
})
|
|
379
380
|
);
|
|
@@ -512,39 +513,43 @@ export class ImportController {
|
|
|
512
513
|
}
|
|
513
514
|
|
|
514
515
|
async executePostImportActions(dbId: string) {
|
|
515
|
-
let actionQueue: Promise<any>[] = [];
|
|
516
|
-
for (const action of this.postImportActionsQueue) {
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
516
|
+
// let actionQueue: Promise<any>[] = [];
|
|
517
|
+
// for (const action of this.postImportActionsQueue) {
|
|
518
|
+
// actionQueue.push(
|
|
519
|
+
// this.importDataActions.executeAfterImportActions(
|
|
520
|
+
// action.finalItem,
|
|
521
|
+
// action.attributeMappings,
|
|
522
|
+
// action.context
|
|
523
|
+
// )
|
|
524
|
+
// );
|
|
525
|
+
// }
|
|
526
|
+
// const BATCH_LIMIT = 5;
|
|
527
|
+
// const splitQueue = _.chunk(actionQueue, BATCH_LIMIT);
|
|
528
|
+
// for (const queue of splitQueue) {
|
|
529
|
+
// const results = await Promise.allSettled(queue);
|
|
530
|
+
// results.forEach((result) => {
|
|
531
|
+
// if (result.status === "rejected") {
|
|
532
|
+
// console.error("An action promise was rejected:", result.reason);
|
|
533
|
+
// logger.error(
|
|
534
|
+
// `An action promise was rejected: ${
|
|
535
|
+
// result.reason
|
|
536
|
+
// } -- ${JSON.stringify(result)}`
|
|
537
|
+
// );
|
|
538
|
+
// }
|
|
539
|
+
// });
|
|
540
|
+
// }
|
|
541
|
+
const collectionActionsPromises = [];
|
|
542
|
+
for (const collection of this.config.collections) {
|
|
543
|
+
collectionActionsPromises.push(
|
|
544
|
+
this.executeActionsInParallel(dbId, collection)
|
|
523
545
|
);
|
|
524
546
|
}
|
|
525
|
-
const results = await Promise.allSettled(
|
|
547
|
+
const results = await Promise.allSettled(collectionActionsPromises);
|
|
526
548
|
results.forEach((result) => {
|
|
527
549
|
if (result.status === "rejected") {
|
|
528
|
-
console.error("
|
|
529
|
-
logger.error(
|
|
530
|
-
`An action promise was rejected: ${result.reason} -- ${JSON.stringify(
|
|
531
|
-
result
|
|
532
|
-
)}`
|
|
533
|
-
);
|
|
550
|
+
console.error("A process batch promise was rejected:", result.reason);
|
|
534
551
|
}
|
|
535
552
|
});
|
|
536
|
-
// const collectionActionsPromises = [];
|
|
537
|
-
// for (const collection of this.config.collections) {
|
|
538
|
-
// collectionActionsPromises.push(
|
|
539
|
-
// this.executeActionsInParallel(dbId, collection)
|
|
540
|
-
// );
|
|
541
|
-
// }
|
|
542
|
-
// const results = await Promise.allSettled(collectionActionsPromises);
|
|
543
|
-
// results.forEach((result) => {
|
|
544
|
-
// if (result.status === "rejected") {
|
|
545
|
-
// console.error("A process batch promise was rejected:", result.reason);
|
|
546
|
-
// }
|
|
547
|
-
// });
|
|
548
553
|
}
|
|
549
554
|
|
|
550
555
|
async executeActionsInParallel(dbId: string, collection: ConfigCollection) {
|
|
@@ -111,11 +111,7 @@ export const setAllPendingAfterImportActionsToReady = async (
|
|
|
111
111
|
) => {
|
|
112
112
|
let lastDocumentId: string | undefined;
|
|
113
113
|
do {
|
|
114
|
-
const query = [
|
|
115
|
-
Query.equal("collectionId", collectionId),
|
|
116
|
-
Query.equal("status", "pending"),
|
|
117
|
-
Query.limit(100),
|
|
118
|
-
];
|
|
114
|
+
const query = [Query.equal("collectionId", collectionId), Query.limit(100)];
|
|
119
115
|
|
|
120
116
|
if (lastDocumentId) {
|
|
121
117
|
query.push(Query.cursorAfter(lastDocumentId));
|