appwrite-utils-cli 0.0.32 → 0.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -97
- package/dist/migrations/backup.d.ts +2 -0
- package/dist/migrations/dataLoader.d.ts +40 -4
- package/dist/migrations/dataLoader.js +119 -126
- package/dist/migrations/importController.js +54 -50
- package/dist/migrations/schemaStrings.js +38 -38
- package/dist/migrations/users.js +4 -0
- package/package.json +54 -54
- package/src/main.ts +83 -83
- package/src/migrations/dataLoader.ts +140 -169
- package/src/migrations/importController.ts +62 -63
- package/src/migrations/openapi.ts +83 -83
- package/src/migrations/schemaStrings.ts +473 -473
- package/src/migrations/users.ts +4 -0
- package/src/utilsController.ts +194 -194
|
@@ -92,7 +92,7 @@ export class DataLoader {
|
|
|
92
92
|
result[key] = updateValue;
|
|
93
93
|
}
|
|
94
94
|
// If the update value is nullish, keep the original value unless it doesn't exist
|
|
95
|
-
else if (sourceValue === undefined) {
|
|
95
|
+
else if (sourceValue === undefined || sourceValue === null) {
|
|
96
96
|
result[key] = updateValue;
|
|
97
97
|
}
|
|
98
98
|
});
|
|
@@ -234,6 +234,10 @@ export class DataLoader {
|
|
|
234
234
|
// Determine if this is the users collection
|
|
235
235
|
let isUsersCollection = this.getCollectionKey(this.config.usersCollectionName) ===
|
|
236
236
|
this.getCollectionKey(collection.name);
|
|
237
|
+
const collectionDefs = collection.importDefs;
|
|
238
|
+
if (!collectionDefs || !collectionDefs.length) {
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
237
241
|
// Process create and update definitions for the collection
|
|
238
242
|
const createDefs = collection.importDefs.filter((def) => def.type === "create" || !def.type);
|
|
239
243
|
const updateDefs = collection.importDefs.filter((def) => def.type === "update");
|
|
@@ -258,8 +262,8 @@ export class DataLoader {
|
|
|
258
262
|
}
|
|
259
263
|
}
|
|
260
264
|
console.log("Running update references");
|
|
261
|
-
|
|
262
|
-
|
|
265
|
+
this.dealWithMergedUsers();
|
|
266
|
+
this.updateOldReferencesForNew();
|
|
263
267
|
console.log("Done running update references");
|
|
264
268
|
}
|
|
265
269
|
// for (const collection of this.config.collections) {
|
|
@@ -272,7 +276,7 @@ export class DataLoader {
|
|
|
272
276
|
this.writeMapsToJsonFile();
|
|
273
277
|
}
|
|
274
278
|
}
|
|
275
|
-
|
|
279
|
+
dealWithMergedUsers() {
|
|
276
280
|
const usersCollectionKey = this.getCollectionKey(this.config.usersCollectionName);
|
|
277
281
|
const usersCollectionPrimaryKeyFields = new Set();
|
|
278
282
|
if (!this.config.collections) {
|
|
@@ -281,7 +285,11 @@ export class DataLoader {
|
|
|
281
285
|
// Collect primary key fields from the users collection definitions
|
|
282
286
|
this.config.collections.forEach((collection) => {
|
|
283
287
|
if (this.getCollectionKey(collection.name) === usersCollectionKey) {
|
|
284
|
-
collection.importDefs
|
|
288
|
+
const collectionImportDefs = collection.importDefs;
|
|
289
|
+
if (!collectionImportDefs || !collectionImportDefs.length) {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
collectionImportDefs.forEach((importDef) => {
|
|
285
293
|
if (importDef.primaryKeyField) {
|
|
286
294
|
usersCollectionPrimaryKeyFields.add(importDef.primaryKeyField);
|
|
287
295
|
}
|
|
@@ -293,19 +301,23 @@ export class DataLoader {
|
|
|
293
301
|
const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
|
|
294
302
|
if (!collectionData || !collectionData.data)
|
|
295
303
|
return;
|
|
296
|
-
collection.importDefs
|
|
304
|
+
const collectionImportDefs = collection.importDefs;
|
|
305
|
+
if (!collectionImportDefs || !collectionImportDefs.length) {
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
collectionImportDefs.forEach((importDef) => {
|
|
297
309
|
importDef.idMappings?.forEach((idMapping) => {
|
|
298
310
|
if (this.getCollectionKey(idMapping.targetCollection) ===
|
|
299
311
|
usersCollectionKey) {
|
|
300
|
-
|
|
312
|
+
const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
|
|
313
|
+
if (usersCollectionPrimaryKeyFields.has(targetFieldKey)) {
|
|
301
314
|
// Process each item in the collection
|
|
302
315
|
collectionData.data.forEach((item) => {
|
|
303
316
|
const oldId = item.context[idMapping.sourceField];
|
|
304
|
-
const newId = this.mergedUserMap.get(oldId);
|
|
317
|
+
const newId = this.mergedUserMap.get(`${oldId}`);
|
|
305
318
|
if (newId) {
|
|
306
319
|
// Update context to use new user ID
|
|
307
|
-
item.
|
|
308
|
-
newId;
|
|
320
|
+
item.finalData[idMapping.fieldToSet || idMapping.sourceField] = newId;
|
|
309
321
|
}
|
|
310
322
|
});
|
|
311
323
|
}
|
|
@@ -333,123 +345,83 @@ export class DataLoader {
|
|
|
333
345
|
for (const idMapping of importDef.idMappings) {
|
|
334
346
|
const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
|
|
335
347
|
const fieldToSetKey = idMapping.fieldToSet || idMapping.sourceField;
|
|
348
|
+
const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
|
|
336
349
|
const valueToMatch = collectionData.data[i].context[idMapping.sourceField];
|
|
350
|
+
// Skip if value to match is missing or empty
|
|
337
351
|
if (!valueToMatch || _.isEmpty(valueToMatch))
|
|
338
352
|
continue;
|
|
353
|
+
const isFieldToSetArray = collectionConfig.attributes.find((attribute) => attribute.key === fieldToSetKey)?.array;
|
|
339
354
|
const targetCollectionData = this.importMap.get(targetCollectionKey);
|
|
340
355
|
if (!targetCollectionData || !targetCollectionData.data)
|
|
341
356
|
continue;
|
|
342
|
-
|
|
343
|
-
|
|
357
|
+
// Find matching data in the target collection
|
|
358
|
+
const foundData = targetCollectionData.data.filter(({ context }) => {
|
|
359
|
+
const targetValue = context[targetFieldKey];
|
|
344
360
|
const isMatch = `${targetValue}` === `${valueToMatch}`;
|
|
345
|
-
//
|
|
346
|
-
|
|
347
|
-
|
|
361
|
+
// Ensure the targetValue is defined and not null
|
|
362
|
+
return (isMatch &&
|
|
363
|
+
targetValue !== undefined &&
|
|
364
|
+
targetValue !== null);
|
|
348
365
|
});
|
|
366
|
+
// Log and skip if no matching data found
|
|
349
367
|
if (!foundData.length) {
|
|
350
|
-
console.log(`No data found for collection: ${targetCollectionKey}
|
|
368
|
+
console.log(`No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`);
|
|
351
369
|
logger.error(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
|
|
352
370
|
continue;
|
|
353
371
|
}
|
|
354
372
|
needsUpdate = true;
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
373
|
+
const getCurrentDataFiltered = (currentData) => {
|
|
374
|
+
if (Array.isArray(currentData.finalData[fieldToSetKey])) {
|
|
375
|
+
return currentData.finalData[fieldToSetKey].filter((data) => `${data}` !== `${valueToMatch}`);
|
|
376
|
+
}
|
|
377
|
+
return currentData.finalData[fieldToSetKey];
|
|
378
|
+
};
|
|
379
|
+
// Get the current data to be updated
|
|
380
|
+
const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
|
|
381
|
+
// Extract the new data to set
|
|
382
|
+
const newData = foundData.map((data) => data.context[idMapping.targetField]);
|
|
383
|
+
// Handle cases where current data is an array
|
|
384
|
+
if (isFieldToSetArray) {
|
|
385
|
+
if (!currentDataFiltered) {
|
|
386
|
+
// Set new data if current data is undefined
|
|
387
|
+
collectionData.data[i].finalData[fieldToSetKey] =
|
|
388
|
+
Array.isArray(newData) ? newData : [newData];
|
|
389
|
+
}
|
|
390
|
+
else {
|
|
391
|
+
// Merge arrays if new data is non-empty array and filter for uniqueness
|
|
392
|
+
collectionData.data[i].finalData[fieldToSetKey] = [
|
|
393
|
+
...new Set([...currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}`)),
|
|
394
|
+
];
|
|
395
|
+
}
|
|
359
396
|
}
|
|
360
397
|
else {
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
}
|
|
369
|
-
if (needsUpdate) {
|
|
370
|
-
this.importMap.set(collectionKey, collectionData);
|
|
371
|
-
}
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
async updateReferencesInRelatedCollections() {
|
|
375
|
-
if (!this.config.collections) {
|
|
376
|
-
return;
|
|
377
|
-
}
|
|
378
|
-
// Iterate over each collection configuration
|
|
379
|
-
for (const collectionConfig of this.config.collections) {
|
|
380
|
-
const collectionKey = this.getCollectionKey(collectionConfig.name);
|
|
381
|
-
const collectionData = this.importMap.get(collectionKey);
|
|
382
|
-
if (!collectionData || !collectionData.data)
|
|
383
|
-
continue;
|
|
384
|
-
console.log(`Updating references for collection: ${collectionConfig.name}`);
|
|
385
|
-
// Iterate over each data item in the current collection
|
|
386
|
-
for (const item of collectionData.data) {
|
|
387
|
-
let needsUpdate = false;
|
|
388
|
-
// Check if the current collection has import definitions with idMappings
|
|
389
|
-
if (collectionConfig.importDefs) {
|
|
390
|
-
for (const importDef of collectionConfig.importDefs) {
|
|
391
|
-
if (importDef.idMappings) {
|
|
392
|
-
// Iterate over each idMapping defined for the current import definition
|
|
393
|
-
for (const idMapping of importDef.idMappings) {
|
|
394
|
-
const oldIds = Array.isArray(item.context[idMapping.sourceField])
|
|
395
|
-
? item.context[idMapping.sourceField]
|
|
396
|
-
: [item.context[idMapping.sourceField]];
|
|
397
|
-
const resolvedNewIds = [];
|
|
398
|
-
oldIds.forEach((oldId) => {
|
|
399
|
-
// Attempt to find a new ID for the old ID
|
|
400
|
-
let newIdForOldId = this.findNewIdForOldId(oldId, idMapping, importDef);
|
|
401
|
-
if (newIdForOldId &&
|
|
402
|
-
!resolvedNewIds.includes(newIdForOldId)) {
|
|
403
|
-
resolvedNewIds.push(newIdForOldId);
|
|
398
|
+
if (!currentDataFiltered) {
|
|
399
|
+
// Set new data if current data is undefined
|
|
400
|
+
collectionData.data[i].finalData[fieldToSetKey] =
|
|
401
|
+
Array.isArray(newData) ? newData[0] : newData;
|
|
404
402
|
}
|
|
405
|
-
else {
|
|
406
|
-
|
|
403
|
+
else if (Array.isArray(newData) && newData.length > 0) {
|
|
404
|
+
// Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
|
|
405
|
+
// and take the first value, because it's an array and the attribute is not an array
|
|
406
|
+
collectionData.data[i].finalData[fieldToSetKey] = [
|
|
407
|
+
...new Set([currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}`)),
|
|
408
|
+
].slice(0, 1)[0];
|
|
409
|
+
}
|
|
410
|
+
else if (!Array.isArray(newData) && newData !== undefined) {
|
|
411
|
+
// Simply update the field if new data is not an array and defined
|
|
412
|
+
collectionData.data[i].finalData[fieldToSetKey] = newData;
|
|
407
413
|
}
|
|
408
|
-
});
|
|
409
|
-
if (resolvedNewIds.length) {
|
|
410
|
-
const targetField = idMapping.fieldToSet || idMapping.targetField;
|
|
411
|
-
const isArray = collectionConfig.attributes.some((attribute) => attribute.key === targetField && attribute.array);
|
|
412
|
-
// Set the target field based on whether it's an array or single value
|
|
413
|
-
item.finalData[targetField] = isArray
|
|
414
|
-
? resolvedNewIds
|
|
415
|
-
: resolvedNewIds[0];
|
|
416
|
-
needsUpdate = true;
|
|
417
414
|
}
|
|
418
415
|
}
|
|
419
416
|
}
|
|
420
417
|
}
|
|
421
418
|
}
|
|
422
|
-
// Update the importMap if changes were made to the item
|
|
423
|
-
if (needsUpdate) {
|
|
424
|
-
this.importMap.set(collectionKey, collectionData);
|
|
425
|
-
logger.info(`Updated item: ${JSON.stringify(item.finalData, undefined, 2)}`);
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
findNewIdForOldId(oldId, idMapping, importDef) {
|
|
431
|
-
// First, check if this ID mapping is related to the users collection.
|
|
432
|
-
const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
|
|
433
|
-
const isUsersCollection = targetCollectionKey ===
|
|
434
|
-
this.getCollectionKey(this.config.usersCollectionName);
|
|
435
|
-
// If handling users, check the mergedUserMap for any existing new ID.
|
|
436
|
-
if (isUsersCollection) {
|
|
437
|
-
for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
|
|
438
|
-
if (oldIds.includes(oldId)) {
|
|
439
|
-
return newUserId;
|
|
440
|
-
}
|
|
441
419
|
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
if (targetCollectionData) {
|
|
446
|
-
const foundEntry = targetCollectionData.data.find((entry) => entry.context[importDef.primaryKeyField] === oldId);
|
|
447
|
-
if (foundEntry) {
|
|
448
|
-
return foundEntry.context.docId; // Assuming `docId` stores the new ID after import
|
|
420
|
+
// Update the import map if any changes were made
|
|
421
|
+
if (needsUpdate) {
|
|
422
|
+
this.importMap.set(collectionKey, collectionData);
|
|
449
423
|
}
|
|
450
424
|
}
|
|
451
|
-
logger.error(`No corresponding new ID found for ${oldId} in ${targetCollectionKey}`);
|
|
452
|
-
return null; // Return null if no new ID is found
|
|
453
425
|
}
|
|
454
426
|
writeMapsToJsonFile() {
|
|
455
427
|
const outputDir = path.resolve(process.cwd());
|
|
@@ -496,7 +468,6 @@ export class DataLoader {
|
|
|
496
468
|
* @returns The transformed item with user-specific keys removed.
|
|
497
469
|
*/
|
|
498
470
|
async prepareUserData(item, attributeMappings, primaryKeyField, newId) {
|
|
499
|
-
// Transform the item data based on the attribute mappings
|
|
500
471
|
let transformedItem = this.transformData(item, attributeMappings);
|
|
501
472
|
const userData = AuthUserCreateSchema.safeParse(transformedItem);
|
|
502
473
|
if (!userData.success) {
|
|
@@ -506,35 +477,58 @@ export class DataLoader {
|
|
|
506
477
|
const email = userData.data.email;
|
|
507
478
|
const phone = userData.data.phone;
|
|
508
479
|
let existingId;
|
|
509
|
-
// Check for duplicate email and
|
|
510
|
-
if (email && email
|
|
511
|
-
|
|
512
|
-
existingId = this.emailToUserIdMap.get(email);
|
|
513
|
-
}
|
|
514
|
-
else {
|
|
515
|
-
this.emailToUserIdMap.set(email, newId);
|
|
516
|
-
}
|
|
480
|
+
// Check for duplicate email and phone
|
|
481
|
+
if (email && this.emailToUserIdMap.has(email)) {
|
|
482
|
+
existingId = this.emailToUserIdMap.get(email);
|
|
517
483
|
}
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
if (this.phoneToUserIdMap.has(phone)) {
|
|
521
|
-
existingId = this.phoneToUserIdMap.get(phone);
|
|
522
|
-
}
|
|
523
|
-
else {
|
|
524
|
-
this.phoneToUserIdMap.set(phone, newId);
|
|
525
|
-
}
|
|
484
|
+
else if (phone && this.phoneToUserIdMap.has(phone)) {
|
|
485
|
+
existingId = this.phoneToUserIdMap.get(phone);
|
|
526
486
|
}
|
|
527
|
-
|
|
528
|
-
|
|
487
|
+
else {
|
|
488
|
+
if (email)
|
|
489
|
+
this.emailToUserIdMap.set(email, newId);
|
|
490
|
+
if (phone)
|
|
491
|
+
this.phoneToUserIdMap.set(phone, newId);
|
|
529
492
|
}
|
|
530
|
-
// If existingId is found, add to mergedUserMap
|
|
531
493
|
if (existingId) {
|
|
532
494
|
userData.data.userId = existingId;
|
|
533
495
|
const mergedUsers = this.mergedUserMap.get(existingId) || [];
|
|
534
496
|
mergedUsers.push(`${item[primaryKeyField]}`);
|
|
535
497
|
this.mergedUserMap.set(existingId, mergedUsers);
|
|
498
|
+
const userFound = this.importMap
|
|
499
|
+
.get(this.getCollectionKey("users"))
|
|
500
|
+
?.data.find((userDataExisting) => {
|
|
501
|
+
let userIdToMatch;
|
|
502
|
+
if (userDataExisting?.finalData?.userId) {
|
|
503
|
+
userIdToMatch = userDataExisting?.finalData?.userId;
|
|
504
|
+
}
|
|
505
|
+
else if (userDataExisting?.finalData?.docId) {
|
|
506
|
+
userIdToMatch = userDataExisting?.finalData?.docId;
|
|
507
|
+
}
|
|
508
|
+
else if (userDataExisting?.context?.userId) {
|
|
509
|
+
userIdToMatch = userDataExisting.context.userId;
|
|
510
|
+
}
|
|
511
|
+
else if (userDataExisting?.context?.docId) {
|
|
512
|
+
userIdToMatch = userDataExisting.context.docId;
|
|
513
|
+
}
|
|
514
|
+
return userIdToMatch === existingId;
|
|
515
|
+
});
|
|
516
|
+
if (userFound) {
|
|
517
|
+
userFound.finalData.userId = existingId;
|
|
518
|
+
}
|
|
519
|
+
return [
|
|
520
|
+
transformedItem,
|
|
521
|
+
existingId,
|
|
522
|
+
{
|
|
523
|
+
rawData: userFound?.rawData,
|
|
524
|
+
finalData: userFound?.finalData,
|
|
525
|
+
},
|
|
526
|
+
];
|
|
527
|
+
}
|
|
528
|
+
else {
|
|
529
|
+
existingId = newId;
|
|
530
|
+
userData.data.userId = existingId;
|
|
536
531
|
}
|
|
537
|
-
// Remove user-specific keys from the transformed item
|
|
538
532
|
const userKeys = ["email", "phone", "name", "labels", "prefs"];
|
|
539
533
|
userKeys.forEach((key) => {
|
|
540
534
|
if (transformedItem.hasOwnProperty(key)) {
|
|
@@ -546,7 +540,6 @@ export class DataLoader {
|
|
|
546
540
|
rawData: item,
|
|
547
541
|
finalData: userData.data,
|
|
548
542
|
};
|
|
549
|
-
// Directly update the importMap with the new user data, without pushing to usersMap.data first
|
|
550
543
|
this.importMap.set(this.getCollectionKey("users"), {
|
|
551
544
|
data: [...(usersMap?.data || []), userDataToAdd],
|
|
552
545
|
});
|
|
@@ -601,12 +594,12 @@ export class DataLoader {
|
|
|
601
594
|
if (!existingId) {
|
|
602
595
|
// No existing user ID, generate a new unique ID
|
|
603
596
|
existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
|
|
604
|
-
transformedItem.
|
|
597
|
+
transformedItem.docId = existingId; // Assign the new ID to the transformed data's docId field
|
|
605
598
|
}
|
|
606
599
|
// Create a context object for the item, including the new ID
|
|
607
600
|
let context = this.createContext(db, collection, item, existingId);
|
|
608
601
|
// Merge the transformed data into the context
|
|
609
|
-
context = { ...context, ...transformedItem };
|
|
602
|
+
context = { ...context, ...transformedItem, ...userData.finalData };
|
|
610
603
|
// If a primary key field is defined, handle the ID mapping and check for duplicates
|
|
611
604
|
if (importDef.primaryKeyField) {
|
|
612
605
|
const oldId = item[importDef.primaryKeyField];
|
|
@@ -86,44 +86,58 @@ export class ImportController {
|
|
|
86
86
|
const usersData = usersDataMap?.data;
|
|
87
87
|
const usersController = new UsersController(this.config, this.database);
|
|
88
88
|
if (usersData) {
|
|
89
|
-
console.log("Found users data");
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
.
|
|
97
|
-
|
|
98
|
-
const userId = userBatch.finalData.userId;
|
|
99
|
-
if (dataLoader.userExistsMap.has(userId)) {
|
|
100
|
-
// We only are storing the existing user ID's as true, so we need to check for that
|
|
101
|
-
if (!(dataLoader.userExistsMap.get(userId) === true)) {
|
|
102
|
-
const userId = userBatch.finalData.userId ||
|
|
103
|
-
userBatch.context.userId ||
|
|
104
|
-
userBatch.context.docId;
|
|
105
|
-
if (!userBatch.finalData.userId) {
|
|
106
|
-
userBatch.finalData.userId = userId;
|
|
107
|
-
}
|
|
108
|
-
return usersController
|
|
109
|
-
.createUserAndReturn(userBatch.finalData)
|
|
110
|
-
.then(() => console.log("Created user"))
|
|
111
|
-
.catch((error) => {
|
|
112
|
-
logger.error("Error creating user:", error, "\nUser data is ", userBatch.finalData);
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
else {
|
|
116
|
-
console.log("Skipped existing user: ", userId);
|
|
117
|
-
return Promise.resolve();
|
|
118
|
-
}
|
|
119
|
-
}
|
|
89
|
+
console.log("Found users data", usersData.length);
|
|
90
|
+
const userDataBatches = createBatches(usersData);
|
|
91
|
+
for (const batch of userDataBatches) {
|
|
92
|
+
console.log("Importing users batch", batch.length);
|
|
93
|
+
const userBatchPromises = batch
|
|
94
|
+
.filter((item) => {
|
|
95
|
+
let itemId;
|
|
96
|
+
if (item.finalData.userId) {
|
|
97
|
+
itemId = item.finalData.userId;
|
|
120
98
|
}
|
|
99
|
+
else if (item.finalData.docId) {
|
|
100
|
+
itemId = item.finalData.docId;
|
|
101
|
+
}
|
|
102
|
+
if (!itemId) {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
return (item &&
|
|
106
|
+
item.finalData &&
|
|
107
|
+
!dataLoader.userExistsMap.has(itemId));
|
|
121
108
|
})
|
|
122
|
-
.
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
109
|
+
.map((item) => {
|
|
110
|
+
return usersController.createUserAndReturn(item.finalData);
|
|
111
|
+
});
|
|
112
|
+
await Promise.all(userBatchPromises);
|
|
113
|
+
for (const item of batch) {
|
|
114
|
+
if (item && item.finalData) {
|
|
115
|
+
dataLoader.userExistsMap.set(item.finalData.userId ||
|
|
116
|
+
item.finalData.docId ||
|
|
117
|
+
item.context.userId ||
|
|
118
|
+
item.context.docId, true);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
console.log("Finished importing users batch", batch.length);
|
|
126
122
|
}
|
|
123
|
+
// for (let i = 0; i < usersData.length; i++) {
|
|
124
|
+
// const user = usersData[i];
|
|
125
|
+
// if (user.finalData) {
|
|
126
|
+
// const userId =
|
|
127
|
+
// user.finalData.userId ||
|
|
128
|
+
// user.context.userId ||
|
|
129
|
+
// user.context.docId;
|
|
130
|
+
// if (!dataLoader.userExistsMap.has(userId)) {
|
|
131
|
+
// if (!user.finalData.userId) {
|
|
132
|
+
// user.finalData.userId = userId;
|
|
133
|
+
// }
|
|
134
|
+
// await usersController.createUserAndReturn(user.finalData);
|
|
135
|
+
// dataLoader.userExistsMap.set(userId, true);
|
|
136
|
+
// } else {
|
|
137
|
+
// console.log("Skipped existing user: ", userId);
|
|
138
|
+
// }
|
|
139
|
+
// }
|
|
140
|
+
// }
|
|
127
141
|
console.log("Finished importing users");
|
|
128
142
|
}
|
|
129
143
|
}
|
|
@@ -146,10 +160,10 @@ export class ImportController {
|
|
|
146
160
|
const batches = dataSplit[i];
|
|
147
161
|
console.log(`Processing batch ${i + 1} of ${dataSplit.length}`);
|
|
148
162
|
const batchPromises = batches.map((item) => {
|
|
149
|
-
const id = item.
|
|
150
|
-
item.
|
|
151
|
-
item.
|
|
152
|
-
item.
|
|
163
|
+
const id = item.finalData.docId ||
|
|
164
|
+
item.finalData.userId ||
|
|
165
|
+
item.context.docId ||
|
|
166
|
+
item.context.userId;
|
|
153
167
|
if (item.finalData.hasOwnProperty("userId")) {
|
|
154
168
|
delete item.finalData.userId;
|
|
155
169
|
}
|
|
@@ -159,17 +173,7 @@ export class ImportController {
|
|
|
159
173
|
if (!item.finalData) {
|
|
160
174
|
return Promise.resolve();
|
|
161
175
|
}
|
|
162
|
-
return this.database
|
|
163
|
-
.createDocument(db.$id, collection.$id, id, item.finalData)
|
|
164
|
-
.then(() => {
|
|
165
|
-
processedItems++;
|
|
166
|
-
console.log("Created item");
|
|
167
|
-
})
|
|
168
|
-
.catch((error) => {
|
|
169
|
-
console.error(`Error creating item in ${collection.name}:`, error, "\nItem data is ", item.finalData);
|
|
170
|
-
throw error;
|
|
171
|
-
// Optionally, log the failed item for retry or review
|
|
172
|
-
});
|
|
176
|
+
return this.database.createDocument(db.$id, collection.$id, id, item.finalData);
|
|
173
177
|
});
|
|
174
178
|
// Wait for all promises in the current batch to resolve
|
|
175
179
|
await Promise.allSettled(batchPromises);
|
|
@@ -17,25 +17,25 @@ export class SchemaGenerator {
|
|
|
17
17
|
const collections = this.config.collections;
|
|
18
18
|
delete this.config.collections;
|
|
19
19
|
const configPath = path.join(this.appwriteFolderPath, "appwriteConfig.ts");
|
|
20
|
-
const configContent = `import { type AppwriteConfig } from "appwrite-utils";
|
|
21
|
-
|
|
22
|
-
const appwriteConfig: AppwriteConfig = {
|
|
23
|
-
appwriteEndpoint: "${this.config.appwriteEndpoint}",
|
|
24
|
-
appwriteProject: "${this.config.appwriteProject}",
|
|
25
|
-
appwriteKey: "${this.config.appwriteKey}",
|
|
26
|
-
enableDevDatabase: ${this.config.enableDevDatabase},
|
|
27
|
-
enableBackups: ${this.config.enableBackups},
|
|
28
|
-
backupInterval: ${this.config.backupInterval},
|
|
29
|
-
backupRetention: ${this.config.backupRetention},
|
|
30
|
-
enableBackupCleanup: ${this.config.enableBackupCleanup},
|
|
31
|
-
enableMockData: ${this.config.enableMockData},
|
|
32
|
-
enableWipeOtherDatabases: ${this.config.enableWipeOtherDatabases},
|
|
33
|
-
documentBucketId: "${this.config.documentBucketId}",
|
|
34
|
-
usersCollectionName: "${this.config.usersCollectionName}",
|
|
35
|
-
databases: ${JSON.stringify(this.config.databases)}
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
export default appwriteConfig;
|
|
20
|
+
const configContent = `import { type AppwriteConfig } from "appwrite-utils";
|
|
21
|
+
|
|
22
|
+
const appwriteConfig: AppwriteConfig = {
|
|
23
|
+
appwriteEndpoint: "${this.config.appwriteEndpoint}",
|
|
24
|
+
appwriteProject: "${this.config.appwriteProject}",
|
|
25
|
+
appwriteKey: "${this.config.appwriteKey}",
|
|
26
|
+
enableDevDatabase: ${this.config.enableDevDatabase},
|
|
27
|
+
enableBackups: ${this.config.enableBackups},
|
|
28
|
+
backupInterval: ${this.config.backupInterval},
|
|
29
|
+
backupRetention: ${this.config.backupRetention},
|
|
30
|
+
enableBackupCleanup: ${this.config.enableBackupCleanup},
|
|
31
|
+
enableMockData: ${this.config.enableMockData},
|
|
32
|
+
enableWipeOtherDatabases: ${this.config.enableWipeOtherDatabases},
|
|
33
|
+
documentBucketId: "${this.config.documentBucketId}",
|
|
34
|
+
usersCollectionName: "${this.config.usersCollectionName}",
|
|
35
|
+
databases: ${JSON.stringify(this.config.databases)}
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export default appwriteConfig;
|
|
39
39
|
`;
|
|
40
40
|
fs.writeFileSync(configPath, configContent, { encoding: "utf-8" });
|
|
41
41
|
const collectionsFolderPath = path.join(this.appwriteFolderPath, "collections");
|
|
@@ -45,19 +45,19 @@ export class SchemaGenerator {
|
|
|
45
45
|
collections?.forEach((collection) => {
|
|
46
46
|
const { databaseId, ...collectionWithoutDbId } = collection; // Destructure to exclude databaseId
|
|
47
47
|
const collectionFilePath = path.join(collectionsFolderPath, `${collection.name}.ts`);
|
|
48
|
-
const collectionContent = `import { type CollectionCreate } from "appwrite-utils";
|
|
49
|
-
|
|
50
|
-
const ${collection.name}Config: Partial<CollectionCreate> = {
|
|
51
|
-
name: "${collection.name}",
|
|
52
|
-
$id: "${collection.$id}",
|
|
53
|
-
enabled: ${collection.enabled},
|
|
54
|
-
documentSecurity: ${collection.documentSecurity},
|
|
55
|
-
$permissions: [
|
|
48
|
+
const collectionContent = `import { type CollectionCreate } from "appwrite-utils";
|
|
49
|
+
|
|
50
|
+
const ${collection.name}Config: Partial<CollectionCreate> = {
|
|
51
|
+
name: "${collection.name}",
|
|
52
|
+
$id: "${collection.$id}",
|
|
53
|
+
enabled: ${collection.enabled},
|
|
54
|
+
documentSecurity: ${collection.documentSecurity},
|
|
55
|
+
$permissions: [
|
|
56
56
|
${collection.$permissions
|
|
57
57
|
.map((permission) => `{ permission: "${permission.permission}", target: "${permission.target}" }`)
|
|
58
|
-
.join(",\n ")}
|
|
59
|
-
],
|
|
60
|
-
attributes: [
|
|
58
|
+
.join(",\n ")}
|
|
59
|
+
],
|
|
60
|
+
attributes: [
|
|
61
61
|
${collection.attributes
|
|
62
62
|
.map((attr) => {
|
|
63
63
|
return `{ ${Object.entries(attr)
|
|
@@ -85,9 +85,9 @@ export class SchemaGenerator {
|
|
|
85
85
|
})
|
|
86
86
|
.join(", ")} }`;
|
|
87
87
|
})
|
|
88
|
-
.join(",\n ")}
|
|
89
|
-
],
|
|
90
|
-
indexes: [
|
|
88
|
+
.join(",\n ")}
|
|
89
|
+
],
|
|
90
|
+
indexes: [
|
|
91
91
|
${(collection.indexes?.map((index) => {
|
|
92
92
|
// Map each attribute to ensure it is properly quoted
|
|
93
93
|
const formattedAttributes = index.attributes.map((attr) => `"${attr}"`).join(", ") ?? "";
|
|
@@ -95,11 +95,11 @@ export class SchemaGenerator {
|
|
|
95
95
|
?.filter((order) => order !== null)
|
|
96
96
|
.map((order) => `"${order}"`)
|
|
97
97
|
.join(", ") ?? ""}] }`;
|
|
98
|
-
}) ?? []).join(",\n ")}
|
|
99
|
-
]
|
|
100
|
-
};
|
|
101
|
-
|
|
102
|
-
export default ${collection.name}Config;
|
|
98
|
+
}) ?? []).join(",\n ")}
|
|
99
|
+
]
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
export default ${collection.name}Config;
|
|
103
103
|
`;
|
|
104
104
|
fs.writeFileSync(collectionFilePath, collectionContent, {
|
|
105
105
|
encoding: "utf-8",
|
package/dist/migrations/users.js
CHANGED
|
@@ -69,6 +69,9 @@ export class UsersController {
|
|
|
69
69
|
async getAllUsers() {
|
|
70
70
|
const allUsers = [];
|
|
71
71
|
const users = await this.users.list([Query.limit(200)]);
|
|
72
|
+
if (users.users.length === 0) {
|
|
73
|
+
return [];
|
|
74
|
+
}
|
|
72
75
|
if (users.users.length === 200) {
|
|
73
76
|
let lastDocumentId = users.users[users.users.length - 1].$id;
|
|
74
77
|
allUsers.push(...users.users);
|
|
@@ -112,6 +115,7 @@ export class UsersController {
|
|
|
112
115
|
if (e instanceof Error) {
|
|
113
116
|
logger.error("FAILED CREATING USER: ", e.message);
|
|
114
117
|
}
|
|
118
|
+
console.log("FAILED CREATING USER: ", e);
|
|
115
119
|
throw e;
|
|
116
120
|
}
|
|
117
121
|
}
|