appwrite-utils-cli 0.0.67 → 0.0.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
@@ -132,6 +132,8 @@ This setup ensures that developers have robust tools at their fingertips to mana
|
|
132
132
|
|
133
133
|
### Changelog
|
134
134
|
|
135
|
+
- 0.0.69: Fixed single ID not getting replaced due to the below change =D also, `nice`
|
136
|
+
- 0.0.68: Fixed the occasional case where, when mapping ID's from old data to new, there would be an array of ID's to match against. `idMappings` now supports arrays.
|
135
137
|
- 0.0.67: Fixed `updates` in `importDef`'s update mappings overwriting postImportActions from the original
|
136
138
|
- 0.0.57: Fixed `dataLoader`'s `idMapping`'s giving me issues
|
137
139
|
- 0.0.55: Added `documentExists` check to batch creation functionality to try to prevent duplicates
|
@@ -229,6 +229,10 @@ export const afterImportActions = {
|
|
229
229
|
// console.log(
|
230
230
|
// `Processing field ${fieldName} in collection ${collId} for document ${docId} in database ${dbId} in bucket ${bucketId} with path ${filePath} and name ${fileName}...`
|
231
231
|
// );
|
232
|
+
if (filePath.length === 0 || fileName.length === 0) {
|
233
|
+
console.error(`File path or name is empty for field ${fieldName} in collection ${collId}, skipping...`);
|
234
|
+
return;
|
235
|
+
}
|
232
236
|
let isArray = false;
|
233
237
|
if (!attribute) {
|
234
238
|
console.log(`Field ${fieldName} not found in collection ${collId}, weird, skipping...`);
|
@@ -74,6 +74,15 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
|
|
74
74
|
error?: string | undefined;
|
75
75
|
xdefault?: number | null | undefined;
|
76
76
|
min?: number | undefined;
|
77
|
+
/**
|
78
|
+
* Prepares the data for creating user collection documents.
|
79
|
+
* This involves loading the data, transforming it according to the import definition,
|
80
|
+
* and handling the creation of new unique IDs for each item.
|
81
|
+
*
|
82
|
+
* @param db - The database configuration.
|
83
|
+
* @param collection - The collection configuration.
|
84
|
+
* @param importDef - The import definition containing the attribute mappings and other relevant info.
|
85
|
+
*/
|
77
86
|
max?: number | undefined;
|
78
87
|
}, {
|
79
88
|
key: string;
|
@@ -441,90 +441,105 @@ export class DataLoader {
|
|
441
441
|
const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
|
442
442
|
const fieldToSetKey = idMapping.fieldToSet || idMapping.sourceField;
|
443
443
|
const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
|
444
|
-
const
|
444
|
+
const sourceValue = this.getValueFromData(collectionData.data[i].finalData, collectionData.data[i].context, idMapping.sourceField);
|
445
445
|
// Skip if value to match is missing or empty
|
446
|
-
if (!
|
447
|
-
_.isEmpty(
|
448
|
-
|
446
|
+
if (!sourceValue ||
|
447
|
+
_.isEmpty(sourceValue) ||
|
448
|
+
sourceValue === null)
|
449
449
|
continue;
|
450
450
|
const isFieldToSetArray = collectionConfig.attributes.find((attribute) => attribute.key === fieldToSetKey)?.array;
|
451
451
|
const targetCollectionData = this.importMap.get(targetCollectionKey);
|
452
452
|
if (!targetCollectionData || !targetCollectionData.data)
|
453
453
|
continue;
|
454
|
-
//
|
455
|
-
const
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
454
|
+
// Handle cases where sourceValue is an array
|
455
|
+
const sourceValues = Array.isArray(sourceValue)
|
456
|
+
? sourceValue.map((sourceValue) => `${sourceValue}`)
|
457
|
+
: [`${sourceValue}`];
|
458
|
+
let newData = [];
|
459
|
+
for (const valueToMatch of sourceValues) {
|
460
|
+
// Find matching data in the target collection
|
461
|
+
const foundData = targetCollectionData.data.filter(({ context, finalData }) => {
|
462
|
+
const targetValue = this.getValueFromData(finalData, context, targetFieldKey);
|
463
|
+
const isMatch = `${targetValue}` === `${valueToMatch}`;
|
464
|
+
// Ensure the targetValue is defined and not null
|
465
|
+
return (isMatch &&
|
466
|
+
targetValue !== undefined &&
|
467
|
+
targetValue !== null);
|
468
|
+
});
|
469
|
+
if (foundData.length) {
|
470
|
+
newData.push(...foundData.map((data) => {
|
471
|
+
const newValue = this.getValueFromData(data.finalData, data.context, idMapping.targetField);
|
472
|
+
return newValue;
|
473
|
+
}));
|
474
|
+
}
|
475
|
+
else {
|
476
|
+
logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
|
477
|
+
}
|
478
|
+
continue;
|
479
|
+
}
|
463
480
|
const getCurrentDataFiltered = (currentData) => {
|
464
481
|
if (Array.isArray(currentData.finalData[fieldToSetKey])) {
|
465
|
-
return currentData.finalData[fieldToSetKey].filter((data) => `${data}`
|
482
|
+
return currentData.finalData[fieldToSetKey].filter((data) => !sourceValues.includes(`${data}`));
|
466
483
|
}
|
467
484
|
return currentData.finalData[fieldToSetKey];
|
468
485
|
};
|
469
486
|
// Get the current data to be updated
|
470
487
|
const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
|
471
|
-
|
472
|
-
|
473
|
-
//
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
488
|
+
if (newData.length) {
|
489
|
+
needsUpdate = true;
|
490
|
+
// Handle cases where current data is an array
|
491
|
+
if (isFieldToSetArray) {
|
492
|
+
if (!currentDataFiltered) {
|
493
|
+
// Set new data if current data is undefined
|
494
|
+
collectionData.data[i].finalData[fieldToSetKey] =
|
495
|
+
Array.isArray(newData) ? newData : [newData];
|
496
|
+
}
|
497
|
+
else {
|
498
|
+
if (Array.isArray(currentDataFiltered)) {
|
499
|
+
// Convert current data to array and merge if new data is non-empty array
|
500
|
+
collectionData.data[i].finalData[fieldToSetKey] = [
|
501
|
+
...new Set([...currentDataFiltered, ...newData].filter((value) => value !== null &&
|
502
|
+
value !== undefined &&
|
503
|
+
value !== "")),
|
504
|
+
];
|
505
|
+
}
|
506
|
+
else {
|
507
|
+
// Merge arrays if new data is non-empty array and filter for uniqueness
|
508
|
+
collectionData.data[i].finalData[fieldToSetKey] = [
|
509
|
+
...new Set([
|
510
|
+
...(Array.isArray(currentDataFiltered)
|
511
|
+
? currentDataFiltered
|
512
|
+
: [currentDataFiltered]),
|
513
|
+
...newData,
|
514
|
+
].filter((value) => value !== null &&
|
515
|
+
value !== undefined &&
|
516
|
+
value !== "" &&
|
517
|
+
!sourceValues.includes(`${value}`))),
|
518
|
+
];
|
519
|
+
}
|
520
|
+
}
|
491
521
|
}
|
492
522
|
else {
|
493
|
-
if (
|
494
|
-
//
|
495
|
-
collectionData.data[i].finalData[fieldToSetKey] =
|
496
|
-
|
497
|
-
];
|
523
|
+
if (!currentDataFiltered) {
|
524
|
+
// Set new data if current data is undefined
|
525
|
+
collectionData.data[i].finalData[fieldToSetKey] =
|
526
|
+
Array.isArray(newData) ? newData[0] : newData;
|
498
527
|
}
|
499
|
-
else {
|
500
|
-
//
|
528
|
+
else if (Array.isArray(newData) && newData.length > 0) {
|
529
|
+
// Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
|
530
|
+
// and take the first value, because it's an array and the attribute is not an array
|
501
531
|
collectionData.data[i].finalData[fieldToSetKey] = [
|
502
|
-
...new Set([
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
532
|
+
...new Set([currentDataFiltered, ...newData].filter((value) => value !== null &&
|
533
|
+
value !== undefined &&
|
534
|
+
value !== "" &&
|
535
|
+
!sourceValues.includes(`${value}`))),
|
536
|
+
].slice(0, 1)[0];
|
537
|
+
}
|
538
|
+
else if (!Array.isArray(newData) &&
|
539
|
+
newData !== undefined) {
|
540
|
+
// Simply update the field if new data is not an array and defined
|
541
|
+
collectionData.data[i].finalData[fieldToSetKey] = newData;
|
509
542
|
}
|
510
|
-
}
|
511
|
-
}
|
512
|
-
else {
|
513
|
-
if (!currentDataFiltered) {
|
514
|
-
// Set new data if current data is undefined
|
515
|
-
collectionData.data[i].finalData[fieldToSetKey] =
|
516
|
-
Array.isArray(newData) ? newData[0] : newData;
|
517
|
-
}
|
518
|
-
else if (Array.isArray(newData) && newData.length > 0) {
|
519
|
-
// Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
|
520
|
-
// and take the first value, because it's an array and the attribute is not an array
|
521
|
-
collectionData.data[i].finalData[fieldToSetKey] = [
|
522
|
-
...new Set([currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}` && value)),
|
523
|
-
].slice(0, 1)[0];
|
524
|
-
}
|
525
|
-
else if (!Array.isArray(newData) && newData !== undefined) {
|
526
|
-
// Simply update the field if new data is not an array and defined
|
527
|
-
collectionData.data[i].finalData[fieldToSetKey] = newData;
|
528
543
|
}
|
529
544
|
}
|
530
545
|
}
|
package/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
{
|
2
2
|
"name": "appwrite-utils-cli",
|
3
3
|
"description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
|
4
|
-
"version": "0.0.
|
4
|
+
"version": "0.0.69",
|
5
5
|
"main": "src/main.ts",
|
6
6
|
"type": "module",
|
7
7
|
"repository": {
|
@@ -33,7 +33,7 @@
|
|
33
33
|
},
|
34
34
|
"dependencies": {
|
35
35
|
"@types/inquirer": "^9.0.7",
|
36
|
-
"appwrite-utils": "^0.3.
|
36
|
+
"appwrite-utils": "^0.3.5",
|
37
37
|
"commander": "^12.0.0",
|
38
38
|
"inquirer": "^9.2.20",
|
39
39
|
"js-yaml": "^4.1.0",
|
@@ -433,6 +433,13 @@ export const afterImportActions = {
|
|
433
433
|
// console.log(
|
434
434
|
// `Processing field ${fieldName} in collection ${collId} for document ${docId} in database ${dbId} in bucket ${bucketId} with path ${filePath} and name ${fileName}...`
|
435
435
|
// );
|
436
|
+
if (filePath.length === 0 || fileName.length === 0) {
|
437
|
+
console.error(
|
438
|
+
`File path or name is empty for field ${fieldName} in collection ${collId}, skipping...`
|
439
|
+
);
|
440
|
+
return;
|
441
|
+
}
|
442
|
+
|
436
443
|
let isArray = false;
|
437
444
|
if (!attribute) {
|
438
445
|
console.log(
|
@@ -542,7 +542,7 @@ export class DataLoader {
|
|
542
542
|
idMapping.fieldToSet || idMapping.sourceField;
|
543
543
|
const targetFieldKey =
|
544
544
|
idMapping.targetFieldToMatch || idMapping.targetField;
|
545
|
-
const
|
545
|
+
const sourceValue = this.getValueFromData(
|
546
546
|
collectionData.data[i].finalData,
|
547
547
|
collectionData.data[i].context,
|
548
548
|
idMapping.sourceField
|
@@ -550,9 +550,9 @@ export class DataLoader {
|
|
550
550
|
|
551
551
|
// Skip if value to match is missing or empty
|
552
552
|
if (
|
553
|
-
!
|
554
|
-
_.isEmpty(
|
555
|
-
|
553
|
+
!sourceValue ||
|
554
|
+
_.isEmpty(sourceValue) ||
|
555
|
+
sourceValue === null
|
556
556
|
)
|
557
557
|
continue;
|
558
558
|
|
@@ -565,28 +565,58 @@ export class DataLoader {
|
|
565
565
|
if (!targetCollectionData || !targetCollectionData.data)
|
566
566
|
continue;
|
567
567
|
|
568
|
-
//
|
569
|
-
const
|
570
|
-
(
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
568
|
+
// Handle cases where sourceValue is an array
|
569
|
+
const sourceValues = Array.isArray(sourceValue)
|
570
|
+
? sourceValue.map((sourceValue) => `${sourceValue}`)
|
571
|
+
: [`${sourceValue}`];
|
572
|
+
let newData = [];
|
573
|
+
|
574
|
+
for (const valueToMatch of sourceValues) {
|
575
|
+
// Find matching data in the target collection
|
576
|
+
const foundData = targetCollectionData.data.filter(
|
577
|
+
({ context, finalData }) => {
|
578
|
+
const targetValue = this.getValueFromData(
|
579
|
+
finalData,
|
580
|
+
context,
|
581
|
+
targetFieldKey
|
582
|
+
);
|
583
|
+
const isMatch = `${targetValue}` === `${valueToMatch}`;
|
584
|
+
// Ensure the targetValue is defined and not null
|
585
|
+
return (
|
586
|
+
isMatch &&
|
587
|
+
targetValue !== undefined &&
|
588
|
+
targetValue !== null
|
589
|
+
);
|
590
|
+
}
|
591
|
+
);
|
592
|
+
|
593
|
+
if (foundData.length) {
|
594
|
+
newData.push(
|
595
|
+
...foundData.map((data) => {
|
596
|
+
const newValue = this.getValueFromData(
|
597
|
+
data.finalData,
|
598
|
+
data.context,
|
599
|
+
idMapping.targetField
|
600
|
+
);
|
601
|
+
return newValue;
|
602
|
+
})
|
575
603
|
);
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
604
|
+
} else {
|
605
|
+
logger.info(
|
606
|
+
`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(
|
607
|
+
idMapping,
|
608
|
+
null,
|
609
|
+
2
|
610
|
+
)}`
|
582
611
|
);
|
583
612
|
}
|
584
|
-
|
613
|
+
continue;
|
614
|
+
}
|
585
615
|
|
586
616
|
const getCurrentDataFiltered = (currentData: any) => {
|
587
617
|
if (Array.isArray(currentData.finalData[fieldToSetKey])) {
|
588
618
|
return currentData.finalData[fieldToSetKey].filter(
|
589
|
-
(data: any) => `${data}`
|
619
|
+
(data: any) => !sourceValues.includes(`${data}`)
|
590
620
|
);
|
591
621
|
}
|
592
622
|
return currentData.finalData[fieldToSetKey];
|
@@ -596,87 +626,76 @@ export class DataLoader {
|
|
596
626
|
const currentDataFiltered = getCurrentDataFiltered(
|
597
627
|
collectionData.data[i]
|
598
628
|
);
|
599
|
-
// Log and skip if no matching data found
|
600
|
-
if (!foundData.length) {
|
601
|
-
// console.log(
|
602
|
-
// `No data found for collection ${collectionConfig.name}: - Target collection: ${targetCollectionKey} - Value to match: ${valueToMatch} - Field to set: ${fieldToSetKey} - Target field to match: ${targetFieldKey} - Target field value: ${idMapping.targetField}`
|
603
|
-
// );
|
604
|
-
logger.info(
|
605
|
-
`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(
|
606
|
-
idMapping,
|
607
|
-
null,
|
608
|
-
2
|
609
|
-
)}`
|
610
|
-
);
|
611
|
-
continue;
|
612
|
-
}
|
613
629
|
|
614
|
-
|
630
|
+
if (newData.length) {
|
631
|
+
needsUpdate = true;
|
615
632
|
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
);
|
623
|
-
return valueFound;
|
624
|
-
});
|
625
|
-
|
626
|
-
// Handle cases where current data is an array
|
627
|
-
if (isFieldToSetArray) {
|
628
|
-
if (!currentDataFiltered) {
|
629
|
-
// Set new data if current data is undefined
|
630
|
-
collectionData.data[i].finalData[fieldToSetKey] =
|
631
|
-
Array.isArray(newData) ? newData : [newData];
|
632
|
-
} else {
|
633
|
-
if (Array.isArray(currentDataFiltered)) {
|
634
|
-
// Convert current data to array and merge if new data is non-empty array
|
635
|
-
collectionData.data[i].finalData[fieldToSetKey] = [
|
636
|
-
...new Set(
|
637
|
-
[...currentDataFiltered, ...newData].filter(
|
638
|
-
(value: any) =>
|
639
|
-
`${value}` !== `${valueToMatch}` && value
|
640
|
-
)
|
641
|
-
),
|
642
|
-
];
|
633
|
+
// Handle cases where current data is an array
|
634
|
+
if (isFieldToSetArray) {
|
635
|
+
if (!currentDataFiltered) {
|
636
|
+
// Set new data if current data is undefined
|
637
|
+
collectionData.data[i].finalData[fieldToSetKey] =
|
638
|
+
Array.isArray(newData) ? newData : [newData];
|
643
639
|
} else {
|
644
|
-
|
640
|
+
if (Array.isArray(currentDataFiltered)) {
|
641
|
+
// Convert current data to array and merge if new data is non-empty array
|
642
|
+
collectionData.data[i].finalData[fieldToSetKey] = [
|
643
|
+
...new Set(
|
644
|
+
[...currentDataFiltered, ...newData].filter(
|
645
|
+
(value: any) =>
|
646
|
+
value !== null &&
|
647
|
+
value !== undefined &&
|
648
|
+
value !== ""
|
649
|
+
)
|
650
|
+
),
|
651
|
+
];
|
652
|
+
} else {
|
653
|
+
// Merge arrays if new data is non-empty array and filter for uniqueness
|
654
|
+
collectionData.data[i].finalData[fieldToSetKey] = [
|
655
|
+
...new Set(
|
656
|
+
[
|
657
|
+
...(Array.isArray(currentDataFiltered)
|
658
|
+
? currentDataFiltered
|
659
|
+
: [currentDataFiltered]),
|
660
|
+
...newData,
|
661
|
+
].filter(
|
662
|
+
(value: any) =>
|
663
|
+
value !== null &&
|
664
|
+
value !== undefined &&
|
665
|
+
value !== "" &&
|
666
|
+
!sourceValues.includes(`${value}`)
|
667
|
+
)
|
668
|
+
),
|
669
|
+
];
|
670
|
+
}
|
671
|
+
}
|
672
|
+
} else {
|
673
|
+
if (!currentDataFiltered) {
|
674
|
+
// Set new data if current data is undefined
|
675
|
+
collectionData.data[i].finalData[fieldToSetKey] =
|
676
|
+
Array.isArray(newData) ? newData[0] : newData;
|
677
|
+
} else if (Array.isArray(newData) && newData.length > 0) {
|
678
|
+
// Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
|
679
|
+
// and take the first value, because it's an array and the attribute is not an array
|
645
680
|
collectionData.data[i].finalData[fieldToSetKey] = [
|
646
681
|
...new Set(
|
647
|
-
[
|
648
|
-
...(Array.isArray(currentDataFiltered)
|
649
|
-
? currentDataFiltered
|
650
|
-
: [currentDataFiltered]),
|
651
|
-
...newData,
|
652
|
-
].filter(
|
682
|
+
[currentDataFiltered, ...newData].filter(
|
653
683
|
(value: any) =>
|
654
|
-
|
684
|
+
value !== null &&
|
685
|
+
value !== undefined &&
|
686
|
+
value !== "" &&
|
687
|
+
!sourceValues.includes(`${value}`)
|
655
688
|
)
|
656
689
|
),
|
657
|
-
];
|
690
|
+
].slice(0, 1)[0];
|
691
|
+
} else if (
|
692
|
+
!Array.isArray(newData) &&
|
693
|
+
newData !== undefined
|
694
|
+
) {
|
695
|
+
// Simply update the field if new data is not an array and defined
|
696
|
+
collectionData.data[i].finalData[fieldToSetKey] = newData;
|
658
697
|
}
|
659
698
|
}
|
660
|
-
} else {
|
661
|
-
if (!currentDataFiltered) {
|
662
|
-
// Set new data if current data is undefined
|
663
|
-
collectionData.data[i].finalData[fieldToSetKey] =
|
664
|
-
Array.isArray(newData) ? newData[0] : newData;
|
665
|
-
} else if (Array.isArray(newData) && newData.length > 0) {
|
666
|
-
// Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
|
667
|
-
// and take the first value, because it's an array and the attribute is not an array
|
668
|
-
collectionData.data[i].finalData[fieldToSetKey] = [
|
669
|
-
...new Set(
|
670
|
-
[currentDataFiltered, ...newData].filter(
|
671
|
-
(value: any) =>
|
672
|
-
`${value}` !== `${valueToMatch}` && value
|
673
|
-
)
|
674
|
-
),
|
675
|
-
].slice(0, 1)[0];
|
676
|
-
} else if (!Array.isArray(newData) && newData !== undefined) {
|
677
|
-
// Simply update the field if new data is not an array and defined
|
678
|
-
collectionData.data[i].finalData[fieldToSetKey] = newData;
|
679
|
-
}
|
680
699
|
}
|
681
700
|
}
|
682
701
|
}
|