appwrite-utils-cli 0.0.66 → 0.0.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/README.md +2 -0
  2. package/dist/migrations/dataLoader.d.ts +31 -13
  3. package/dist/migrations/dataLoader.js +97 -76
  4. package/package.json +2 -2
  5. package/src/migrations/dataLoader.ts +135 -102
package/README.md CHANGED
@@ -132,6 +132,8 @@ This setup ensures that developers have robust tools at their fingertips to mana
132
132
 
133
133
  ### Changelog
134
134
 
135
+ - 0.0.68: Fixed the occasional case where, when mapping ID's from old data to new, there would be an array of ID's to match against. `idMappings` now supports arrays.
136
+ - 0.0.67: Fixed `updates` in `importDef`'s update mappings overwriting postImportActions from the original
135
137
  - 0.0.57: Fixed `dataLoader`'s `idMapping`'s giving me issues
136
138
  - 0.0.55: Added `documentExists` check to batch creation functionality to try to prevent duplicates
137
139
  - 0.0.54: Various fixes in here
package/dist/migrations/dataLoader.d.ts CHANGED
@@ -366,16 +366,7 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
366
366
  fieldToSet: z.ZodOptional<z.ZodString>;
367
367
  targetFieldToMatch: z.ZodOptional<z.ZodString>;
368
368
  targetField: z.ZodString;
369
- targetCollection: z.ZodString; /**
370
- * Prepares the data for updating documents within a collection.
371
- * This method loads the raw data based on the import definition, transforms it according to the attribute mappings,
372
- * finds the new ID for each item based on the primary key or update mapping, and then validates the transformed data.
373
- * If the data is valid, it updates the import definition with any post-import actions and adds the item to the current collection data.
374
- *
375
- * @param db - The database configuration.
376
- * @param collection - The collection configuration.
377
- * @param importDef - The import definition containing the attribute mappings and other relevant info.
378
- */
369
+ targetCollection: z.ZodString;
379
370
  }, "strip", z.ZodTypeAny, {
380
371
  sourceField: string;
381
372
  targetField: string;
@@ -675,7 +666,16 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
675
666
  converters?: string[] | undefined;
676
667
  validationActions?: {
677
668
  params: string[];
678
- action: string;
669
+ action: string; /**
670
+ * Generates attribute mappings with post-import actions based on the provided attribute mappings.
671
+ * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
672
+ * and update the field with the file's ID if necessary.
673
+ *
674
+ * @param attributeMappings - The attribute mappings from the import definition.
675
+ * @param context - The context object containing information about the database, collection, and document.
676
+ * @param item - The item being imported, used for resolving template paths in fileData mappings.
677
+ * @returns The attribute mappings updated with any necessary post-import actions.
678
+ */
679
679
  }[] | undefined;
680
680
  postImportActions?: {
681
681
  params: (string | Record<string, any>)[];
@@ -1295,7 +1295,16 @@ export declare const CollectionImportDataSchema: z.ZodObject<{
1295
1295
  converters?: string[] | undefined;
1296
1296
  validationActions?: {
1297
1297
  params: string[];
1298
- action: string;
1298
+ action: string; /**
1299
+ * Generates attribute mappings with post-import actions based on the provided attribute mappings.
1300
+ * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
1301
+ * and update the field with the file's ID if necessary.
1302
+ *
1303
+ * @param attributeMappings - The attribute mappings from the import definition.
1304
+ * @param context - The context object containing information about the database, collection, and document.
1305
+ * @param item - The item being imported, used for resolving template paths in fileData mappings.
1306
+ * @returns The attribute mappings updated with any necessary post-import actions.
1307
+ */
1299
1308
  }[] | undefined;
1300
1309
  postImportActions?: {
1301
1310
  params: (string | Record<string, any>)[];
@@ -1699,7 +1708,16 @@ export declare class DataLoader {
1699
1708
  converters?: string[] | undefined;
1700
1709
  validationActions?: {
1701
1710
  params: string[];
1702
- action: string;
1711
+ action: string; /**
1712
+ * Generates attribute mappings with post-import actions based on the provided attribute mappings.
1713
+ * This method checks each mapping for a fileData attribute and adds a post-import action to create a file
1714
+ * and update the field with the file's ID if necessary.
1715
+ *
1716
+ * @param attributeMappings - The attribute mappings from the import definition.
1717
+ * @param context - The context object containing information about the database, collection, and document.
1718
+ * @param item - The item being imported, used for resolving template paths in fileData mappings.
1719
+ * @returns The attribute mappings updated with any necessary post-import actions.
1720
+ */
1703
1721
  }[] | undefined;
1704
1722
  postImportActions?: {
1705
1723
  params: (string | Record<string, any>)[];
package/dist/migrations/dataLoader.js CHANGED
@@ -441,88 +441,101 @@ export class DataLoader {
441
441
  const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
442
442
  const fieldToSetKey = idMapping.fieldToSet || idMapping.sourceField;
443
443
  const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
444
- const valueToMatch = this.getValueFromData(collectionData.data[i].finalData, collectionData.data[i].context, idMapping.sourceField);
444
+ const sourceValue = this.getValueFromData(collectionData.data[i].finalData, collectionData.data[i].context, idMapping.sourceField);
445
445
  // Skip if value to match is missing or empty
446
- if (!valueToMatch ||
447
- _.isEmpty(valueToMatch) ||
448
- valueToMatch === null)
446
+ if (!sourceValue ||
447
+ _.isEmpty(sourceValue) ||
448
+ sourceValue === null)
449
449
  continue;
450
450
  const isFieldToSetArray = collectionConfig.attributes.find((attribute) => attribute.key === fieldToSetKey)?.array;
451
451
  const targetCollectionData = this.importMap.get(targetCollectionKey);
452
452
  if (!targetCollectionData || !targetCollectionData.data)
453
453
  continue;
454
- // Find matching data in the target collection
455
- const foundData = targetCollectionData.data.filter(({ context, finalData }) => {
456
- const targetValue = this.getValueFromData(finalData, context, targetFieldKey);
457
- const isMatch = `${targetValue}` === `${valueToMatch}`;
458
- // Ensure the targetValue is defined and not null
459
- return (isMatch &&
460
- targetValue !== undefined &&
461
- targetValue !== null);
462
- });
454
+ // Handle cases where sourceValue is an array
455
+ const sourceValues = Array.isArray(sourceValue)
456
+ ? sourceValue
457
+ : [sourceValue];
458
+ const newData = [];
459
+ for (const valueToMatch of sourceValues) {
460
+ // Find matching data in the target collection
461
+ const foundData = targetCollectionData.data.filter(({ context, finalData }) => {
462
+ const targetValue = this.getValueFromData(finalData, context, targetFieldKey);
463
+ const isMatch = `${targetValue}` === `${valueToMatch}`;
464
+ // Ensure the targetValue is defined and not null
465
+ return (isMatch &&
466
+ targetValue !== undefined &&
467
+ targetValue !== null);
468
+ });
469
+ if (foundData.length) {
470
+ newData.push(...foundData.map((data) => {
471
+ return this.getValueFromData(data.finalData, data.context, idMapping.targetField);
472
+ }));
473
+ }
474
+ else {
475
+ logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
476
+ }
477
+ }
463
478
  const getCurrentDataFiltered = (currentData) => {
464
479
  if (Array.isArray(currentData.finalData[fieldToSetKey])) {
465
- return currentData.finalData[fieldToSetKey].filter((data) => `${data}` !== `${valueToMatch}`);
480
+ return currentData.finalData[fieldToSetKey].filter((data) => !sourceValues.includes(`${data}`));
466
481
  }
467
482
  return currentData.finalData[fieldToSetKey];
468
483
  };
469
484
  // Get the current data to be updated
470
485
  const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
471
- // Log and skip if no matching data found
472
- if (!foundData.length) {
473
- console.log(`No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`);
474
- logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
475
- continue;
476
- }
477
- needsUpdate = true;
478
- // Extract the new data to set
479
- const newData = foundData.map((data) => {
480
- const valueFound = this.getValueFromData(data.finalData, data.context, idMapping.targetField);
481
- return valueFound;
482
- });
483
- // Handle cases where current data is an array
484
- if (isFieldToSetArray) {
485
- if (!currentDataFiltered) {
486
- // Set new data if current data is undefined
487
- collectionData.data[i].finalData[fieldToSetKey] =
488
- Array.isArray(newData) ? newData : [newData];
486
+ if (newData.length) {
487
+ needsUpdate = true;
488
+ // Handle cases where current data is an array
489
+ if (isFieldToSetArray) {
490
+ if (!currentDataFiltered) {
491
+ // Set new data if current data is undefined
492
+ collectionData.data[i].finalData[fieldToSetKey] =
493
+ Array.isArray(newData) ? newData : [newData];
494
+ }
495
+ else {
496
+ if (Array.isArray(currentDataFiltered)) {
497
+ // Convert current data to array and merge if new data is non-empty array
498
+ collectionData.data[i].finalData[fieldToSetKey] = [
499
+ ...new Set([...currentDataFiltered, ...newData].filter((value) => value !== null &&
500
+ value !== undefined &&
501
+ value !== "")),
502
+ ];
503
+ }
504
+ else {
505
+ // Merge arrays if new data is non-empty array and filter for uniqueness
506
+ collectionData.data[i].finalData[fieldToSetKey] = [
507
+ ...new Set([
508
+ ...(Array.isArray(currentDataFiltered)
509
+ ? currentDataFiltered
510
+ : [currentDataFiltered]),
511
+ ...newData,
512
+ ].filter((value) => value !== null &&
513
+ value !== undefined &&
514
+ value !== "")),
515
+ ];
516
+ }
517
+ }
489
518
  }
490
519
  else {
491
- if (Array.isArray(currentDataFiltered)) {
492
- // Convert current data to array and merge if new data is non-empty array
493
- collectionData.data[i].finalData[fieldToSetKey] = [
494
- ...new Set([...currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}` && value)),
495
- ];
520
+ if (!currentDataFiltered) {
521
+ // Set new data if current data is undefined
522
+ collectionData.data[i].finalData[fieldToSetKey] =
523
+ Array.isArray(newData) ? newData[0] : newData;
496
524
  }
497
- else {
498
- // Merge arrays if new data is non-empty array and filter for uniqueness
525
+ else if (Array.isArray(newData) && newData.length > 0) {
526
+ // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
527
+ // and take the first value, because it's an array and the attribute is not an array
499
528
  collectionData.data[i].finalData[fieldToSetKey] = [
500
- ...new Set([
501
- ...(Array.isArray(currentDataFiltered)
502
- ? currentDataFiltered
503
- : [currentDataFiltered]),
504
- ...newData,
505
- ].filter((value) => `${value}` !== `${valueToMatch}` && value)),
506
- ];
529
+ ...new Set([currentDataFiltered, ...newData].filter((value) => value !== null &&
530
+ value !== undefined &&
531
+ value !== "")),
532
+ ].slice(0, 1)[0];
533
+ }
534
+ else if (!Array.isArray(newData) &&
535
+ newData !== undefined) {
536
+ // Simply update the field if new data is not an array and defined
537
+ collectionData.data[i].finalData[fieldToSetKey] = newData;
507
538
  }
508
- }
509
- }
510
- else {
511
- if (!currentDataFiltered) {
512
- // Set new data if current data is undefined
513
- collectionData.data[i].finalData[fieldToSetKey] =
514
- Array.isArray(newData) ? newData[0] : newData;
515
- }
516
- else if (Array.isArray(newData) && newData.length > 0) {
517
- // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
518
- // and take the first value, because it's an array and the attribute is not an array
519
- collectionData.data[i].finalData[fieldToSetKey] = [
520
- ...new Set([currentDataFiltered, ...newData].filter((value) => `${value}` !== `${valueToMatch}` && value)),
521
- ].slice(0, 1)[0];
522
- }
523
- else if (!Array.isArray(newData) && newData !== undefined) {
524
- // Simply update the field if new data is not an array and defined
525
- collectionData.data[i].finalData[fieldToSetKey] = newData;
526
539
  }
527
540
  }
528
541
  }
@@ -978,17 +991,14 @@ export class DataLoader {
978
991
  item[importDef.updateMapping.originalIdField] ||
979
992
  transformedData[importDef.updateMapping.originalIdField];
980
993
  if (oldId) {
981
- itemDataToUpdate = currentData?.data.find(({ context, rawData, finalData }) => {
994
+ itemDataToUpdate = currentData?.data.find(({ context, finalData }) => {
982
995
  const targetField = importDef.updateMapping.targetField ??
983
996
  importDef.updateMapping.originalIdField;
984
997
  return (`${context[targetField]}` === `${oldId}` ||
985
- `${rawData[targetField]}` === `${oldId}` ||
986
998
  `${finalData[targetField]}` === `${oldId}`);
987
999
  });
988
1000
  if (itemDataToUpdate) {
989
- newId =
990
- itemDataToUpdate.finalData.docId ||
991
- itemDataToUpdate.context.docId;
1001
+ newId = itemDataToUpdate.context.docId;
992
1002
  }
993
1003
  }
994
1004
  }
@@ -1036,10 +1046,10 @@ export class DataLoader {
1036
1046
  }
1037
1047
  transformedData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
1038
1048
  // Create a context object for the item, including the new ID and transformed data
1039
- let context = this.createContext(db, collection, item, newId);
1049
+ let context = itemDataToUpdate.context;
1040
1050
  context = this.mergeObjects(context, transformedData);
1041
1051
  // Validate the item before proceeding
1042
- const isValid = await this.importDataActions.validateItem(item, importDef.attributeMappings, context);
1052
+ const isValid = this.importDataActions.validateItem(item, importDef.attributeMappings, context);
1043
1053
  if (!isValid) {
1044
1054
  logger.info(`Skipping item: ${JSON.stringify(item, null, 2)} because it's invalid`);
1045
1055
  continue;
@@ -1049,15 +1059,26 @@ export class DataLoader {
1049
1059
  // Update the import definition with the new attribute mappings
1050
1060
  const newImportDef = {
1051
1061
  ...importDef,
1052
- attributeMappings: [
1053
- ...importDef.attributeMappings,
1054
- ...mappingsWithActions,
1055
- ],
1062
+ attributeMappings: mappingsWithActions,
1056
1063
  };
1057
1064
  if (itemDataToUpdate) {
1058
1065
  itemDataToUpdate.finalData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
1059
1066
  itemDataToUpdate.context = context;
1060
- itemDataToUpdate.importDef = newImportDef;
1067
+ // Merge existing importDef with new importDef, focusing only on postImportActions
1068
+ itemDataToUpdate.importDef = {
1069
+ ...itemDataToUpdate.importDef,
1070
+ attributeMappings: itemDataToUpdate.importDef?.attributeMappings.map((attrMapping, index) => ({
1071
+ ...attrMapping,
1072
+ postImportActions: [
1073
+ ...(attrMapping.postImportActions || []),
1074
+ ...(newImportDef.attributeMappings[index]
1075
+ ?.postImportActions || []),
1076
+ ],
1077
+ })) || [],
1078
+ };
1079
+ if (collection.name.toLowerCase() === "councils") {
1080
+ console.log(`Mappings in update councils: ${JSON.stringify(itemDataToUpdate.importDef.attributeMappings, null, 2)}`);
1081
+ }
1061
1082
  }
1062
1083
  else {
1063
1084
  currentData.data.push({
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "appwrite-utils-cli",
3
3
  "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
4
- "version": "0.0.66",
4
+ "version": "0.0.68",
5
5
  "main": "src/main.ts",
6
6
  "type": "module",
7
7
  "repository": {
@@ -33,7 +33,7 @@
33
33
  },
34
34
  "dependencies": {
35
35
  "@types/inquirer": "^9.0.7",
36
- "appwrite-utils": "^0.3.3",
36
+ "appwrite-utils": "^0.3.5",
37
37
  "commander": "^12.0.0",
38
38
  "inquirer": "^9.2.20",
39
39
  "js-yaml": "^4.1.0",
package/src/migrations/dataLoader.ts CHANGED
@@ -542,7 +542,7 @@ export class DataLoader {
542
542
  idMapping.fieldToSet || idMapping.sourceField;
543
543
  const targetFieldKey =
544
544
  idMapping.targetFieldToMatch || idMapping.targetField;
545
- const valueToMatch = this.getValueFromData(
545
+ const sourceValue = this.getValueFromData(
546
546
  collectionData.data[i].finalData,
547
547
  collectionData.data[i].context,
548
548
  idMapping.sourceField
@@ -550,9 +550,9 @@ export class DataLoader {
550
550
 
551
551
  // Skip if value to match is missing or empty
552
552
  if (
553
- !valueToMatch ||
554
- _.isEmpty(valueToMatch) ||
555
- valueToMatch === null
553
+ !sourceValue ||
554
+ _.isEmpty(sourceValue) ||
555
+ sourceValue === null
556
556
  )
557
557
  continue;
558
558
 
@@ -565,28 +565,56 @@ export class DataLoader {
565
565
  if (!targetCollectionData || !targetCollectionData.data)
566
566
  continue;
567
567
 
568
- // Find matching data in the target collection
569
- const foundData = targetCollectionData.data.filter(
570
- ({ context, finalData }) => {
571
- const targetValue = this.getValueFromData(
572
- finalData,
573
- context,
574
- targetFieldKey
568
+ // Handle cases where sourceValue is an array
569
+ const sourceValues = Array.isArray(sourceValue)
570
+ ? sourceValue
571
+ : [sourceValue];
572
+ const newData = [];
573
+
574
+ for (const valueToMatch of sourceValues) {
575
+ // Find matching data in the target collection
576
+ const foundData = targetCollectionData.data.filter(
577
+ ({ context, finalData }) => {
578
+ const targetValue = this.getValueFromData(
579
+ finalData,
580
+ context,
581
+ targetFieldKey
582
+ );
583
+ const isMatch = `${targetValue}` === `${valueToMatch}`;
584
+ // Ensure the targetValue is defined and not null
585
+ return (
586
+ isMatch &&
587
+ targetValue !== undefined &&
588
+ targetValue !== null
589
+ );
590
+ }
591
+ );
592
+
593
+ if (foundData.length) {
594
+ newData.push(
595
+ ...foundData.map((data) => {
596
+ return this.getValueFromData(
597
+ data.finalData,
598
+ data.context,
599
+ idMapping.targetField
600
+ );
601
+ })
575
602
  );
576
- const isMatch = `${targetValue}` === `${valueToMatch}`;
577
- // Ensure the targetValue is defined and not null
578
- return (
579
- isMatch &&
580
- targetValue !== undefined &&
581
- targetValue !== null
603
+ } else {
604
+ logger.info(
605
+ `No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(
606
+ idMapping,
607
+ null,
608
+ 2
609
+ )}`
582
610
  );
583
611
  }
584
- );
612
+ }
585
613
 
586
614
  const getCurrentDataFiltered = (currentData: any) => {
587
615
  if (Array.isArray(currentData.finalData[fieldToSetKey])) {
588
616
  return currentData.finalData[fieldToSetKey].filter(
589
- (data: any) => `${data}` !== `${valueToMatch}`
617
+ (data: any) => !sourceValues.includes(`${data}`)
590
618
  );
591
619
  }
592
620
  return currentData.finalData[fieldToSetKey];
@@ -596,87 +624,74 @@ export class DataLoader {
596
624
  const currentDataFiltered = getCurrentDataFiltered(
597
625
  collectionData.data[i]
598
626
  );
599
- // Log and skip if no matching data found
600
- if (!foundData.length) {
601
- console.log(
602
- `No data found for collection ${collectionConfig.name}:\nTarget collection: ${targetCollectionKey}\nValue to match: ${valueToMatch}\nField to set: ${fieldToSetKey}\nTarget field to match: ${targetFieldKey}\nTarget field value: ${idMapping.targetField}`
603
- );
604
- logger.info(
605
- `No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(
606
- idMapping,
607
- null,
608
- 2
609
- )}`
610
- );
611
- continue;
612
- }
613
627
 
614
- needsUpdate = true;
628
+ if (newData.length) {
629
+ needsUpdate = true;
615
630
 
616
- // Extract the new data to set
617
- const newData = foundData.map((data) => {
618
- const valueFound = this.getValueFromData(
619
- data.finalData,
620
- data.context,
621
- idMapping.targetField
622
- );
623
- return valueFound;
624
- });
625
-
626
- // Handle cases where current data is an array
627
- if (isFieldToSetArray) {
628
- if (!currentDataFiltered) {
629
- // Set new data if current data is undefined
630
- collectionData.data[i].finalData[fieldToSetKey] =
631
- Array.isArray(newData) ? newData : [newData];
632
- } else {
633
- if (Array.isArray(currentDataFiltered)) {
634
- // Convert current data to array and merge if new data is non-empty array
635
- collectionData.data[i].finalData[fieldToSetKey] = [
636
- ...new Set(
637
- [...currentDataFiltered, ...newData].filter(
638
- (value: any) =>
639
- `${value}` !== `${valueToMatch}` && value
640
- )
641
- ),
642
- ];
631
+ // Handle cases where current data is an array
632
+ if (isFieldToSetArray) {
633
+ if (!currentDataFiltered) {
634
+ // Set new data if current data is undefined
635
+ collectionData.data[i].finalData[fieldToSetKey] =
636
+ Array.isArray(newData) ? newData : [newData];
643
637
  } else {
644
- // Merge arrays if new data is non-empty array and filter for uniqueness
638
+ if (Array.isArray(currentDataFiltered)) {
639
+ // Convert current data to array and merge if new data is non-empty array
640
+ collectionData.data[i].finalData[fieldToSetKey] = [
641
+ ...new Set(
642
+ [...currentDataFiltered, ...newData].filter(
643
+ (value: any) =>
644
+ value !== null &&
645
+ value !== undefined &&
646
+ value !== ""
647
+ )
648
+ ),
649
+ ];
650
+ } else {
651
+ // Merge arrays if new data is non-empty array and filter for uniqueness
652
+ collectionData.data[i].finalData[fieldToSetKey] = [
653
+ ...new Set(
654
+ [
655
+ ...(Array.isArray(currentDataFiltered)
656
+ ? currentDataFiltered
657
+ : [currentDataFiltered]),
658
+ ...newData,
659
+ ].filter(
660
+ (value: any) =>
661
+ value !== null &&
662
+ value !== undefined &&
663
+ value !== ""
664
+ )
665
+ ),
666
+ ];
667
+ }
668
+ }
669
+ } else {
670
+ if (!currentDataFiltered) {
671
+ // Set new data if current data is undefined
672
+ collectionData.data[i].finalData[fieldToSetKey] =
673
+ Array.isArray(newData) ? newData[0] : newData;
674
+ } else if (Array.isArray(newData) && newData.length > 0) {
675
+ // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
676
+ // and take the first value, because it's an array and the attribute is not an array
645
677
  collectionData.data[i].finalData[fieldToSetKey] = [
646
678
  ...new Set(
647
- [
648
- ...(Array.isArray(currentDataFiltered)
649
- ? currentDataFiltered
650
- : [currentDataFiltered]),
651
- ...newData,
652
- ].filter(
679
+ [currentDataFiltered, ...newData].filter(
653
680
  (value: any) =>
654
- `${value}` !== `${valueToMatch}` && value
681
+ value !== null &&
682
+ value !== undefined &&
683
+ value !== ""
655
684
  )
656
685
  ),
657
- ];
686
+ ].slice(0, 1)[0];
687
+ } else if (
688
+ !Array.isArray(newData) &&
689
+ newData !== undefined
690
+ ) {
691
+ // Simply update the field if new data is not an array and defined
692
+ collectionData.data[i].finalData[fieldToSetKey] = newData;
658
693
  }
659
694
  }
660
- } else {
661
- if (!currentDataFiltered) {
662
- // Set new data if current data is undefined
663
- collectionData.data[i].finalData[fieldToSetKey] =
664
- Array.isArray(newData) ? newData[0] : newData;
665
- } else if (Array.isArray(newData) && newData.length > 0) {
666
- // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
667
- // and take the first value, because it's an array and the attribute is not an array
668
- collectionData.data[i].finalData[fieldToSetKey] = [
669
- ...new Set(
670
- [currentDataFiltered, ...newData].filter(
671
- (value: any) =>
672
- `${value}` !== `${valueToMatch}` && value
673
- )
674
- ),
675
- ].slice(0, 1)[0];
676
- } else if (!Array.isArray(newData) && newData !== undefined) {
677
- // Simply update the field if new data is not an array and defined
678
- collectionData.data[i].finalData[fieldToSetKey] = newData;
679
- }
680
695
  }
681
696
  }
682
697
  }
@@ -1297,23 +1312,20 @@ export class DataLoader {
1297
1312
  transformedData[importDef.updateMapping.originalIdField];
1298
1313
  if (oldId) {
1299
1314
  itemDataToUpdate = currentData?.data.find(
1300
- ({ context, rawData, finalData }) => {
1315
+ ({ context, finalData }) => {
1301
1316
  const targetField =
1302
1317
  importDef.updateMapping!.targetField ??
1303
1318
  importDef.updateMapping!.originalIdField;
1304
1319
 
1305
1320
  return (
1306
1321
  `${context[targetField]}` === `${oldId}` ||
1307
- `${rawData[targetField]}` === `${oldId}` ||
1308
1322
  `${finalData[targetField]}` === `${oldId}`
1309
1323
  );
1310
1324
  }
1311
1325
  );
1312
1326
 
1313
1327
  if (itemDataToUpdate) {
1314
- newId =
1315
- itemDataToUpdate.finalData.docId ||
1316
- itemDataToUpdate.context.docId;
1328
+ newId = itemDataToUpdate.context.docId;
1317
1329
  }
1318
1330
  }
1319
1331
  }
@@ -1407,11 +1419,11 @@ export class DataLoader {
1407
1419
  );
1408
1420
 
1409
1421
  // Create a context object for the item, including the new ID and transformed data
1410
- let context = this.createContext(db, collection, item, newId);
1422
+ let context = itemDataToUpdate.context;
1411
1423
  context = this.mergeObjects(context, transformedData);
1412
1424
 
1413
1425
  // Validate the item before proceeding
1414
- const isValid = await this.importDataActions.validateItem(
1426
+ const isValid = this.importDataActions.validateItem(
1415
1427
  item,
1416
1428
  importDef.attributeMappings,
1417
1429
  context
@@ -1434,10 +1446,7 @@ export class DataLoader {
1434
1446
  // Update the import definition with the new attribute mappings
1435
1447
  const newImportDef = {
1436
1448
  ...importDef,
1437
- attributeMappings: [
1438
- ...importDef.attributeMappings,
1439
- ...mappingsWithActions,
1440
- ],
1449
+ attributeMappings: mappingsWithActions,
1441
1450
  };
1442
1451
 
1443
1452
  if (itemDataToUpdate) {
@@ -1446,7 +1455,31 @@ export class DataLoader {
1446
1455
  transformedData
1447
1456
  );
1448
1457
  itemDataToUpdate.context = context;
1449
- itemDataToUpdate.importDef = newImportDef;
1458
+ // Merge existing importDef with new importDef, focusing only on postImportActions
1459
+ itemDataToUpdate.importDef = {
1460
+ ...itemDataToUpdate.importDef,
1461
+ attributeMappings:
1462
+ itemDataToUpdate.importDef?.attributeMappings.map(
1463
+ (attrMapping, index) => ({
1464
+ ...attrMapping,
1465
+ postImportActions: [
1466
+ ...(attrMapping.postImportActions || []),
1467
+ ...(newImportDef.attributeMappings[index]
1468
+ ?.postImportActions || []),
1469
+ ],
1470
+ })
1471
+ ) || [],
1472
+ } as ImportDef;
1473
+
1474
+ if (collection.name.toLowerCase() === "councils") {
1475
+ console.log(
1476
+ `Mappings in update councils: ${JSON.stringify(
1477
+ itemDataToUpdate.importDef.attributeMappings,
1478
+ null,
1479
+ 2
1480
+ )}`
1481
+ );
1482
+ }
1450
1483
  } else {
1451
1484
  currentData!.data.push({
1452
1485
  rawData: item,