mcdev 4.1.12 → 4.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/.eslintrc.json +1 -1
  2. package/.github/ISSUE_TEMPLATE/bug.yml +1 -0
  3. package/.github/PULL_REQUEST_TEMPLATE.md +2 -3
  4. package/.nycrc.json +5 -0
  5. package/README.md +34 -1528
  6. package/boilerplate/config.json +2 -6
  7. package/boilerplate/files/.vscode/extensions.json +1 -0
  8. package/boilerplate/files/.vscode/settings.json +3 -0
  9. package/docs/dist/documentation.md +437 -31
  10. package/lib/Deployer.js +10 -8
  11. package/lib/MetadataTypeDefinitions.js +3 -0
  12. package/lib/MetadataTypeInfo.js +3 -0
  13. package/lib/Retriever.js +14 -7
  14. package/lib/cli.js +1 -0
  15. package/lib/index.js +6 -6
  16. package/lib/metadataTypes/AccountUser.js +2 -2
  17. package/lib/metadataTypes/Asset.js +45 -35
  18. package/lib/metadataTypes/Automation.js +4 -4
  19. package/lib/metadataTypes/DataExtension.js +14 -8
  20. package/lib/metadataTypes/DataExtensionField.js +44 -9
  21. package/lib/metadataTypes/Discovery.js +5 -5
  22. package/lib/metadataTypes/Folder.js +30 -6
  23. package/lib/metadataTypes/List.js +115 -17
  24. package/lib/metadataTypes/MetadataType.js +73 -40
  25. package/lib/metadataTypes/Query.js +2 -2
  26. package/lib/metadataTypes/Script.js +2 -2
  27. package/lib/metadataTypes/TransactionalEmail.js +163 -0
  28. package/lib/metadataTypes/TransactionalMessage.js +127 -0
  29. package/lib/metadataTypes/TransactionalPush.js +77 -0
  30. package/lib/metadataTypes/TransactionalSMS.js +354 -0
  31. package/lib/metadataTypes/TriggeredSendDefinition.js +11 -9
  32. package/lib/metadataTypes/definitions/TransactionalEmail.definition.js +145 -0
  33. package/lib/metadataTypes/definitions/TransactionalPush.definition.js +109 -0
  34. package/lib/metadataTypes/definitions/TransactionalSMS.definition.js +103 -0
  35. package/lib/metadataTypes/definitions/TriggeredSendDefinition.definition.js +36 -36
  36. package/lib/util/auth.js +2 -2
  37. package/lib/util/businessUnit.js +1 -1
  38. package/lib/util/cli.js +19 -20
  39. package/lib/util/config.js +13 -12
  40. package/lib/util/devops.js +4 -4
  41. package/lib/util/init.config.js +7 -7
  42. package/lib/util/init.git.js +11 -23
  43. package/lib/util/init.js +67 -3
  44. package/lib/util/util.js +20 -12
  45. package/package.json +19 -12
  46. package/test/dataExtension.test.js +36 -19
  47. package/test/mockRoot/.mcdevrc.json +13 -2
  48. package/test/mockRoot/deploy/testInstance/testBU/dataExtension/childBU_dataextension_test.dataExtension-meta.json +27 -7
  49. package/test/mockRoot/deploy/testInstance/testBU/query/testExistingQuery.query-meta.json +1 -1
  50. package/test/mockRoot/deploy/testInstance/testBU/query/testExistingQuery.query-meta.sql +3 -1
  51. package/test/mockRoot/deploy/testInstance/testBU/query/{testQuery.query-meta.json → testNewQuery.query-meta.json} +3 -3
  52. package/test/mockRoot/deploy/testInstance/testBU/query/{testQuery.query-meta.sql → testNewQuery.query-meta.sql} +0 -0
  53. package/test/mockRoot/deploy/testInstance/testBU/transactionalEmail/testExisting_temail.transactionalEmail-meta.json +24 -0
  54. package/test/mockRoot/deploy/testInstance/testBU/transactionalEmail/testNew_temail.transactionalEmail-meta.json +24 -0
  55. package/test/mockRoot/deploy/testInstance/testBU/transactionalPush/testExisting_tpush.transactionalPush-meta.json +18 -0
  56. package/test/mockRoot/deploy/testInstance/testBU/transactionalPush/testNew_tpush.transactionalPush-meta.json +18 -0
  57. package/test/mockRoot/deploy/testInstance/testBU/transactionalSMS/testExisting_tsms.transactionalSMS-meta.amp +4 -0
  58. package/test/mockRoot/deploy/testInstance/testBU/transactionalSMS/testExisting_tsms.transactionalSMS-meta.json +15 -0
  59. package/test/mockRoot/deploy/testInstance/testBU/transactionalSMS/testNew_tsms.transactionalSMS-meta.amp +4 -0
  60. package/test/mockRoot/deploy/testInstance/testBU/transactionalSMS/testNew_tsms.transactionalSMS-meta.json +15 -0
  61. package/test/query.test.js +57 -23
  62. package/test/resources/1111111/businessUnit/retrieve-response.xml +33 -0
  63. package/test/resources/1111111/list/retrieve-response.xml +39 -0
  64. package/test/resources/9999999/asset/v1/content/assets/query/post-response.json +72 -0
  65. package/test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat/patch-response.json +2 -2
  66. package/test/resources/9999999/automation/v1/queries/get-response.json +2 -2
  67. package/test/resources/9999999/automation/v1/queries/post-response.json +3 -3
  68. package/test/resources/9999999/dataExtension/build-expected.json +2 -2
  69. package/test/resources/9999999/dataFolder/retrieve-response.xml +95 -2
  70. package/test/resources/9999999/interaction/v1/interactions/get-response.json +296 -0
  71. package/test/resources/9999999/legacy/v1/beta/mobile/code/get-response.json +32 -0
  72. package/test/resources/9999999/legacy/v1/beta/mobile/keyword/get-response.json +46 -0
  73. package/test/resources/9999999/list/retrieve-response.xml +78 -0
  74. package/test/resources/9999999/messaging/v1/email/definitions/get-response.json +15 -0
  75. package/test/resources/9999999/messaging/v1/email/definitions/post-response.json +19 -0
  76. package/test/resources/9999999/messaging/v1/email/definitions/testExisting_temail/get-response.json +26 -0
  77. package/test/resources/9999999/messaging/v1/email/definitions/testExisting_temail/patch-response.json +26 -0
  78. package/test/resources/9999999/messaging/v1/push/definitions/get-response.json +15 -0
  79. package/test/resources/9999999/messaging/v1/push/definitions/post-response.json +13 -0
  80. package/test/resources/9999999/messaging/v1/push/definitions/testExisting_tpush/get-response.json +13 -0
  81. package/test/resources/9999999/messaging/v1/push/definitions/testExisting_tpush/patch-response.json +13 -0
  82. package/test/resources/9999999/messaging/v1/sms/definitions/get-response.json +15 -0
  83. package/test/resources/9999999/messaging/v1/sms/definitions/post-response.json +17 -0
  84. package/test/resources/9999999/messaging/v1/sms/definitions/testExisting_tsms/get-response.json +18 -0
  85. package/test/resources/9999999/messaging/v1/sms/definitions/testExisting_tsms/patch-response.json +18 -0
  86. package/test/resources/9999999/query/build-expected.json +2 -2
  87. package/test/resources/9999999/query/build-expected.sql +6 -0
  88. package/test/resources/9999999/query/get-expected.json +1 -1
  89. package/test/resources/9999999/query/get-expected.sql +6 -0
  90. package/test/resources/9999999/query/patch-expected.json +1 -1
  91. package/test/resources/9999999/query/patch-expected.sql +6 -0
  92. package/test/resources/9999999/query/post-expected.json +3 -3
  93. package/test/resources/9999999/query/post-expected.sql +4 -0
  94. package/test/resources/9999999/query/template-expected.json +1 -1
  95. package/test/resources/9999999/query/template-expected.sql +6 -0
  96. package/test/resources/9999999/transactionalEmail/build-expected.json +22 -0
  97. package/test/resources/9999999/transactionalEmail/get-expected.json +24 -0
  98. package/test/resources/9999999/transactionalEmail/patch-expected.json +24 -0
  99. package/test/resources/9999999/transactionalEmail/post-expected.json +24 -0
  100. package/test/resources/9999999/transactionalEmail/template-expected.json +22 -0
  101. package/test/resources/9999999/transactionalPush/build-expected.json +8 -0
  102. package/test/resources/9999999/transactionalPush/get-expected.json +11 -0
  103. package/test/resources/9999999/transactionalPush/patch-expected.json +16 -0
  104. package/test/resources/9999999/transactionalPush/post-expected.json +16 -0
  105. package/test/resources/9999999/transactionalPush/template-expected.json +8 -0
  106. package/test/resources/9999999/transactionalSMS/build-expected.amp +4 -0
  107. package/test/resources/9999999/transactionalSMS/build-expected.json +13 -0
  108. package/test/resources/9999999/transactionalSMS/get-expected.amp +4 -0
  109. package/test/resources/9999999/transactionalSMS/get-expected.json +15 -0
  110. package/test/resources/9999999/transactionalSMS/patch-expected.amp +4 -0
  111. package/test/resources/9999999/transactionalSMS/patch-expected.json +15 -0
  112. package/test/resources/9999999/transactionalSMS/post-expected.amp +4 -0
  113. package/test/resources/9999999/transactionalSMS/post-expected.json +15 -0
  114. package/test/resources/9999999/transactionalSMS/template-expected.amp +4 -0
  115. package/test/resources/9999999/transactionalSMS/template-expected.json +13 -0
  116. package/test/transactionalEmail.test.js +120 -0
  117. package/test/transactionalPush.test.js +120 -0
  118. package/test/transactionalSMS.test.js +149 -0
  119. package/test/utils.js +57 -8
package/lib/Deployer.js CHANGED
@@ -116,15 +116,15 @@ class Deployer {
116
116
  ) {
117
117
  const buObject = await Cli.getCredentialObject(
118
118
  properties,
119
- cred !== null ? cred + '/' + bu : null,
119
+ cred === null ? null : cred + '/' + bu,
120
120
  null,
121
121
  true
122
122
  );
123
- if (buObject !== null) {
123
+ if (buObject === null) {
124
+ return;
125
+ } else {
124
126
  cred = buObject.credential;
125
127
  bu = buObject.businessUnit;
126
- } else {
127
- return;
128
128
  }
129
129
  }
130
130
 
@@ -328,11 +328,11 @@ class Deployer {
328
328
  const allowedDeFolderContentTypes = ['dataextension', 'shared_dataextension'];
329
329
  for (const metadataType of metadataTypeArr) {
330
330
  if (!MetadataTypeInfo[metadataType].definition.dependencies.includes('folder')) {
331
- Util.logger.debug(` ☇ skipping ${metadataType}: folder not a dependency`);
331
+ Util.logger.debug(` ☇ skipping ${metadataType} folders: folder not a dependency`);
332
332
  continue;
333
333
  }
334
334
  if (!MetadataTypeInfo[metadataType].definition.folderType) {
335
- Util.logger.debug(` ☇ skipping ${metadataType}: folderType not set`);
335
+ Util.logger.debug(` ☇ skipping ${metadataType} folders: folderType not set`);
336
336
  continue;
337
337
  }
338
338
  if (
@@ -341,11 +341,13 @@ class Deployer {
341
341
  )
342
342
  ) {
343
343
  Util.logger.warn(
344
- ` ☇ skipping ${metadataType}: folderType ${MetadataTypeInfo[metadataType].definition.folderType} not supported for deployment`
344
+ ` ☇ skipping ${metadataType} folders: folderType ${MetadataTypeInfo[metadataType].definition.folderType} not supported for deployment. Please consider creating folders for this type manually as a pre-deployment step, if you see errors about missing dependent folders for this type later in this log.`
345
345
  );
346
346
  continue;
347
347
  }
348
- Util.logger.debug(`Creating relevant folders for ${metadataType} in deploy dir`);
348
+ Util.logger.debug(
349
+ ` - create ${metadataType} folders: Creating relevant folders in deploy dir`
350
+ );
349
351
 
350
352
  const allFolders = Object.keys(metadata[metadataType])
351
353
  .filter(
@@ -32,6 +32,9 @@ const MetadataTypeDefinitions = {
32
32
  role: require('./metadataTypes/definitions/Role.definition'),
33
33
  script: require('./metadataTypes/definitions/Script.definition'),
34
34
  setDefinition: require('./metadataTypes/definitions/SetDefinition.definition'),
35
+ transactionalEmail: require('./metadataTypes/definitions/TransactionalEmail.definition'),
36
+ transactionalPush: require('./metadataTypes/definitions/TransactionalPush.definition'),
37
+ transactionalSMS: require('./metadataTypes/definitions/TransactionalSMS.definition'),
35
38
  triggeredSendDefinition: require('./metadataTypes/definitions/TriggeredSendDefinition.definition'),
36
39
  };
37
40
 
@@ -32,6 +32,9 @@ const MetadataTypeInfo = {
32
32
  role: require('./metadataTypes/Role'),
33
33
  script: require('./metadataTypes/Script'),
34
34
  setDefinition: require('./metadataTypes/SetDefinition'),
35
+ transactionalEmail: require('./metadataTypes/TransactionalEmail'),
36
+ transactionalPush: require('./metadataTypes/TransactionalPush'),
37
+ transactionalSMS: require('./metadataTypes/TransactionalSMS'),
35
38
  triggeredSendDefinition: require('./metadataTypes/TriggeredSendDefinition'),
36
39
  };
37
40
 
package/lib/Retriever.js CHANGED
@@ -113,11 +113,11 @@ class Retriever {
113
113
  }
114
114
  Util.logger.info(
115
115
  `Retrieving: ${metadataType}` +
116
- (typeKeyMap[metadataType][0] !== null
117
- ? ` ${color.dim}(Keys: ${typeKeyMap[metadataType].join(', ')})${
116
+ (typeKeyMap[metadataType][0] === null
117
+ ? ''
118
+ : ` ${color.dim}(Keys: ${typeKeyMap[metadataType].join(', ')})${
118
119
  color.reset
119
- }`
120
- : '')
120
+ }`)
121
121
  );
122
122
  result = await (changelogOnly
123
123
  ? MetadataTypeInfo[type].retrieveChangelog(this.buObject, null, subType)
@@ -163,9 +163,16 @@ class Retriever {
163
163
  }
164
164
  }
165
165
  } catch (ex) {
166
- Util.logger.errorStack(ex, `Retrieving ${metadataType} failed`);
167
- // do not continue retrieving if one type failed. simply skip processing the rest of the for-loop
168
- break;
166
+ if (
167
+ ex.code === 'invalid_client' ||
168
+ ex.message.startsWith('Client authentication failed.')
169
+ ) {
170
+ // do not continue retrieving if we logged an authentication issue
171
+ Util.logger.error(ex.message);
172
+ break;
173
+ } else {
174
+ Util.logger.errorStack(ex, ` - Retrieving ${metadataType} failed`);
175
+ }
169
176
  }
170
177
  }
171
178
  return retrieveChangelog;
package/lib/cli.js CHANGED
@@ -392,6 +392,7 @@ yargs
392
392
  * @returns {string[]} values split into an array.
393
393
  */
394
394
  function csvToArray(csv) {
395
+ // eslint-disable-next-line unicorn/no-negated-condition
395
396
  return !csv
396
397
  ? null
397
398
  : csv.includes(',')
package/lib/index.js CHANGED
@@ -151,15 +151,15 @@ class Mcdev {
151
151
  ) {
152
152
  const buObject = await Cli.getCredentialObject(
153
153
  properties,
154
- cred !== null ? cred + '/' + bu : null,
154
+ cred === null ? null : cred + '/' + bu,
155
155
  null,
156
156
  true
157
157
  );
158
- if (buObject !== null) {
158
+ if (buObject === null) {
159
+ return;
160
+ } else {
159
161
  cred = buObject.credential;
160
162
  bu = buObject.businessUnit;
161
- } else {
162
- return;
163
163
  }
164
164
  }
165
165
 
@@ -206,7 +206,7 @@ class Mcdev {
206
206
  }
207
207
  const buObject = await Cli.getCredentialObject(
208
208
  properties,
209
- cred !== null ? cred + '/' + bu : null,
209
+ cred === null ? null : cred + '/' + bu,
210
210
  null,
211
211
  true
212
212
  );
@@ -290,7 +290,7 @@ class Mcdev {
290
290
  */
291
291
  static async initProject(credentialsName) {
292
292
  Util.logger.info('mcdev:: Setting up project');
293
- const properties = await config.getProperties(!!credentialsName);
293
+ const properties = await config.getProperties(!!credentialsName, true);
294
294
  await Init.initProject(properties, credentialsName);
295
295
  }
296
296
 
@@ -112,7 +112,7 @@ class AccountUser extends MetadataType {
112
112
  },
113
113
  };
114
114
  }
115
-
115
+ Util.logger.info(` - Loading ${this.definition.type}. This might take a while...`);
116
116
  return super.retrieveSOAP(retrieveDir, buObject, requestParams);
117
117
  }
118
118
  /**
@@ -377,7 +377,7 @@ class AccountUser extends MetadataType {
377
377
  metadata.AssociatedBusinessUnits__c = this.userIdBuMap[metadata.ID] || [];
378
378
 
379
379
  let roles;
380
- if (metadata.Roles.Role) {
380
+ if (metadata.Roles?.Role) {
381
381
  // normalize to always use array
382
382
  if (!metadata.Roles.Role.length) {
383
383
  metadata.Roles.Role = [metadata.Roles.Role];
@@ -169,7 +169,7 @@ class Asset extends MetadataType {
169
169
  const subtypeIds = subTypeArray?.map(
170
170
  (subTypeItemName) => Asset.definition.typeMapping[subTypeItemName]
171
171
  );
172
- const uri = 'asset/v1/content/assets/query';
172
+ const uri = '/asset/v1/content/assets/query';
173
173
  const payload = {
174
174
  page: {
175
175
  page: 1,
@@ -270,6 +270,15 @@ class Asset extends MetadataType {
270
270
 
271
271
  // only when we save results do we need the complete metadata or files. caching can skip these
272
272
  if (retrieveDir && items.length > 0) {
273
+ for (const item of items) {
274
+ if (item.customerKey.trim() !== item.customerKey) {
275
+ Util.logger.warn(
276
+ ` - ${this.definition.type} ${item[this.definition.nameField]} (${
277
+ item[this.definition.keyField]
278
+ }) has leading or trailing spaces in customerKey. Please remove them in SFMC.`
279
+ );
280
+ }
281
+ }
273
282
  // we have to wait on execution or it potentially causes memory reference issues when changing between BUs
274
283
  await this.requestAndSaveExtended(items, subType, retrieveDir, templateVariables);
275
284
  Util.logger.debug(`Downloaded asset-${subType}: ${items.length}`);
@@ -486,7 +495,11 @@ class Asset extends MetadataType {
486
495
  * @returns {TYPE.CodeExtractItem} metadata
487
496
  */
488
497
  static postRetrieveTasks(metadata) {
489
- return this.parseMetadata(metadata);
498
+ // folder
499
+ this.setFolderPath(metadata);
500
+ // extract HTML for selected subtypes and convert payload for easier processing in MetadataType.saveResults()
501
+ metadata = this._extractCode(metadata);
502
+ return metadata;
490
503
  }
491
504
 
492
505
  /**
@@ -778,19 +791,6 @@ class Asset extends MetadataType {
778
791
  }
779
792
  }
780
793
 
781
- /**
782
- * parses retrieved Metadata before saving
783
- *
784
- * @param {TYPE.AssetItem} metadata a single asset definition
785
- * @returns {TYPE.CodeExtractItem} parsed metadata definition
786
- */
787
- static parseMetadata(metadata) {
788
- // folder
789
- this.setFolderPath(metadata);
790
- // extract HTML for selected subtypes and convert payload for easier processing in MetadataType.saveResults()
791
- metadata = this._extractCode(metadata);
792
- return metadata;
793
- }
794
794
  /**
795
795
  * helper for {@link preDeployTasks} that loads extracted code content back into JSON
796
796
  *
@@ -806,13 +806,15 @@ class Asset extends MetadataType {
806
806
  const fileList = [];
807
807
  let subDirArr;
808
808
  let readDirArr;
809
+ // unfortunately, asset's key can contain spaces at beginning/end which can break the file system when folders are created with it
810
+ const customerKey = metadata.customerKey.trim();
809
811
  switch (metadata.assetType.name) {
810
812
  case 'templatebasedemail': // message
811
813
  case 'htmlemail': {
812
814
  // message
813
815
  // this complex type always creates its own subdir per asset
814
816
  subDirArr = [this.definition.type, subType];
815
- readDirArr = [deployDir, ...subDirArr, templateName || metadata.customerKey];
817
+ readDirArr = [deployDir, ...subDirArr, templateName || customerKey];
816
818
 
817
819
  // metadata.views.html.content (mandatory)
818
820
  // the main content can be empty (=not set up yet) hence check if we did extract sth or else readFile() will print error msgs
@@ -833,7 +835,7 @@ class Asset extends MetadataType {
833
835
  if (templateName) {
834
836
  // to use this method in templating, store a copy of the info in fileList
835
837
  fileList.push({
836
- subFolder: [...subDirArr, metadata.customerKey],
838
+ subFolder: [...subDirArr, customerKey],
837
839
  fileName: 'index' + subtypeExtension,
838
840
  fileExt: 'html',
839
841
  content: metadata.views.html.content,
@@ -850,7 +852,7 @@ class Asset extends MetadataType {
850
852
  subtypeExtension,
851
853
  subDirArr,
852
854
  fileList,
853
- metadata.customerKey,
855
+ customerKey,
854
856
  templateName,
855
857
  fileListOnly
856
858
  );
@@ -866,9 +868,7 @@ class Asset extends MetadataType {
866
868
  await File.pathExists(
867
869
  File.normalizePath([
868
870
  ...readDirArr,
869
- `${
870
- templateName || metadata.customerKey // TODO check why this could be templateName
871
- }${subtypeExtension}.html`,
871
+ `${templateName || customerKey}${subtypeExtension}.html`,
872
872
  ])
873
873
  )
874
874
  ) {
@@ -876,7 +876,7 @@ class Asset extends MetadataType {
876
876
  if (!fileListOnly) {
877
877
  metadata.views.text.content = await File.readFilteredFilename(
878
878
  readDirArr,
879
- (templateName || metadata.customerKey) + subtypeExtension,
879
+ (templateName || customerKey) + subtypeExtension,
880
880
  'html'
881
881
  );
882
882
  }
@@ -884,7 +884,7 @@ class Asset extends MetadataType {
884
884
  // to use this method in templating, store a copy of the info in fileList
885
885
  fileList.push({
886
886
  subFolder: subDirArr,
887
- fileName: metadata.customerKey + subtypeExtension,
887
+ fileName: customerKey + subtypeExtension,
888
888
  fileExt: 'html',
889
889
  content: metadata.views.text.content,
890
890
  });
@@ -896,7 +896,7 @@ class Asset extends MetadataType {
896
896
  // asset
897
897
  // this complex type always creates its own subdir per asset
898
898
  subDirArr = [this.definition.type, subType];
899
- readDirArr = [deployDir, ...subDirArr, templateName || metadata.customerKey];
899
+ readDirArr = [deployDir, ...subDirArr, templateName || customerKey];
900
900
 
901
901
  // metadata.views.html.slots.<>.blocks.<>.content (optional) (pre & post 20222)
902
902
  if (metadata?.views?.html?.slots) {
@@ -907,7 +907,7 @@ class Asset extends MetadataType {
907
907
  subtypeExtension,
908
908
  subDirArr,
909
909
  fileList,
910
- metadata.customerKey,
910
+ customerKey,
911
911
  templateName,
912
912
  fileListOnly
913
913
  );
@@ -934,7 +934,7 @@ class Asset extends MetadataType {
934
934
  if (templateName) {
935
935
  // to use this method in templating, store a copy of the info in fileList
936
936
  fileList.push({
937
- subFolder: [...subDirArr, metadata.customerKey],
937
+ subFolder: [...subDirArr, customerKey],
938
938
  fileName: 'views.html.content' + subtypeExtension,
939
939
  fileExt: 'html',
940
940
  content: metadata.views.html.content,
@@ -960,7 +960,7 @@ class Asset extends MetadataType {
960
960
  if (templateName) {
961
961
  // to use this method in templating, store a copy of the info in fileList
962
962
  fileList.push({
963
- subFolder: [...subDirArr, metadata.customerKey],
963
+ subFolder: [...subDirArr, customerKey],
964
964
  fileName: 'content' + subtypeExtension,
965
965
  fileExt: 'html',
966
966
  content: metadata.views.html.content,
@@ -993,7 +993,7 @@ class Asset extends MetadataType {
993
993
  await File.pathExists(
994
994
  File.normalizePath([
995
995
  ...readDirArr,
996
- `${templateName || metadata.customerKey}${subtypeExtension}.${ext}`,
996
+ `${templateName || customerKey}${subtypeExtension}.${ext}`,
997
997
  ])
998
998
  )
999
999
  ) {
@@ -1001,7 +1001,7 @@ class Asset extends MetadataType {
1001
1001
  if (!fileListOnly) {
1002
1002
  metadata.content = await File.readFilteredFilename(
1003
1003
  readDirArr,
1004
- (templateName || metadata.customerKey) + subtypeExtension,
1004
+ (templateName || customerKey) + subtypeExtension,
1005
1005
  ext
1006
1006
  );
1007
1007
  }
@@ -1009,7 +1009,7 @@ class Asset extends MetadataType {
1009
1009
  // to use this method in templating, store a copy of the info in fileList
1010
1010
  fileList.push({
1011
1011
  subFolder: subDirArr,
1012
- fileName: (templateName || metadata.customerKey) + subtypeExtension,
1012
+ fileName: (templateName || customerKey) + subtypeExtension,
1013
1013
  fileExt: ext,
1014
1014
  content: metadata.content,
1015
1015
  });
@@ -1104,7 +1104,7 @@ class Asset extends MetadataType {
1104
1104
  }
1105
1105
  }
1106
1106
  /**
1107
- * helper for {@link parseMetadata} that finds code content in JSON and extracts it
1107
+ * helper for {@link postRetrieveTasks} that finds code content in JSON and extracts it
1108
1108
  * to allow saving that separately and formatted
1109
1109
  *
1110
1110
  * @param {TYPE.AssetItem} metadata a single asset definition
@@ -1113,6 +1113,8 @@ class Asset extends MetadataType {
1113
1113
  static _extractCode(metadata) {
1114
1114
  const codeArr = [];
1115
1115
  let subType;
1116
+ // unfortunately, asset's key can contain spaces at beginning/end which can break the file system when folders are created with it
1117
+ const customerKey = metadata.customerKey.trim();
1116
1118
  switch (metadata.assetType.name) {
1117
1119
  case 'templatebasedemail': // message
1118
1120
  case 'htmlemail': {
@@ -1133,7 +1135,11 @@ class Asset extends MetadataType {
1133
1135
  this._extractCode_slots('views.html.slots', metadata.views.html.slots, codeArr);
1134
1136
  }
1135
1137
 
1136
- return { json: metadata, codeArr: codeArr, subFolder: [metadata.customerKey] };
1138
+ return {
1139
+ json: metadata,
1140
+ codeArr: codeArr,
1141
+ subFolder: [customerKey],
1142
+ };
1137
1143
  }
1138
1144
  case 'textonlyemail': {
1139
1145
  // message
@@ -1141,7 +1147,7 @@ class Asset extends MetadataType {
1141
1147
  if (metadata.views?.text?.content?.length) {
1142
1148
  codeArr.push({
1143
1149
  subFolder: null,
1144
- fileName: metadata.customerKey,
1150
+ fileName: customerKey,
1145
1151
  fileExt: 'html',
1146
1152
  content: metadata.views.text.content,
1147
1153
  });
@@ -1179,7 +1185,11 @@ class Asset extends MetadataType {
1179
1185
  });
1180
1186
  delete metadata.content;
1181
1187
  }
1182
- return { json: metadata, codeArr: codeArr, subFolder: [metadata.customerKey] };
1188
+ return {
1189
+ json: metadata,
1190
+ codeArr: codeArr,
1191
+ subFolder: [customerKey],
1192
+ };
1183
1193
  }
1184
1194
  case 'buttonblock': // block - Button Block
1185
1195
  case 'freeformblock': // block
@@ -1202,7 +1212,7 @@ class Asset extends MetadataType {
1202
1212
  if (metadata?.content?.length) {
1203
1213
  codeArr.push({
1204
1214
  subFolder: null,
1205
- fileName: metadata.customerKey,
1215
+ fileName: customerKey,
1206
1216
  fileExt: fileExt,
1207
1217
  content: metadata.content,
1208
1218
  });
@@ -937,10 +937,7 @@ class Automation extends MetadataType {
937
937
  * @returns {string[]} list of all files that need to be committed in a flat array ['path/file1.ext', 'path/file2.ext']
938
938
  */
939
939
  static getFilesToCommit(keyArr) {
940
- if (!this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type)) {
941
- // document automation is not active upon retrieve, run default method instead
942
- return super.getFilesToCommit(keyArr);
943
- } else {
940
+ if (this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type)) {
944
941
  // document automation is active. assume we want to commit the MD file as well
945
942
  const path = File.normalizePath([
946
943
  this.properties.directories.retrieve,
@@ -954,6 +951,9 @@ class Automation extends MetadataType {
954
951
  File.normalizePath([path, `${key}.${this.definition.type}-doc.md`]),
955
952
  ]);
956
953
  return fileList;
954
+ } else {
955
+ // document automation is not active upon retrieve, run default method instead
956
+ return super.getFilesToCommit(keyArr);
957
957
  }
958
958
  }
959
959
  }
@@ -360,7 +360,13 @@ class DataExtension extends MetadataType {
360
360
  Util.logger.info(' - Caching dependent Metadata: folder (shared via _ParentBU_)');
361
361
  Folder.client = this.client;
362
362
  Folder.properties = this.properties;
363
- const result = await Folder.retrieveForCache(buObjectParentBu);
363
+ const result = await Folder.retrieveForCache(buObjectParentBu, [
364
+ 'shared_data',
365
+ 'synchronizeddataextension',
366
+ 'salesforcedataextension',
367
+ 'shared_dataextension',
368
+ 'dataextension',
369
+ ]);
364
370
  cache.mergeMetadata('folder', result.metadata, buObject.eid);
365
371
 
366
372
  // get the types and clean out non-shared ones
@@ -1025,10 +1031,10 @@ class DataExtension extends MetadataType {
1025
1031
  const { metadata } = await super.retrieveSOAP(null, null, options, additionalFields);
1026
1032
  for (const key in metadata) {
1027
1033
  // some system data extensions do not have CategoryID which throws errors in other places. These do not need to be parsed
1028
- if (!metadata[key].CategoryID) {
1029
- delete metadata[key];
1030
- } else {
1034
+ if (metadata[key].CategoryID) {
1031
1035
  metadata[key].Fields = [];
1036
+ } else {
1037
+ delete metadata[key];
1032
1038
  }
1033
1039
  }
1034
1040
  return metadata;
@@ -1041,10 +1047,7 @@ class DataExtension extends MetadataType {
1041
1047
  * @returns {string[]} list of all files that need to be committed in a flat array ['path/file1.ext', 'path/file2.ext']
1042
1048
  */
1043
1049
  static getFilesToCommit(keyArr) {
1044
- if (!this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type)) {
1045
- // document dataExtension is not active upon retrieve, run default method instead
1046
- return super.getFilesToCommit(keyArr);
1047
- } else {
1050
+ if (this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type)) {
1048
1051
  // document dataExtension is active. assume we want to commit the MD file as well
1049
1052
  const path = File.normalizePath([
1050
1053
  this.properties.directories.retrieve,
@@ -1058,6 +1061,9 @@ class DataExtension extends MetadataType {
1058
1061
  File.normalizePath([path, `${key}.${this.definition.type}-doc.md`]),
1059
1062
  ]);
1060
1063
  return fileList;
1064
+ } else {
1065
+ // document dataExtension is not active upon retrieve, run default method instead
1066
+ return super.getFilesToCommit(keyArr);
1061
1067
  }
1062
1068
  }
1063
1069
  }
@@ -100,6 +100,19 @@ class DataExtensionField extends MetadataType {
100
100
  * @returns {Object.<string, TYPE.DataExtensionFieldItem>} existing fields by their original name to allow re-adding FieldType after update
101
101
  */
102
102
  static async prepareDeployColumnsOnUpdate(deployColumns, deKey) {
103
+ // get row count to know which field restrictions apply
104
+ let hasData = false;
105
+ try {
106
+ const rowset = await this.client.rest.get(
107
+ `/data/v1/customobjectdata/key/${deKey}/rowset?$page=1&$pagesize=1`
108
+ );
109
+ const rowCount = rowset.count;
110
+ hasData = rowCount > 0;
111
+ Util.logger.debug(`dataExtension ${deKey} row count: ${rowCount}`);
112
+ } catch (ex) {
113
+ Util.logger.debug(`Could not retrieve rowcount for ${deKey}: ${ex.message}`);
114
+ }
115
+
103
116
  // retrieve existing fields to enable updating them
104
117
  const response = await this.retrieveForCache(
105
118
  {
@@ -129,6 +142,7 @@ class DataExtensionField extends MetadataType {
129
142
 
130
143
  // Updating to a new FieldType will result in an error; warn & afterwards remove it
131
144
  if (itemOld.FieldType !== item.FieldType) {
145
+ // applicable: with or without data but simply ignored by API
132
146
  Util.logger.warn(
133
147
  ` - The Field Type of an existing field cannot be changed. Keeping the original value for [${deKey}].[${item.Name}]: '${itemOld.FieldType}'`
134
148
  );
@@ -141,14 +155,16 @@ class DataExtensionField extends MetadataType {
141
155
  delete item.FieldType;
142
156
 
143
157
  if (itemOld.MaxLength > item.MaxLength) {
158
+ // applicable: with or without data (Code 310007)
144
159
  Util.logger.warn(
145
160
  ` - The length of an existing field cannot be decreased. Keeping the original value for [${deKey}].[${item.Name}]: '${itemOld.MaxLength}'`
146
161
  );
147
162
  item.MaxLength = itemOld.MaxLength;
148
163
  }
149
164
  if (Util.isFalse(itemOld.IsRequired) && Util.isTrue(item.IsRequired)) {
165
+ // applicable: with or without data (Code 310007)
150
166
  Util.logger.warn(
151
- ` - A field cannot be changed to be required on update after it was created to allow nulls: [${deKey}].[${item.Name}]`
167
+ ` - A field cannot be changed to be required on update after it was created to allow nulls. Resetting to not equired: [${deKey}].[${item.Name}]`
152
168
  );
153
169
  item.IsRequired = itemOld.IsRequired;
154
170
  }
@@ -179,11 +195,20 @@ class DataExtensionField extends MetadataType {
179
195
  item.ObjectID = itemOld.ObjectID;
180
196
  } else {
181
197
  // field is getting added ---
182
- if (Util.isTrue(item.IsRequired) && item.DefaultValue === '') {
183
- Util.logger.warn(
184
- ` - Adding new fields to an existing table requires that these fields are either not-required (nullable) or have a default value set. Changing [${deKey}].[${item.Name}] to be not-required`
185
- );
186
- item.IsRequired = false;
198
+
199
+ if (hasData && Util.isTrue(item.IsRequired) && item.DefaultValue === '') {
200
+ // applicable: with data only
201
+ if (Util.isFalse(item.IsPrimaryKey)) {
202
+ Util.logger.warn(
203
+ ` - Adding new fields to an existing table requires that these fields are either not-required (nullable) or have a default value set. Changing [${deKey}].[${item.Name}] to be not-required`
204
+ );
205
+ item.IsRequired = false;
206
+ } else {
207
+ Util.logger.error(
208
+ `- You cannot add a new primary key field to an existing table that has data. Removing [${deKey}].[${item.Name}] from deployment`
209
+ );
210
+ deployColumns.splice(i, 1);
211
+ }
187
212
  }
188
213
  if (item.Name_new) {
189
214
  Util.logger.warn(
@@ -194,6 +219,15 @@ class DataExtensionField extends MetadataType {
194
219
  // Field doesn't exist in target, therefore Remove ObjectID if present
195
220
  delete item.ObjectID;
196
221
  }
222
+ if (Util.isTrue(item.IsPrimaryKey) && Util.isFalse(item.IsRequired)) {
223
+ // applicable: with or without data
224
+ Util.logger.warn(
225
+ `- Primary Key field [${deKey}].[${item.Name}] cannot be not-required (nullable). Changing field to be required!`
226
+ );
227
+ item.IsRequired = true;
228
+ }
229
+
230
+ // filter bad manual changes to the json
197
231
  if (!Util.isTrue(item.IsRequired) && !Util.isFalse(item.IsRequired)) {
198
232
  Util.logger.error(
199
233
  `- Invalid value for 'IsRequired' of [${deKey}].[${item.Name}]. Found '${item.IsRequired}' instead of 'true'/'false'. Removing field from deploy!`
@@ -207,6 +241,7 @@ class DataExtensionField extends MetadataType {
207
241
  deployColumns.splice(i, 1);
208
242
  }
209
243
  }
244
+
210
245
  Util.logger.debug(
211
246
  `${deployColumns.length} Fields added/updated for [${deKey}]${
212
247
  deployColumns.length ? ': ' : ''
@@ -277,15 +312,15 @@ class DataExtensionField extends MetadataType {
277
312
  this.postDeleteTasks(buObject, customerKey);
278
313
  return true;
279
314
  } catch (ex) {
280
- if (!handleOutside) {
315
+ if (handleOutside) {
316
+ throw ex;
317
+ } else {
281
318
  const errorMsg = ex.results?.length
282
319
  ? `${ex.results[0].StatusMessage} (Code ${ex.results[0].ErrorCode})`
283
320
  : ex.message;
284
321
  Util.logger.error(
285
322
  `- error deleting ${this.definition.type} '${customerKey}': ${errorMsg}`
286
323
  );
287
- } else {
288
- throw ex;
289
324
  }
290
325
 
291
326
  return false;
@@ -25,11 +25,7 @@ class Discovery extends MetadataType {
25
25
  if (key) {
26
26
  Util.logger.error('Discovery.retrieve() does not support key parameter');
27
27
  }
28
- if (buObject.eid !== buObject.mid) {
29
- // don't run for BUs other than Parent BU
30
- Util.logger.warn(' - Skipping Discovery retrieval on non-parent BU');
31
- return;
32
- } else {
28
+ if (buObject.eid === buObject.mid) {
33
29
  const res = await this.client.rest.getCollection(
34
30
  Object.keys(this.definition.endPointMapping).map(
35
31
  (endpoint) => this.definition.endPointMapping[endpoint]
@@ -44,6 +40,10 @@ class Discovery extends MetadataType {
44
40
  await super.saveResults(metadataStructure, retrieveDir, null);
45
41
  Util.logger.info('Downloaded: ' + this.definition.type);
46
42
  return { metadata: metadataStructure, type: this.definition.type };
43
+ } else {
44
+ // don't run for BUs other than Parent BU
45
+ Util.logger.warn(' - Skipping Discovery retrieval on non-parent BU');
46
+ return;
47
47
  }
48
48
  }
49
49
  }