@contentstack/cli-cm-export 1.20.2 → 2.0.0-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +1 -1
  2. package/lib/commands/cm/stacks/export.js +15 -6
  3. package/lib/config/index.js +0 -2
  4. package/lib/export/module-exporter.js +37 -38
  5. package/lib/export/modules/assets.js +71 -14
  6. package/lib/export/modules/base-class.d.ts +17 -0
  7. package/lib/export/modules/base-class.js +45 -0
  8. package/lib/export/modules/content-types.js +30 -8
  9. package/lib/export/modules/custom-roles.js +61 -21
  10. package/lib/export/modules/entries.d.ts +2 -0
  11. package/lib/export/modules/entries.js +137 -37
  12. package/lib/export/modules/environments.js +45 -20
  13. package/lib/export/modules/extensions.js +42 -17
  14. package/lib/export/modules/global-fields.d.ts +1 -1
  15. package/lib/export/modules/global-fields.js +24 -6
  16. package/lib/export/modules/index.d.ts +1 -0
  17. package/lib/export/modules/index.js +1 -0
  18. package/lib/export/modules/labels.js +43 -16
  19. package/lib/export/modules/locales.js +31 -12
  20. package/lib/export/modules/marketplace-apps.d.ts +5 -2
  21. package/lib/export/modules/marketplace-apps.js +95 -25
  22. package/lib/export/modules/personalize.d.ts +12 -2
  23. package/lib/export/modules/personalize.js +181 -46
  24. package/lib/export/modules/stack.js +83 -28
  25. package/lib/export/modules/taxonomies.d.ts +4 -9
  26. package/lib/export/modules/taxonomies.js +123 -75
  27. package/lib/export/modules/webhooks.js +40 -17
  28. package/lib/export/modules/workflows.js +57 -20
  29. package/lib/types/default-config.d.ts +0 -2
  30. package/lib/types/index.d.ts +1 -1
  31. package/lib/utils/common-helper.d.ts +1 -2
  32. package/lib/utils/common-helper.js +1 -12
  33. package/lib/utils/constants.d.ts +147 -0
  34. package/lib/utils/constants.js +160 -0
  35. package/lib/utils/export-config-handler.js +2 -0
  36. package/lib/utils/index.d.ts +1 -0
  37. package/lib/utils/index.js +6 -1
  38. package/lib/utils/marketplace-app-helper.d.ts +1 -0
  39. package/lib/utils/marketplace-app-helper.js +21 -12
  40. package/lib/utils/progress-strategy-registry.d.ts +7 -0
  41. package/lib/utils/progress-strategy-registry.js +92 -0
  42. package/messages/index.json +4 -2
  43. package/oclif.manifest.json +1 -1
  44. package/package.json +2 -2
  45. package/lib/export/modules-js/assets.d.ts +0 -43
  46. package/lib/export/modules-js/assets.js +0 -396
  47. package/lib/export/modules-js/content-types.d.ts +0 -21
  48. package/lib/export/modules-js/content-types.js +0 -76
  49. package/lib/export/modules-js/custom-roles.d.ts +0 -21
  50. package/lib/export/modules-js/custom-roles.js +0 -76
  51. package/lib/export/modules-js/entries.d.ts +0 -18
  52. package/lib/export/modules-js/entries.js +0 -143
  53. package/lib/export/modules-js/environments.d.ts +0 -16
  54. package/lib/export/modules-js/environments.js +0 -62
  55. package/lib/export/modules-js/extensions.d.ts +0 -18
  56. package/lib/export/modules-js/extensions.js +0 -57
  57. package/lib/export/modules-js/global-fields.d.ts +0 -22
  58. package/lib/export/modules-js/global-fields.js +0 -108
  59. package/lib/export/modules-js/index.d.ts +0 -2
  60. package/lib/export/modules-js/index.js +0 -31
  61. package/lib/export/modules-js/labels.d.ts +0 -14
  62. package/lib/export/modules-js/labels.js +0 -56
  63. package/lib/export/modules-js/locales.d.ts +0 -23
  64. package/lib/export/modules-js/locales.js +0 -68
  65. package/lib/export/modules-js/marketplace-apps.d.ts +0 -21
  66. package/lib/export/modules-js/marketplace-apps.js +0 -132
  67. package/lib/export/modules-js/stack.d.ts +0 -18
  68. package/lib/export/modules-js/stack.js +0 -91
  69. package/lib/export/modules-js/webhooks.d.ts +0 -18
  70. package/lib/export/modules-js/webhooks.js +0 -60
  71. package/lib/export/modules-js/workflows.d.ts +0 -16
  72. package/lib/export/modules-js/workflows.js +0 -89
@@ -0,0 +1,92 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const constants_1 = require("./constants");
4
+ /**
5
+ * Progress Strategy Registrations for Export Modules
6
+ * This file registers progress calculation strategies for all export modules
7
+ * to ensure correct item counts in the final summary.
8
+ */
9
+ const cli_utilities_1 = require("@contentstack/cli-utilities");
10
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.CONTENT_TYPES], new cli_utilities_1.DefaultProgressStrategy());
11
+ // Register strategy for Assets - custom strategy to avoid double counting
12
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.ASSETS], new cli_utilities_1.CustomProgressStrategy((processes) => {
13
+ // Both ASSET_METADATA and ASSET_DOWNLOADS represent the same assets
14
+ // Count only the downloads process to avoid double counting in summary
15
+ const downloadsProcess = processes.get(constants_1.PROCESS_NAMES.ASSET_DOWNLOADS);
16
+ if (downloadsProcess) {
17
+ return {
18
+ total: downloadsProcess.total,
19
+ success: downloadsProcess.successCount,
20
+ failures: downloadsProcess.failureCount,
21
+ };
22
+ }
23
+ // Fallback to metadata process if downloads don't exist
24
+ const metadataProcess = processes.get(constants_1.PROCESS_NAMES.ASSET_METADATA);
25
+ if (metadataProcess) {
26
+ return {
27
+ total: metadataProcess.total,
28
+ success: metadataProcess.successCount,
29
+ failures: metadataProcess.failureCount,
30
+ };
31
+ }
32
+ return null; // Fall back to default aggregation
33
+ }));
34
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.GLOBAL_FIELDS], new cli_utilities_1.DefaultProgressStrategy());
35
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.EXTENSIONS], new cli_utilities_1.DefaultProgressStrategy());
36
+ // Register strategy for Environments - simple module
37
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.ENVIRONMENTS], new cli_utilities_1.DefaultProgressStrategy());
38
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.LOCALES], new cli_utilities_1.DefaultProgressStrategy());
39
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.LABELS], new cli_utilities_1.DefaultProgressStrategy());
40
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.WEBHOOKS], new cli_utilities_1.DefaultProgressStrategy());
41
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.WORKFLOWS], new cli_utilities_1.DefaultProgressStrategy());
42
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.CUSTOM_ROLES], new cli_utilities_1.DefaultProgressStrategy());
43
+ // Register strategy for Taxonomies - use Taxonomies & Terms as primary process
44
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.TAXONOMIES], new cli_utilities_1.PrimaryProcessStrategy(constants_1.PROCESS_NAMES.EXPORT_TAXONOMIES_TERMS));
45
+ // Register strategy for Marketplace Apps - complex module with app installations
46
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.MARKETPLACE_APPS], new cli_utilities_1.CustomProgressStrategy((processes) => {
47
+ // For marketplace apps, count the actual apps exported
48
+ const appsExport = processes.get(constants_1.PROCESS_NAMES.FETCH_APPS);
49
+ if (appsExport) {
50
+ return {
51
+ total: appsExport.total,
52
+ success: appsExport.successCount,
53
+ failures: appsExport.failureCount,
54
+ };
55
+ }
56
+ const setup = processes.get(constants_1.PROCESS_NAMES.FETCH_CONFIG_MANIFEST);
57
+ if (setup) {
58
+ return {
59
+ total: setup.total,
60
+ success: setup.successCount,
61
+ failures: setup.failureCount,
62
+ };
63
+ }
64
+ return null;
65
+ }));
66
+ // Register strategy for Stack Settings - use Settings as primary process
67
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.STACK], new cli_utilities_1.PrimaryProcessStrategy(constants_1.PROCESS_NAMES.STACK_SETTINGS));
68
+ // Register strategy for Personalize - complex module with projects/experiences
69
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.PERSONALIZE], new cli_utilities_1.CustomProgressStrategy((processes) => {
70
+ // For personalize, we want to count projects as the main metric
71
+ const projectExport = processes.get(constants_1.PROCESS_NAMES.PERSONALIZE_PROJECTS);
72
+ if (projectExport) {
73
+ return {
74
+ total: projectExport.total,
75
+ success: projectExport.successCount,
76
+ failures: projectExport.failureCount,
77
+ };
78
+ }
79
+ // Fallback to any other main process
80
+ const mainProcess = Array.from(processes.values())[0];
81
+ if (mainProcess) {
82
+ return {
83
+ total: mainProcess.total,
84
+ success: mainProcess.successCount,
85
+ failures: mainProcess.failureCount,
86
+ };
87
+ }
88
+ return null;
89
+ }));
90
+ // Register strategy for Entries - use Entries as primary process
91
+ cli_utilities_1.ProgressStrategyRegistry.register(constants_1.MODULE_NAMES[constants_1.MODULE_CONTEXTS.ENTRIES], new cli_utilities_1.PrimaryProcessStrategy(constants_1.PROCESS_NAMES.ENTRIES));
92
+ exports.default = cli_utilities_1.ProgressStrategyRegistry;
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "ASSET_EXPORT_COMPLETE": "Asset export process completed successfully",
3
- "ASSET_FOLDERS_EXPORT_COMPLETE": "Asset folder structure exported successfully",
3
+ "ASSET_FOLDERS_EXPORT_COMPLETE": "Asset folder structure exported successfully with %s folder(s)",
4
4
  "ASSET_METADATA_EXPORT_COMPLETE": "Asset metadata exported successfully",
5
5
  "ASSET_VERSIONED_METADATA_EXPORT_COMPLETE": "Versioned asset metadata exported successfully",
6
6
  "ASSET_DOWNLOAD_COMPLETE": "Asset download completed successfully",
@@ -65,5 +65,7 @@
65
65
  "BRANCH_EXPORT_FAILED": "Failed to export contents from branch (UID: %s)",
66
66
 
67
67
  "ROLES_NO_CUSTOM_ROLES": "No custom roles found in the current stack",
68
- "ROLES_EXPORTING_ROLE": "Exporting role '%s'"
68
+ "ROLES_EXPORTING_ROLE": "Exporting role '%s'",
69
+
70
+ "GLOBAL_FIELDS_NOT_FOUND": "No global fields found in the current stack"
69
71
  }
@@ -159,5 +159,5 @@
159
159
  ]
160
160
  }
161
161
  },
162
- "version": "1.20.2"
162
+ "version": "2.0.0-beta"
163
163
  }
package/package.json CHANGED
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@contentstack/cli-cm-export",
3
3
  "description": "Contentstack CLI plugin to export content from stack",
4
- "version": "1.20.2",
4
+ "version": "2.0.0-beta",
5
5
  "author": "Contentstack",
6
6
  "bugs": "https://github.com/contentstack/cli/issues",
7
7
  "dependencies": {
8
8
  "@contentstack/cli-command": "~1.6.1",
9
- "@contentstack/cli-variants": "~1.3.3",
9
+ "@contentstack/cli-variants": "~2.0.0-beta",
10
10
  "@oclif/core": "^4.3.3",
11
11
  "@contentstack/cli-utilities": "~1.14.1",
12
12
  "async": "^3.2.6",
@@ -1,43 +0,0 @@
1
- export = ExportAssets;
2
- declare class ExportAssets {
3
- constructor(exportConfig: any, stackAPIClient: any);
4
- config: any;
5
- bLimit: number;
6
- vLimit: any;
7
- invalidKeys: string[];
8
- folderJSONPath: any;
9
- folderData: any[];
10
- assetsFolderPath: any;
11
- assetContents: {};
12
- httpClient: HttpClient;
13
- assetConfig: {
14
- dirName: string;
15
- fileName: string;
16
- batchLimit: number;
17
- host: string;
18
- invalidKeys: string[];
19
- chunkFileSize: number;
20
- downloadLimit: number;
21
- fetchConcurrency: number;
22
- assetsMetaKeys: string[];
23
- securedAssets: boolean;
24
- displayExecutionTime: boolean;
25
- enableDownloadStatus: boolean;
26
- includeVersionedAssets: boolean;
27
- dependencies?: import("../../types").Modules[];
28
- };
29
- stackAPIClient: any;
30
- assetDownloadRetry: {};
31
- assetDownloadRetryLimit: number;
32
- start(): any;
33
- assetContentsFile: string;
34
- exportFolders(): any;
35
- getFolderJSON(skip: any, fCount: any): any;
36
- getAssetCount(folder: any): any;
37
- getAssetJSON(skip: any): any;
38
- getVersionedAssetJSON(uid: any, version: any, bucket: any): any;
39
- downloadAsset(asset: any): any;
40
- getFolders(): any;
41
- getFolderDetails(skip: any, tCount: any): any;
42
- }
43
- import { HttpClient } from "@contentstack/cli-utilities/lib/http-client/client";
@@ -1,396 +0,0 @@
1
- /*!
2
- * Contentstack Export
3
- * Copyright (c) 2024 Contentstack LLC
4
- * MIT Licensed
5
- */
6
- const mkdirp = require('mkdirp');
7
- const path = require('path');
8
- const fs = require('fs');
9
- const Promise = require('bluebird');
10
- const _ = require('lodash');
11
- const chalk = require('chalk');
12
- const progress = require('progress-stream');
13
- const { HttpClient, configHandler, validateUids, sanitizePath, validateFileName } = require('@contentstack/cli-utilities');
14
- const { fileHelper, log, formatError } = require('../../utils');
15
- let { default: config } = require('../../config');
16
- module.exports = class ExportAssets {
17
- constructor(exportConfig, stackAPIClient) {
18
- this.folderData = [];
19
- this.assetContents = {};
20
- this.httpClient = HttpClient.create();
21
- this.assetConfig = config.modules.assets;
22
- this.stackAPIClient = stackAPIClient;
23
- this.config = _.merge(config, exportConfig);
24
- this.folderData = [];
25
- this.assetContents = {};
26
- this.assetDownloadRetry = {};
27
- this.assetDownloadRetryLimit = 3;
28
- this.invalidKeys = this.assetConfig.invalidKeys;
29
- this.bLimit = this.assetConfig.batchLimit || 15;
30
- this.vLimit = this.assetConfig.downloadLimit || this.config.fetchConcurrency || 3;
31
- }
32
- start() {
33
- const self = this;
34
- this.assetsFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.assetConfig.dirName);
35
- this.assetContentsFile = path.resolve(this.assetsFolderPath, 'assets.json');
36
- this.folderJSONPath = path.resolve(this.assetsFolderPath, 'folders.json');
37
- log(this.config, 'Starting assets export', 'success');
38
- // Create asset folder
39
- mkdirp.sync(this.assetsFolderPath);
40
- return new Promise((resolve, reject) => {
41
- // TBD: getting all the assets should have optimized
42
- return self
43
- .getAssetCount()
44
- .then((count) => {
45
- const assetBatches = [];
46
- if (typeof count !== 'number' || count === 0) {
47
- log(self.config, 'No assets found', 'success');
48
- return resolve();
49
- }
50
- for (let i = 0; i <= count; i += self.bLimit) {
51
- assetBatches.push(i);
52
- }
53
- return Promise.map(assetBatches, (batch) => {
54
- return self
55
- .getAssetJSON(batch)
56
- .then((assetsJSON) => {
57
- return Promise.map(assetsJSON, (assetJSON) => {
58
- if (self.assetConfig.downloadVersionAssets) {
59
- return self
60
- .getVersionedAssetJSON(assetJSON.uid, assetJSON._version)
61
- .then(() => {
62
- self.assetContents[assetJSON.uid] = assetJSON;
63
- })
64
- .catch((error) => {
65
- log(self.config, `Asset '${assetJSON.uid}' failed to download.\n ${formatError(error)}`, 'error');
66
- log(self.config, error, 'error');
67
- });
68
- }
69
- else {
70
- return self
71
- .downloadAsset(assetJSON)
72
- .then(() => {
73
- self.assetContents[assetJSON.uid] = assetJSON;
74
- })
75
- .catch((err) => {
76
- log(self.config, `Asset '${assetJSON.uid}' download failed. ${formatError(err)}`, 'error');
77
- return err;
78
- });
79
- }
80
- }, { concurrency: self.vLimit })
81
- .then(() => {
82
- log(self.config, 'Batch no ' + (batch + 1) + ' of assets is complete', 'success');
83
- // fileHelper.writeFileSync(this.assetContentsFile, self.assetContents)
84
- })
85
- .catch((error) => {
86
- log(self.config, `Asset batch ${batch + 1} failed to download`, 'error');
87
- log(self.config, formatError(error), 'error');
88
- log(self.config, error, 'error');
89
- });
90
- })
91
- .catch((error) => {
92
- log(self.config, error, 'error');
93
- reject(error);
94
- });
95
- }, { concurrency: self.assetConfig.concurrencyLimit || 1 })
96
- .then(() => {
97
- fileHelper.writeFileSync(self.assetContentsFile, self.assetContents);
98
- return self
99
- .exportFolders()
100
- .then(() => {
101
- log(self.config, chalk.green('Asset export completed successfully'), 'success');
102
- return resolve();
103
- })
104
- .catch((error) => {
105
- log(self.config, error, 'error');
106
- reject(error);
107
- });
108
- })
109
- .catch((error) => {
110
- fileHelper.writeFileSync(self.assetContentsFile, self.assetContents);
111
- log(self.config, `Asset export failed. ${formatError(error)}`, 'error');
112
- log(self.config, error, 'error');
113
- reject(error);
114
- });
115
- })
116
- .catch((error) => {
117
- log(self.config, error, 'error');
118
- reject(error);
119
- });
120
- });
121
- }
122
- exportFolders() {
123
- const self = this;
124
- return new Promise((resolve, reject) => {
125
- return self
126
- .getAssetCount(true)
127
- .then((fCount) => {
128
- if (fCount === 0) {
129
- log(self.config, 'No folders were found in the stack!', 'success');
130
- return resolve();
131
- }
132
- return self
133
- .getFolderJSON(0, fCount)
134
- .then(() => {
135
- // asset folders have been successfully exported
136
- log(self.config, 'Asset-folders have been successfully exported!', 'success');
137
- return resolve();
138
- })
139
- .catch((error) => {
140
- log(self.config, `Error while exporting asset-folders!\n ${formatError(error)}`, 'error');
141
- return reject(error);
142
- });
143
- })
144
- .catch((error) => {
145
- log(self.config, error, 'error');
146
- // error while fetching asset folder count
147
- return reject(error);
148
- });
149
- });
150
- }
151
- getFolderJSON(skip, fCount) {
152
- const self = this;
153
- return new Promise((resolve, reject) => {
154
- if (typeof skip !== 'number') {
155
- skip = 0;
156
- }
157
- if (skip >= fCount) {
158
- fileHelper.writeFileSync(self.folderJSONPath, self.folderData);
159
- return resolve();
160
- }
161
- const queryRequestObj = {
162
- skip,
163
- include_folders: true,
164
- query: { is_dir: true },
165
- };
166
- self.stackAPIClient
167
- .asset()
168
- .query(queryRequestObj)
169
- .find()
170
- .then((response) => {
171
- skip += 100;
172
- self.folderData.push(...response.items);
173
- return self.getFolderJSON(skip, fCount).then(resolve).catch(reject);
174
- })
175
- .catch((error) => reject(error));
176
- });
177
- }
178
- getAssetCount(folder) {
179
- const self = this;
180
- return new Promise((resolve, reject) => {
181
- if (folder && typeof folder === 'boolean') {
182
- const queryOptions = {
183
- skip: 99999990,
184
- include_count: true,
185
- include_folders: true,
186
- query: { is_dir: true },
187
- };
188
- self.stackAPIClient
189
- .asset()
190
- .query(queryOptions)
191
- .find()
192
- .then((asset) => {
193
- return resolve(asset.count);
194
- })
195
- .catch((error) => {
196
- log(self.config, error, 'error');
197
- });
198
- }
199
- else {
200
- const queryOptions = { skip: 99999990, include_count: true };
201
- self.stackAPIClient
202
- .asset()
203
- .query(queryOptions)
204
- .find()
205
- .then(({ count }) => resolve(count))
206
- .catch((error) => {
207
- log(self.config, error, 'error');
208
- reject(error);
209
- });
210
- }
211
- });
212
- }
213
- getAssetJSON(skip) {
214
- const self = this;
215
- return new Promise((resolve, reject) => {
216
- if (typeof skip !== 'number') {
217
- skip = 0;
218
- }
219
- const queryRequestObj = {
220
- skip: skip,
221
- limit: self.bLimit,
222
- include_publish_details: true,
223
- except: {
224
- BASE: self.invalidKeys,
225
- },
226
- };
227
- self.stackAPIClient
228
- .asset()
229
- .query(queryRequestObj)
230
- .find()
231
- .then(({ items }) => resolve(items))
232
- .catch((error) => {
233
- log(self.config, error, 'error');
234
- return reject();
235
- });
236
- });
237
- }
238
- getVersionedAssetJSON(uid, version, bucket) {
239
- const self = this;
240
- const assetVersionInfo = bucket || [];
241
- return new Promise((resolve, reject) => {
242
- if (self.assetDownloadRetry[uid + version] > self.assetDownloadRetryLimit) {
243
- console.log('Reached max', self.assetDownloadRetry[uid + version]);
244
- return reject(new Error('Asset Max download retry limit exceeded! ' + uid));
245
- }
246
- if (version <= 0) {
247
- if (validateUids(uid)) {
248
- const assetVersionInfoFile = path.resolve(sanitizePath(self.assetsFolderPath), sanitizePath(uid), '_contentstack_' + sanitizePath(uid) + '.json');
249
- fileHelper.writeFileSync(assetVersionInfoFile, assetVersionInfo);
250
- return resolve();
251
- }
252
- }
253
- const queryrequestOption = {
254
- version: version,
255
- include_publish_details: true,
256
- except: {
257
- BASE: self.invalidKeys,
258
- },
259
- };
260
- self.stackAPIClient
261
- .asset(uid)
262
- .fetch(queryrequestOption)
263
- .then((versionedAssetJSONResponse) => {
264
- self
265
- .downloadAsset(versionedAssetJSONResponse)
266
- .then(() => {
267
- assetVersionInfo.splice(0, 0, versionedAssetJSONResponse);
268
- // Remove duplicates
269
- assetVersionInfo = _.uniqWith(assetVersionInfo, _.isEqual);
270
- self.getVersionedAssetJSON(uid, --version, assetVersionInfo).then(resolve).catch(reject);
271
- })
272
- .catch(reject);
273
- })
274
- .catch((error) => {
275
- log(self.config, error, 'error');
276
- if (error.status === 408) {
277
- console.log('retrying', uid);
278
- // retrying when timeout
279
- self.assetDownloadRetry[uid + version]
280
- ? ++self.assetDownloadRetry[uid + version]
281
- : (self.assetDownloadRetry[uid + version] = 1);
282
- return self.getVersionedAssetJSON(uid, version, assetVersionInfo).then(resolve).catch(reject);
283
- }
284
- reject(error);
285
- });
286
- });
287
- }
288
- downloadAsset(asset) {
289
- const self = this;
290
- return new Promise(async (resolve, reject) => {
291
- if (!validateUids(asset.uid) && !validateFileName(asset.filename)) {
292
- reject(`UIDs not valid`);
293
- }
294
- const assetFolderPath = path.resolve(sanitizePath(self.assetsFolderPath), sanitizePath(asset.uid));
295
- const assetFilePath = path.resolve(sanitizePath(assetFolderPath), sanitizePath(asset.filename));
296
- if (fs.existsSync(assetFilePath)) {
297
- log(self.config, 'Skipping download of { title: ' + asset.filename + ', uid: ' + asset.uid + ' }, as they already exist', 'success');
298
- return resolve();
299
- }
300
- self.assetStream = {
301
- url: self.config.securedAssets ? `${asset.url}?authtoken=${configHandler.get('authtoken')}` : asset.url,
302
- };
303
- fileHelper.makeDirectory(assetFolderPath);
304
- const assetFileStream = fs.createWriteStream(assetFilePath);
305
- self.assetStream.url = encodeURI(self.assetStream.url);
306
- self.httpClient
307
- .options({ responseType: 'stream' })
308
- .get(self.assetStream.url)
309
- .then(({ data: assetStreamRequest }) => {
310
- if (self.assetConfig.enableDownloadStatus) {
311
- const str = progress({
312
- time: 5000,
313
- length: assetStreamRequest.headers['content-length'],
314
- });
315
- str.on('progress', (progressData) => {
316
- console.log(`${asset.filename}: ${Math.round(progressData.percentage)}%`);
317
- });
318
- assetStreamRequest.pipe(str).pipe(assetFileStream);
319
- }
320
- assetStreamRequest.pipe(assetFileStream);
321
- })
322
- .catch((error) => {
323
- log(self.config, error, 'error');
324
- reject(error);
325
- });
326
- assetFileStream
327
- .on('close', function () {
328
- log(self.config, 'Downloaded ' + asset.filename + ': ' + asset.uid + ' successfully!', 'success');
329
- return resolve();
330
- })
331
- .on('error', (error) => {
332
- log(self.config, `Download ${asset.filename}: ${asset.uid} failed!`, 'error');
333
- log(self.config, error, 'error');
334
- reject(error);
335
- });
336
- });
337
- }
338
- getFolders() {
339
- const self = this;
340
- return new Promise((resolve, reject) => {
341
- return self
342
- .getAssetCount(true)
343
- .then((count) => {
344
- if (count === 0) {
345
- log(self.config, 'No folders were found in the stack', 'success');
346
- return resolve();
347
- }
348
- return self
349
- .getFolderDetails(0, count)
350
- .then(function () {
351
- log(self.config, chalk.green('Exported asset-folders successfully!'), 'success');
352
- return resolve();
353
- })
354
- .catch(function (error) {
355
- log(self.config, error, 'error');
356
- reject(error);
357
- });
358
- })
359
- .catch(function (error) {
360
- log(self.config, error, 'error');
361
- reject(error);
362
- });
363
- });
364
- }
365
- getFolderDetails(skip, tCount) {
366
- const self = this;
367
- return new Promise((resolve, reject) => {
368
- if (typeof skip !== 'number') {
369
- skip = 0;
370
- }
371
- if (skip > tCount) {
372
- fileHelper.writeFileSync(self.folderJSONPath, self.folderContents);
373
- return resolve();
374
- }
375
- const queryRequestObj = {
376
- skip: skip,
377
- include_folders: true,
378
- query: { is_dir: true },
379
- };
380
- self.stackAPIClient
381
- .asset()
382
- .query(queryRequestObj)
383
- .find()
384
- .then((folderDetailsResponse) => {
385
- for (let i in folderDetailsResponse.items) {
386
- self.folderContents.push(folderDetailsResponse.items[i]);
387
- }
388
- skip += 100;
389
- return self.getFolderDetails(skip, tCount).then(resolve).catch(reject);
390
- })
391
- .catch((error) => {
392
- log(self.config, error, 'error');
393
- });
394
- });
395
- }
396
- };
@@ -1,21 +0,0 @@
1
- export = ContentTypesExport;
2
- declare class ContentTypesExport {
3
- constructor(exportConfig: any, stackAPIClient: any);
4
- stackAPIClient: any;
5
- exportConfig: any;
6
- contentTypesConfig: any;
7
- qs: {
8
- include_count: boolean;
9
- asc: string;
10
- limit: any;
11
- include_global_field_schema: boolean;
12
- };
13
- contentTypesPath: string;
14
- contentTypes: any[];
15
- fetchConcurrency: any;
16
- writeConcurrency: any;
17
- start(): Promise<void>;
18
- getContentTypes(skip?: number): any;
19
- sanitizeAttribs(contentTypes: any): any[];
20
- writeContentTypes(contentTypes: any): Promise<any>;
21
- }
@@ -1,76 +0,0 @@
1
- const path = require('path');
2
- const chalk = require('chalk');
3
- const { fileHelper, executeTask, formatError, log } = require('../../utils');
4
- const { sanitizePath } = require('@contentstack/cli-utilities');
5
- class ContentTypesExport {
6
- constructor(exportConfig, stackAPIClient) {
7
- this.stackAPIClient = stackAPIClient;
8
- this.exportConfig = exportConfig;
9
- this.contentTypesConfig = exportConfig.modules.content_types;
10
- this.qs = {
11
- include_count: true,
12
- asc: 'updated_at',
13
- limit: this.contentTypesConfig.limit,
14
- include_global_field_schema: true,
15
- };
16
- // If content type id is provided then use it as part of query
17
- if (Array.isArray(this.exportConfig.contentTypes) && this.exportConfig.contentTypes.length > 0) {
18
- this.qs.uid = { $in: this.exportConfig.contentTypes };
19
- }
20
- this.contentTypesPath = path.resolve(sanitizePath(exportConfig.data), sanitizePath(exportConfig.branchName) || '', sanitizePath(this.contentTypesConfig.dirName));
21
- this.contentTypes = [];
22
- this.fetchConcurrency = this.contentTypesConfig.fetchConcurrency || this.exportConfig.fetchConcurrency;
23
- this.writeConcurrency = this.contentTypesConfig.writeConcurrency || this.exportConfig.writeConcurrency;
24
- }
25
- async start() {
26
- try {
27
- log(this.exportConfig, 'Starting content type export', 'success');
28
- await fileHelper.makeDirectory(this.contentTypesPath);
29
- await this.getContentTypes();
30
- await this.writeContentTypes(this.contentTypes);
31
- log(this.exportConfig, chalk.green('Content type(s) exported successfully'), 'success');
32
- }
33
- catch (error) {
34
- log(this.exportConfig, `Failed to export content types ${formatError(error)}`, 'error');
35
- throw new Error('Failed to export content types');
36
- }
37
- }
38
- async getContentTypes(skip = 0) {
39
- if (skip) {
40
- this.qs.skip = skip;
41
- }
42
- const contentTypeSearchResponse = await this.stackAPIClient.contentType().query(this.qs).find();
43
- if (Array.isArray(contentTypeSearchResponse.items) && contentTypeSearchResponse.items.length > 0) {
44
- let updatedContentTypes = this.sanitizeAttribs(contentTypeSearchResponse.items);
45
- this.contentTypes.push(...updatedContentTypes);
46
- skip += this.contentTypesConfig.limit;
47
- if (skip > contentTypeSearchResponse.count) {
48
- return;
49
- }
50
- return await this.getContentTypes(skip);
51
- }
52
- else {
53
- log(this.exportConfig, 'No content types returned for the given query', 'info');
54
- }
55
- }
56
- sanitizeAttribs(contentTypes) {
57
- let updatedContentTypes = [];
58
- contentTypes.forEach((contentType) => {
59
- for (let key in contentType) {
60
- if (this.contentTypesConfig.validKeys.indexOf(key) === -1) {
61
- delete contentType[key];
62
- }
63
- }
64
- updatedContentTypes.push(contentType);
65
- });
66
- return updatedContentTypes;
67
- }
68
- async writeContentTypes(contentTypes) {
69
- function write(contentType) {
70
- return fileHelper.writeFile(path.join(sanitizePath(this.contentTypesPath), `${sanitizePath(contentType.uid === 'schema' ? 'schema|1' : contentType.uid)}.json`), contentType);
71
- }
72
- await executeTask(contentTypes, write.bind(this), { concurrency: this.writeConcurrency });
73
- return fileHelper.writeFile(path.join(this.contentTypesPath, 'schema.json'), contentTypes);
74
- }
75
- }
76
- module.exports = ContentTypesExport;