@contentstack/cli-cm-import 1.5.11 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +6 -14
  2. package/bin/dev +17 -0
  3. package/bin/dev.cmd +3 -0
  4. package/bin/run +6 -0
  5. package/bin/run.cmd +3 -0
  6. package/lib/commands/cm/stacks/import.d.ts +10 -0
  7. package/lib/commands/cm/stacks/import.js +111 -0
  8. package/lib/config/index.d.ts +3 -0
  9. package/lib/config/index.js +372 -0
  10. package/lib/import/index.d.ts +1 -0
  11. package/lib/import/index.js +8 -0
  12. package/lib/import/module-importer.d.ts +13 -0
  13. package/lib/import/module-importer.js +70 -0
  14. package/lib/import/modules/assets.d.ts +63 -0
  15. package/lib/import/modules/assets.js +265 -0
  16. package/lib/import/modules/base-class.d.ts +69 -0
  17. package/lib/import/modules/base-class.js +165 -0
  18. package/lib/import/modules/index.d.ts +2 -0
  19. package/lib/import/modules/index.js +19 -0
  20. package/lib/import/modules/locales.d.ts +31 -0
  21. package/lib/import/modules/locales.js +152 -0
  22. package/lib/import/modules-js/assets.d.ts +33 -0
  23. package/lib/import/modules-js/assets.js +415 -0
  24. package/lib/import/modules-js/content-types.d.ts +33 -0
  25. package/lib/import/modules-js/content-types.js +176 -0
  26. package/lib/import/modules-js/custom-roles.d.ts +15 -0
  27. package/lib/import/modules-js/custom-roles.js +141 -0
  28. package/lib/import/modules-js/entries.d.ts +54 -0
  29. package/lib/import/modules-js/entries.js +1260 -0
  30. package/lib/import/modules-js/environments.d.ts +13 -0
  31. package/lib/import/modules-js/environments.js +85 -0
  32. package/lib/import/modules-js/extensions.d.ts +17 -0
  33. package/lib/import/modules-js/extensions.js +86 -0
  34. package/lib/import/modules-js/global-fields.d.ts +13 -0
  35. package/lib/import/modules-js/global-fields.js +109 -0
  36. package/lib/import/modules-js/index.d.ts +1 -0
  37. package/lib/import/modules-js/index.js +33 -0
  38. package/lib/import/modules-js/labels.d.ts +20 -0
  39. package/lib/import/modules-js/labels.js +148 -0
  40. package/lib/import/modules-js/locales.d.ts +24 -0
  41. package/lib/import/modules-js/locales.js +196 -0
  42. package/lib/import/modules-js/marketplace-apps.d.ts +60 -0
  43. package/lib/import/modules-js/marketplace-apps.js +409 -0
  44. package/lib/import/modules-js/webhooks.d.ts +17 -0
  45. package/lib/import/modules-js/webhooks.js +85 -0
  46. package/lib/import/modules-js/workflows.d.ts +18 -0
  47. package/lib/import/modules-js/workflows.js +132 -0
  48. package/lib/types/default-config.d.ts +130 -0
  49. package/lib/types/default-config.js +2 -0
  50. package/lib/types/import-config.d.ts +51 -0
  51. package/lib/types/import-config.js +2 -0
  52. package/lib/types/index.d.ts +30 -0
  53. package/lib/types/index.js +2 -0
  54. package/lib/utils/asset-helper.d.ts +4 -0
  55. package/lib/utils/asset-helper.js +387 -0
  56. package/lib/utils/backup-handler.d.ts +2 -0
  57. package/lib/utils/backup-handler.js +31 -0
  58. package/lib/utils/common-helper.d.ts +20 -0
  59. package/lib/utils/common-helper.js +244 -0
  60. package/lib/utils/content-type-helper.d.ts +49 -0
  61. package/lib/utils/content-type-helper.js +143 -0
  62. package/lib/utils/entries-helper.d.ts +4 -0
  63. package/lib/utils/entries-helper.js +252 -0
  64. package/lib/utils/extension-helper.d.ts +10 -0
  65. package/lib/utils/extension-helper.js +72 -0
  66. package/lib/utils/file-helper.d.ts +14 -0
  67. package/lib/utils/file-helper.js +140 -0
  68. package/lib/utils/import-config-handler.d.ts +3 -0
  69. package/lib/utils/import-config-handler.js +73 -0
  70. package/lib/utils/index.d.ts +12 -0
  71. package/lib/utils/index.js +29 -0
  72. package/lib/utils/interactive.d.ts +2 -0
  73. package/lib/utils/interactive.js +23 -0
  74. package/lib/utils/logger.d.ts +8 -0
  75. package/lib/utils/logger.js +154 -0
  76. package/lib/utils/login-handler.d.ts +8 -0
  77. package/lib/utils/login-handler.js +53 -0
  78. package/lib/utils/marketplace-app-helper.d.ts +4 -0
  79. package/lib/utils/marketplace-app-helper.js +31 -0
  80. package/messages/index.json +1 -7
  81. package/oclif.manifest.json +2 -2
  82. package/package.json +47 -21
  83. package/src/app.js +0 -217
  84. package/src/commands/cm/stacks/import.js +0 -161
  85. package/src/config/default.js +0 -352
  86. package/src/lib/import/assets.js +0 -495
  87. package/src/lib/import/content-types.js +0 -201
  88. package/src/lib/import/custom-roles.js +0 -169
  89. package/src/lib/import/entries.js +0 -1495
  90. package/src/lib/import/environments.js +0 -106
  91. package/src/lib/import/extensions.js +0 -108
  92. package/src/lib/import/global-fields.js +0 -135
  93. package/src/lib/import/labels.js +0 -175
  94. package/src/lib/import/locales.js +0 -216
  95. package/src/lib/import/marketplace-apps.js +0 -542
  96. package/src/lib/import/webhooks.js +0 -113
  97. package/src/lib/import/workflows.js +0 -166
  98. package/src/lib/util/extensionsUidReplace.js +0 -67
  99. package/src/lib/util/fs.js +0 -124
  100. package/src/lib/util/import-flags.js +0 -187
  101. package/src/lib/util/index.js +0 -222
  102. package/src/lib/util/log.js +0 -144
  103. package/src/lib/util/login.js +0 -58
  104. package/src/lib/util/lookupReplaceAssets.js +0 -366
  105. package/src/lib/util/lookupReplaceEntries.js +0 -250
  106. package/src/lib/util/marketplace-app-helper.js +0 -31
  107. package/src/lib/util/removeReferenceFields.js +0 -59
  108. package/src/lib/util/schemaTemplate.js +0 -38
  109. package/src/lib/util/supress-mandatory-fields.js +0 -34
  110. package/src/lib/util/upload.js +0 -56
@@ -0,0 +1,70 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const tslib_1 = require("tslib");
4
+ const utils_1 = require("../utils");
5
+ const modules_1 = tslib_1.__importDefault(require("./modules"));
6
+ const modules_js_1 = tslib_1.__importDefault(require("./modules-js"));
7
+ class ModuleImporter {
8
+ constructor(managementAPIClient, importConfig) {
9
+ this.managementAPIClient = managementAPIClient;
10
+ this.stackAPIClient = this.managementAPIClient.stack({
11
+ api_key: importConfig.apiKey,
12
+ management_token: importConfig.management_token,
13
+ });
14
+ this.importConfig = importConfig;
15
+ }
16
+ async start() {
17
+ if (this.importConfig.branchName) {
18
+ await (0, utils_1.validateBranch)(this.stackAPIClient, this.importConfig, this.importConfig.branchName);
19
+ }
20
+ if (!this.importConfig.master_locale) {
21
+ let masterLocalResponse = await (0, utils_1.masterLocalDetails)(this.stackAPIClient);
22
+ this.importConfig['master_locale'] = { code: masterLocalResponse.code };
23
+ this.importConfig.masterLocale = { code: masterLocalResponse.code };
24
+ }
25
+ const backupDir = await (0, utils_1.backupHandler)(this.importConfig);
26
+ if (backupDir) {
27
+ this.importConfig.backupDir = backupDir;
28
+ // To support the old config
29
+ this.importConfig.data = backupDir;
30
+ }
31
+ await (0, utils_1.sanitizeStack)(this.stackAPIClient);
32
+ return this.import();
33
+ }
34
+ async import() {
35
+ // checks for single module or all modules
36
+ if (this.importConfig.singleModuleImport) {
37
+ return this.importByModuleByName(this.importConfig.moduleName);
38
+ }
39
+ return this.importAllModules();
40
+ }
41
+ async importByModuleByName(moduleName) {
42
+ (0, utils_1.log)(this.importConfig, `Starting import of ${moduleName} module`, 'info');
43
+ const basePath = `${this.importConfig.backupDir}/${moduleName}`;
44
+ // import the modules by name
45
+ // calls the module runner which inturn calls the module itself
46
+ // Todo: Implement a mechanism to determine whether module is new or old
47
+ if (this.importConfig.useNewModuleStructure &&
48
+ this.importConfig.updatedModules.indexOf(moduleName) !== -1
49
+ //&& new FsUtility({ basePath }).isNewFsStructure
50
+ ) {
51
+ return (0, modules_1.default)({
52
+ stackAPIClient: this.stackAPIClient,
53
+ importConfig: this.importConfig,
54
+ moduleName,
55
+ });
56
+ }
57
+ return (0, modules_js_1.default)({
58
+ stackAPIClient: this.stackAPIClient,
59
+ importConfig: this.importConfig,
60
+ moduleName,
61
+ });
62
+ }
63
+ async importAllModules() {
64
+ // use the algorithm to determine the parallel and sequential execution of modules
65
+ for (let moduleName of this.importConfig.modules.types) {
66
+ await this.importByModuleByName(moduleName);
67
+ }
68
+ }
69
+ }
70
+ exports.default = ModuleImporter;
@@ -0,0 +1,63 @@
1
+ import BaseClass, { ApiOptions } from './base-class';
2
+ import { ModuleClassParams } from '../../types';
3
+ export default class ImportAssets extends BaseClass {
4
+ private fs;
5
+ private assetsPath;
6
+ private mapperDirPath;
7
+ private assetsRootPath;
8
+ private assetUidMapperPath;
9
+ private assetUrlMapperPath;
10
+ private assetFolderUidMapperPath;
11
+ assetConfig: {
12
+ dirName: string;
13
+ assetBatchLimit: number;
14
+ publishAssets: boolean;
15
+ fileName: string;
16
+ importSameStructure: boolean;
17
+ uploadAssetsConcurrency: number;
18
+ displayExecutionTime: boolean;
19
+ importFoldersConcurrency: number;
20
+ includeVersionedAssets: boolean;
21
+ host: string;
22
+ folderValidKeys: string[];
23
+ validKeys: string[];
24
+ };
25
+ private environments;
26
+ private assetsUidMap;
27
+ private assetsUrlMap;
28
+ private assetsFolderMap;
29
+ constructor({ importConfig, stackAPIClient }: ModuleClassParams);
30
+ /**
31
+ * @method start
32
+ * @returns {Promise<void>} Promise<any>
33
+ */
34
+ start(): Promise<void>;
35
+ /**
36
+ * @method importFolders
37
+ * @returns {Promise<any>} Promise<any>
38
+ */
39
+ importFolders(): Promise<any>;
40
+ /**
41
+ * @method importAssets
42
+ * @param {boolean} isVersion boolean
43
+ * @returns {Promise<void>} Promise<void>
44
+ */
45
+ importAssets(isVersion?: boolean): Promise<void>;
46
+ /**
47
+ * @method serializeAssets
48
+ * @param {ApiOptions} apiOptions ApiOptions
49
+ * @returns {ApiOptions} ApiOptions
50
+ */
51
+ serializeAssets(apiOptions: ApiOptions): ApiOptions;
52
+ /**
53
+ * @method publish
54
+ * @returns {Promise<void>} Promise<void>
55
+ */
56
+ publish(): Promise<void>;
57
+ /**
58
+ * @method constructFolderImportOrder
59
+ * @param {Record<string, any>[]} folders object
60
+ * @returns {Array<Record<string, any>>} Array<Record<string, any>>
61
+ */
62
+ constructFolderImportOrder(folders: any): Array<Record<string, any>>;
63
+ }
@@ -0,0 +1,265 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const tslib_1 = require("tslib");
4
+ const map_1 = tslib_1.__importDefault(require("lodash/map"));
5
+ const values_1 = tslib_1.__importDefault(require("lodash/values"));
6
+ const filter_1 = tslib_1.__importDefault(require("lodash/filter"));
7
+ const unionBy_1 = tslib_1.__importDefault(require("lodash/unionBy"));
8
+ const orderBy_1 = tslib_1.__importDefault(require("lodash/orderBy"));
9
+ const isEmpty_1 = tslib_1.__importDefault(require("lodash/isEmpty"));
10
+ const node_fs_1 = require("node:fs");
11
+ const includes_1 = tslib_1.__importDefault(require("lodash/includes"));
12
+ const node_path_1 = require("node:path");
13
+ const cli_utilities_1 = require("@contentstack/cli-utilities");
14
+ const config_1 = tslib_1.__importDefault(require("../../config"));
15
+ const utils_1 = require("../../utils");
16
+ const base_class_1 = tslib_1.__importDefault(require("./base-class"));
17
+ class ImportAssets extends base_class_1.default {
18
+ constructor({ importConfig, stackAPIClient }) {
19
+ super({ importConfig, stackAPIClient });
20
+ this.assetConfig = config_1.default.modules.assets;
21
+ this.environments = {};
22
+ this.assetsUidMap = {};
23
+ this.assetsUrlMap = {};
24
+ this.assetsFolderMap = {};
25
+ this.assetsPath = (0, node_path_1.join)(this.importConfig.backupDir, 'assets');
26
+ this.mapperDirPath = (0, node_path_1.join)(this.importConfig.backupDir, 'mapper', 'assets');
27
+ this.assetUidMapperPath = (0, node_path_1.join)(this.mapperDirPath, 'uid-mapping.json');
28
+ this.assetUrlMapperPath = (0, node_path_1.join)(this.mapperDirPath, 'url-mapping.json');
29
+ this.assetFolderUidMapperPath = (0, node_path_1.join)(this.mapperDirPath, 'folder-mapping.json');
30
+ this.assetsRootPath = (0, node_path_1.join)(this.importConfig.backupDir, this.assetConfig.dirName);
31
+ this.fs = new cli_utilities_1.FsUtility({ basePath: this.mapperDirPath });
32
+ this.environments = this.fs.readFile((0, node_path_1.join)(this.importConfig.backupDir, 'environments', 'environments.json'), true);
33
+ }
34
+ /**
35
+ * @method start
36
+ * @returns {Promise<void>} Promise<any>
37
+ */
38
+ async start() {
39
+ // NOTE Step 1: Import folders and create uid mapping file
40
+ await this.importFolders();
41
+ // NOTE Step 2: Import versioned assets and create it mapping files (uid, url)
42
+ if (this.assetConfig.includeVersionedAssets) {
43
+ if ((0, node_fs_1.existsSync)(`${this.assetsPath}/versions`))
44
+ await this.importAssets(true);
45
+ else
46
+ (0, utils_1.log)(this.importConfig, 'No Versioned assets found to import', 'info');
47
+ }
48
+ // NOTE Step 3: Import Assets and create it mapping files (uid, url)
49
+ await this.importAssets();
50
+ // NOTE Step 4: Publish assets
51
+ if (this.assetConfig.publishAssets)
52
+ await this.publish();
53
+ }
54
+ /**
55
+ * @method importFolders
56
+ * @returns {Promise<any>} Promise<any>
57
+ */
58
+ async importFolders() {
59
+ const folders = this.fs.readFile((0, node_path_1.resolve)(this.assetsRootPath, 'folders.json'));
60
+ if ((0, isEmpty_1.default)(folders)) {
61
+ (0, utils_1.log)(this.importConfig, 'No folders found to import', 'info');
62
+ return;
63
+ }
64
+ const batches = this.constructFolderImportOrder(folders);
65
+ const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }) => {
66
+ this.assetsFolderMap[uid] = response.uid;
67
+ (0, utils_1.log)(this.importConfig, `Created folder: '${name}'`, 'success');
68
+ };
69
+ const onReject = ({ error, apiData: { name } = { name: '' } }) => {
70
+ (0, utils_1.log)(this.importConfig, `${name} folder creation failed.!`, 'error');
71
+ (0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
72
+ };
73
+ const serializeData = (apiOptions) => {
74
+ if (apiOptions.apiData.parent_uid) {
75
+ apiOptions.apiData.parent_uid = this.assetsFolderMap[apiOptions.apiData.parent_uid];
76
+ }
77
+ return apiOptions;
78
+ };
79
+ const batch = (0, map_1.default)((0, unionBy_1.default)(batches, 'parent_uid'), 'parent_uid');
80
+ for (const parent_uid of batch) {
81
+ // NOTE create parent folders
82
+ /* eslint-disable no-await-in-loop */
83
+ await this.makeConcurrentCall({
84
+ apiContent: (0, orderBy_1.default)((0, filter_1.default)(batches, { parent_uid }), 'created_at'),
85
+ processName: 'import assets folders',
86
+ apiParams: {
87
+ serializeData,
88
+ reject: onReject,
89
+ resolve: onSuccess,
90
+ entity: 'create-assets-folder',
91
+ includeParamOnCompletion: true,
92
+ },
93
+ concurrencyLimit: this.assetConfig.importFoldersConcurrency,
94
+ }, undefined, false);
95
+ }
96
+ if (!(0, isEmpty_1.default)(this.assetsFolderMap)) {
97
+ this.fs.writeFile(this.assetFolderUidMapperPath, this.assetsFolderMap);
98
+ }
99
+ }
100
+ /**
101
+ * @method importAssets
102
+ * @param {boolean} isVersion boolean
103
+ * @returns {Promise<void>} Promise<void>
104
+ */
105
+ async importAssets(isVersion = false) {
106
+ const processName = isVersion ? 'import versioned assets' : 'import assets';
107
+ const indexFileName = isVersion ? 'versioned-assets.json' : 'assets.json';
108
+ const basePath = isVersion ? (0, node_path_1.join)(this.assetsPath, 'versions') : this.assetsPath;
109
+ const fs = new cli_utilities_1.FsUtility({ basePath, indexFileName });
110
+ const indexer = fs.indexFileContent;
111
+ const indexerCount = (0, values_1.default)(indexer).length;
112
+ const onSuccess = ({ response = {}, apiData: { uid, url, title } = undefined }) => {
113
+ this.assetsUidMap[uid] = response.uid;
114
+ this.assetsUrlMap[url] = response.url;
115
+ (0, utils_1.log)(this.importConfig, `Created asset: '${title}'`, 'info');
116
+ };
117
+ const onReject = ({ error, apiData: { title } = undefined }) => {
118
+ (0, utils_1.log)(this.importConfig, `${title} asset upload failed.!`, 'error');
119
+ (0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
120
+ };
121
+ /* eslint-disable @typescript-eslint/no-unused-vars, guard-for-in */
122
+ for (const index in indexer) {
123
+ const chunk = await fs.readChunkFiles.next().catch((error) => {
124
+ (0, utils_1.log)(this.importConfig, error, 'error');
125
+ });
126
+ if (chunk) {
127
+ let apiContent = (0, orderBy_1.default)((0, values_1.default)(chunk), '_version');
128
+ if (isVersion && this.assetConfig.importSameStructure) {
129
+ // NOTE to create same structure it must have seed assets/version 1 asset to be created first
130
+ await this.makeConcurrentCall({
131
+ processName,
132
+ indexerCount,
133
+ currentIndexer: +index,
134
+ apiContent: (0, filter_1.default)(apiContent, ({ _version }) => _version === 1),
135
+ apiParams: {
136
+ reject: onReject,
137
+ resolve: onSuccess,
138
+ entity: 'create-assets',
139
+ includeParamOnCompletion: true,
140
+ serializeData: this.serializeAssets.bind(this),
141
+ },
142
+ concurrencyLimit: this.assetConfig.uploadAssetsConcurrency,
143
+ });
144
+ apiContent = (0, filter_1.default)(apiContent, ({ _version }) => _version > 1);
145
+ }
146
+ await this.makeConcurrentCall({
147
+ apiContent,
148
+ processName,
149
+ indexerCount,
150
+ currentIndexer: +index,
151
+ apiParams: {
152
+ reject: onReject,
153
+ resolve: onSuccess,
154
+ entity: 'create-assets',
155
+ includeParamOnCompletion: true,
156
+ serializeData: this.serializeAssets.bind(this),
157
+ },
158
+ concurrencyLimit: this.assetConfig.uploadAssetsConcurrency,
159
+ }, undefined, !isVersion);
160
+ }
161
+ }
162
+ if (!isVersion && (!(0, isEmpty_1.default)(this.assetsUidMap) || !(0, isEmpty_1.default)(this.assetsUrlMap))) {
163
+ this.fs.writeFile(this.assetUidMapperPath, this.assetsUidMap);
164
+ this.fs.writeFile(this.assetUrlMapperPath, this.assetsUrlMap);
165
+ }
166
+ }
167
+ /**
168
+ * @method serializeAssets
169
+ * @param {ApiOptions} apiOptions ApiOptions
170
+ * @returns {ApiOptions} ApiOptions
171
+ */
172
+ serializeAssets(apiOptions) {
173
+ const { apiData: asset } = apiOptions;
174
+ if (!this.assetConfig.importSameStructure &&
175
+ !this.assetConfig.includeVersionedAssets &&
176
+ /* eslint-disable @typescript-eslint/no-unused-vars, no-prototype-builtins */
177
+ this.assetsUidMap.hasOwnProperty(asset.uid)) {
178
+ (0, utils_1.log)(this.importConfig, `Skipping upload of asset: ${asset.uid}. Its mapped to: ${this.assetsUidMap[asset.uid]}`, 'success');
179
+ apiOptions.entity = undefined;
180
+ return apiOptions;
181
+ }
182
+ asset.upload = (0, node_path_1.join)(this.assetsPath, 'files', asset.uid, asset.filename);
183
+ if (asset.parent_uid) {
184
+ asset.parent_uid = this.assetsFolderMap[asset.parent_uid];
185
+ }
186
+ apiOptions.apiData = asset;
187
+ if (this.assetsUidMap[asset.uid] && this.assetConfig.importSameStructure) {
188
+ apiOptions.entity = 'replace-assets';
189
+ apiOptions.uid = this.assetsUidMap[asset.uid];
190
+ }
191
+ return apiOptions;
192
+ }
193
+ /**
194
+ * @method publish
195
+ * @returns {Promise<void>} Promise<void>
196
+ */
197
+ async publish() {
198
+ const fs = new cli_utilities_1.FsUtility({ basePath: this.assetsPath, indexFileName: 'assets.json' });
199
+ if ((0, isEmpty_1.default)(this.assetsUidMap)) {
200
+ this.assetsUidMap = fs.readFile(this.assetUidMapperPath, true);
201
+ }
202
+ const indexer = fs.indexFileContent;
203
+ const indexerCount = (0, values_1.default)(indexer).length;
204
+ const onSuccess = ({ apiData: { uid, title } = undefined }) => {
205
+ (0, utils_1.log)(this.importConfig, `Asset '${uid}: ${title}' published successfully`, 'success');
206
+ };
207
+ const onReject = ({ error, apiData: { uid, title } = undefined }) => {
208
+ (0, utils_1.log)(this.importConfig, `Asset '${uid}: ${title}' not published`, 'error');
209
+ (0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
210
+ };
211
+ const serializeData = (apiOptions) => {
212
+ const { apiData: asset } = apiOptions;
213
+ const publishDetails = (0, filter_1.default)(asset.publish_details, 'environment');
214
+ const locales = (0, map_1.default)(publishDetails, 'locale');
215
+ const environments = (0, map_1.default)(publishDetails, ({ environment }) => this.environments[environment].name);
216
+ asset.locales = locales;
217
+ asset.environments = environments;
218
+ apiOptions.uid = this.assetsUidMap[asset.uid];
219
+ apiOptions.apiData.publishDetails = { locales, environments };
220
+ if (!apiOptions.uid)
221
+ apiOptions.entity = undefined;
222
+ return apiOptions;
223
+ };
224
+ /* eslint-disable @typescript-eslint/no-unused-vars */
225
+ for (const index in indexer) {
226
+ const apiContent = (0, filter_1.default)((0, values_1.default)(await fs.readChunkFiles.next()), ({ publish_details }) => !(0, isEmpty_1.default)(publish_details));
227
+ await this.makeConcurrentCall({
228
+ apiContent,
229
+ indexerCount,
230
+ currentIndexer: +index,
231
+ processName: 'assets publish',
232
+ apiParams: {
233
+ serializeData,
234
+ reject: onReject,
235
+ resolve: onSuccess,
236
+ entity: 'publish-assets',
237
+ includeParamOnCompletion: true,
238
+ },
239
+ concurrencyLimit: this.assetConfig.uploadAssetsConcurrency,
240
+ });
241
+ }
242
+ }
243
+ /**
244
+ * @method constructFolderImportOrder
245
+ * @param {Record<string, any>[]} folders object
246
+ * @returns {Array<Record<string, any>>} Array<Record<string, any>>
247
+ */
248
+ constructFolderImportOrder(folders) {
249
+ let parentUid = [];
250
+ // NOTE: Read root folder
251
+ const importOrder = (0, filter_1.default)(folders, { parent_uid: null }).map(({ uid, name, parent_uid, created_at }) => {
252
+ parentUid.push(uid);
253
+ return { uid, name, parent_uid, created_at };
254
+ });
255
+ while (!(0, isEmpty_1.default)(parentUid)) {
256
+ // NOTE: Read nested folders every iteration until we find empty folders
257
+ parentUid = (0, filter_1.default)(folders, ({ parent_uid }) => (0, includes_1.default)(parentUid, parent_uid)).map(({ uid, name, parent_uid, created_at }) => {
258
+ importOrder.push({ uid, name, parent_uid, created_at });
259
+ return uid;
260
+ });
261
+ }
262
+ return importOrder;
263
+ }
264
+ }
265
+ exports.default = ImportAssets;
@@ -0,0 +1,69 @@
1
+ import { Stack } from '@contentstack/management/types/stack';
2
+ import { ImportConfig, ModuleClassParams } from '../../types';
3
+ export type AdditionalKeys = {
4
+ backupDir: string;
5
+ };
6
+ export type ApiModuleType = 'create-assets' | 'replace-assets' | 'publish-assets' | 'create-assets-folder' | 'create-locale' | 'update-locale';
7
+ export type ApiOptions = {
8
+ uid?: string;
9
+ url?: string;
10
+ entity: ApiModuleType;
11
+ apiData?: Record<any, any>;
12
+ resolve: (value: any) => void;
13
+ reject: (error: any) => void;
14
+ additionalInfo?: Record<any, any>;
15
+ includeParamOnCompletion?: boolean;
16
+ serializeData?: (input: ApiOptions) => any;
17
+ };
18
+ export type EnvType = {
19
+ processName: string;
20
+ totalCount?: number;
21
+ indexerCount?: number;
22
+ currentIndexer?: number;
23
+ apiParams?: ApiOptions;
24
+ concurrencyLimit?: number;
25
+ apiContent: Record<string, any>[];
26
+ };
27
+ export type CustomPromiseHandlerInput = {
28
+ index: number;
29
+ batchIndex: number;
30
+ apiParams?: ApiOptions;
31
+ isLastRequest: boolean;
32
+ };
33
+ export type CustomPromiseHandler = (input: CustomPromiseHandlerInput) => Promise<any>;
34
+ export default abstract class BaseClass {
35
+ readonly client: Stack;
36
+ importConfig: ImportConfig;
37
+ modulesConfig: any;
38
+ constructor({ importConfig, stackAPIClient }: Omit<ModuleClassParams, 'moduleName'>);
39
+ get stack(): Stack;
40
+ /**
41
+ * @method delay
42
+ * @param {number} ms number
43
+ * @returns {Promise} Promise<void>
44
+ */
45
+ delay(ms: number): Promise<void>;
46
+ /**
47
+ * @method makeConcurrentCall
48
+ * @param {Record<string, any>} env EnvType
49
+ * @param {CustomPromiseHandler} promisifyHandler CustomPromiseHandler
50
+ * @param {boolean} logBatchCompletionMsg boolean
51
+ * @returns {Promise} Promise<void>
52
+ */
53
+ makeConcurrentCall(env: EnvType, promisifyHandler?: CustomPromiseHandler, logBatchCompletionMsg?: boolean): Promise<void>;
54
+ /**
55
+ * @method logMsgAndWaitIfRequired
56
+ * @param {string} processName string
57
+ * @param {number} start number
58
+ * @param {number} batchNo - number
59
+ * @returns {Promise} Promise<void>
60
+ */
61
+ logMsgAndWaitIfRequired(processName: string, start: number, totelBatches: number, batchNo: number, logBatchCompletionMsg?: boolean, indexerCount?: number, currentIndexer?: number): Promise<void>;
62
+ /**
63
+ * @method makeAPICall
64
+ * @param {Record<string, any>} apiOptions - Api related params
65
+ * @param {Record<string, any>} isLastRequest - Boolean
66
+ * @return {Promise} Promise<void>
67
+ */
68
+ makeAPICall(apiOptions: ApiOptions, isLastRequest?: boolean): Promise<void>;
69
+ }
@@ -0,0 +1,165 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const tslib_1 = require("tslib");
4
+ const pick_1 = tslib_1.__importDefault(require("lodash/pick"));
5
+ const last_1 = tslib_1.__importDefault(require("lodash/last"));
6
+ const chunk_1 = tslib_1.__importDefault(require("lodash/chunk"));
7
+ const isEmpty_1 = tslib_1.__importDefault(require("lodash/isEmpty"));
8
+ const entries_1 = tslib_1.__importDefault(require("lodash/entries"));
9
+ const isEqual_1 = tslib_1.__importDefault(require("lodash/isEqual"));
10
+ const utils_1 = require("../../utils");
11
+ class BaseClass {
12
+ constructor({ importConfig, stackAPIClient }) {
13
+ this.client = stackAPIClient;
14
+ this.importConfig = importConfig;
15
+ this.modulesConfig = importConfig.modules;
16
+ }
17
+ get stack() {
18
+ return this.client;
19
+ }
20
+ /**
21
+ * @method delay
22
+ * @param {number} ms number
23
+ * @returns {Promise} Promise<void>
24
+ */
25
+ delay(ms) {
26
+ /* eslint-disable no-promise-executor-return */
27
+ return new Promise((resolve) => setTimeout(resolve, ms <= 0 ? 0 : ms));
28
+ }
29
+ /**
30
+ * @method makeConcurrentCall
31
+ * @param {Record<string, any>} env EnvType
32
+ * @param {CustomPromiseHandler} promisifyHandler CustomPromiseHandler
33
+ * @param {boolean} logBatchCompletionMsg boolean
34
+ * @returns {Promise} Promise<void>
35
+ */
36
+ makeConcurrentCall(env, promisifyHandler, logBatchCompletionMsg = true) {
37
+ const { apiParams, apiContent, processName, indexerCount, currentIndexer, concurrencyLimit = this.importConfig.modules.apiConcurrency, } = env;
38
+ /* eslint-disable no-async-promise-executor */
39
+ return new Promise(async (resolve) => {
40
+ let batchNo = 0;
41
+ let isLastRequest = false;
42
+ const batches = (0, chunk_1.default)(apiContent, concurrencyLimit);
43
+ /* eslint-disable no-promise-executor-return */
44
+ if ((0, isEmpty_1.default)(batches))
45
+ return resolve();
46
+ for (const [batchIndex, batch] of (0, entries_1.default)(batches)) {
47
+ batchNo += 1;
48
+ const allPromise = [];
49
+ const start = Date.now();
50
+ for (const [index, element] of (0, entries_1.default)(batch)) {
51
+ let promise = Promise.resolve();
52
+ isLastRequest = (0, isEqual_1.default)((0, last_1.default)(batch), element) && (0, isEqual_1.default)((0, last_1.default)(batches), batch);
53
+ if (promisifyHandler instanceof Function) {
54
+ promise = promisifyHandler({
55
+ apiParams,
56
+ isLastRequest,
57
+ index: Number(index),
58
+ batchIndex: Number(batchIndex),
59
+ });
60
+ }
61
+ else if (apiParams) {
62
+ apiParams.apiData = element;
63
+ promise = this.makeAPICall(apiParams, isLastRequest);
64
+ }
65
+ allPromise.push(promise);
66
+ }
67
+ /* eslint-disable no-await-in-loop */
68
+ await Promise.allSettled(allPromise);
69
+ /* eslint-disable no-await-in-loop */
70
+ await this.logMsgAndWaitIfRequired(processName, start, batches.length, batchNo, logBatchCompletionMsg, indexerCount, currentIndexer);
71
+ if (isLastRequest)
72
+ resolve();
73
+ }
74
+ });
75
+ }
76
+ /**
77
+ * @method logMsgAndWaitIfRequired
78
+ * @param {string} processName string
79
+ * @param {number} start number
80
+ * @param {number} batchNo - number
81
+ * @returns {Promise} Promise<void>
82
+ */
83
+ async logMsgAndWaitIfRequired(processName, start, totelBatches, batchNo, logBatchCompletionMsg = true, indexerCount, currentIndexer) {
84
+ const end = Date.now();
85
+ const exeTime = end - start;
86
+ if (logBatchCompletionMsg) {
87
+ let batchMsg = '';
88
+ // info: Batch No. 20 of import assets is complete
89
+ if (currentIndexer)
90
+ batchMsg += `Current chunk processing is (${currentIndexer}/${indexerCount})`;
91
+ (0, utils_1.log)(this.importConfig, `Batch No. (${batchNo}/${totelBatches}) of ${processName} is complete. ${batchMsg}`, 'success');
92
+ }
93
+ if (this.importConfig.modules.assets.displayExecutionTime) {
94
+ console.log(`Time taken to execute: ${exeTime} milliseconds; wait time: ${exeTime < 1000 ? 1000 - exeTime : 0} milliseconds`);
95
+ }
96
+ if (exeTime < 1000)
97
+ await this.delay(1000 - exeTime);
98
+ }
99
+ /**
100
+ * @method makeAPICall
101
+ * @param {Record<string, any>} apiOptions - Api related params
102
+ * @param {Record<string, any>} isLastRequest - Boolean
103
+ * @return {Promise} Promise<void>
104
+ */
105
+ makeAPICall(apiOptions, isLastRequest = false) {
106
+ if (apiOptions.serializeData instanceof Function) {
107
+ apiOptions = apiOptions.serializeData(apiOptions);
108
+ }
109
+ const { uid, entity, reject, resolve, apiData, additionalInfo, includeParamOnCompletion } = apiOptions;
110
+ const onSuccess = (response) => resolve({
111
+ response,
112
+ isLastRequest,
113
+ additionalInfo,
114
+ apiData: includeParamOnCompletion ? apiData : undefined,
115
+ });
116
+ const onReject = (error) => reject({
117
+ error,
118
+ isLastRequest,
119
+ additionalInfo,
120
+ apiData: includeParamOnCompletion ? apiData : undefined,
121
+ });
122
+ switch (entity) {
123
+ case 'create-assets-folder':
124
+ return this.stack
125
+ .asset()
126
+ .folder()
127
+ .create({ asset: (0, pick_1.default)(apiData, this.modulesConfig.assets.folderValidKeys) })
128
+ .then(onSuccess)
129
+ .catch(onReject);
130
+ case 'create-assets':
131
+ return this.stack
132
+ .asset()
133
+ .create((0, pick_1.default)(apiData, [...this.modulesConfig.assets.validKeys, 'upload']))
134
+ .then(onSuccess)
135
+ .catch(onReject);
136
+ case 'replace-assets':
137
+ return this.stack
138
+ .asset(uid)
139
+ .replace((0, pick_1.default)(apiData, [...this.modulesConfig.assets.validKeys, 'upload']))
140
+ .then(onSuccess)
141
+ .catch(onReject);
142
+ case 'publish-assets':
143
+ return this.stack
144
+ .asset(uid)
145
+ .publish((0, pick_1.default)(apiData, ['publishDetails']))
146
+ .then(onSuccess)
147
+ .catch(onReject);
148
+ case 'create-locale':
149
+ return this.stack
150
+ .locale()
151
+ .create({ locale: (0, pick_1.default)(apiData, ['name', 'code']) })
152
+ .then(onSuccess)
153
+ .catch(onReject);
154
+ case 'update-locale':
155
+ return this.stack
156
+ .locale(apiData.code)
157
+ .update({ locale: (0, pick_1.default)(apiData, [...this.modulesConfig.locales.requiredKeys]) })
158
+ .then(onSuccess)
159
+ .catch(onReject);
160
+ default:
161
+ return Promise.resolve();
162
+ }
163
+ }
164
+ }
165
+ exports.default = BaseClass;
@@ -0,0 +1,2 @@
1
+ import { ModuleClassParams } from '../../types';
2
+ export default function startModuleImport(modulePayload: ModuleClassParams): Promise<any>;
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ var _a;
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ const tslib_1 = require("tslib");
5
+ const cli_utilities_1 = require("@contentstack/cli-utilities");
6
+ const node_path_1 = require("node:path");
7
+ const modules_js_1 = tslib_1.__importDefault(require("../modules-js"));
8
+ async function startModuleImport(modulePayload) {
9
+ var _a;
10
+ // Todo: Remove below code when auto detect mechanism implemented for old and new module
11
+ if (modulePayload.moduleName === 'assets' &&
12
+ !new cli_utilities_1.FsUtility({ basePath: (0, node_path_1.join)((_a = modulePayload.importConfig) === null || _a === void 0 ? void 0 : _a.backupDir, 'assets') }).isNewFsStructure) {
13
+ return (0, modules_js_1.default)(modulePayload);
14
+ }
15
+ const { default: ModuleRunner } = await (_a = `./${modulePayload.moduleName}`, Promise.resolve().then(() => tslib_1.__importStar(require(_a))));
16
+ const moduleRunner = new ModuleRunner(modulePayload);
17
+ return moduleRunner.start();
18
+ }
19
+ exports.default = startModuleImport;