@contentstack/cli-cm-import 1.5.10 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -14
- package/bin/dev +17 -0
- package/bin/dev.cmd +3 -0
- package/bin/run +6 -0
- package/bin/run.cmd +3 -0
- package/lib/commands/cm/stacks/import.d.ts +10 -0
- package/lib/commands/cm/stacks/import.js +111 -0
- package/lib/config/index.d.ts +3 -0
- package/lib/config/index.js +372 -0
- package/lib/import/index.d.ts +1 -0
- package/lib/import/index.js +8 -0
- package/lib/import/module-importer.d.ts +13 -0
- package/lib/import/module-importer.js +70 -0
- package/lib/import/modules/assets.d.ts +63 -0
- package/lib/import/modules/assets.js +265 -0
- package/lib/import/modules/base-class.d.ts +69 -0
- package/lib/import/modules/base-class.js +165 -0
- package/lib/import/modules/index.d.ts +2 -0
- package/lib/import/modules/index.js +19 -0
- package/lib/import/modules/locales.d.ts +31 -0
- package/lib/import/modules/locales.js +152 -0
- package/lib/import/modules-js/assets.d.ts +33 -0
- package/lib/import/modules-js/assets.js +415 -0
- package/lib/import/modules-js/content-types.d.ts +33 -0
- package/lib/import/modules-js/content-types.js +176 -0
- package/lib/import/modules-js/custom-roles.d.ts +15 -0
- package/lib/import/modules-js/custom-roles.js +141 -0
- package/lib/import/modules-js/entries.d.ts +54 -0
- package/lib/import/modules-js/entries.js +1260 -0
- package/lib/import/modules-js/environments.d.ts +13 -0
- package/lib/import/modules-js/environments.js +85 -0
- package/lib/import/modules-js/extensions.d.ts +17 -0
- package/lib/import/modules-js/extensions.js +86 -0
- package/lib/import/modules-js/global-fields.d.ts +13 -0
- package/lib/import/modules-js/global-fields.js +109 -0
- package/lib/import/modules-js/index.d.ts +1 -0
- package/lib/import/modules-js/index.js +33 -0
- package/lib/import/modules-js/labels.d.ts +20 -0
- package/lib/import/modules-js/labels.js +148 -0
- package/lib/import/modules-js/locales.d.ts +24 -0
- package/lib/import/modules-js/locales.js +196 -0
- package/lib/import/modules-js/marketplace-apps.d.ts +60 -0
- package/lib/import/modules-js/marketplace-apps.js +409 -0
- package/lib/import/modules-js/webhooks.d.ts +17 -0
- package/lib/import/modules-js/webhooks.js +85 -0
- package/lib/import/modules-js/workflows.d.ts +18 -0
- package/lib/import/modules-js/workflows.js +132 -0
- package/lib/types/default-config.d.ts +130 -0
- package/lib/types/default-config.js +2 -0
- package/lib/types/import-config.d.ts +51 -0
- package/lib/types/import-config.js +2 -0
- package/lib/types/index.d.ts +30 -0
- package/lib/types/index.js +2 -0
- package/lib/utils/asset-helper.d.ts +4 -0
- package/lib/utils/asset-helper.js +387 -0
- package/lib/utils/backup-handler.d.ts +2 -0
- package/lib/utils/backup-handler.js +31 -0
- package/lib/utils/common-helper.d.ts +20 -0
- package/lib/utils/common-helper.js +244 -0
- package/lib/utils/content-type-helper.d.ts +49 -0
- package/lib/utils/content-type-helper.js +143 -0
- package/lib/utils/entries-helper.d.ts +4 -0
- package/lib/utils/entries-helper.js +252 -0
- package/lib/utils/extension-helper.d.ts +10 -0
- package/lib/utils/extension-helper.js +72 -0
- package/lib/utils/file-helper.d.ts +14 -0
- package/lib/utils/file-helper.js +140 -0
- package/lib/utils/import-config-handler.d.ts +3 -0
- package/lib/utils/import-config-handler.js +73 -0
- package/lib/utils/index.d.ts +12 -0
- package/lib/utils/index.js +29 -0
- package/lib/utils/interactive.d.ts +2 -0
- package/lib/utils/interactive.js +23 -0
- package/lib/utils/logger.d.ts +8 -0
- package/lib/utils/logger.js +154 -0
- package/lib/utils/login-handler.d.ts +8 -0
- package/lib/utils/login-handler.js +53 -0
- package/lib/utils/marketplace-app-helper.d.ts +4 -0
- package/lib/utils/marketplace-app-helper.js +31 -0
- package/messages/index.json +1 -7
- package/oclif.manifest.json +2 -2
- package/package.json +47 -21
- package/src/app.js +0 -217
- package/src/commands/cm/stacks/import.js +0 -161
- package/src/config/default.js +0 -352
- package/src/lib/import/assets.js +0 -495
- package/src/lib/import/content-types.js +0 -201
- package/src/lib/import/custom-roles.js +0 -169
- package/src/lib/import/entries.js +0 -1480
- package/src/lib/import/environments.js +0 -106
- package/src/lib/import/extensions.js +0 -108
- package/src/lib/import/global-fields.js +0 -135
- package/src/lib/import/labels.js +0 -175
- package/src/lib/import/locales.js +0 -216
- package/src/lib/import/marketplace-apps.js +0 -542
- package/src/lib/import/webhooks.js +0 -113
- package/src/lib/import/workflows.js +0 -166
- package/src/lib/util/extensionsUidReplace.js +0 -67
- package/src/lib/util/fs.js +0 -124
- package/src/lib/util/import-flags.js +0 -187
- package/src/lib/util/index.js +0 -222
- package/src/lib/util/log.js +0 -144
- package/src/lib/util/login.js +0 -58
- package/src/lib/util/lookupReplaceAssets.js +0 -366
- package/src/lib/util/lookupReplaceEntries.js +0 -250
- package/src/lib/util/marketplace-app-helper.js +0 -31
- package/src/lib/util/removeReferenceFields.js +0 -59
- package/src/lib/util/schemaTemplate.js +0 -38
- package/src/lib/util/supress-mandatory-fields.js +0 -34
- package/src/lib/util/upload.js +0 -56
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const tslib_1 = require("tslib");
|
|
4
|
+
const utils_1 = require("../utils");
|
|
5
|
+
const modules_1 = tslib_1.__importDefault(require("./modules"));
|
|
6
|
+
const modules_js_1 = tslib_1.__importDefault(require("./modules-js"));
|
|
7
|
+
class ModuleImporter {
|
|
8
|
+
constructor(managementAPIClient, importConfig) {
|
|
9
|
+
this.managementAPIClient = managementAPIClient;
|
|
10
|
+
this.stackAPIClient = this.managementAPIClient.stack({
|
|
11
|
+
api_key: importConfig.apiKey,
|
|
12
|
+
management_token: importConfig.management_token,
|
|
13
|
+
});
|
|
14
|
+
this.importConfig = importConfig;
|
|
15
|
+
}
|
|
16
|
+
async start() {
|
|
17
|
+
if (this.importConfig.branchName) {
|
|
18
|
+
await (0, utils_1.validateBranch)(this.stackAPIClient, this.importConfig, this.importConfig.branchName);
|
|
19
|
+
}
|
|
20
|
+
if (!this.importConfig.master_locale) {
|
|
21
|
+
let masterLocalResponse = await (0, utils_1.masterLocalDetails)(this.stackAPIClient);
|
|
22
|
+
this.importConfig['master_locale'] = { code: masterLocalResponse.code };
|
|
23
|
+
this.importConfig.masterLocale = { code: masterLocalResponse.code };
|
|
24
|
+
}
|
|
25
|
+
const backupDir = await (0, utils_1.backupHandler)(this.importConfig);
|
|
26
|
+
if (backupDir) {
|
|
27
|
+
this.importConfig.backupDir = backupDir;
|
|
28
|
+
// To support the old config
|
|
29
|
+
this.importConfig.data = backupDir;
|
|
30
|
+
}
|
|
31
|
+
await (0, utils_1.sanitizeStack)(this.stackAPIClient);
|
|
32
|
+
return this.import();
|
|
33
|
+
}
|
|
34
|
+
async import() {
|
|
35
|
+
// checks for single module or all modules
|
|
36
|
+
if (this.importConfig.singleModuleImport) {
|
|
37
|
+
return this.importByModuleByName(this.importConfig.moduleName);
|
|
38
|
+
}
|
|
39
|
+
return this.importAllModules();
|
|
40
|
+
}
|
|
41
|
+
async importByModuleByName(moduleName) {
|
|
42
|
+
(0, utils_1.log)(this.importConfig, `Starting import of ${moduleName} module`, 'info');
|
|
43
|
+
const basePath = `${this.importConfig.backupDir}/${moduleName}`;
|
|
44
|
+
// import the modules by name
|
|
45
|
+
// calls the module runner which inturn calls the module itself
|
|
46
|
+
// Todo: Implement a mechanism to determine whether module is new or old
|
|
47
|
+
if (this.importConfig.useNewModuleStructure &&
|
|
48
|
+
this.importConfig.updatedModules.indexOf(moduleName) !== -1
|
|
49
|
+
//&& new FsUtility({ basePath }).isNewFsStructure
|
|
50
|
+
) {
|
|
51
|
+
return (0, modules_1.default)({
|
|
52
|
+
stackAPIClient: this.stackAPIClient,
|
|
53
|
+
importConfig: this.importConfig,
|
|
54
|
+
moduleName,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
return (0, modules_js_1.default)({
|
|
58
|
+
stackAPIClient: this.stackAPIClient,
|
|
59
|
+
importConfig: this.importConfig,
|
|
60
|
+
moduleName,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
async importAllModules() {
|
|
64
|
+
// use the algorithm to determine the parallel and sequential execution of modules
|
|
65
|
+
for (let moduleName of this.importConfig.modules.types) {
|
|
66
|
+
await this.importByModuleByName(moduleName);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
exports.default = ModuleImporter;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import BaseClass, { ApiOptions } from './base-class';
|
|
2
|
+
import { ModuleClassParams } from '../../types';
|
|
3
|
+
export default class ImportAssets extends BaseClass {
|
|
4
|
+
private fs;
|
|
5
|
+
private assetsPath;
|
|
6
|
+
private mapperDirPath;
|
|
7
|
+
private assetsRootPath;
|
|
8
|
+
private assetUidMapperPath;
|
|
9
|
+
private assetUrlMapperPath;
|
|
10
|
+
private assetFolderUidMapperPath;
|
|
11
|
+
assetConfig: {
|
|
12
|
+
dirName: string;
|
|
13
|
+
assetBatchLimit: number;
|
|
14
|
+
publishAssets: boolean;
|
|
15
|
+
fileName: string;
|
|
16
|
+
importSameStructure: boolean;
|
|
17
|
+
uploadAssetsConcurrency: number;
|
|
18
|
+
displayExecutionTime: boolean;
|
|
19
|
+
importFoldersConcurrency: number;
|
|
20
|
+
includeVersionedAssets: boolean;
|
|
21
|
+
host: string;
|
|
22
|
+
folderValidKeys: string[];
|
|
23
|
+
validKeys: string[];
|
|
24
|
+
};
|
|
25
|
+
private environments;
|
|
26
|
+
private assetsUidMap;
|
|
27
|
+
private assetsUrlMap;
|
|
28
|
+
private assetsFolderMap;
|
|
29
|
+
constructor({ importConfig, stackAPIClient }: ModuleClassParams);
|
|
30
|
+
/**
|
|
31
|
+
* @method start
|
|
32
|
+
* @returns {Promise<void>} Promise<any>
|
|
33
|
+
*/
|
|
34
|
+
start(): Promise<void>;
|
|
35
|
+
/**
|
|
36
|
+
* @method importFolders
|
|
37
|
+
* @returns {Promise<any>} Promise<any>
|
|
38
|
+
*/
|
|
39
|
+
importFolders(): Promise<any>;
|
|
40
|
+
/**
|
|
41
|
+
* @method importAssets
|
|
42
|
+
* @param {boolean} isVersion boolean
|
|
43
|
+
* @returns {Promise<void>} Promise<void>
|
|
44
|
+
*/
|
|
45
|
+
importAssets(isVersion?: boolean): Promise<void>;
|
|
46
|
+
/**
|
|
47
|
+
* @method serializeAssets
|
|
48
|
+
* @param {ApiOptions} apiOptions ApiOptions
|
|
49
|
+
* @returns {ApiOptions} ApiOptions
|
|
50
|
+
*/
|
|
51
|
+
serializeAssets(apiOptions: ApiOptions): ApiOptions;
|
|
52
|
+
/**
|
|
53
|
+
* @method publish
|
|
54
|
+
* @returns {Promise<void>} Promise<void>
|
|
55
|
+
*/
|
|
56
|
+
publish(): Promise<void>;
|
|
57
|
+
/**
|
|
58
|
+
* @method constructFolderImportOrder
|
|
59
|
+
* @param {Record<string, any>[]} folders object
|
|
60
|
+
* @returns {Array<Record<string, any>>} Array<Record<string, any>>
|
|
61
|
+
*/
|
|
62
|
+
constructFolderImportOrder(folders: any): Array<Record<string, any>>;
|
|
63
|
+
}
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const tslib_1 = require("tslib");
|
|
4
|
+
const map_1 = tslib_1.__importDefault(require("lodash/map"));
|
|
5
|
+
const values_1 = tslib_1.__importDefault(require("lodash/values"));
|
|
6
|
+
const filter_1 = tslib_1.__importDefault(require("lodash/filter"));
|
|
7
|
+
const unionBy_1 = tslib_1.__importDefault(require("lodash/unionBy"));
|
|
8
|
+
const orderBy_1 = tslib_1.__importDefault(require("lodash/orderBy"));
|
|
9
|
+
const isEmpty_1 = tslib_1.__importDefault(require("lodash/isEmpty"));
|
|
10
|
+
const node_fs_1 = require("node:fs");
|
|
11
|
+
const includes_1 = tslib_1.__importDefault(require("lodash/includes"));
|
|
12
|
+
const node_path_1 = require("node:path");
|
|
13
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
14
|
+
const config_1 = tslib_1.__importDefault(require("../../config"));
|
|
15
|
+
const utils_1 = require("../../utils");
|
|
16
|
+
const base_class_1 = tslib_1.__importDefault(require("./base-class"));
|
|
17
|
+
class ImportAssets extends base_class_1.default {
|
|
18
|
+
constructor({ importConfig, stackAPIClient }) {
|
|
19
|
+
super({ importConfig, stackAPIClient });
|
|
20
|
+
this.assetConfig = config_1.default.modules.assets;
|
|
21
|
+
this.environments = {};
|
|
22
|
+
this.assetsUidMap = {};
|
|
23
|
+
this.assetsUrlMap = {};
|
|
24
|
+
this.assetsFolderMap = {};
|
|
25
|
+
this.assetsPath = (0, node_path_1.join)(this.importConfig.backupDir, 'assets');
|
|
26
|
+
this.mapperDirPath = (0, node_path_1.join)(this.importConfig.backupDir, 'mapper', 'assets');
|
|
27
|
+
this.assetUidMapperPath = (0, node_path_1.join)(this.mapperDirPath, 'uid-mapping.json');
|
|
28
|
+
this.assetUrlMapperPath = (0, node_path_1.join)(this.mapperDirPath, 'url-mapping.json');
|
|
29
|
+
this.assetFolderUidMapperPath = (0, node_path_1.join)(this.mapperDirPath, 'folder-mapping.json');
|
|
30
|
+
this.assetsRootPath = (0, node_path_1.join)(this.importConfig.backupDir, this.assetConfig.dirName);
|
|
31
|
+
this.fs = new cli_utilities_1.FsUtility({ basePath: this.mapperDirPath });
|
|
32
|
+
this.environments = this.fs.readFile((0, node_path_1.join)(this.importConfig.backupDir, 'environments', 'environments.json'), true);
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* @method start
|
|
36
|
+
* @returns {Promise<void>} Promise<any>
|
|
37
|
+
*/
|
|
38
|
+
async start() {
|
|
39
|
+
// NOTE Step 1: Import folders and create uid mapping file
|
|
40
|
+
await this.importFolders();
|
|
41
|
+
// NOTE Step 2: Import versioned assets and create it mapping files (uid, url)
|
|
42
|
+
if (this.assetConfig.includeVersionedAssets) {
|
|
43
|
+
if ((0, node_fs_1.existsSync)(`${this.assetsPath}/versions`))
|
|
44
|
+
await this.importAssets(true);
|
|
45
|
+
else
|
|
46
|
+
(0, utils_1.log)(this.importConfig, 'No Versioned assets found to import', 'info');
|
|
47
|
+
}
|
|
48
|
+
// NOTE Step 3: Import Assets and create it mapping files (uid, url)
|
|
49
|
+
await this.importAssets();
|
|
50
|
+
// NOTE Step 4: Publish assets
|
|
51
|
+
if (this.assetConfig.publishAssets)
|
|
52
|
+
await this.publish();
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* @method importFolders
|
|
56
|
+
* @returns {Promise<any>} Promise<any>
|
|
57
|
+
*/
|
|
58
|
+
async importFolders() {
|
|
59
|
+
const folders = this.fs.readFile((0, node_path_1.resolve)(this.assetsRootPath, 'folders.json'));
|
|
60
|
+
if ((0, isEmpty_1.default)(folders)) {
|
|
61
|
+
(0, utils_1.log)(this.importConfig, 'No folders found to import', 'info');
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const batches = this.constructFolderImportOrder(folders);
|
|
65
|
+
const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }) => {
|
|
66
|
+
this.assetsFolderMap[uid] = response.uid;
|
|
67
|
+
(0, utils_1.log)(this.importConfig, `Created folder: '${name}'`, 'success');
|
|
68
|
+
};
|
|
69
|
+
const onReject = ({ error, apiData: { name } = { name: '' } }) => {
|
|
70
|
+
(0, utils_1.log)(this.importConfig, `${name} folder creation failed.!`, 'error');
|
|
71
|
+
(0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
|
|
72
|
+
};
|
|
73
|
+
const serializeData = (apiOptions) => {
|
|
74
|
+
if (apiOptions.apiData.parent_uid) {
|
|
75
|
+
apiOptions.apiData.parent_uid = this.assetsFolderMap[apiOptions.apiData.parent_uid];
|
|
76
|
+
}
|
|
77
|
+
return apiOptions;
|
|
78
|
+
};
|
|
79
|
+
const batch = (0, map_1.default)((0, unionBy_1.default)(batches, 'parent_uid'), 'parent_uid');
|
|
80
|
+
for (const parent_uid of batch) {
|
|
81
|
+
// NOTE create parent folders
|
|
82
|
+
/* eslint-disable no-await-in-loop */
|
|
83
|
+
await this.makeConcurrentCall({
|
|
84
|
+
apiContent: (0, orderBy_1.default)((0, filter_1.default)(batches, { parent_uid }), 'created_at'),
|
|
85
|
+
processName: 'import assets folders',
|
|
86
|
+
apiParams: {
|
|
87
|
+
serializeData,
|
|
88
|
+
reject: onReject,
|
|
89
|
+
resolve: onSuccess,
|
|
90
|
+
entity: 'create-assets-folder',
|
|
91
|
+
includeParamOnCompletion: true,
|
|
92
|
+
},
|
|
93
|
+
concurrencyLimit: this.assetConfig.importFoldersConcurrency,
|
|
94
|
+
}, undefined, false);
|
|
95
|
+
}
|
|
96
|
+
if (!(0, isEmpty_1.default)(this.assetsFolderMap)) {
|
|
97
|
+
this.fs.writeFile(this.assetFolderUidMapperPath, this.assetsFolderMap);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* @method importAssets
|
|
102
|
+
* @param {boolean} isVersion boolean
|
|
103
|
+
* @returns {Promise<void>} Promise<void>
|
|
104
|
+
*/
|
|
105
|
+
async importAssets(isVersion = false) {
|
|
106
|
+
const processName = isVersion ? 'import versioned assets' : 'import assets';
|
|
107
|
+
const indexFileName = isVersion ? 'versioned-assets.json' : 'assets.json';
|
|
108
|
+
const basePath = isVersion ? (0, node_path_1.join)(this.assetsPath, 'versions') : this.assetsPath;
|
|
109
|
+
const fs = new cli_utilities_1.FsUtility({ basePath, indexFileName });
|
|
110
|
+
const indexer = fs.indexFileContent;
|
|
111
|
+
const indexerCount = (0, values_1.default)(indexer).length;
|
|
112
|
+
const onSuccess = ({ response = {}, apiData: { uid, url, title } = undefined }) => {
|
|
113
|
+
this.assetsUidMap[uid] = response.uid;
|
|
114
|
+
this.assetsUrlMap[url] = response.url;
|
|
115
|
+
(0, utils_1.log)(this.importConfig, `Created asset: '${title}'`, 'info');
|
|
116
|
+
};
|
|
117
|
+
const onReject = ({ error, apiData: { title } = undefined }) => {
|
|
118
|
+
(0, utils_1.log)(this.importConfig, `${title} asset upload failed.!`, 'error');
|
|
119
|
+
(0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
|
|
120
|
+
};
|
|
121
|
+
/* eslint-disable @typescript-eslint/no-unused-vars, guard-for-in */
|
|
122
|
+
for (const index in indexer) {
|
|
123
|
+
const chunk = await fs.readChunkFiles.next().catch((error) => {
|
|
124
|
+
(0, utils_1.log)(this.importConfig, error, 'error');
|
|
125
|
+
});
|
|
126
|
+
if (chunk) {
|
|
127
|
+
let apiContent = (0, orderBy_1.default)((0, values_1.default)(chunk), '_version');
|
|
128
|
+
if (isVersion && this.assetConfig.importSameStructure) {
|
|
129
|
+
// NOTE to create same structure it must have seed assets/version 1 asset to be created first
|
|
130
|
+
await this.makeConcurrentCall({
|
|
131
|
+
processName,
|
|
132
|
+
indexerCount,
|
|
133
|
+
currentIndexer: +index,
|
|
134
|
+
apiContent: (0, filter_1.default)(apiContent, ({ _version }) => _version === 1),
|
|
135
|
+
apiParams: {
|
|
136
|
+
reject: onReject,
|
|
137
|
+
resolve: onSuccess,
|
|
138
|
+
entity: 'create-assets',
|
|
139
|
+
includeParamOnCompletion: true,
|
|
140
|
+
serializeData: this.serializeAssets.bind(this),
|
|
141
|
+
},
|
|
142
|
+
concurrencyLimit: this.assetConfig.uploadAssetsConcurrency,
|
|
143
|
+
});
|
|
144
|
+
apiContent = (0, filter_1.default)(apiContent, ({ _version }) => _version > 1);
|
|
145
|
+
}
|
|
146
|
+
await this.makeConcurrentCall({
|
|
147
|
+
apiContent,
|
|
148
|
+
processName,
|
|
149
|
+
indexerCount,
|
|
150
|
+
currentIndexer: +index,
|
|
151
|
+
apiParams: {
|
|
152
|
+
reject: onReject,
|
|
153
|
+
resolve: onSuccess,
|
|
154
|
+
entity: 'create-assets',
|
|
155
|
+
includeParamOnCompletion: true,
|
|
156
|
+
serializeData: this.serializeAssets.bind(this),
|
|
157
|
+
},
|
|
158
|
+
concurrencyLimit: this.assetConfig.uploadAssetsConcurrency,
|
|
159
|
+
}, undefined, !isVersion);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
if (!isVersion && (!(0, isEmpty_1.default)(this.assetsUidMap) || !(0, isEmpty_1.default)(this.assetsUrlMap))) {
|
|
163
|
+
this.fs.writeFile(this.assetUidMapperPath, this.assetsUidMap);
|
|
164
|
+
this.fs.writeFile(this.assetUrlMapperPath, this.assetsUrlMap);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* @method serializeAssets
|
|
169
|
+
* @param {ApiOptions} apiOptions ApiOptions
|
|
170
|
+
* @returns {ApiOptions} ApiOptions
|
|
171
|
+
*/
|
|
172
|
+
serializeAssets(apiOptions) {
|
|
173
|
+
const { apiData: asset } = apiOptions;
|
|
174
|
+
if (!this.assetConfig.importSameStructure &&
|
|
175
|
+
!this.assetConfig.includeVersionedAssets &&
|
|
176
|
+
/* eslint-disable @typescript-eslint/no-unused-vars, no-prototype-builtins */
|
|
177
|
+
this.assetsUidMap.hasOwnProperty(asset.uid)) {
|
|
178
|
+
(0, utils_1.log)(this.importConfig, `Skipping upload of asset: ${asset.uid}. Its mapped to: ${this.assetsUidMap[asset.uid]}`, 'success');
|
|
179
|
+
apiOptions.entity = undefined;
|
|
180
|
+
return apiOptions;
|
|
181
|
+
}
|
|
182
|
+
asset.upload = (0, node_path_1.join)(this.assetsPath, 'files', asset.uid, asset.filename);
|
|
183
|
+
if (asset.parent_uid) {
|
|
184
|
+
asset.parent_uid = this.assetsFolderMap[asset.parent_uid];
|
|
185
|
+
}
|
|
186
|
+
apiOptions.apiData = asset;
|
|
187
|
+
if (this.assetsUidMap[asset.uid] && this.assetConfig.importSameStructure) {
|
|
188
|
+
apiOptions.entity = 'replace-assets';
|
|
189
|
+
apiOptions.uid = this.assetsUidMap[asset.uid];
|
|
190
|
+
}
|
|
191
|
+
return apiOptions;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* @method publish
|
|
195
|
+
* @returns {Promise<void>} Promise<void>
|
|
196
|
+
*/
|
|
197
|
+
async publish() {
|
|
198
|
+
const fs = new cli_utilities_1.FsUtility({ basePath: this.assetsPath, indexFileName: 'assets.json' });
|
|
199
|
+
if ((0, isEmpty_1.default)(this.assetsUidMap)) {
|
|
200
|
+
this.assetsUidMap = fs.readFile(this.assetUidMapperPath, true);
|
|
201
|
+
}
|
|
202
|
+
const indexer = fs.indexFileContent;
|
|
203
|
+
const indexerCount = (0, values_1.default)(indexer).length;
|
|
204
|
+
const onSuccess = ({ apiData: { uid, title } = undefined }) => {
|
|
205
|
+
(0, utils_1.log)(this.importConfig, `Asset '${uid}: ${title}' published successfully`, 'success');
|
|
206
|
+
};
|
|
207
|
+
const onReject = ({ error, apiData: { uid, title } = undefined }) => {
|
|
208
|
+
(0, utils_1.log)(this.importConfig, `Asset '${uid}: ${title}' not published`, 'error');
|
|
209
|
+
(0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
|
|
210
|
+
};
|
|
211
|
+
const serializeData = (apiOptions) => {
|
|
212
|
+
const { apiData: asset } = apiOptions;
|
|
213
|
+
const publishDetails = (0, filter_1.default)(asset.publish_details, 'environment');
|
|
214
|
+
const locales = (0, map_1.default)(publishDetails, 'locale');
|
|
215
|
+
const environments = (0, map_1.default)(publishDetails, ({ environment }) => this.environments[environment].name);
|
|
216
|
+
asset.locales = locales;
|
|
217
|
+
asset.environments = environments;
|
|
218
|
+
apiOptions.uid = this.assetsUidMap[asset.uid];
|
|
219
|
+
apiOptions.apiData.publishDetails = { locales, environments };
|
|
220
|
+
if (!apiOptions.uid)
|
|
221
|
+
apiOptions.entity = undefined;
|
|
222
|
+
return apiOptions;
|
|
223
|
+
};
|
|
224
|
+
/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
225
|
+
for (const index in indexer) {
|
|
226
|
+
const apiContent = (0, filter_1.default)((0, values_1.default)(await fs.readChunkFiles.next()), ({ publish_details }) => !(0, isEmpty_1.default)(publish_details));
|
|
227
|
+
await this.makeConcurrentCall({
|
|
228
|
+
apiContent,
|
|
229
|
+
indexerCount,
|
|
230
|
+
currentIndexer: +index,
|
|
231
|
+
processName: 'assets publish',
|
|
232
|
+
apiParams: {
|
|
233
|
+
serializeData,
|
|
234
|
+
reject: onReject,
|
|
235
|
+
resolve: onSuccess,
|
|
236
|
+
entity: 'publish-assets',
|
|
237
|
+
includeParamOnCompletion: true,
|
|
238
|
+
},
|
|
239
|
+
concurrencyLimit: this.assetConfig.uploadAssetsConcurrency,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* @method constructFolderImportOrder
|
|
245
|
+
* @param {Record<string, any>[]} folders object
|
|
246
|
+
* @returns {Array<Record<string, any>>} Array<Record<string, any>>
|
|
247
|
+
*/
|
|
248
|
+
constructFolderImportOrder(folders) {
|
|
249
|
+
let parentUid = [];
|
|
250
|
+
// NOTE: Read root folder
|
|
251
|
+
const importOrder = (0, filter_1.default)(folders, { parent_uid: null }).map(({ uid, name, parent_uid, created_at }) => {
|
|
252
|
+
parentUid.push(uid);
|
|
253
|
+
return { uid, name, parent_uid, created_at };
|
|
254
|
+
});
|
|
255
|
+
while (!(0, isEmpty_1.default)(parentUid)) {
|
|
256
|
+
// NOTE: Read nested folders every iteration until we find empty folders
|
|
257
|
+
parentUid = (0, filter_1.default)(folders, ({ parent_uid }) => (0, includes_1.default)(parentUid, parent_uid)).map(({ uid, name, parent_uid, created_at }) => {
|
|
258
|
+
importOrder.push({ uid, name, parent_uid, created_at });
|
|
259
|
+
return uid;
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
return importOrder;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
exports.default = ImportAssets;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { Stack } from '@contentstack/management/types/stack';
|
|
2
|
+
import { ImportConfig, ModuleClassParams } from '../../types';
|
|
3
|
+
export type AdditionalKeys = {
|
|
4
|
+
backupDir: string;
|
|
5
|
+
};
|
|
6
|
+
export type ApiModuleType = 'create-assets' | 'replace-assets' | 'publish-assets' | 'create-assets-folder' | 'create-locale' | 'update-locale';
|
|
7
|
+
export type ApiOptions = {
|
|
8
|
+
uid?: string;
|
|
9
|
+
url?: string;
|
|
10
|
+
entity: ApiModuleType;
|
|
11
|
+
apiData?: Record<any, any>;
|
|
12
|
+
resolve: (value: any) => void;
|
|
13
|
+
reject: (error: any) => void;
|
|
14
|
+
additionalInfo?: Record<any, any>;
|
|
15
|
+
includeParamOnCompletion?: boolean;
|
|
16
|
+
serializeData?: (input: ApiOptions) => any;
|
|
17
|
+
};
|
|
18
|
+
export type EnvType = {
|
|
19
|
+
processName: string;
|
|
20
|
+
totalCount?: number;
|
|
21
|
+
indexerCount?: number;
|
|
22
|
+
currentIndexer?: number;
|
|
23
|
+
apiParams?: ApiOptions;
|
|
24
|
+
concurrencyLimit?: number;
|
|
25
|
+
apiContent: Record<string, any>[];
|
|
26
|
+
};
|
|
27
|
+
export type CustomPromiseHandlerInput = {
|
|
28
|
+
index: number;
|
|
29
|
+
batchIndex: number;
|
|
30
|
+
apiParams?: ApiOptions;
|
|
31
|
+
isLastRequest: boolean;
|
|
32
|
+
};
|
|
33
|
+
export type CustomPromiseHandler = (input: CustomPromiseHandlerInput) => Promise<any>;
|
|
34
|
+
export default abstract class BaseClass {
|
|
35
|
+
readonly client: Stack;
|
|
36
|
+
importConfig: ImportConfig;
|
|
37
|
+
modulesConfig: any;
|
|
38
|
+
constructor({ importConfig, stackAPIClient }: Omit<ModuleClassParams, 'moduleName'>);
|
|
39
|
+
get stack(): Stack;
|
|
40
|
+
/**
|
|
41
|
+
* @method delay
|
|
42
|
+
* @param {number} ms number
|
|
43
|
+
* @returns {Promise} Promise<void>
|
|
44
|
+
*/
|
|
45
|
+
delay(ms: number): Promise<void>;
|
|
46
|
+
/**
|
|
47
|
+
* @method makeConcurrentCall
|
|
48
|
+
* @param {Record<string, any>} env EnvType
|
|
49
|
+
* @param {CustomPromiseHandler} promisifyHandler CustomPromiseHandler
|
|
50
|
+
* @param {boolean} logBatchCompletionMsg boolean
|
|
51
|
+
* @returns {Promise} Promise<void>
|
|
52
|
+
*/
|
|
53
|
+
makeConcurrentCall(env: EnvType, promisifyHandler?: CustomPromiseHandler, logBatchCompletionMsg?: boolean): Promise<void>;
|
|
54
|
+
/**
|
|
55
|
+
* @method logMsgAndWaitIfRequired
|
|
56
|
+
* @param {string} processName string
|
|
57
|
+
* @param {number} start number
|
|
58
|
+
* @param {number} batchNo - number
|
|
59
|
+
* @returns {Promise} Promise<void>
|
|
60
|
+
*/
|
|
61
|
+
logMsgAndWaitIfRequired(processName: string, start: number, totelBatches: number, batchNo: number, logBatchCompletionMsg?: boolean, indexerCount?: number, currentIndexer?: number): Promise<void>;
|
|
62
|
+
/**
|
|
63
|
+
* @method makeAPICall
|
|
64
|
+
* @param {Record<string, any>} apiOptions - Api related params
|
|
65
|
+
* @param {Record<string, any>} isLastRequest - Boolean
|
|
66
|
+
* @return {Promise} Promise<void>
|
|
67
|
+
*/
|
|
68
|
+
makeAPICall(apiOptions: ApiOptions, isLastRequest?: boolean): Promise<void>;
|
|
69
|
+
}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const tslib_1 = require("tslib");
|
|
4
|
+
const pick_1 = tslib_1.__importDefault(require("lodash/pick"));
|
|
5
|
+
const last_1 = tslib_1.__importDefault(require("lodash/last"));
|
|
6
|
+
const chunk_1 = tslib_1.__importDefault(require("lodash/chunk"));
|
|
7
|
+
const isEmpty_1 = tslib_1.__importDefault(require("lodash/isEmpty"));
|
|
8
|
+
const entries_1 = tslib_1.__importDefault(require("lodash/entries"));
|
|
9
|
+
const isEqual_1 = tslib_1.__importDefault(require("lodash/isEqual"));
|
|
10
|
+
const utils_1 = require("../../utils");
|
|
11
|
+
class BaseClass {
|
|
12
|
+
constructor({ importConfig, stackAPIClient }) {
|
|
13
|
+
this.client = stackAPIClient;
|
|
14
|
+
this.importConfig = importConfig;
|
|
15
|
+
this.modulesConfig = importConfig.modules;
|
|
16
|
+
}
|
|
17
|
+
get stack() {
|
|
18
|
+
return this.client;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* @method delay
|
|
22
|
+
* @param {number} ms number
|
|
23
|
+
* @returns {Promise} Promise<void>
|
|
24
|
+
*/
|
|
25
|
+
delay(ms) {
|
|
26
|
+
/* eslint-disable no-promise-executor-return */
|
|
27
|
+
return new Promise((resolve) => setTimeout(resolve, ms <= 0 ? 0 : ms));
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* @method makeConcurrentCall
|
|
31
|
+
* @param {Record<string, any>} env EnvType
|
|
32
|
+
* @param {CustomPromiseHandler} promisifyHandler CustomPromiseHandler
|
|
33
|
+
* @param {boolean} logBatchCompletionMsg boolean
|
|
34
|
+
* @returns {Promise} Promise<void>
|
|
35
|
+
*/
|
|
36
|
+
makeConcurrentCall(env, promisifyHandler, logBatchCompletionMsg = true) {
|
|
37
|
+
const { apiParams, apiContent, processName, indexerCount, currentIndexer, concurrencyLimit = this.importConfig.modules.apiConcurrency, } = env;
|
|
38
|
+
/* eslint-disable no-async-promise-executor */
|
|
39
|
+
return new Promise(async (resolve) => {
|
|
40
|
+
let batchNo = 0;
|
|
41
|
+
let isLastRequest = false;
|
|
42
|
+
const batches = (0, chunk_1.default)(apiContent, concurrencyLimit);
|
|
43
|
+
/* eslint-disable no-promise-executor-return */
|
|
44
|
+
if ((0, isEmpty_1.default)(batches))
|
|
45
|
+
return resolve();
|
|
46
|
+
for (const [batchIndex, batch] of (0, entries_1.default)(batches)) {
|
|
47
|
+
batchNo += 1;
|
|
48
|
+
const allPromise = [];
|
|
49
|
+
const start = Date.now();
|
|
50
|
+
for (const [index, element] of (0, entries_1.default)(batch)) {
|
|
51
|
+
let promise = Promise.resolve();
|
|
52
|
+
isLastRequest = (0, isEqual_1.default)((0, last_1.default)(batch), element) && (0, isEqual_1.default)((0, last_1.default)(batches), batch);
|
|
53
|
+
if (promisifyHandler instanceof Function) {
|
|
54
|
+
promise = promisifyHandler({
|
|
55
|
+
apiParams,
|
|
56
|
+
isLastRequest,
|
|
57
|
+
index: Number(index),
|
|
58
|
+
batchIndex: Number(batchIndex),
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
else if (apiParams) {
|
|
62
|
+
apiParams.apiData = element;
|
|
63
|
+
promise = this.makeAPICall(apiParams, isLastRequest);
|
|
64
|
+
}
|
|
65
|
+
allPromise.push(promise);
|
|
66
|
+
}
|
|
67
|
+
/* eslint-disable no-await-in-loop */
|
|
68
|
+
await Promise.allSettled(allPromise);
|
|
69
|
+
/* eslint-disable no-await-in-loop */
|
|
70
|
+
await this.logMsgAndWaitIfRequired(processName, start, batches.length, batchNo, logBatchCompletionMsg, indexerCount, currentIndexer);
|
|
71
|
+
if (isLastRequest)
|
|
72
|
+
resolve();
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* @method logMsgAndWaitIfRequired
|
|
78
|
+
* @param {string} processName string
|
|
79
|
+
* @param {number} start number
|
|
80
|
+
* @param {number} batchNo - number
|
|
81
|
+
* @returns {Promise} Promise<void>
|
|
82
|
+
*/
|
|
83
|
+
async logMsgAndWaitIfRequired(processName, start, totelBatches, batchNo, logBatchCompletionMsg = true, indexerCount, currentIndexer) {
|
|
84
|
+
const end = Date.now();
|
|
85
|
+
const exeTime = end - start;
|
|
86
|
+
if (logBatchCompletionMsg) {
|
|
87
|
+
let batchMsg = '';
|
|
88
|
+
// info: Batch No. 20 of import assets is complete
|
|
89
|
+
if (currentIndexer)
|
|
90
|
+
batchMsg += `Current chunk processing is (${currentIndexer}/${indexerCount})`;
|
|
91
|
+
(0, utils_1.log)(this.importConfig, `Batch No. (${batchNo}/${totelBatches}) of ${processName} is complete. ${batchMsg}`, 'success');
|
|
92
|
+
}
|
|
93
|
+
if (this.importConfig.modules.assets.displayExecutionTime) {
|
|
94
|
+
console.log(`Time taken to execute: ${exeTime} milliseconds; wait time: ${exeTime < 1000 ? 1000 - exeTime : 0} milliseconds`);
|
|
95
|
+
}
|
|
96
|
+
if (exeTime < 1000)
|
|
97
|
+
await this.delay(1000 - exeTime);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* @method makeAPICall
|
|
101
|
+
* @param {Record<string, any>} apiOptions - Api related params
|
|
102
|
+
* @param {Record<string, any>} isLastRequest - Boolean
|
|
103
|
+
* @return {Promise} Promise<void>
|
|
104
|
+
*/
|
|
105
|
+
makeAPICall(apiOptions, isLastRequest = false) {
|
|
106
|
+
if (apiOptions.serializeData instanceof Function) {
|
|
107
|
+
apiOptions = apiOptions.serializeData(apiOptions);
|
|
108
|
+
}
|
|
109
|
+
const { uid, entity, reject, resolve, apiData, additionalInfo, includeParamOnCompletion } = apiOptions;
|
|
110
|
+
const onSuccess = (response) => resolve({
|
|
111
|
+
response,
|
|
112
|
+
isLastRequest,
|
|
113
|
+
additionalInfo,
|
|
114
|
+
apiData: includeParamOnCompletion ? apiData : undefined,
|
|
115
|
+
});
|
|
116
|
+
const onReject = (error) => reject({
|
|
117
|
+
error,
|
|
118
|
+
isLastRequest,
|
|
119
|
+
additionalInfo,
|
|
120
|
+
apiData: includeParamOnCompletion ? apiData : undefined,
|
|
121
|
+
});
|
|
122
|
+
switch (entity) {
|
|
123
|
+
case 'create-assets-folder':
|
|
124
|
+
return this.stack
|
|
125
|
+
.asset()
|
|
126
|
+
.folder()
|
|
127
|
+
.create({ asset: (0, pick_1.default)(apiData, this.modulesConfig.assets.folderValidKeys) })
|
|
128
|
+
.then(onSuccess)
|
|
129
|
+
.catch(onReject);
|
|
130
|
+
case 'create-assets':
|
|
131
|
+
return this.stack
|
|
132
|
+
.asset()
|
|
133
|
+
.create((0, pick_1.default)(apiData, [...this.modulesConfig.assets.validKeys, 'upload']))
|
|
134
|
+
.then(onSuccess)
|
|
135
|
+
.catch(onReject);
|
|
136
|
+
case 'replace-assets':
|
|
137
|
+
return this.stack
|
|
138
|
+
.asset(uid)
|
|
139
|
+
.replace((0, pick_1.default)(apiData, [...this.modulesConfig.assets.validKeys, 'upload']))
|
|
140
|
+
.then(onSuccess)
|
|
141
|
+
.catch(onReject);
|
|
142
|
+
case 'publish-assets':
|
|
143
|
+
return this.stack
|
|
144
|
+
.asset(uid)
|
|
145
|
+
.publish((0, pick_1.default)(apiData, ['publishDetails']))
|
|
146
|
+
.then(onSuccess)
|
|
147
|
+
.catch(onReject);
|
|
148
|
+
case 'create-locale':
|
|
149
|
+
return this.stack
|
|
150
|
+
.locale()
|
|
151
|
+
.create({ locale: (0, pick_1.default)(apiData, ['name', 'code']) })
|
|
152
|
+
.then(onSuccess)
|
|
153
|
+
.catch(onReject);
|
|
154
|
+
case 'update-locale':
|
|
155
|
+
return this.stack
|
|
156
|
+
.locale(apiData.code)
|
|
157
|
+
.update({ locale: (0, pick_1.default)(apiData, [...this.modulesConfig.locales.requiredKeys]) })
|
|
158
|
+
.then(onSuccess)
|
|
159
|
+
.catch(onReject);
|
|
160
|
+
default:
|
|
161
|
+
return Promise.resolve();
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
exports.default = BaseClass;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var _a;
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
6
|
+
const node_path_1 = require("node:path");
|
|
7
|
+
const modules_js_1 = tslib_1.__importDefault(require("../modules-js"));
|
|
8
|
+
async function startModuleImport(modulePayload) {
|
|
9
|
+
var _a;
|
|
10
|
+
// Todo: Remove below code when auto detect mechanism implemented for old and new module
|
|
11
|
+
if (modulePayload.moduleName === 'assets' &&
|
|
12
|
+
!new cli_utilities_1.FsUtility({ basePath: (0, node_path_1.join)((_a = modulePayload.importConfig) === null || _a === void 0 ? void 0 : _a.backupDir, 'assets') }).isNewFsStructure) {
|
|
13
|
+
return (0, modules_js_1.default)(modulePayload);
|
|
14
|
+
}
|
|
15
|
+
const { default: ModuleRunner } = await (_a = `./${modulePayload.moduleName}`, Promise.resolve().then(() => tslib_1.__importStar(require(_a))));
|
|
16
|
+
const moduleRunner = new ModuleRunner(modulePayload);
|
|
17
|
+
return moduleRunner.start();
|
|
18
|
+
}
|
|
19
|
+
exports.default = startModuleImport;
|