@contentstack/cli-cm-export 1.17.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/commands/cm/stacks/export.d.ts +2 -0
- package/lib/commands/cm/stacks/export.js +43 -15
- package/lib/export/module-exporter.js +16 -10
- package/lib/export/modules/assets.d.ts +0 -5
- package/lib/export/modules/assets.js +47 -32
- package/lib/export/modules/base-class.d.ts +1 -0
- package/lib/export/modules/base-class.js +16 -2
- package/lib/export/modules/content-types.d.ts +1 -1
- package/lib/export/modules/content-types.js +25 -12
- package/lib/export/modules/custom-roles.js +50 -9
- package/lib/export/modules/entries.d.ts +1 -1
- package/lib/export/modules/entries.js +63 -16
- package/lib/export/modules/environments.js +29 -7
- package/lib/export/modules/extensions.js +30 -7
- package/lib/export/modules/global-fields.js +27 -7
- package/lib/export/modules/index.js +10 -3
- package/lib/export/modules/labels.js +29 -7
- package/lib/export/modules/locales.d.ts +1 -1
- package/lib/export/modules/locales.js +29 -7
- package/lib/export/modules/marketplace-apps.d.ts +2 -1
- package/lib/export/modules/marketplace-apps.js +62 -15
- package/lib/export/modules/personalize.js +23 -6
- package/lib/export/modules/stack.js +57 -12
- package/lib/export/modules/taxonomies.d.ts +0 -1
- package/lib/export/modules/taxonomies.js +40 -30
- package/lib/export/modules/webhooks.js +29 -7
- package/lib/export/modules/workflows.js +38 -10
- package/lib/types/export-config.d.ts +6 -1
- package/lib/types/index.d.ts +11 -0
- package/lib/utils/basic-login.js +7 -8
- package/lib/utils/common-helper.js +3 -4
- package/lib/utils/export-config-handler.js +44 -0
- package/lib/utils/marketplace-app-helper.js +1 -2
- package/lib/utils/setup-branches.js +1 -1
- package/lib/utils/setup-export-dir.js +1 -1
- package/messages/index.json +69 -1
- package/oclif.manifest.json +9 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -48,7 +48,7 @@ $ npm install -g @contentstack/cli-cm-export
|
|
|
48
48
|
$ csdx COMMAND
|
|
49
49
|
running command...
|
|
50
50
|
$ csdx (--version)
|
|
51
|
-
@contentstack/cli-cm-export/1.
|
|
51
|
+
@contentstack/cli-cm-export/1.18.0 linux-x64 node-v22.17.1
|
|
52
52
|
$ csdx --help [COMMAND]
|
|
53
53
|
USAGE
|
|
54
54
|
$ csdx COMMAND
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const tslib_1 = require("tslib");
|
|
4
|
-
const path_1 = tslib_1.__importDefault(require("path"));
|
|
5
3
|
const cli_command_1 = require("@contentstack/cli-command");
|
|
6
4
|
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
7
5
|
const export_1 = require("../../../export");
|
|
@@ -12,14 +10,13 @@ class ExportCommand extends cli_command_1.Command {
|
|
|
12
10
|
let exportDir = (0, cli_utilities_1.pathValidator)('logs');
|
|
13
11
|
try {
|
|
14
12
|
const { flags } = await this.parse(ExportCommand);
|
|
15
|
-
|
|
16
|
-
//
|
|
17
|
-
|
|
18
|
-
exportConfig.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
exportConfig.modules.personalize.baseURL[exportConfig.region.name] = this.personalizeUrl;
|
|
13
|
+
const exportConfig = await (0, utils_1.setupExportConfig)(flags);
|
|
14
|
+
// Prepare the context object
|
|
15
|
+
const context = this.createExportContext(exportConfig.apiKey, exportConfig.authenticationMethod);
|
|
16
|
+
exportConfig.context = Object.assign({}, context);
|
|
17
|
+
//log.info(`Using Cli Version: ${this.context?.cliVersion}`, exportConfig.context);
|
|
18
|
+
// Assign exportConfig variables
|
|
19
|
+
this.assignExportConfig(exportConfig);
|
|
23
20
|
exportDir = (0, cli_utilities_1.sanitizePath)(exportConfig.cliLogsPath || exportConfig.data || exportConfig.exportDir);
|
|
24
21
|
const managementAPIClient = await (0, cli_utilities_1.managementSDKClient)(exportConfig);
|
|
25
22
|
const moduleExporter = new export_1.ModuleExporter(managementAPIClient, exportConfig);
|
|
@@ -27,12 +24,39 @@ class ExportCommand extends cli_command_1.Command {
|
|
|
27
24
|
if (!((_a = exportConfig.branches) === null || _a === void 0 ? void 0 : _a.length)) {
|
|
28
25
|
(0, utils_1.writeExportMetaFile)(exportConfig);
|
|
29
26
|
}
|
|
30
|
-
|
|
31
|
-
|
|
27
|
+
cli_utilities_1.log.success(`The content of the stack ${exportConfig.apiKey} has been exported successfully!`, exportConfig.context);
|
|
28
|
+
cli_utilities_1.log.info(`The exported content has been stored at '${exportDir}'`, exportConfig.context);
|
|
29
|
+
cli_utilities_1.log.success(`The log has been stored at '${(0, cli_utilities_1.getLogPath)()}'`, exportConfig.context);
|
|
32
30
|
}
|
|
33
31
|
catch (error) {
|
|
34
|
-
(0,
|
|
35
|
-
|
|
32
|
+
(0, cli_utilities_1.handleAndLogError)(error);
|
|
33
|
+
cli_utilities_1.log.info(`The log has been stored at '${(0, cli_utilities_1.getLogPath)()}'`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
// Create export context object
|
|
37
|
+
createExportContext(apiKey, authenticationMethod) {
|
|
38
|
+
var _a, _b, _c;
|
|
39
|
+
return {
|
|
40
|
+
command: ((_b = (_a = this.context) === null || _a === void 0 ? void 0 : _a.info) === null || _b === void 0 ? void 0 : _b.command) || 'cm:stacks:export',
|
|
41
|
+
module: '',
|
|
42
|
+
userId: cli_utilities_1.configHandler.get('userUid') || '',
|
|
43
|
+
email: cli_utilities_1.configHandler.get('email') || '',
|
|
44
|
+
sessionId: ((_c = this.context) === null || _c === void 0 ? void 0 : _c.sessionId) || '',
|
|
45
|
+
apiKey: apiKey || '',
|
|
46
|
+
orgId: cli_utilities_1.configHandler.get('oauthOrgUid') || '',
|
|
47
|
+
authenticationMethod: authenticationMethod || 'Basic Auth',
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
// Assign values to exportConfig
|
|
51
|
+
assignExportConfig(exportConfig) {
|
|
52
|
+
// Note setting host to create cma client
|
|
53
|
+
exportConfig.host = this.cmaHost;
|
|
54
|
+
exportConfig.region = this.region;
|
|
55
|
+
if (this.developerHubUrl) {
|
|
56
|
+
exportConfig.developerHubBaseUrl = this.developerHubUrl;
|
|
57
|
+
}
|
|
58
|
+
if (this.personalizeUrl) {
|
|
59
|
+
exportConfig.modules.personalize.baseURL[exportConfig.region.name] = this.personalizeUrl;
|
|
36
60
|
}
|
|
37
61
|
}
|
|
38
62
|
}
|
|
@@ -101,7 +125,7 @@ ExportCommand.flags = {
|
|
|
101
125
|
branch: cli_utilities_1.flags.string({
|
|
102
126
|
char: 'B',
|
|
103
127
|
// default: 'main',
|
|
104
|
-
description:
|
|
128
|
+
description: "[optional] The name of the branch where you want to export your content. If you don't mention the branch name, then by default the content will be exported from all the branches of your stack.",
|
|
105
129
|
parse: (0, cli_utilities_1.printFlagDeprecation)(['-B'], ['--branch']),
|
|
106
130
|
}),
|
|
107
131
|
'secured-assets': cli_utilities_1.flags.boolean({
|
|
@@ -112,5 +136,9 @@ ExportCommand.flags = {
|
|
|
112
136
|
required: false,
|
|
113
137
|
description: '[optional] Force override all Marketplace prompts.',
|
|
114
138
|
}),
|
|
139
|
+
query: cli_utilities_1.flags.string({
|
|
140
|
+
description: '[optional] Query object (inline JSON or file path) to filter module exports.',
|
|
141
|
+
hidden: true,
|
|
142
|
+
}),
|
|
115
143
|
};
|
|
116
144
|
ExportCommand.aliases = ['cm:export'];
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const tslib_1 = require("tslib");
|
|
4
4
|
const path = tslib_1.__importStar(require("path"));
|
|
5
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
5
6
|
const utils_1 = require("../utils");
|
|
6
7
|
const modules_1 = tslib_1.__importDefault(require("./modules"));
|
|
7
8
|
const modules_js_1 = tslib_1.__importDefault(require("./modules-js"));
|
|
@@ -32,19 +33,19 @@ class ModuleExporter {
|
|
|
32
33
|
this.exportConfig.branchName = branch.uid;
|
|
33
34
|
this.stackAPIClient.stackHeaders.branch = branch.uid;
|
|
34
35
|
this.exportConfig.branchDir = path.join(this.exportConfig.exportDir, branch.uid);
|
|
35
|
-
|
|
36
|
+
cli_utilities_1.log.info(`Exporting content from branch ${branch.uid}`, this.exportConfig.context);
|
|
36
37
|
(0, utils_1.writeExportMetaFile)(this.exportConfig, this.exportConfig.branchDir);
|
|
37
38
|
await this.export();
|
|
38
|
-
|
|
39
|
+
cli_utilities_1.log.success(`The content of branch ${branch.uid} has been exported successfully!`, this.exportConfig.context);
|
|
39
40
|
}
|
|
40
41
|
catch (error) {
|
|
41
|
-
(0,
|
|
42
|
-
throw new Error(
|
|
42
|
+
(0, cli_utilities_1.handleAndLogError)(error, Object.assign(Object.assign({}, this.exportConfig.context), { branch: branch.uid }), cli_utilities_1.messageHandler.parse('FAILED_EXPORT_CONTENT_BRANCH', { branch: branch.uid }));
|
|
43
|
+
throw new Error(cli_utilities_1.messageHandler.parse('FAILED_EXPORT_CONTENT_BRANCH', { branch: branch.uid }));
|
|
43
44
|
}
|
|
44
45
|
}
|
|
45
46
|
}
|
|
46
47
|
async export() {
|
|
47
|
-
|
|
48
|
+
cli_utilities_1.log.info(`Started to export content, version is ${this.exportConfig.contentVersion}`, this.exportConfig.context);
|
|
48
49
|
// checks for single module or all modules
|
|
49
50
|
if (this.exportConfig.singleModuleExport) {
|
|
50
51
|
return this.exportSingleModule(this.exportConfig.moduleName);
|
|
@@ -52,7 +53,7 @@ class ModuleExporter {
|
|
|
52
53
|
return this.exportAllModules();
|
|
53
54
|
}
|
|
54
55
|
async exportByModuleByName(moduleName) {
|
|
55
|
-
|
|
56
|
+
cli_utilities_1.log.info(`Exporting module: ${moduleName}`, this.exportConfig.context);
|
|
56
57
|
// export the modules by name
|
|
57
58
|
// calls the module runner which inturn calls the module itself
|
|
58
59
|
let exportedModuleResponse;
|
|
@@ -80,10 +81,15 @@ class ModuleExporter {
|
|
|
80
81
|
}
|
|
81
82
|
async exportSingleModule(moduleName) {
|
|
82
83
|
// Note stack is always exported
|
|
83
|
-
let exportModules = [
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
84
|
+
let exportModules = [];
|
|
85
|
+
if (!this.exportConfig.skipStackSettings) {
|
|
86
|
+
exportModules.push('stack');
|
|
87
|
+
}
|
|
88
|
+
if (!this.exportConfig.skipDependencies) {
|
|
89
|
+
const { modules: { [moduleName]: { dependencies = [] } = {} }, } = this.exportConfig;
|
|
90
|
+
if (dependencies.length > 0) {
|
|
91
|
+
exportModules = exportModules.concat(dependencies);
|
|
92
|
+
}
|
|
87
93
|
}
|
|
88
94
|
exportModules.push(moduleName);
|
|
89
95
|
for (const moduleName of exportModules) {
|
|
@@ -40,11 +40,6 @@ export default class ExportAssets extends BaseClass {
|
|
|
40
40
|
* @returns Promise<any|void>
|
|
41
41
|
*/
|
|
42
42
|
getVersionedAssets(): Promise<any | void>;
|
|
43
|
-
/**
|
|
44
|
-
* @method getAssetsCount
|
|
45
|
-
* @param isDir boolean
|
|
46
|
-
* @returns Promise<number|undefined>
|
|
47
|
-
*/
|
|
48
43
|
getAssetsCount(isDir?: boolean): Promise<number | void>;
|
|
49
44
|
/**
|
|
50
45
|
* @method downloadAssets
|
|
@@ -16,7 +16,6 @@ const node_fs_1 = require("node:fs");
|
|
|
16
16
|
const node_path_1 = require("node:path");
|
|
17
17
|
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
18
18
|
const config_1 = tslib_1.__importDefault(require("../../config"));
|
|
19
|
-
const utils_1 = require("../../utils");
|
|
20
19
|
const base_class_1 = tslib_1.__importDefault(require("./base-class"));
|
|
21
20
|
class ExportAssets extends base_class_1.default {
|
|
22
21
|
constructor({ exportConfig, stackAPIClient }) {
|
|
@@ -24,6 +23,7 @@ class ExportAssets extends base_class_1.default {
|
|
|
24
23
|
this.assetConfig = config_1.default.modules.assets;
|
|
25
24
|
this.assetsFolder = [];
|
|
26
25
|
this.versionedAssets = [];
|
|
26
|
+
this.exportConfig.context.module = 'assets';
|
|
27
27
|
}
|
|
28
28
|
get commonQueryParam() {
|
|
29
29
|
return {
|
|
@@ -34,17 +34,22 @@ class ExportAssets extends base_class_1.default {
|
|
|
34
34
|
}
|
|
35
35
|
async start() {
|
|
36
36
|
this.assetsRootPath = (0, node_path_1.resolve)(this.exportConfig.data, this.exportConfig.branchName || '', this.assetConfig.dirName);
|
|
37
|
+
cli_utilities_1.log.debug(`Assets root path resolved to: ${this.assetsRootPath}`, this.exportConfig.context);
|
|
38
|
+
cli_utilities_1.log.debug('Fetching assets and folders count...', this.exportConfig.context);
|
|
37
39
|
// NOTE step 1: Get assets and it's folder count in parallel
|
|
38
40
|
const [assetsCount, assetsFolderCount] = await Promise.all([this.getAssetsCount(), this.getAssetsCount(true)]);
|
|
41
|
+
cli_utilities_1.log.debug('Fetching assets and folders data...', this.exportConfig.context);
|
|
39
42
|
// NOTE step 2: Get assets and it's folder data in parallel
|
|
40
43
|
await Promise.all([this.getAssetsFolders(assetsFolderCount), this.getAssets(assetsCount)]);
|
|
41
44
|
// NOTE step 3: Get versioned assets
|
|
42
45
|
if (!(0, isEmpty_1.default)(this.versionedAssets) && this.assetConfig.includeVersionedAssets) {
|
|
46
|
+
cli_utilities_1.log.debug('Fetching versioned assets metadata...', this.exportConfig.context);
|
|
43
47
|
await this.getVersionedAssets();
|
|
44
48
|
}
|
|
49
|
+
cli_utilities_1.log.debug('Starting download of all assets...', this.exportConfig.context);
|
|
45
50
|
// NOTE step 4: Download all assets
|
|
46
51
|
await this.downloadAssets();
|
|
47
|
-
|
|
52
|
+
cli_utilities_1.log.success(cli_utilities_1.messageHandler.parse('ASSET_EXPORT_COMPLETE'), this.exportConfig.context);
|
|
48
53
|
}
|
|
49
54
|
/**
|
|
50
55
|
* @method getAssetsFolders
|
|
@@ -55,13 +60,14 @@ class ExportAssets extends base_class_1.default {
|
|
|
55
60
|
if (!totalCount)
|
|
56
61
|
return Promise.resolve();
|
|
57
62
|
const queryParam = Object.assign(Object.assign({}, this.commonQueryParam), { query: { is_dir: true } });
|
|
63
|
+
cli_utilities_1.log.debug(`Fetching asset folders with query: ${JSON.stringify(queryParam)}`, this.exportConfig.context);
|
|
58
64
|
const onSuccess = ({ response: { items } }) => {
|
|
65
|
+
cli_utilities_1.log.debug(`Fetched ${(items === null || items === void 0 ? void 0 : items.length) || 0} asset folders`, this.exportConfig.context);
|
|
59
66
|
if (!(0, isEmpty_1.default)(items))
|
|
60
67
|
this.assetsFolder.push(...items);
|
|
61
68
|
};
|
|
62
69
|
const onReject = ({ error }) => {
|
|
63
|
-
(0,
|
|
64
|
-
(0, utils_1.log)(this.exportConfig, error, 'error');
|
|
70
|
+
(0, cli_utilities_1.handleAndLogError)(error, Object.assign({}, this.exportConfig.context));
|
|
65
71
|
};
|
|
66
72
|
return this.makeConcurrentCall({
|
|
67
73
|
totalCount,
|
|
@@ -75,9 +81,11 @@ class ExportAssets extends base_class_1.default {
|
|
|
75
81
|
concurrencyLimit: this.assetConfig.fetchConcurrency,
|
|
76
82
|
}).then(() => {
|
|
77
83
|
if (!(0, isEmpty_1.default)(this.assetsFolder)) {
|
|
78
|
-
|
|
84
|
+
const path = (0, node_path_1.resolve)(this.assetsRootPath, 'folders.json');
|
|
85
|
+
cli_utilities_1.log.debug(`Writing asset folders to ${path}`, this.exportConfig.context);
|
|
86
|
+
new cli_utilities_1.FsUtility({ basePath: this.assetsRootPath }).writeFile(path, this.assetsFolder);
|
|
79
87
|
}
|
|
80
|
-
|
|
88
|
+
cli_utilities_1.log.info(cli_utilities_1.messageHandler.parse('ASSET_FOLDERS_EXPORT_COMPLETE', this.assetsFolder.length), this.exportConfig.context);
|
|
81
89
|
});
|
|
82
90
|
}
|
|
83
91
|
/**
|
|
@@ -88,12 +96,15 @@ class ExportAssets extends base_class_1.default {
|
|
|
88
96
|
getAssets(totalCount) {
|
|
89
97
|
if (!totalCount)
|
|
90
98
|
return Promise.resolve();
|
|
99
|
+
cli_utilities_1.log.debug(`Fetching ${totalCount} assets...`, this.exportConfig.context);
|
|
91
100
|
let fs;
|
|
92
101
|
let metaHandler;
|
|
93
102
|
const queryParam = Object.assign(Object.assign({}, this.commonQueryParam), { include_publish_details: true, except: { BASE: this.assetConfig.invalidKeys } });
|
|
103
|
+
this.applyQueryFilters(queryParam, 'assets');
|
|
94
104
|
if (this.assetConfig.includeVersionedAssets) {
|
|
95
105
|
const customHandler = (array) => {
|
|
96
106
|
const versionAssets = (0, filter_1.default)(array, ({ _version }) => _version > 1);
|
|
107
|
+
cli_utilities_1.log.debug(`Found ${versionAssets.length} versioned assets`, this.exportConfig.context);
|
|
97
108
|
if (!(0, isEmpty_1.default)(versionAssets)) {
|
|
98
109
|
this.versionedAssets.push(...(0, map_1.default)(versionAssets, ({ uid, _version }) => ({
|
|
99
110
|
[uid]: _version,
|
|
@@ -103,11 +114,12 @@ class ExportAssets extends base_class_1.default {
|
|
|
103
114
|
metaHandler = customHandler;
|
|
104
115
|
}
|
|
105
116
|
const onReject = ({ error }) => {
|
|
106
|
-
(0,
|
|
107
|
-
(0, utils_1.log)(this.exportConfig, error.message, 'error');
|
|
117
|
+
(0, cli_utilities_1.handleAndLogError)(error, Object.assign({}, this.exportConfig.context), cli_utilities_1.messageHandler.parse('ASSET_QUERY_FAILED'));
|
|
108
118
|
};
|
|
109
119
|
const onSuccess = ({ response: { items } }) => {
|
|
120
|
+
cli_utilities_1.log.debug(`Fetched ${(items === null || items === void 0 ? void 0 : items.length) || 0} assets`, this.exportConfig.context);
|
|
110
121
|
if (!fs && !(0, isEmpty_1.default)(items)) {
|
|
122
|
+
cli_utilities_1.log.debug('Initializing FsUtility for writing assets metadata', this.exportConfig.context);
|
|
111
123
|
fs = new cli_utilities_1.FsUtility({
|
|
112
124
|
metaHandler,
|
|
113
125
|
moduleName: 'assets',
|
|
@@ -117,8 +129,10 @@ class ExportAssets extends base_class_1.default {
|
|
|
117
129
|
metaPickKeys: (0, merge_1.default)(['uid', 'url', 'filename', 'parent_uid'], this.assetConfig.assetsMetaKeys),
|
|
118
130
|
});
|
|
119
131
|
}
|
|
120
|
-
if (!(0, isEmpty_1.default)(items))
|
|
132
|
+
if (!(0, isEmpty_1.default)(items)) {
|
|
133
|
+
cli_utilities_1.log.debug(`Writing ${items.length} assets into file`, this.exportConfig.context);
|
|
121
134
|
fs === null || fs === void 0 ? void 0 : fs.writeIntoFile(items, { mapKeyVal: true });
|
|
135
|
+
}
|
|
122
136
|
};
|
|
123
137
|
return this.makeConcurrentCall({
|
|
124
138
|
module: 'assets',
|
|
@@ -132,7 +146,7 @@ class ExportAssets extends base_class_1.default {
|
|
|
132
146
|
concurrencyLimit: this.assetConfig.fetchConcurrency,
|
|
133
147
|
}).then(() => {
|
|
134
148
|
fs === null || fs === void 0 ? void 0 : fs.completeFile(true);
|
|
135
|
-
|
|
149
|
+
cli_utilities_1.log.info(cli_utilities_1.messageHandler.parse('ASSET_METADATA_EXPORT_COMPLETE'), this.exportConfig.context);
|
|
136
150
|
});
|
|
137
151
|
}
|
|
138
152
|
/**
|
|
@@ -140,6 +154,7 @@ class ExportAssets extends base_class_1.default {
|
|
|
140
154
|
* @returns Promise<any|void>
|
|
141
155
|
*/
|
|
142
156
|
getVersionedAssets() {
|
|
157
|
+
cli_utilities_1.log.debug('Preparing to fetch versioned assets...', this.exportConfig.context);
|
|
143
158
|
let fs;
|
|
144
159
|
const queryParam = Object.assign(Object.assign({}, this.commonQueryParam), { include_publish_details: true, except: { BASE: this.assetConfig.invalidKeys } });
|
|
145
160
|
const versionedAssets = (0, map_1.default)(this.versionedAssets, (element) => {
|
|
@@ -150,11 +165,13 @@ class ExportAssets extends base_class_1.default {
|
|
|
150
165
|
}
|
|
151
166
|
return batch;
|
|
152
167
|
}).flat();
|
|
168
|
+
cli_utilities_1.log.debug(`Prepared ${versionedAssets.length} versioned asset queries`, this.exportConfig.context);
|
|
153
169
|
const apiBatches = (0, chunk_1.default)(versionedAssets, this.assetConfig.fetchConcurrency);
|
|
154
170
|
const promisifyHandler = (input) => {
|
|
155
171
|
const { index, batchIndex, apiParams, isLastRequest } = input;
|
|
156
172
|
const batch = apiBatches[batchIndex][index];
|
|
157
173
|
const [uid, version] = (0, first_1.default)((0, entries_1.default)(batch));
|
|
174
|
+
cli_utilities_1.log.debug(`Fetching versioned asset [UID: ${uid}, Version: ${version}]`, this.exportConfig.context);
|
|
158
175
|
if (apiParams === null || apiParams === void 0 ? void 0 : apiParams.queryParam) {
|
|
159
176
|
apiParams.uid = uid;
|
|
160
177
|
apiParams.queryParam.version = version;
|
|
@@ -172,15 +189,13 @@ class ExportAssets extends base_class_1.default {
|
|
|
172
189
|
metaPickKeys: (0, merge_1.default)(['uid', 'url', 'filename', '_version', 'parent_uid'], this.assetConfig.assetsMetaKeys),
|
|
173
190
|
});
|
|
174
191
|
}
|
|
175
|
-
if (!(0, isEmpty_1.default)(response))
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
});
|
|
192
|
+
if (!(0, isEmpty_1.default)(response)) {
|
|
193
|
+
cli_utilities_1.log.debug(`Writing versioned asset: UID=${response.uid}, Version=${response._version}`, this.exportConfig.context);
|
|
194
|
+
fs === null || fs === void 0 ? void 0 : fs.writeIntoFile([response], { mapKeyVal: true, keyName: ['uid', '_version'] });
|
|
195
|
+
}
|
|
180
196
|
};
|
|
181
197
|
const onReject = ({ error }) => {
|
|
182
|
-
(0,
|
|
183
|
-
(0, utils_1.log)(this.exportConfig, error, 'error');
|
|
198
|
+
(0, cli_utilities_1.handleAndLogError)(error, Object.assign({}, this.exportConfig.context), cli_utilities_1.messageHandler.parse('ASSET_VERSIONED_QUERY_FAILED'));
|
|
184
199
|
};
|
|
185
200
|
return this.makeConcurrentCall({
|
|
186
201
|
apiBatches,
|
|
@@ -195,26 +210,24 @@ class ExportAssets extends base_class_1.default {
|
|
|
195
210
|
concurrencyLimit: this.assetConfig.fetchConcurrency,
|
|
196
211
|
}, promisifyHandler).then(() => {
|
|
197
212
|
fs === null || fs === void 0 ? void 0 : fs.completeFile(true);
|
|
198
|
-
|
|
213
|
+
cli_utilities_1.log.info(cli_utilities_1.messageHandler.parse('ASSET_VERSIONED_METADATA_EXPORT_COMPLETE'), this.exportConfig.context);
|
|
199
214
|
});
|
|
200
215
|
}
|
|
201
|
-
/**
|
|
202
|
-
* @method getAssetsCount
|
|
203
|
-
* @param isDir boolean
|
|
204
|
-
* @returns Promise<number|undefined>
|
|
205
|
-
*/
|
|
206
216
|
getAssetsCount(isDir = false) {
|
|
207
217
|
const queryParam = Object.assign(Object.assign({ limit: 1 }, this.commonQueryParam), { skip: 10 ** 100 });
|
|
208
218
|
if (isDir)
|
|
209
219
|
queryParam.query = { is_dir: true };
|
|
220
|
+
cli_utilities_1.log.debug(`Querying count of assets${isDir ? ' (folders only)' : ''} with params: ${JSON.stringify(queryParam)}`, this.exportConfig.context);
|
|
210
221
|
return this.stack
|
|
211
222
|
.asset()
|
|
212
223
|
.query(queryParam)
|
|
213
224
|
.count()
|
|
214
|
-
.then(({ assets }) =>
|
|
225
|
+
.then(({ assets }) => {
|
|
226
|
+
cli_utilities_1.log.debug(`Received asset count: ${assets}`, this.exportConfig.context);
|
|
227
|
+
return assets;
|
|
228
|
+
})
|
|
215
229
|
.catch((error) => {
|
|
216
|
-
(0,
|
|
217
|
-
(0, utils_1.log)(this.exportConfig, error, 'error');
|
|
230
|
+
(0, cli_utilities_1.handleAndLogError)(error, Object.assign({}, this.exportConfig.context), cli_utilities_1.messageHandler.parse('ASSET_COUNT_QUERY_FAILED'));
|
|
218
231
|
});
|
|
219
232
|
}
|
|
220
233
|
/**
|
|
@@ -227,6 +240,7 @@ class ExportAssets extends base_class_1.default {
|
|
|
227
240
|
createDirIfNotExist: false,
|
|
228
241
|
basePath: this.assetsRootPath,
|
|
229
242
|
});
|
|
243
|
+
cli_utilities_1.log.debug('Reading asset metadata for download...', this.exportConfig.context);
|
|
230
244
|
const assetsMetaData = fs.getPlainMeta();
|
|
231
245
|
let listOfAssets = (0, values_1.default)(assetsMetaData).flat();
|
|
232
246
|
if (this.assetConfig.includeVersionedAssets) {
|
|
@@ -234,19 +248,20 @@ class ExportAssets extends base_class_1.default {
|
|
|
234
248
|
listOfAssets.push(...(0, values_1.default)(versionedAssetsMetaData).flat());
|
|
235
249
|
}
|
|
236
250
|
listOfAssets = (0, uniqBy_1.default)(listOfAssets, 'url');
|
|
251
|
+
cli_utilities_1.log.debug(`Total unique assets to download: ${listOfAssets.length}`, this.exportConfig.context);
|
|
237
252
|
const apiBatches = (0, chunk_1.default)(listOfAssets, this.assetConfig.downloadLimit);
|
|
238
253
|
const downloadedAssetsDirs = await (0, cli_utilities_1.getDirectories)((0, node_path_1.resolve)(this.assetsRootPath, 'files'));
|
|
239
254
|
const onSuccess = ({ response: { data }, additionalInfo }) => {
|
|
240
255
|
const { asset } = additionalInfo;
|
|
241
256
|
const assetFolderPath = (0, node_path_1.resolve)(this.assetsRootPath, 'files', asset.uid);
|
|
242
257
|
const assetFilePath = (0, node_path_1.resolve)(assetFolderPath, asset.filename);
|
|
258
|
+
cli_utilities_1.log.debug(`Saving asset to: ${assetFilePath}`, this.exportConfig.context);
|
|
243
259
|
if (!(0, includes_1.default)(downloadedAssetsDirs, asset.uid)) {
|
|
244
260
|
fs.createFolderIfNotExist(assetFolderPath);
|
|
245
261
|
}
|
|
246
262
|
const assetWriterStream = (0, node_fs_1.createWriteStream)(assetFilePath);
|
|
247
263
|
assetWriterStream.on('error', (error) => {
|
|
248
|
-
(0,
|
|
249
|
-
(0, utils_1.log)(this.exportConfig, error, 'error');
|
|
264
|
+
(0, cli_utilities_1.handleAndLogError)(error, Object.assign(Object.assign({}, this.exportConfig.context), { uid: asset.uid, filename: asset.fileName }), cli_utilities_1.messageHandler.parse('ASSET_DOWNLOAD_FAILED', asset.filename, asset.uid));
|
|
250
265
|
});
|
|
251
266
|
/**
|
|
252
267
|
* NOTE if pipe not working as expected add the following code below to fix the issue
|
|
@@ -269,12 +284,11 @@ class ExportAssets extends base_class_1.default {
|
|
|
269
284
|
else {
|
|
270
285
|
data.pipe(assetWriterStream);
|
|
271
286
|
}
|
|
272
|
-
|
|
287
|
+
cli_utilities_1.log.success(cli_utilities_1.messageHandler.parse('ASSET_DOWNLOAD_SUCCESS', asset.filename, asset.uid), this.exportConfig.context);
|
|
273
288
|
};
|
|
274
289
|
const onReject = ({ error, additionalInfo }) => {
|
|
275
290
|
const { asset } = additionalInfo;
|
|
276
|
-
(0,
|
|
277
|
-
(0, utils_1.log)(this.exportConfig, error, 'error');
|
|
291
|
+
(0, cli_utilities_1.handleAndLogError)(error, Object.assign(Object.assign({}, this.exportConfig.context), { uid: asset.uid, filename: asset.filename }), cli_utilities_1.messageHandler.parse('ASSET_DOWNLOAD_FAILED', asset.filename, asset.uid));
|
|
278
292
|
};
|
|
279
293
|
const promisifyHandler = (input) => {
|
|
280
294
|
const { index, batchIndex } = input;
|
|
@@ -282,6 +296,7 @@ class ExportAssets extends base_class_1.default {
|
|
|
282
296
|
const url = this.assetConfig.securedAssets
|
|
283
297
|
? `${asset.url}?authtoken=${cli_utilities_1.configHandler.get('authtoken')}`
|
|
284
298
|
: asset.url;
|
|
299
|
+
cli_utilities_1.log.debug(`Preparing to download asset: ${asset.filename} (UID: ${asset.uid}) from URL: ${url}`, this.exportConfig.context);
|
|
285
300
|
return this.makeAPICall({
|
|
286
301
|
reject: onReject,
|
|
287
302
|
resolve: onSuccess,
|
|
@@ -296,7 +311,7 @@ class ExportAssets extends base_class_1.default {
|
|
|
296
311
|
totalCount: listOfAssets.length,
|
|
297
312
|
concurrencyLimit: this.assetConfig.downloadLimit,
|
|
298
313
|
}, promisifyHandler).then(() => {
|
|
299
|
-
|
|
314
|
+
cli_utilities_1.log.success(cli_utilities_1.messageHandler.parse('ASSET_DOWNLOAD_COMPLETE'), this.exportConfig.context);
|
|
300
315
|
});
|
|
301
316
|
}
|
|
302
317
|
}
|
|
@@ -46,4 +46,5 @@ export default abstract class BaseClass {
|
|
|
46
46
|
* @returns Promise<any>
|
|
47
47
|
*/
|
|
48
48
|
makeAPICall({ module: moduleName, reject, resolve, url, uid, additionalInfo, queryParam }: ApiOptions, isLastRequest?: boolean): Promise<any>;
|
|
49
|
+
protected applyQueryFilters(requestObject: any, moduleName: string): any;
|
|
49
50
|
}
|
|
@@ -8,7 +8,7 @@ const chunk_1 = tslib_1.__importDefault(require("lodash/chunk"));
|
|
|
8
8
|
const isEmpty_1 = tslib_1.__importDefault(require("lodash/isEmpty"));
|
|
9
9
|
const entries_1 = tslib_1.__importDefault(require("lodash/entries"));
|
|
10
10
|
const isEqual_1 = tslib_1.__importDefault(require("lodash/isEqual"));
|
|
11
|
-
const
|
|
11
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
12
12
|
class BaseClass {
|
|
13
13
|
constructor({ exportConfig, stackAPIClient }) {
|
|
14
14
|
this.client = stackAPIClient;
|
|
@@ -76,7 +76,7 @@ class BaseClass {
|
|
|
76
76
|
async logMsgAndWaitIfRequired(module, start, batchNo) {
|
|
77
77
|
const end = Date.now();
|
|
78
78
|
const exeTime = end - start;
|
|
79
|
-
|
|
79
|
+
cli_utilities_1.log.success(`Batch No. ${batchNo} of ${module} is complete. Time taken: ${exeTime} milliseconds`, this.exportConfig.context);
|
|
80
80
|
if (this.exportConfig.modules.assets.displayExecutionTime) {
|
|
81
81
|
console.log(`Time taken to execute: ${exeTime} milliseconds; wait time: ${exeTime < 1000 ? 1000 - exeTime : 0} milliseconds`);
|
|
82
82
|
}
|
|
@@ -120,5 +120,19 @@ class BaseClass {
|
|
|
120
120
|
return Promise.resolve();
|
|
121
121
|
}
|
|
122
122
|
}
|
|
123
|
+
applyQueryFilters(requestObject, moduleName) {
|
|
124
|
+
var _a, _b;
|
|
125
|
+
if ((_b = (_a = this.exportConfig.query) === null || _a === void 0 ? void 0 : _a.modules) === null || _b === void 0 ? void 0 : _b[moduleName]) {
|
|
126
|
+
const moduleQuery = this.exportConfig.query.modules[moduleName];
|
|
127
|
+
// Merge the query parameters with existing requestObject
|
|
128
|
+
if (moduleQuery) {
|
|
129
|
+
if (!requestObject.query) {
|
|
130
|
+
requestObject.query = moduleQuery;
|
|
131
|
+
}
|
|
132
|
+
Object.assign(requestObject.query, moduleQuery);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return requestObject;
|
|
136
|
+
}
|
|
123
137
|
}
|
|
124
138
|
exports.default = BaseClass;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { ExportConfig, ModuleClassParams } from '../../types';
|
|
2
1
|
import BaseClass from './base-class';
|
|
2
|
+
import { ExportConfig, ModuleClassParams } from '../../types';
|
|
3
3
|
export default class ContentTypesExport extends BaseClass {
|
|
4
4
|
private stackAPIClient;
|
|
5
5
|
exportConfig: ExportConfig;
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const tslib_1 = require("tslib");
|
|
4
4
|
const path = tslib_1.__importStar(require("path"));
|
|
5
|
-
const utils_1 = require("../../utils");
|
|
6
|
-
const base_class_1 = tslib_1.__importDefault(require("./base-class"));
|
|
7
5
|
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
6
|
+
const base_class_1 = tslib_1.__importDefault(require("./base-class"));
|
|
7
|
+
const utils_1 = require("../../utils");
|
|
8
8
|
class ContentTypesExport extends base_class_1.default {
|
|
9
9
|
constructor({ exportConfig, stackAPIClient }) {
|
|
10
10
|
super({ exportConfig, stackAPIClient });
|
|
@@ -20,29 +20,36 @@ class ContentTypesExport extends base_class_1.default {
|
|
|
20
20
|
if (Array.isArray(this.exportConfig.contentTypes) && this.exportConfig.contentTypes.length > 0) {
|
|
21
21
|
this.qs.uid = { $in: this.exportConfig.contentTypes };
|
|
22
22
|
}
|
|
23
|
+
// Add after existing qs setup and before contentTypesDirPath
|
|
24
|
+
this.applyQueryFilters(this.qs, 'content-types');
|
|
23
25
|
this.contentTypesDirPath = path.resolve((0, cli_utilities_1.sanitizePath)(exportConfig.data), (0, cli_utilities_1.sanitizePath)(exportConfig.branchName || ''), (0, cli_utilities_1.sanitizePath)(this.contentTypesConfig.dirName));
|
|
24
26
|
this.contentTypes = [];
|
|
27
|
+
this.exportConfig.context.module = 'content-types';
|
|
25
28
|
}
|
|
26
29
|
async start() {
|
|
27
30
|
try {
|
|
28
|
-
|
|
31
|
+
cli_utilities_1.log.debug('Starting content types export process...', this.exportConfig.context);
|
|
29
32
|
await utils_1.fsUtil.makeDirectory(this.contentTypesDirPath);
|
|
33
|
+
cli_utilities_1.log.debug(`Created directory at path: ${this.contentTypesDirPath}`, this.exportConfig.context);
|
|
30
34
|
await this.getContentTypes();
|
|
31
35
|
await this.writeContentTypes(this.contentTypes);
|
|
32
|
-
|
|
36
|
+
cli_utilities_1.log.success(cli_utilities_1.messageHandler.parse('CONTENT_TYPE_EXPORT_COMPLETE'), this.exportConfig.context);
|
|
33
37
|
}
|
|
34
38
|
catch (error) {
|
|
35
|
-
(0,
|
|
36
|
-
throw new Error('Failed to export content types');
|
|
39
|
+
(0, cli_utilities_1.handleAndLogError)(error, Object.assign({}, this.exportConfig.context));
|
|
37
40
|
}
|
|
38
41
|
}
|
|
39
42
|
async getContentTypes(skip = 0) {
|
|
43
|
+
var _a;
|
|
40
44
|
if (skip) {
|
|
41
45
|
this.qs.skip = skip;
|
|
46
|
+
cli_utilities_1.log.debug(`Fetching content types with skip: ${skip}`, this.exportConfig.context);
|
|
42
47
|
}
|
|
48
|
+
cli_utilities_1.log.debug(`Querying content types with parameters: ${JSON.stringify(this.qs, null, 2)}`, this.exportConfig.context);
|
|
43
49
|
const contentTypeSearchResponse = await this.stackAPIClient.contentType().query(this.qs).find();
|
|
50
|
+
cli_utilities_1.log.debug(`Fetched ${((_a = contentTypeSearchResponse.items) === null || _a === void 0 ? void 0 : _a.length) || 0} content types out of total ${contentTypeSearchResponse.count}`, this.exportConfig.context);
|
|
44
51
|
if (Array.isArray(contentTypeSearchResponse.items) && contentTypeSearchResponse.items.length > 0) {
|
|
45
|
-
|
|
52
|
+
const updatedContentTypes = this.sanitizeAttribs(contentTypeSearchResponse.items);
|
|
46
53
|
this.contentTypes.push(...updatedContentTypes);
|
|
47
54
|
skip += this.contentTypesConfig.limit || 100;
|
|
48
55
|
if (skip >= contentTypeSearchResponse.count) {
|
|
@@ -51,13 +58,14 @@ class ContentTypesExport extends base_class_1.default {
|
|
|
51
58
|
return await this.getContentTypes(skip);
|
|
52
59
|
}
|
|
53
60
|
else {
|
|
54
|
-
|
|
61
|
+
cli_utilities_1.log.info(cli_utilities_1.messageHandler.parse('CONTENT_TYPE_NO_TYPES'), this.exportConfig.context);
|
|
55
62
|
}
|
|
56
63
|
}
|
|
57
64
|
sanitizeAttribs(contentTypes) {
|
|
58
|
-
|
|
65
|
+
cli_utilities_1.log.debug(`Sanitizing ${contentTypes === null || contentTypes === void 0 ? void 0 : contentTypes.length} content types`, this.exportConfig.context);
|
|
66
|
+
const updatedContentTypes = [];
|
|
59
67
|
contentTypes.forEach((contentType) => {
|
|
60
|
-
for (
|
|
68
|
+
for (const key in contentType) {
|
|
61
69
|
if (this.contentTypesConfig.validKeys.indexOf(key) === -1) {
|
|
62
70
|
delete contentType[key];
|
|
63
71
|
}
|
|
@@ -67,11 +75,16 @@ class ContentTypesExport extends base_class_1.default {
|
|
|
67
75
|
return updatedContentTypes;
|
|
68
76
|
}
|
|
69
77
|
async writeContentTypes(contentTypes) {
|
|
78
|
+
cli_utilities_1.log.debug(`Writing ${contentTypes === null || contentTypes === void 0 ? void 0 : contentTypes.length} content types to disk`, this.exportConfig.context);
|
|
70
79
|
function write(contentType) {
|
|
71
80
|
return utils_1.fsUtil.writeFile(path.join((0, cli_utilities_1.sanitizePath)(this.contentTypesDirPath), (0, cli_utilities_1.sanitizePath)(`${contentType.uid === 'schema' ? 'schema|1' : contentType.uid}.json`)), contentType);
|
|
72
81
|
}
|
|
73
|
-
await (0, utils_1.executeTask)(contentTypes, write.bind(this), {
|
|
74
|
-
|
|
82
|
+
await (0, utils_1.executeTask)(contentTypes, write.bind(this), {
|
|
83
|
+
concurrency: this.exportConfig.writeConcurrency,
|
|
84
|
+
});
|
|
85
|
+
const schemaFilePath = path.join(this.contentTypesDirPath, 'schema.json');
|
|
86
|
+
cli_utilities_1.log.debug(`Writing aggregate schema to: ${schemaFilePath}`, this.exportConfig.context);
|
|
87
|
+
return utils_1.fsUtil.writeFile(schemaFilePath, contentTypes);
|
|
75
88
|
}
|
|
76
89
|
}
|
|
77
90
|
exports.default = ContentTypesExport;
|