@contentstack/cli-cm-import 1.5.11 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -14
- package/bin/dev +17 -0
- package/bin/dev.cmd +3 -0
- package/bin/run +6 -0
- package/bin/run.cmd +3 -0
- package/lib/commands/cm/stacks/import.d.ts +10 -0
- package/lib/commands/cm/stacks/import.js +111 -0
- package/lib/config/index.d.ts +3 -0
- package/lib/config/index.js +395 -0
- package/lib/import/index.d.ts +1 -0
- package/lib/import/index.js +8 -0
- package/lib/import/module-importer.d.ts +13 -0
- package/lib/import/module-importer.js +70 -0
- package/lib/import/modules/assets.d.ts +63 -0
- package/lib/import/modules/assets.js +265 -0
- package/lib/import/modules/base-class.d.ts +70 -0
- package/lib/import/modules/base-class.js +218 -0
- package/lib/import/modules/content-types.d.ts +56 -0
- package/lib/import/modules/content-types.js +186 -0
- package/lib/import/modules/custom-roles.d.ts +37 -0
- package/lib/import/modules/custom-roles.js +171 -0
- package/lib/import/modules/environments.d.ts +27 -0
- package/lib/import/modules/environments.js +106 -0
- package/lib/import/modules/extensions.d.ts +27 -0
- package/lib/import/modules/extensions.js +106 -0
- package/lib/import/modules/global-fields.d.ts +34 -0
- package/lib/import/modules/global-fields.js +99 -0
- package/lib/import/modules/index.d.ts +2 -0
- package/lib/import/modules/index.js +19 -0
- package/lib/import/modules/labels.d.ts +34 -0
- package/lib/import/modules/labels.js +171 -0
- package/lib/import/modules/locales.d.ts +31 -0
- package/lib/import/modules/locales.js +144 -0
- package/lib/import/modules/marketplace-apps.d.ts +51 -0
- package/lib/import/modules/marketplace-apps.js +297 -0
- package/lib/import/modules/webhooks.d.ts +27 -0
- package/lib/import/modules/webhooks.js +110 -0
- package/lib/import/modules-js/assets.d.ts +33 -0
- package/lib/import/modules-js/assets.js +415 -0
- package/lib/import/modules-js/content-types.d.ts +33 -0
- package/lib/import/modules-js/content-types.js +176 -0
- package/lib/import/modules-js/custom-roles.d.ts +15 -0
- package/lib/import/modules-js/custom-roles.js +143 -0
- package/lib/import/modules-js/entries.d.ts +54 -0
- package/lib/import/modules-js/entries.js +1260 -0
- package/lib/import/modules-js/environments.d.ts +13 -0
- package/lib/import/modules-js/environments.js +85 -0
- package/lib/import/modules-js/extensions.d.ts +18 -0
- package/lib/import/modules-js/extensions.js +86 -0
- package/lib/import/modules-js/global-fields.d.ts +13 -0
- package/lib/import/modules-js/global-fields.js +109 -0
- package/lib/import/modules-js/index.d.ts +1 -0
- package/lib/import/modules-js/index.js +33 -0
- package/lib/import/modules-js/labels.d.ts +20 -0
- package/lib/import/modules-js/labels.js +148 -0
- package/lib/import/modules-js/locales.d.ts +24 -0
- package/lib/import/modules-js/locales.js +196 -0
- package/lib/import/modules-js/marketplace-apps.d.ts +60 -0
- package/lib/import/modules-js/marketplace-apps.js +409 -0
- package/lib/import/modules-js/webhooks.d.ts +17 -0
- package/lib/import/modules-js/webhooks.js +85 -0
- package/lib/import/modules-js/workflows.d.ts +18 -0
- package/lib/import/modules-js/workflows.js +132 -0
- package/lib/types/default-config.d.ts +143 -0
- package/lib/types/default-config.js +2 -0
- package/lib/types/import-config.d.ts +52 -0
- package/lib/types/import-config.js +2 -0
- package/lib/types/index.d.ts +63 -0
- package/lib/types/index.js +4 -0
- package/lib/utils/asset-helper.d.ts +4 -0
- package/lib/utils/asset-helper.js +387 -0
- package/lib/utils/backup-handler.d.ts +2 -0
- package/lib/utils/backup-handler.js +31 -0
- package/lib/utils/common-helper.d.ts +20 -0
- package/lib/utils/common-helper.js +244 -0
- package/lib/utils/content-type-helper.d.ts +51 -0
- package/lib/utils/content-type-helper.js +145 -0
- package/lib/utils/entries-helper.d.ts +4 -0
- package/lib/utils/entries-helper.js +252 -0
- package/lib/utils/extension-helper.d.ts +5 -0
- package/lib/utils/extension-helper.js +84 -0
- package/lib/utils/file-helper.d.ts +14 -0
- package/lib/utils/file-helper.js +140 -0
- package/lib/utils/import-config-handler.d.ts +3 -0
- package/lib/utils/import-config-handler.js +73 -0
- package/lib/utils/index.d.ts +12 -0
- package/lib/utils/index.js +39 -0
- package/lib/utils/interactive.d.ts +7 -0
- package/lib/utils/interactive.js +88 -0
- package/lib/utils/logger.d.ts +8 -0
- package/lib/utils/logger.js +154 -0
- package/lib/utils/login-handler.d.ts +8 -0
- package/lib/utils/login-handler.js +53 -0
- package/lib/utils/marketplace-app-helper.d.ts +16 -0
- package/lib/utils/marketplace-app-helper.js +143 -0
- package/messages/index.json +1 -7
- package/oclif.manifest.json +2 -2
- package/package.json +46 -20
- package/src/app.js +0 -217
- package/src/commands/cm/stacks/import.js +0 -161
- package/src/config/default.js +0 -352
- package/src/lib/import/assets.js +0 -495
- package/src/lib/import/content-types.js +0 -201
- package/src/lib/import/custom-roles.js +0 -169
- package/src/lib/import/entries.js +0 -1495
- package/src/lib/import/environments.js +0 -106
- package/src/lib/import/extensions.js +0 -108
- package/src/lib/import/global-fields.js +0 -135
- package/src/lib/import/labels.js +0 -175
- package/src/lib/import/locales.js +0 -216
- package/src/lib/import/marketplace-apps.js +0 -542
- package/src/lib/import/webhooks.js +0 -113
- package/src/lib/import/workflows.js +0 -166
- package/src/lib/util/extensionsUidReplace.js +0 -67
- package/src/lib/util/fs.js +0 -124
- package/src/lib/util/import-flags.js +0 -187
- package/src/lib/util/index.js +0 -222
- package/src/lib/util/log.js +0 -144
- package/src/lib/util/login.js +0 -58
- package/src/lib/util/lookupReplaceAssets.js +0 -366
- package/src/lib/util/lookupReplaceEntries.js +0 -250
- package/src/lib/util/marketplace-app-helper.js +0 -31
- package/src/lib/util/removeReferenceFields.js +0 -59
- package/src/lib/util/schemaTemplate.js +0 -38
- package/src/lib/util/supress-mandatory-fields.js +0 -34
- package/src/lib/util/upload.js +0 -56
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Contentstack Import
|
|
3
|
+
* Copyright (c) 2019 Contentstack LLC
|
|
4
|
+
* MIT Licensed
|
|
5
|
+
*/
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const _ = require('lodash');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
const chalk = require('chalk');
|
|
10
|
+
const mkdirp = require('mkdirp');
|
|
11
|
+
const Promise = require('bluebird');
|
|
12
|
+
let { default: config } = require('../../config');
|
|
13
|
+
const { fileHelper, log, uploadAssetHelper } = require('../../utils');
|
|
14
|
+
module.exports = class ImportAssets {
|
|
15
|
+
constructor(importConfig, stackAPIClient) {
|
|
16
|
+
this.fails = [];
|
|
17
|
+
this.uidMapping = {};
|
|
18
|
+
this.urlMapping = {};
|
|
19
|
+
this.assetBucket = [];
|
|
20
|
+
this.folderBucket = [];
|
|
21
|
+
this.folderDetails = [];
|
|
22
|
+
this.mappedFolderUids = {};
|
|
23
|
+
this.config = _.merge(config, importConfig);
|
|
24
|
+
this.stackAPIClient = stackAPIClient;
|
|
25
|
+
this.assetConfig = config.modules['assets-old'];
|
|
26
|
+
this.assetBatchLimit =
|
|
27
|
+
this.assetConfig.hasOwnProperty('assetBatchLimit') && typeof this.assetConfig.assetBatchLimit === 'number'
|
|
28
|
+
? this.assetConfig.assetBatchLimit
|
|
29
|
+
: 2;
|
|
30
|
+
}
|
|
31
|
+
start() {
|
|
32
|
+
let self = this;
|
|
33
|
+
log(self.config, 'Migrating assets', 'success');
|
|
34
|
+
this.assetsFolderPath = path.join(this.config.data, this.assetConfig.dirName);
|
|
35
|
+
this.mapperDirPath = path.resolve(this.config.data, 'mapper', 'assets');
|
|
36
|
+
this.environmentPath = path.resolve(this.config.data, 'environments', 'environments.json');
|
|
37
|
+
this.uidMapperPath = path.join(this.mapperDirPath, 'uid-mapping.json');
|
|
38
|
+
this.urlMapperPath = path.join(this.mapperDirPath, 'url-mapping.json');
|
|
39
|
+
this.failsPath = path.join(this.mapperDirPath, 'fail.json');
|
|
40
|
+
this.assets = fileHelper.readFileSync(path.join(this.assetsFolderPath, this.assetConfig.fileName));
|
|
41
|
+
this.environment = fileHelper.readFileSync(this.environmentPath);
|
|
42
|
+
if (fs.existsSync(this.uidMapperPath)) {
|
|
43
|
+
this.uidMapping = fileHelper.readFileSync(this.uidMapperPath);
|
|
44
|
+
}
|
|
45
|
+
if (fs.existsSync(this.urlMapperPath)) {
|
|
46
|
+
this.urlMapping = fileHelper.readFileSync(this.urlMapperPath);
|
|
47
|
+
}
|
|
48
|
+
mkdirp.sync(this.mapperDirPath);
|
|
49
|
+
return new Promise(function (resolve, reject) {
|
|
50
|
+
if (self.assets === undefined || _.isEmpty(self.assets)) {
|
|
51
|
+
log(self.config, 'No Assets Found', 'success');
|
|
52
|
+
return resolve({ empty: true });
|
|
53
|
+
}
|
|
54
|
+
let batches = [];
|
|
55
|
+
let assetUids = Object.keys(self.assets);
|
|
56
|
+
for (let i = 0; i < assetUids.length; i += self.assetBatchLimit) {
|
|
57
|
+
batches.push(assetUids.slice(i, i + self.assetBatchLimit));
|
|
58
|
+
}
|
|
59
|
+
return self
|
|
60
|
+
.importFolders()
|
|
61
|
+
.then(function () {
|
|
62
|
+
return Promise.map(batches, async function (batch, index) {
|
|
63
|
+
return Promise.map(batch, function (assetUid) {
|
|
64
|
+
if (self.uidMapping.hasOwnProperty(assetUid)) {
|
|
65
|
+
log(self.config, 'Skipping upload of asset: ' + assetUid + '. Its mapped to: ' + self.uidMapping[assetUid], 'success');
|
|
66
|
+
// the asset has been already imported
|
|
67
|
+
return void 0;
|
|
68
|
+
}
|
|
69
|
+
let currentAssetFolderPath = path.join(self.assetsFolderPath, assetUid);
|
|
70
|
+
if (fs.existsSync(currentAssetFolderPath)) {
|
|
71
|
+
// if this is true, means, the exported asset data is versioned
|
|
72
|
+
// hence, upload each asset with its version
|
|
73
|
+
if (self.config.versioning) {
|
|
74
|
+
return self.uploadVersionedAssets(assetUid, currentAssetFolderPath).catch(function (error) {
|
|
75
|
+
log(self.config, 'Asset upload failed \n' + error, 'error');
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
let uidContainer = {};
|
|
79
|
+
let urlContainer = {};
|
|
80
|
+
let assetPath = path.resolve(currentAssetFolderPath, self.assets[assetUid].filename);
|
|
81
|
+
if (self.assets[assetUid].parent_uid && typeof self.assets[assetUid].parent_uid === 'string') {
|
|
82
|
+
if (self.mappedFolderUids.hasOwnProperty(self.assets[assetUid].parent_uid)) {
|
|
83
|
+
self.assets[assetUid].parent_uid = self.mappedFolderUids[self.assets[assetUid].parent_uid];
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
log(self.config, `'${self.assets[assetUid].parent_uid}' parent_uid was not found! Thus, setting it as 'null'`, 'error');
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return self
|
|
90
|
+
.uploadAsset(assetPath, self.assets[assetUid], uidContainer, urlContainer)
|
|
91
|
+
.then(async function () {
|
|
92
|
+
self.uidMapping[assetUid] = uidContainer[assetUid];
|
|
93
|
+
self.urlMapping[self.assets[assetUid].url] = urlContainer[self.assets[assetUid].url];
|
|
94
|
+
if (self.config.entriesPublish && self.assets[assetUid].publish_details.length > 0) {
|
|
95
|
+
let assetsUid = uidContainer[assetUid];
|
|
96
|
+
try {
|
|
97
|
+
return await self.publish(assetsUid, self.assets[assetUid]);
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
return error;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// assetUid has been successfully uploaded
|
|
104
|
+
// log them onto /mapper/assets/success.json
|
|
105
|
+
})
|
|
106
|
+
.catch(function (error) {
|
|
107
|
+
log(self.config, 'Asset upload failed \n' + error, 'error');
|
|
108
|
+
return error;
|
|
109
|
+
// asset failed to upload
|
|
110
|
+
// log them onto /mapper/assets/fail.json
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
log(self.config, `'${currentAssetFolderPath}' does not exist!`, 'error');
|
|
114
|
+
}, { concurrency: self.assetConfig.assetBatchLimit }).then(function () {
|
|
115
|
+
fileHelper.writeFileSync(self.uidMapperPath, self.uidMapping);
|
|
116
|
+
fileHelper.writeFileSync(self.urlMapperPath, self.urlMapping);
|
|
117
|
+
// completed uploading assets
|
|
118
|
+
log(self.config, 'Completed asset import of batch no: ' + (index + 1), 'success');
|
|
119
|
+
return index + 1;
|
|
120
|
+
// TODO: if there are failures, retry
|
|
121
|
+
});
|
|
122
|
+
}, { concurrency: 1 })
|
|
123
|
+
.then(function () {
|
|
124
|
+
let numberOfSuccessfulAssetUploads = Object.keys(self.uidMapping).length;
|
|
125
|
+
if (numberOfSuccessfulAssetUploads > 0) {
|
|
126
|
+
log(self.config, chalk.green(numberOfSuccessfulAssetUploads + ' assets uploaded successfully!'), 'success');
|
|
127
|
+
}
|
|
128
|
+
// TODO: if there are failures, retry
|
|
129
|
+
return resolve();
|
|
130
|
+
})
|
|
131
|
+
.catch(function (error) {
|
|
132
|
+
log(self.config, error, 'error');
|
|
133
|
+
return reject(error);
|
|
134
|
+
});
|
|
135
|
+
})
|
|
136
|
+
.catch(function (error) {
|
|
137
|
+
log(self.config, error, 'error');
|
|
138
|
+
return reject(error);
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
uploadVersionedAssets(uid, assetFolderPath) {
|
|
143
|
+
let self = this;
|
|
144
|
+
return new Promise(function (resolve, reject) {
|
|
145
|
+
let versionedAssetMetadata = fileHelper.readFileSync(path.join(assetFolderPath, '_contentstack_' + uid + '.json'));
|
|
146
|
+
// using last version, find asset's parent
|
|
147
|
+
let lastVersion = versionedAssetMetadata[versionedAssetMetadata.length - 1];
|
|
148
|
+
if (typeof lastVersion.parent_uid === 'string') {
|
|
149
|
+
if (self.mappedFolderUids.hasOwnProperty(lastVersion.parent_uid)) {
|
|
150
|
+
// update each version of that asset with the last version's parent_uid
|
|
151
|
+
versionedAssetMetadata.forEach(function (assetMetadata) {
|
|
152
|
+
assetMetadata.parent_uid = self.mappedFolderUids[lastVersion.parent_uid];
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
log(self.config, (lastVersion.parent_uid + " parent_uid was not found! Thus, setting it as 'null'", 'error'));
|
|
157
|
+
versionedAssetMetadata.forEach(function (assetMetadata) {
|
|
158
|
+
assetMetadata.parent_uid = null;
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
let counter = 0;
|
|
163
|
+
let uidContainer = {};
|
|
164
|
+
let urlContainer = {};
|
|
165
|
+
let filesStreamed = [];
|
|
166
|
+
return Promise.map(versionedAssetMetadata, function () {
|
|
167
|
+
let assetMetadata = versionedAssetMetadata[counter];
|
|
168
|
+
let assetPath = path.join(assetFolderPath, assetMetadata.filename);
|
|
169
|
+
if (++counter === 1) {
|
|
170
|
+
return self
|
|
171
|
+
.uploadAsset(assetPath, assetMetadata, uidContainer, urlContainer)
|
|
172
|
+
.then(function () {
|
|
173
|
+
filesStreamed.push(assetMetadata.filename);
|
|
174
|
+
})
|
|
175
|
+
.catch((error) => {
|
|
176
|
+
log(self.config, error, 'error');
|
|
177
|
+
reject(error);
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
return self
|
|
181
|
+
.updateAsset(assetPath, assetMetadata, filesStreamed, uidContainer, urlContainer)
|
|
182
|
+
.then(function () {
|
|
183
|
+
filesStreamed.push(assetMetadata.filename);
|
|
184
|
+
})
|
|
185
|
+
.catch((error) => {
|
|
186
|
+
log(self.config, error, 'error');
|
|
187
|
+
});
|
|
188
|
+
}, { concurrency: self.assetConfig.uploadAssetsConcurrency })
|
|
189
|
+
.then(function () {
|
|
190
|
+
self.uidMapping[uid] = uidContainer[uid];
|
|
191
|
+
for (let url in urlContainer) {
|
|
192
|
+
self.urlMapping[url] = urlContainer[url];
|
|
193
|
+
}
|
|
194
|
+
// completed uploading all the versions of the asset
|
|
195
|
+
return resolve();
|
|
196
|
+
})
|
|
197
|
+
.catch(function (error) {
|
|
198
|
+
// failed to upload asset
|
|
199
|
+
// write it on fail logs, but do not stop the process
|
|
200
|
+
log(self.config, 'Failed to upload asset\n' + error, 'error');
|
|
201
|
+
return resolve();
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
updateAsset(assetPath, metadata, filesStreamed, _uidContainer, urlContainer) {
|
|
206
|
+
const self = this;
|
|
207
|
+
return new Promise(function (resolve, reject) {
|
|
208
|
+
let requestOption = {};
|
|
209
|
+
if (filesStreamed && filesStreamed.indexOf(metadata.filename) !== -1) {
|
|
210
|
+
log(self.config, 'Skipping re-upload/streaming of ' + metadata.uid + '/' + metadata.filename, 'success');
|
|
211
|
+
requestOption.formData = {};
|
|
212
|
+
return resolve();
|
|
213
|
+
}
|
|
214
|
+
log(self.config, 'Streaming: ' + metadata.uid + '/' + metadata.filename, 'success');
|
|
215
|
+
requestOption.formData = {};
|
|
216
|
+
if (metadata.hasOwnProperty('parent_uid') && typeof metadata.parent_uid === 'string') {
|
|
217
|
+
requestOption.formData['asset[parent_uid]'] = metadata.parent_uid;
|
|
218
|
+
}
|
|
219
|
+
if (metadata.hasOwnProperty('description') && typeof metadata.description === 'string') {
|
|
220
|
+
requestOption.formData['asset[description]'] = metadata.description;
|
|
221
|
+
}
|
|
222
|
+
if (metadata.hasOwnProperty('tags') && Array.isArray(metadata.tags)) {
|
|
223
|
+
requestOption.formData['asset[tags]'] = metadata.tags;
|
|
224
|
+
}
|
|
225
|
+
if (metadata.hasOwnProperty('title') && typeof metadata.title === 'string') {
|
|
226
|
+
requestOption.formData['asset[title]'] = metadata.title;
|
|
227
|
+
}
|
|
228
|
+
return uploadAssetHelper(self.config, requestOption, assetPath)
|
|
229
|
+
.then(function (response) {
|
|
230
|
+
urlContainer[metadata.url] = response.url;
|
|
231
|
+
return resolve();
|
|
232
|
+
})
|
|
233
|
+
.catch(function (error) {
|
|
234
|
+
log(self.config, error, 'error');
|
|
235
|
+
return reject(error);
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
uploadAsset(assetPath, metadata, uidContainer, urlContainer) {
|
|
240
|
+
const self = this;
|
|
241
|
+
return new Promise(function (resolve, reject) {
|
|
242
|
+
let requestOption = {};
|
|
243
|
+
if (metadata.hasOwnProperty('parent_uid') && typeof metadata.parent_uid === 'string') {
|
|
244
|
+
requestOption.parent_uid = metadata.parent_uid;
|
|
245
|
+
}
|
|
246
|
+
if (metadata.hasOwnProperty('description') && typeof metadata.description === 'string') {
|
|
247
|
+
requestOption.description = metadata.description;
|
|
248
|
+
}
|
|
249
|
+
// eslint-disable-next-line no-prototype-builtins
|
|
250
|
+
if (metadata.hasOwnProperty('tags') && Array.isArray(metadata.tags)) {
|
|
251
|
+
requestOption.tags = metadata.tags;
|
|
252
|
+
}
|
|
253
|
+
if (metadata.hasOwnProperty('title') && typeof metadata.title === 'string') {
|
|
254
|
+
requestOption.title = metadata.title;
|
|
255
|
+
}
|
|
256
|
+
return uploadAssetHelper(self.config, requestOption, assetPath)
|
|
257
|
+
.then(function (response) {
|
|
258
|
+
uidContainer[metadata.uid] = response.uid;
|
|
259
|
+
urlContainer[metadata.url] = response.url;
|
|
260
|
+
return resolve();
|
|
261
|
+
})
|
|
262
|
+
.catch(function (error) {
|
|
263
|
+
log(self.config, error, 'error');
|
|
264
|
+
return reject(error);
|
|
265
|
+
});
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
importFolders() {
|
|
269
|
+
let self = this;
|
|
270
|
+
return new Promise(function (resolve, reject) {
|
|
271
|
+
let mappedFolderPath = path.resolve(self.config.data, 'mapper', 'assets', 'folder-mapping.json');
|
|
272
|
+
self.folderDetails = fileHelper.readFileSync(path.resolve(self.assetsFolderPath, 'folders.json'));
|
|
273
|
+
if (_.isEmpty(self.folderDetails)) {
|
|
274
|
+
log(self.config, 'No folders were found at: ' + path.join(self.assetsFolderPath, 'folders.json'), 'success');
|
|
275
|
+
return resolve();
|
|
276
|
+
}
|
|
277
|
+
let tree = self.buildTree(_.cloneDeep(self.folderDetails));
|
|
278
|
+
let createdFolders = {};
|
|
279
|
+
let createdFolderUids = [];
|
|
280
|
+
// if a few folders have already been created, skip re-creating them
|
|
281
|
+
if (fs.existsSync(mappedFolderPath)) {
|
|
282
|
+
createdFolders = fileHelper.readFileSync(mappedFolderPath);
|
|
283
|
+
// check if the read file has mapped objects
|
|
284
|
+
if (_.isPlainObject(createdFolders)) {
|
|
285
|
+
createdFolderUids = Object.keys(createdFolders);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
self.buildFolderReqObjs(createdFolderUids, tree, null);
|
|
289
|
+
let idx = 0;
|
|
290
|
+
return Promise.map(self.folderBucket, function () {
|
|
291
|
+
let folder = self.folderBucket[idx];
|
|
292
|
+
if (createdFolders.hasOwnProperty(folder.json.asset.parent_uid)) {
|
|
293
|
+
// replace old uid with new
|
|
294
|
+
folder.json.asset.parent_uid = createdFolders[folder.json.asset.parent_uid];
|
|
295
|
+
}
|
|
296
|
+
return self.stackAPIClient
|
|
297
|
+
.asset()
|
|
298
|
+
.folder()
|
|
299
|
+
.create(folder.json)
|
|
300
|
+
.then((response) => {
|
|
301
|
+
log(self.config, "Created folder: '" + folder.json.asset.name + "'", 'success');
|
|
302
|
+
// assigning newUid to oldUid
|
|
303
|
+
createdFolders[folder.oldUid] = response.uid;
|
|
304
|
+
fileHelper.writeFileSync(mappedFolderPath, createdFolders);
|
|
305
|
+
idx++;
|
|
306
|
+
})
|
|
307
|
+
.catch(function (err) {
|
|
308
|
+
let error = JSON.parse(err.message);
|
|
309
|
+
if (error.errors.authorization || error.errors.api_key) {
|
|
310
|
+
log(self.config, 'Api_key or management_token is not valid', 'error');
|
|
311
|
+
return reject(error);
|
|
312
|
+
}
|
|
313
|
+
log(self.config, err, 'error');
|
|
314
|
+
return error;
|
|
315
|
+
});
|
|
316
|
+
}, { concurrency: self.assetConfig.importFoldersConcurrency })
|
|
317
|
+
.then(function () {
|
|
318
|
+
self.mappedFolderUids = fileHelper.readFileSync(mappedFolderPath);
|
|
319
|
+
// completed creating folders
|
|
320
|
+
return resolve();
|
|
321
|
+
})
|
|
322
|
+
.catch(function (error) {
|
|
323
|
+
log(self.config, error, 'error');
|
|
324
|
+
return reject(error);
|
|
325
|
+
});
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
buildFolderReqObjs(createdFolderUids, tree, parent_uid) {
|
|
329
|
+
let self = this;
|
|
330
|
+
for (let leaf in tree) {
|
|
331
|
+
// if the folder is already created, skip
|
|
332
|
+
if (createdFolderUids.indexOf(leaf) !== -1) {
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
let folderObj = _.find(self.folderDetails, { uid: leaf });
|
|
336
|
+
let requestOption = {
|
|
337
|
+
json: {
|
|
338
|
+
asset: {
|
|
339
|
+
name: folderObj.name,
|
|
340
|
+
parent_uid: parent_uid || null,
|
|
341
|
+
},
|
|
342
|
+
},
|
|
343
|
+
oldUid: leaf,
|
|
344
|
+
};
|
|
345
|
+
self.folderBucket.push(requestOption);
|
|
346
|
+
if (Object.keys(tree[leaf]).length > 0) {
|
|
347
|
+
self.buildFolderReqObjs(createdFolderUids, tree[leaf], leaf);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
buildTree(coll) {
|
|
352
|
+
let tree = {};
|
|
353
|
+
for (let i = 0; i < coll.length; i++) {
|
|
354
|
+
if (coll[i].parent_uid === null || !coll[i].hasOwnProperty('parent_uid')) {
|
|
355
|
+
tree[coll[i].uid] = {};
|
|
356
|
+
coll.splice(i, 1);
|
|
357
|
+
i--;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
this.findBranches(tree, _.keys(tree), coll);
|
|
361
|
+
return tree;
|
|
362
|
+
}
|
|
363
|
+
findBranches(tree, branches, coll) {
|
|
364
|
+
let self = this;
|
|
365
|
+
_.forEach(branches, (branch) => {
|
|
366
|
+
for (const element of coll) {
|
|
367
|
+
if (branch === element.parent_uid) {
|
|
368
|
+
let childUid = element.uid;
|
|
369
|
+
tree[branch][childUid] = {};
|
|
370
|
+
self.findBranches(tree[branch], [childUid], coll);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
publish(assetUid, assetObject) {
|
|
376
|
+
let envId = [];
|
|
377
|
+
let self = this;
|
|
378
|
+
let locales = [];
|
|
379
|
+
let requestObject = { json: { asset: {} } };
|
|
380
|
+
return new Promise(function (resolve, reject) {
|
|
381
|
+
_.forEach(assetObject.publish_details, function (pubObject) {
|
|
382
|
+
if (self.environment.hasOwnProperty(pubObject.environment)) {
|
|
383
|
+
envId.push(self.environment[pubObject.environment].name);
|
|
384
|
+
let idx = _.indexOf(locales, pubObject.locale);
|
|
385
|
+
if (idx === -1) {
|
|
386
|
+
locales.push(pubObject.locale);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
requestObject.json.asset.environments = envId;
|
|
391
|
+
requestObject.json.asset.locales = locales;
|
|
392
|
+
return self.stackAPIClient
|
|
393
|
+
.asset(assetUid)
|
|
394
|
+
.publish({ publishDetails: requestObject.json.asset })
|
|
395
|
+
.then(function () {
|
|
396
|
+
log(self.config, 'Asset ' + assetUid + ' published successfully', 'success');
|
|
397
|
+
return resolve();
|
|
398
|
+
})
|
|
399
|
+
.catch(function (err) {
|
|
400
|
+
if (err && err.message) {
|
|
401
|
+
let error;
|
|
402
|
+
try {
|
|
403
|
+
error = JSON.parse(err.message);
|
|
404
|
+
}
|
|
405
|
+
catch (cError) {
|
|
406
|
+
error = { errorMessage: err.message };
|
|
407
|
+
}
|
|
408
|
+
log(self.config, 'Asset ' + assetUid + ' not published, ' + error.errorMessage, 'error');
|
|
409
|
+
return reject(err);
|
|
410
|
+
}
|
|
411
|
+
return reject(err);
|
|
412
|
+
});
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export = ContentTypesImport;
|
|
2
|
+
declare class ContentTypesImport {
|
|
3
|
+
constructor(importConfig: any, stackAPIClient: any);
|
|
4
|
+
stackAPIClient: any;
|
|
5
|
+
importConfig: any;
|
|
6
|
+
contentTypeConfig: any;
|
|
7
|
+
globalFieldConfig: any;
|
|
8
|
+
importConcurrency: any;
|
|
9
|
+
writeConcurrency: any;
|
|
10
|
+
contentTypesFolderPath: string;
|
|
11
|
+
mapperFolderPath: string;
|
|
12
|
+
existingContentTypesPath: string;
|
|
13
|
+
globalFieldsFolderPath: string;
|
|
14
|
+
globalFieldMapperFolderPath: string;
|
|
15
|
+
globalFieldPendingPath: string;
|
|
16
|
+
ignoredFilesInContentTypesFolder: Map<string, string>;
|
|
17
|
+
contentTypes: any[];
|
|
18
|
+
existingContentTypesUIds: any[];
|
|
19
|
+
titleToUIdMap: Map<any, any>;
|
|
20
|
+
requestOptions: {
|
|
21
|
+
json: {};
|
|
22
|
+
};
|
|
23
|
+
fieldRules: any[];
|
|
24
|
+
installedExtensions: any[];
|
|
25
|
+
globalFields: any[];
|
|
26
|
+
existingGlobalFields: any[];
|
|
27
|
+
pendingGlobalFields: any[];
|
|
28
|
+
start(): Promise<void>;
|
|
29
|
+
seedContentType(contentType: any): Promise<boolean>;
|
|
30
|
+
updateContentType(contentType: any): Promise<void>;
|
|
31
|
+
updateGlobalFields(uid: any): Promise<boolean>;
|
|
32
|
+
mapUidToTitle(): Promise<void>;
|
|
33
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const chalk = require('chalk');
|
|
4
|
+
const { cloneDeep, find, findIndex } = require('lodash');
|
|
5
|
+
const { fileHelper, log, executeTask, formatError, schemaTemplate, lookupExtension } = require('../../utils');
|
|
6
|
+
class ContentTypesImport {
|
|
7
|
+
constructor(importConfig, stackAPIClient) {
|
|
8
|
+
this.stackAPIClient = stackAPIClient;
|
|
9
|
+
this.importConfig = importConfig;
|
|
10
|
+
this.contentTypeConfig = importConfig.modules.content_types;
|
|
11
|
+
this.globalFieldConfig = importConfig.modules.globalfields;
|
|
12
|
+
this.importConcurrency = this.contentTypeConfig.importConcurrency || this.importConfig.importConcurrency;
|
|
13
|
+
this.writeConcurrency = this.contentTypeConfig.writeConcurrency || this.importConfig.writeConcurrency;
|
|
14
|
+
this.contentTypesFolderPath = path.join(this.importConfig.data, this.contentTypeConfig.dirName);
|
|
15
|
+
this.mapperFolderPath = path.join(this.importConfig.data, 'mapper', 'content_types');
|
|
16
|
+
this.existingContentTypesPath = path.join(this.mapperFolderPath, 'success.json');
|
|
17
|
+
this.globalFieldsFolderPath = path.resolve(this.importConfig.data, this.globalFieldConfig.dirName);
|
|
18
|
+
this.globalFieldMapperFolderPath = path.join(importConfig.data, 'mapper', 'global_fields', 'success.json');
|
|
19
|
+
this.globalFieldPendingPath = path.join(importConfig.data, 'mapper', 'global_fields', 'pending_global_fields.js');
|
|
20
|
+
this.ignoredFilesInContentTypesFolder = new Map([
|
|
21
|
+
['__master.json', 'true'],
|
|
22
|
+
['__priority.json', 'true'],
|
|
23
|
+
['schema.json', 'true'],
|
|
24
|
+
['.DS_Store', 'true'],
|
|
25
|
+
]);
|
|
26
|
+
this.contentTypes = [];
|
|
27
|
+
this.existingContentTypesUIds = [];
|
|
28
|
+
this.titleToUIdMap = new Map();
|
|
29
|
+
this.requestOptions = {
|
|
30
|
+
json: {},
|
|
31
|
+
};
|
|
32
|
+
this.fieldRules = [];
|
|
33
|
+
this.installedExtensions = [];
|
|
34
|
+
this.globalFields = [];
|
|
35
|
+
this.existingGlobalFields = [];
|
|
36
|
+
this.pendingGlobalFields = [];
|
|
37
|
+
}
|
|
38
|
+
async start() {
|
|
39
|
+
try {
|
|
40
|
+
const appMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json');
|
|
41
|
+
this.installedExtensions = ((await fileHelper.readFileSync(appMapperPath)) || { extension_uid: {} }).extension_uid;
|
|
42
|
+
// read content types
|
|
43
|
+
// remove content types already existing
|
|
44
|
+
if (fs.existsSync(this.existingContentTypesPath)) {
|
|
45
|
+
this.existingContentTypesUIds = fileHelper.readFileSync(this.existingContentTypesPath) || [];
|
|
46
|
+
this.existingContentTypesUIds = new Map(this.existingContentTypesUIds.map((id) => [id, 'true']));
|
|
47
|
+
}
|
|
48
|
+
const contentTypeFiles = fileHelper.readdirSync(this.contentTypesFolderPath);
|
|
49
|
+
for (let contentTypeName of contentTypeFiles) {
|
|
50
|
+
if (!this.ignoredFilesInContentTypesFolder.has(contentTypeName)) {
|
|
51
|
+
const contentTypePath = path.join(this.contentTypesFolderPath, contentTypeName);
|
|
52
|
+
const contentType = await fileHelper.readFile(contentTypePath);
|
|
53
|
+
if (!this.existingContentTypesUIds.length || !this.existingContentTypesUIds.has(contentType.uid)) {
|
|
54
|
+
this.contentTypes.push(await fileHelper.readFile(contentTypePath));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
// seed content type
|
|
59
|
+
log(this.importConfig, 'Started to seed content types', 'info');
|
|
60
|
+
await executeTask(this.contentTypes, this.seedContentType.bind(this), { concurrency: this.importConcurrency });
|
|
61
|
+
log(this.importConfig, 'Created content types', 'success');
|
|
62
|
+
log(this.importConfig, 'Started to update content types with references', 'info');
|
|
63
|
+
await executeTask(this.contentTypes, this.updateContentType.bind(this), { concurrency: this.importConcurrency });
|
|
64
|
+
log(this.importConfig, 'Updated content types with references', 'success');
|
|
65
|
+
// global field update
|
|
66
|
+
this.pendingGlobalFields = fileHelper.readFileSync(this.globalFieldPendingPath);
|
|
67
|
+
if (Array.isArray(this.pendingGlobalFields) && this.pendingGlobalFields.length > 0) {
|
|
68
|
+
this.globalFields = fileHelper.readFileSync(path.resolve(this.globalFieldsFolderPath, this.globalFieldConfig.fileName));
|
|
69
|
+
this.existingGlobalFields = fileHelper.readFileSync(this.globalFieldMapperFolderPath);
|
|
70
|
+
try {
|
|
71
|
+
log(this.importConfig, 'Started to update pending global field with content type references', 'info');
|
|
72
|
+
await executeTask(this.pendingGlobalFields, this.updateGlobalFields.bind(this), {
|
|
73
|
+
concurrency: this.importConcurrency,
|
|
74
|
+
});
|
|
75
|
+
log(this.importConfig, 'Updated pending global fields with content type with references', 'success');
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
log(this.importConfig, `Failed to updates global fields with content type reference ${formatError(error)}`, 'error');
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// write field rules
|
|
82
|
+
if (this.fieldRules.length > 0) {
|
|
83
|
+
try {
|
|
84
|
+
await fileHelper.writeFile(path.join(this.contentTypesFolderPath, 'field_rules_uid.json'), this.fieldRules);
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
log(this.importConfig, `Failed to write field rules ${formatError(error)}`, 'success');
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
log(this.importConfig, chalk.green('Content types imported successfully'), 'success');
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
let message_content_type = "";
|
|
94
|
+
if (error.request !== undefined && JSON.parse(error.request.data).content_type !== undefined) {
|
|
95
|
+
if (JSON.parse(error.request.data).content_type.uid) {
|
|
96
|
+
message_content_type =
|
|
97
|
+
' Update the content type with content_type_uid - ' + JSON.parse(error.request.data).content_type.uid;
|
|
98
|
+
}
|
|
99
|
+
else if (JSON.parse(error.request.data).content_type.title) {
|
|
100
|
+
message_content_type =
|
|
101
|
+
' Update the content type with content_type_title - ' + JSON.parse(error.request.data).content_type.title;
|
|
102
|
+
}
|
|
103
|
+
error.errorMessage = error.errorMessage + message_content_type;
|
|
104
|
+
}
|
|
105
|
+
log(this.importConfig, formatError(error.errorMessage), 'error');
|
|
106
|
+
log(this.importConfig, formatError(error), 'error');
|
|
107
|
+
throw new Error('Failed to import content types');
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
async seedContentType(contentType) {
|
|
111
|
+
const body = cloneDeep(schemaTemplate);
|
|
112
|
+
body.content_type.uid = contentType.uid;
|
|
113
|
+
body.content_type.title = contentType.title;
|
|
114
|
+
const requestObject = cloneDeep(this.requestOptions);
|
|
115
|
+
requestObject.json = body;
|
|
116
|
+
try {
|
|
117
|
+
await this.stackAPIClient.contentType().create(requestObject.json);
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
if (error.errorCode === 115 && (error.errors.uid || error.errors.title)) {
|
|
121
|
+
// content type uid already exists
|
|
122
|
+
// _.remove(self.contentTypes, { uid: uid });
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
throw error;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
async updateContentType(contentType) {
|
|
129
|
+
if (typeof contentType !== 'object')
|
|
130
|
+
return;
|
|
131
|
+
const requestObject = cloneDeep(this.requestOptions);
|
|
132
|
+
if (contentType.field_rules) {
|
|
133
|
+
this.fieldRules.push(contentType.uid);
|
|
134
|
+
delete contentType.field_rules;
|
|
135
|
+
}
|
|
136
|
+
lookupExtension(this.importConfig, contentType.schema, this.importConfig.preserveStackVersion, this.installedExtensions);
|
|
137
|
+
requestObject.json.content_type = contentType;
|
|
138
|
+
const contentTypeResponse = this.stackAPIClient.contentType(contentType.uid);
|
|
139
|
+
Object.assign(contentTypeResponse, cloneDeep(contentType));
|
|
140
|
+
await contentTypeResponse.update();
|
|
141
|
+
log(this.importConfig, contentType.uid + ' updated with references', 'success');
|
|
142
|
+
}
|
|
143
|
+
async updateGlobalFields(uid) {
|
|
144
|
+
const globalField = find(this.globalFields, { uid });
|
|
145
|
+
if (globalField) {
|
|
146
|
+
lookupExtension(this.importConfig, globalField.schema, this.importConfig.preserveStackVersion, this.installedExtensions);
|
|
147
|
+
let globalFieldObj = this.stackAPIClient.globalField(globalField.uid);
|
|
148
|
+
Object.assign(globalFieldObj, cloneDeep(globalField));
|
|
149
|
+
try {
|
|
150
|
+
const globalFieldResponse = await globalFieldObj.update();
|
|
151
|
+
const existingGlobalField = findIndex(this.existingGlobalFields, (existingGlobalFieldUId) => {
|
|
152
|
+
return globalFieldResponse.uid === existingGlobalFieldUId;
|
|
153
|
+
});
|
|
154
|
+
// Improve write the updated global fields once all updates are completed
|
|
155
|
+
this.existingGlobalFields.splice(existingGlobalField, 1, globalField);
|
|
156
|
+
await fileHelper.writeFile(this.globalFieldMapperFolderPath, this.existingGlobalFields).catch((error) => {
|
|
157
|
+
log(this.importConfig, `failed to write updated the global field ${uid} ${formatError(error)}`);
|
|
158
|
+
});
|
|
159
|
+
log(this.importConfig, `Updated the global field ${uid} with content type references `);
|
|
160
|
+
return true;
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
log(this.importConfig, `failed to update the global field ${uid} ${formatError(error)}`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
log(this.importConfig, `Global field ${uid} does not exist, and hence failed to update.`);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
async mapUidToTitle() {
|
|
171
|
+
this.contentTypes.forEach((ct) => {
|
|
172
|
+
this.titleToUIdMap.set(ct.uid, ct.title);
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
module.exports = ContentTypesImport;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export = ImportCustomRoles;
|
|
2
|
+
declare class ImportCustomRoles {
|
|
3
|
+
constructor(importConfig: any, stackAPIClient: any);
|
|
4
|
+
fails: any[];
|
|
5
|
+
labelUids: any[];
|
|
6
|
+
customRolesUidMapper: {};
|
|
7
|
+
customRolesConfig: {
|
|
8
|
+
dirName: string;
|
|
9
|
+
fileName: string;
|
|
10
|
+
customRolesLocalesFileName: string;
|
|
11
|
+
};
|
|
12
|
+
config: any;
|
|
13
|
+
stackAPIClient: any;
|
|
14
|
+
start(): Promise<void>;
|
|
15
|
+
}
|