@contentstack/cli-cm-import 1.5.11 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -14
- package/bin/dev +17 -0
- package/bin/dev.cmd +3 -0
- package/bin/run +6 -0
- package/bin/run.cmd +3 -0
- package/lib/commands/cm/stacks/import.d.ts +10 -0
- package/lib/commands/cm/stacks/import.js +111 -0
- package/lib/config/index.d.ts +3 -0
- package/lib/config/index.js +372 -0
- package/lib/import/index.d.ts +1 -0
- package/lib/import/index.js +8 -0
- package/lib/import/module-importer.d.ts +13 -0
- package/lib/import/module-importer.js +70 -0
- package/lib/import/modules/assets.d.ts +63 -0
- package/lib/import/modules/assets.js +265 -0
- package/lib/import/modules/base-class.d.ts +69 -0
- package/lib/import/modules/base-class.js +165 -0
- package/lib/import/modules/index.d.ts +2 -0
- package/lib/import/modules/index.js +19 -0
- package/lib/import/modules/locales.d.ts +31 -0
- package/lib/import/modules/locales.js +152 -0
- package/lib/import/modules-js/assets.d.ts +33 -0
- package/lib/import/modules-js/assets.js +415 -0
- package/lib/import/modules-js/content-types.d.ts +33 -0
- package/lib/import/modules-js/content-types.js +176 -0
- package/lib/import/modules-js/custom-roles.d.ts +15 -0
- package/lib/import/modules-js/custom-roles.js +141 -0
- package/lib/import/modules-js/entries.d.ts +54 -0
- package/lib/import/modules-js/entries.js +1260 -0
- package/lib/import/modules-js/environments.d.ts +13 -0
- package/lib/import/modules-js/environments.js +85 -0
- package/lib/import/modules-js/extensions.d.ts +17 -0
- package/lib/import/modules-js/extensions.js +86 -0
- package/lib/import/modules-js/global-fields.d.ts +13 -0
- package/lib/import/modules-js/global-fields.js +109 -0
- package/lib/import/modules-js/index.d.ts +1 -0
- package/lib/import/modules-js/index.js +33 -0
- package/lib/import/modules-js/labels.d.ts +20 -0
- package/lib/import/modules-js/labels.js +148 -0
- package/lib/import/modules-js/locales.d.ts +24 -0
- package/lib/import/modules-js/locales.js +196 -0
- package/lib/import/modules-js/marketplace-apps.d.ts +60 -0
- package/lib/import/modules-js/marketplace-apps.js +409 -0
- package/lib/import/modules-js/webhooks.d.ts +17 -0
- package/lib/import/modules-js/webhooks.js +85 -0
- package/lib/import/modules-js/workflows.d.ts +18 -0
- package/lib/import/modules-js/workflows.js +132 -0
- package/lib/types/default-config.d.ts +130 -0
- package/lib/types/default-config.js +2 -0
- package/lib/types/import-config.d.ts +51 -0
- package/lib/types/import-config.js +2 -0
- package/lib/types/index.d.ts +30 -0
- package/lib/types/index.js +2 -0
- package/lib/utils/asset-helper.d.ts +4 -0
- package/lib/utils/asset-helper.js +387 -0
- package/lib/utils/backup-handler.d.ts +2 -0
- package/lib/utils/backup-handler.js +31 -0
- package/lib/utils/common-helper.d.ts +20 -0
- package/lib/utils/common-helper.js +244 -0
- package/lib/utils/content-type-helper.d.ts +49 -0
- package/lib/utils/content-type-helper.js +143 -0
- package/lib/utils/entries-helper.d.ts +4 -0
- package/lib/utils/entries-helper.js +252 -0
- package/lib/utils/extension-helper.d.ts +10 -0
- package/lib/utils/extension-helper.js +72 -0
- package/lib/utils/file-helper.d.ts +14 -0
- package/lib/utils/file-helper.js +140 -0
- package/lib/utils/import-config-handler.d.ts +3 -0
- package/lib/utils/import-config-handler.js +73 -0
- package/lib/utils/index.d.ts +12 -0
- package/lib/utils/index.js +29 -0
- package/lib/utils/interactive.d.ts +2 -0
- package/lib/utils/interactive.js +23 -0
- package/lib/utils/logger.d.ts +8 -0
- package/lib/utils/logger.js +154 -0
- package/lib/utils/login-handler.d.ts +8 -0
- package/lib/utils/login-handler.js +53 -0
- package/lib/utils/marketplace-app-helper.d.ts +4 -0
- package/lib/utils/marketplace-app-helper.js +31 -0
- package/messages/index.json +1 -7
- package/oclif.manifest.json +2 -2
- package/package.json +47 -21
- package/src/app.js +0 -217
- package/src/commands/cm/stacks/import.js +0 -161
- package/src/config/default.js +0 -352
- package/src/lib/import/assets.js +0 -495
- package/src/lib/import/content-types.js +0 -201
- package/src/lib/import/custom-roles.js +0 -169
- package/src/lib/import/entries.js +0 -1495
- package/src/lib/import/environments.js +0 -106
- package/src/lib/import/extensions.js +0 -108
- package/src/lib/import/global-fields.js +0 -135
- package/src/lib/import/labels.js +0 -175
- package/src/lib/import/locales.js +0 -216
- package/src/lib/import/marketplace-apps.js +0 -542
- package/src/lib/import/webhooks.js +0 -113
- package/src/lib/import/workflows.js +0 -166
- package/src/lib/util/extensionsUidReplace.js +0 -67
- package/src/lib/util/fs.js +0 -124
- package/src/lib/util/import-flags.js +0 -187
- package/src/lib/util/index.js +0 -222
- package/src/lib/util/log.js +0 -144
- package/src/lib/util/login.js +0 -58
- package/src/lib/util/lookupReplaceAssets.js +0 -366
- package/src/lib/util/lookupReplaceEntries.js +0 -250
- package/src/lib/util/marketplace-app-helper.js +0 -31
- package/src/lib/util/removeReferenceFields.js +0 -59
- package/src/lib/util/schemaTemplate.js +0 -38
- package/src/lib/util/supress-mandatory-fields.js +0 -34
- package/src/lib/util/upload.js +0 -56
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Contentstack Import
|
|
3
|
+
* Copyright (c) 2019 Contentstack LLC
|
|
4
|
+
* MIT Licensed
|
|
5
|
+
*/
|
|
6
|
+
import { ModuleClassParams } from '../../types';
|
|
7
|
+
import BaseClass from './base-class';
|
|
8
|
+
export default class ImportLocales extends BaseClass {
|
|
9
|
+
private langMapperPath;
|
|
10
|
+
private langFolderPath;
|
|
11
|
+
private langFailsPath;
|
|
12
|
+
private langSuccessPath;
|
|
13
|
+
private langUidMapperPath;
|
|
14
|
+
private languages;
|
|
15
|
+
private config;
|
|
16
|
+
private stackAPIClient;
|
|
17
|
+
private failedLocales;
|
|
18
|
+
private createdLocales;
|
|
19
|
+
private langUidMapper;
|
|
20
|
+
private localeConfig;
|
|
21
|
+
client: any;
|
|
22
|
+
private reqConcurrency;
|
|
23
|
+
private masterLanguage;
|
|
24
|
+
private masterLanguageConfig;
|
|
25
|
+
private sourceMasterLanguage;
|
|
26
|
+
constructor({ importConfig, stackAPIClient }: ModuleClassParams);
|
|
27
|
+
start(): Promise<any>;
|
|
28
|
+
checkAndUpdateMasterLocale(): Promise<any>;
|
|
29
|
+
createLocales(): Promise<any>;
|
|
30
|
+
updateLocales(): Promise<unknown>;
|
|
31
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/* eslint-disable no-prototype-builtins */
|
|
3
|
+
/*!
|
|
4
|
+
* Contentstack Import
|
|
5
|
+
* Copyright (c) 2019 Contentstack LLC
|
|
6
|
+
* MIT Licensed
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
const tslib_1 = require("tslib");
|
|
10
|
+
const path = tslib_1.__importStar(require("path"));
|
|
11
|
+
const lodash_1 = require("lodash");
|
|
12
|
+
const cli_utilities_1 = require("@contentstack/cli-utilities");
|
|
13
|
+
const utils_1 = require("../../utils");
|
|
14
|
+
const base_class_1 = tslib_1.__importDefault(require("./base-class"));
|
|
15
|
+
class ImportLocales extends base_class_1.default {
|
|
16
|
+
constructor({ importConfig, stackAPIClient }) {
|
|
17
|
+
super({ importConfig, stackAPIClient });
|
|
18
|
+
this.config = importConfig;
|
|
19
|
+
this.localeConfig = importConfig.modules.locales;
|
|
20
|
+
this.masterLanguage = importConfig.masterLocale;
|
|
21
|
+
this.masterLanguageConfig = importConfig.modules.masterLocale;
|
|
22
|
+
this.stackAPIClient = stackAPIClient;
|
|
23
|
+
this.languages = [];
|
|
24
|
+
this.langUidMapper = {};
|
|
25
|
+
this.createdLocales = [];
|
|
26
|
+
this.failedLocales = [];
|
|
27
|
+
this.reqConcurrency = this.localeConfig.writeConcurrency || this.config.writeConcurrency;
|
|
28
|
+
this.langMapperPath = path.resolve(this.config.data, 'mapper', 'languages');
|
|
29
|
+
this.langFolderPath = path.resolve(this.config.data, this.localeConfig.dirName);
|
|
30
|
+
this.langFailsPath = path.resolve(this.config.data, 'mapper', 'languages', 'fails.json');
|
|
31
|
+
this.langSuccessPath = path.resolve(this.config.data, 'mapper', 'languages', 'success.json');
|
|
32
|
+
this.langUidMapperPath = path.resolve(this.config.data, 'mapper', 'languages', 'uid-mapper.json');
|
|
33
|
+
}
|
|
34
|
+
async start() {
|
|
35
|
+
/**
|
|
36
|
+
* read locales
|
|
37
|
+
* create a mapper dir
|
|
38
|
+
* read / create a locale uid mapper
|
|
39
|
+
* update master language details if it same
|
|
40
|
+
* create locales
|
|
41
|
+
* update locales
|
|
42
|
+
*/
|
|
43
|
+
this.languages = utils_1.fsUtil.readFile(path.join(this.langFolderPath, this.localeConfig.fileName));
|
|
44
|
+
if (!this.languages || (0, lodash_1.isEmpty)(this.languages)) {
|
|
45
|
+
(0, utils_1.log)(this.config, 'No languages found to import', 'info');
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
this.sourceMasterLanguage = utils_1.fsUtil.readFile(path.join(this.langFolderPath, this.masterLanguageConfig.fileName));
|
|
49
|
+
await utils_1.fileHelper.makeDirectory(this.langMapperPath);
|
|
50
|
+
if (utils_1.fileHelper.fileExistsSync(this.langUidMapperPath)) {
|
|
51
|
+
this.langUidMapper = utils_1.fsUtil.readFile(this.langUidMapperPath) || {};
|
|
52
|
+
}
|
|
53
|
+
await this.checkAndUpdateMasterLocale().catch((error) => {
|
|
54
|
+
(0, utils_1.log)(this.config, (0, utils_1.formatError)(error), 'error');
|
|
55
|
+
});
|
|
56
|
+
await this.createLocales().catch((error) => {
|
|
57
|
+
(0, utils_1.log)(this.config, (0, utils_1.formatError)(error), 'error');
|
|
58
|
+
Promise.reject('Failed to import locales');
|
|
59
|
+
});
|
|
60
|
+
utils_1.fsUtil.writeFile(this.langFailsPath, this.failedLocales);
|
|
61
|
+
await this.updateLocales().catch((error) => {
|
|
62
|
+
(0, utils_1.log)(this.config, (0, utils_1.formatError)(error), 'error');
|
|
63
|
+
Promise.reject('Failed to update locales');
|
|
64
|
+
});
|
|
65
|
+
(0, utils_1.log)(this.config, 'Languages have been imported successfully!', 'success');
|
|
66
|
+
}
|
|
67
|
+
async checkAndUpdateMasterLocale() {
|
|
68
|
+
var _a, _b, _c, _d;
|
|
69
|
+
let sourceMasterLangDetails = (this.sourceMasterLanguage && Object.values(this.sourceMasterLanguage)) || [];
|
|
70
|
+
if (((_a = sourceMasterLangDetails === null || sourceMasterLangDetails === void 0 ? void 0 : sourceMasterLangDetails[0]) === null || _a === void 0 ? void 0 : _a.code) === ((_b = this.masterLanguage) === null || _b === void 0 ? void 0 : _b.code)) {
|
|
71
|
+
let masterLangDetails = await this.stackAPIClient
|
|
72
|
+
.locale(this.masterLanguage['code'])
|
|
73
|
+
.fetch()
|
|
74
|
+
.catch((error) => {
|
|
75
|
+
(0, utils_1.log)(this.config, (0, utils_1.formatError)(error), 'error');
|
|
76
|
+
});
|
|
77
|
+
if (((_c = masterLangDetails === null || masterLangDetails === void 0 ? void 0 : masterLangDetails.name) === null || _c === void 0 ? void 0 : _c.toString().toUpperCase()) !==
|
|
78
|
+
((_d = sourceMasterLangDetails[0]['name']) === null || _d === void 0 ? void 0 : _d.toString().toUpperCase())) {
|
|
79
|
+
cli_utilities_1.cliux.print('WARNING!!! The master language name for the source and destination is different.', {
|
|
80
|
+
color: 'yellow',
|
|
81
|
+
});
|
|
82
|
+
cli_utilities_1.cliux.print(`Old Master language name: ${masterLangDetails['name']}`, { color: 'red' });
|
|
83
|
+
cli_utilities_1.cliux.print(`New Master language name: ${sourceMasterLangDetails[0]['name']}`, { color: 'green' });
|
|
84
|
+
const langUpdateConfirmation = await cli_utilities_1.cliux.inquire({
|
|
85
|
+
type: 'confirm',
|
|
86
|
+
message: 'Are you sure you want to update name of master language?',
|
|
87
|
+
name: 'confirmation',
|
|
88
|
+
});
|
|
89
|
+
if (langUpdateConfirmation) {
|
|
90
|
+
let langUid = sourceMasterLangDetails[0] && sourceMasterLangDetails[0]['uid'];
|
|
91
|
+
let sourceMasterLanguage = this.sourceMasterLanguage[langUid];
|
|
92
|
+
if (!sourceMasterLanguage) {
|
|
93
|
+
(0, utils_1.log)(this.config, `Master language details not found with id ${langUid} to update`, 'warn');
|
|
94
|
+
}
|
|
95
|
+
const langUpdateRequest = this.stackAPIClient.locale(sourceMasterLanguage.code);
|
|
96
|
+
langUpdateRequest.name = sourceMasterLanguage.name;
|
|
97
|
+
await langUpdateRequest.update().catch(function (error) {
|
|
98
|
+
(0, utils_1.log)(this.config, (0, utils_1.formatError)(error), 'error');
|
|
99
|
+
});
|
|
100
|
+
(0, utils_1.log)(this.config, 'Master Languages name have been updated successfully!', 'success');
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
async createLocales() {
|
|
106
|
+
const onSuccess = ({ response = {}, apiData: { uid, code } = undefined }) => {
|
|
107
|
+
this.langUidMapper[uid] = response.uid;
|
|
108
|
+
this.createdLocales.push((0, lodash_1.pick)(response, [...this.localeConfig.requiredKeys]));
|
|
109
|
+
(0, utils_1.log)(this.importConfig, `Created locale: '${code}'`, 'info');
|
|
110
|
+
utils_1.fsUtil.writeFile(this.langUidMapperPath, this.langUidMapper);
|
|
111
|
+
};
|
|
112
|
+
const onReject = ({ error, apiData: { uid, code } = undefined }) => {
|
|
113
|
+
(0, utils_1.log)(this.importConfig, `Language '${code}' failed to import`, 'error');
|
|
114
|
+
(0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
|
|
115
|
+
this.failedLocales.push({ uid, code });
|
|
116
|
+
};
|
|
117
|
+
return await this.makeConcurrentCall({
|
|
118
|
+
processName: 'Import locales',
|
|
119
|
+
apiContent: (0, lodash_1.filter)((0, lodash_1.values)(this.languages), (lang) => lang.code !== this.masterLanguage.code),
|
|
120
|
+
apiParams: {
|
|
121
|
+
reject: onReject.bind(this),
|
|
122
|
+
resolve: onSuccess.bind(this),
|
|
123
|
+
entity: 'create-locale',
|
|
124
|
+
includeParamOnCompletion: true,
|
|
125
|
+
},
|
|
126
|
+
concurrencyLimit: this.reqConcurrency,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
async updateLocales() {
|
|
130
|
+
const onSuccess = ({ response = {}, apiData: { uid, code } = undefined }) => {
|
|
131
|
+
(0, utils_1.log)(this.importConfig, `Updated locale: '${code}'`, 'info');
|
|
132
|
+
utils_1.fsUtil.writeFile(this.langSuccessPath, this.createdLocales);
|
|
133
|
+
};
|
|
134
|
+
const onReject = ({ error, apiData: { uid, code } = undefined }) => {
|
|
135
|
+
(0, utils_1.log)(this.importConfig, `Language '${code}' failed to update`, 'error');
|
|
136
|
+
(0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
|
|
137
|
+
utils_1.fsUtil.writeFile(this.langFailsPath, this.failedLocales);
|
|
138
|
+
};
|
|
139
|
+
return await this.makeConcurrentCall({
|
|
140
|
+
processName: 'Update locales',
|
|
141
|
+
apiContent: (0, lodash_1.values)(this.languages),
|
|
142
|
+
apiParams: {
|
|
143
|
+
reject: onReject.bind(this),
|
|
144
|
+
resolve: onSuccess.bind(this),
|
|
145
|
+
entity: 'update-locale',
|
|
146
|
+
includeParamOnCompletion: true,
|
|
147
|
+
},
|
|
148
|
+
concurrencyLimit: this.reqConcurrency,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
exports.default = ImportLocales;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export = ImportAssets;
|
|
2
|
+
declare class ImportAssets {
|
|
3
|
+
constructor(importConfig: any, stackAPIClient: any);
|
|
4
|
+
assets: any;
|
|
5
|
+
fails: any[];
|
|
6
|
+
assetConfig: any;
|
|
7
|
+
mapperDirPath: any;
|
|
8
|
+
assetBatchLimit: any;
|
|
9
|
+
uidMapping: {};
|
|
10
|
+
urlMapping: {};
|
|
11
|
+
environmentPath: any;
|
|
12
|
+
assetBucket: any[];
|
|
13
|
+
assetsFolderPath: any;
|
|
14
|
+
folderBucket: any[];
|
|
15
|
+
folderDetails: any[];
|
|
16
|
+
mappedFolderUids: {};
|
|
17
|
+
config: any;
|
|
18
|
+
stackAPIClient: any;
|
|
19
|
+
start(): Promise<any>;
|
|
20
|
+
uidMapperPath: string;
|
|
21
|
+
urlMapperPath: string;
|
|
22
|
+
failsPath: string;
|
|
23
|
+
environment: any;
|
|
24
|
+
uploadVersionedAssets(uid: any, assetFolderPath: any): Promise<any>;
|
|
25
|
+
updateAsset(assetPath: any, metadata: any, filesStreamed: any, _uidContainer: any, urlContainer: any): Promise<any>;
|
|
26
|
+
uploadAsset(assetPath: any, metadata: any, uidContainer: any, urlContainer: any): Promise<any>;
|
|
27
|
+
importFolders(): Promise<any>;
|
|
28
|
+
buildFolderReqObjs(createdFolderUids: any, tree: any, parent_uid: any): void;
|
|
29
|
+
buildTree(coll: any): {};
|
|
30
|
+
findBranches(tree: any, branches: any, coll: any): void;
|
|
31
|
+
publish(assetUid: any, assetObject: any): Promise<any>;
|
|
32
|
+
}
|
|
33
|
+
import Promise = require("bluebird");
|
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Contentstack Import
|
|
3
|
+
* Copyright (c) 2019 Contentstack LLC
|
|
4
|
+
* MIT Licensed
|
|
5
|
+
*/
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const _ = require('lodash');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
const chalk = require('chalk');
|
|
10
|
+
const mkdirp = require('mkdirp');
|
|
11
|
+
const Promise = require('bluebird');
|
|
12
|
+
let { default: config } = require('../../config');
|
|
13
|
+
const { fileHelper, log, uploadAssetHelper } = require('../../utils');
|
|
14
|
+
module.exports = class ImportAssets {
|
|
15
|
+
constructor(importConfig, stackAPIClient) {
|
|
16
|
+
this.fails = [];
|
|
17
|
+
this.uidMapping = {};
|
|
18
|
+
this.urlMapping = {};
|
|
19
|
+
this.assetBucket = [];
|
|
20
|
+
this.folderBucket = [];
|
|
21
|
+
this.folderDetails = [];
|
|
22
|
+
this.mappedFolderUids = {};
|
|
23
|
+
this.config = _.merge(config, importConfig);
|
|
24
|
+
this.stackAPIClient = stackAPIClient;
|
|
25
|
+
this.assetConfig = config.modules['assets-old'];
|
|
26
|
+
this.assetBatchLimit =
|
|
27
|
+
this.assetConfig.hasOwnProperty('assetBatchLimit') && typeof this.assetConfig.assetBatchLimit === 'number'
|
|
28
|
+
? this.assetConfig.assetBatchLimit
|
|
29
|
+
: 2;
|
|
30
|
+
}
|
|
31
|
+
start() {
|
|
32
|
+
let self = this;
|
|
33
|
+
log(self.config, 'Migrating assets', 'success');
|
|
34
|
+
this.assetsFolderPath = path.join(this.config.data, this.assetConfig.dirName);
|
|
35
|
+
this.mapperDirPath = path.resolve(this.config.data, 'mapper', 'assets');
|
|
36
|
+
this.environmentPath = path.resolve(this.config.data, 'environments', 'environments.json');
|
|
37
|
+
this.uidMapperPath = path.join(this.mapperDirPath, 'uid-mapping.json');
|
|
38
|
+
this.urlMapperPath = path.join(this.mapperDirPath, 'url-mapping.json');
|
|
39
|
+
this.failsPath = path.join(this.mapperDirPath, 'fail.json');
|
|
40
|
+
this.assets = fileHelper.readFileSync(path.join(this.assetsFolderPath, this.assetConfig.fileName));
|
|
41
|
+
this.environment = fileHelper.readFileSync(this.environmentPath);
|
|
42
|
+
if (fs.existsSync(this.uidMapperPath)) {
|
|
43
|
+
this.uidMapping = fileHelper.readFileSync(this.uidMapperPath);
|
|
44
|
+
}
|
|
45
|
+
if (fs.existsSync(this.urlMapperPath)) {
|
|
46
|
+
this.urlMapping = fileHelper.readFileSync(this.urlMapperPath);
|
|
47
|
+
}
|
|
48
|
+
mkdirp.sync(this.mapperDirPath);
|
|
49
|
+
return new Promise(function (resolve, reject) {
|
|
50
|
+
if (self.assets === undefined || _.isEmpty(self.assets)) {
|
|
51
|
+
log(self.config, 'No Assets Found', 'success');
|
|
52
|
+
return resolve({ empty: true });
|
|
53
|
+
}
|
|
54
|
+
let batches = [];
|
|
55
|
+
let assetUids = Object.keys(self.assets);
|
|
56
|
+
for (let i = 0; i < assetUids.length; i += self.assetBatchLimit) {
|
|
57
|
+
batches.push(assetUids.slice(i, i + self.assetBatchLimit));
|
|
58
|
+
}
|
|
59
|
+
return self
|
|
60
|
+
.importFolders()
|
|
61
|
+
.then(function () {
|
|
62
|
+
return Promise.map(batches, async function (batch, index) {
|
|
63
|
+
return Promise.map(batch, function (assetUid) {
|
|
64
|
+
if (self.uidMapping.hasOwnProperty(assetUid)) {
|
|
65
|
+
log(self.config, 'Skipping upload of asset: ' + assetUid + '. Its mapped to: ' + self.uidMapping[assetUid], 'success');
|
|
66
|
+
// the asset has been already imported
|
|
67
|
+
return void 0;
|
|
68
|
+
}
|
|
69
|
+
let currentAssetFolderPath = path.join(self.assetsFolderPath, assetUid);
|
|
70
|
+
if (fs.existsSync(currentAssetFolderPath)) {
|
|
71
|
+
// if this is true, means, the exported asset data is versioned
|
|
72
|
+
// hence, upload each asset with its version
|
|
73
|
+
if (self.config.versioning) {
|
|
74
|
+
return self.uploadVersionedAssets(assetUid, currentAssetFolderPath).catch(function (error) {
|
|
75
|
+
log(self.config, 'Asset upload failed \n' + error, 'error');
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
let uidContainer = {};
|
|
79
|
+
let urlContainer = {};
|
|
80
|
+
let assetPath = path.resolve(currentAssetFolderPath, self.assets[assetUid].filename);
|
|
81
|
+
if (self.assets[assetUid].parent_uid && typeof self.assets[assetUid].parent_uid === 'string') {
|
|
82
|
+
if (self.mappedFolderUids.hasOwnProperty(self.assets[assetUid].parent_uid)) {
|
|
83
|
+
self.assets[assetUid].parent_uid = self.mappedFolderUids[self.assets[assetUid].parent_uid];
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
log(self.config, `'${self.assets[assetUid].parent_uid}' parent_uid was not found! Thus, setting it as 'null'`, 'error');
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return self
|
|
90
|
+
.uploadAsset(assetPath, self.assets[assetUid], uidContainer, urlContainer)
|
|
91
|
+
.then(async function () {
|
|
92
|
+
self.uidMapping[assetUid] = uidContainer[assetUid];
|
|
93
|
+
self.urlMapping[self.assets[assetUid].url] = urlContainer[self.assets[assetUid].url];
|
|
94
|
+
if (self.config.entriesPublish && self.assets[assetUid].publish_details.length > 0) {
|
|
95
|
+
let assetsUid = uidContainer[assetUid];
|
|
96
|
+
try {
|
|
97
|
+
return await self.publish(assetsUid, self.assets[assetUid]);
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
return error;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// assetUid has been successfully uploaded
|
|
104
|
+
// log them onto /mapper/assets/success.json
|
|
105
|
+
})
|
|
106
|
+
.catch(function (error) {
|
|
107
|
+
log(self.config, 'Asset upload failed \n' + error, 'error');
|
|
108
|
+
return error;
|
|
109
|
+
// asset failed to upload
|
|
110
|
+
// log them onto /mapper/assets/fail.json
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
log(self.config, `'${currentAssetFolderPath}' does not exist!`, 'error');
|
|
114
|
+
}, { concurrency: self.assetConfig.assetBatchLimit }).then(function () {
|
|
115
|
+
fileHelper.writeFileSync(self.uidMapperPath, self.uidMapping);
|
|
116
|
+
fileHelper.writeFileSync(self.urlMapperPath, self.urlMapping);
|
|
117
|
+
// completed uploading assets
|
|
118
|
+
log(self.config, 'Completed asset import of batch no: ' + (index + 1), 'success');
|
|
119
|
+
return index + 1;
|
|
120
|
+
// TODO: if there are failures, retry
|
|
121
|
+
});
|
|
122
|
+
}, { concurrency: 1 })
|
|
123
|
+
.then(function () {
|
|
124
|
+
let numberOfSuccessfulAssetUploads = Object.keys(self.uidMapping).length;
|
|
125
|
+
if (numberOfSuccessfulAssetUploads > 0) {
|
|
126
|
+
log(self.config, chalk.green(numberOfSuccessfulAssetUploads + ' assets uploaded successfully!'), 'success');
|
|
127
|
+
}
|
|
128
|
+
// TODO: if there are failures, retry
|
|
129
|
+
return resolve();
|
|
130
|
+
})
|
|
131
|
+
.catch(function (error) {
|
|
132
|
+
log(self.config, error, 'error');
|
|
133
|
+
return reject(error);
|
|
134
|
+
});
|
|
135
|
+
})
|
|
136
|
+
.catch(function (error) {
|
|
137
|
+
log(self.config, error, 'error');
|
|
138
|
+
return reject(error);
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
uploadVersionedAssets(uid, assetFolderPath) {
|
|
143
|
+
let self = this;
|
|
144
|
+
return new Promise(function (resolve, reject) {
|
|
145
|
+
let versionedAssetMetadata = fileHelper.readFileSync(path.join(assetFolderPath, '_contentstack_' + uid + '.json'));
|
|
146
|
+
// using last version, find asset's parent
|
|
147
|
+
let lastVersion = versionedAssetMetadata[versionedAssetMetadata.length - 1];
|
|
148
|
+
if (typeof lastVersion.parent_uid === 'string') {
|
|
149
|
+
if (self.mappedFolderUids.hasOwnProperty(lastVersion.parent_uid)) {
|
|
150
|
+
// update each version of that asset with the last version's parent_uid
|
|
151
|
+
versionedAssetMetadata.forEach(function (assetMetadata) {
|
|
152
|
+
assetMetadata.parent_uid = self.mappedFolderUids[lastVersion.parent_uid];
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
log(self.config, (lastVersion.parent_uid + " parent_uid was not found! Thus, setting it as 'null'", 'error'));
|
|
157
|
+
versionedAssetMetadata.forEach(function (assetMetadata) {
|
|
158
|
+
assetMetadata.parent_uid = null;
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
let counter = 0;
|
|
163
|
+
let uidContainer = {};
|
|
164
|
+
let urlContainer = {};
|
|
165
|
+
let filesStreamed = [];
|
|
166
|
+
return Promise.map(versionedAssetMetadata, function () {
|
|
167
|
+
let assetMetadata = versionedAssetMetadata[counter];
|
|
168
|
+
let assetPath = path.join(assetFolderPath, assetMetadata.filename);
|
|
169
|
+
if (++counter === 1) {
|
|
170
|
+
return self
|
|
171
|
+
.uploadAsset(assetPath, assetMetadata, uidContainer, urlContainer)
|
|
172
|
+
.then(function () {
|
|
173
|
+
filesStreamed.push(assetMetadata.filename);
|
|
174
|
+
})
|
|
175
|
+
.catch((error) => {
|
|
176
|
+
log(self.config, error, 'error');
|
|
177
|
+
reject(error);
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
return self
|
|
181
|
+
.updateAsset(assetPath, assetMetadata, filesStreamed, uidContainer, urlContainer)
|
|
182
|
+
.then(function () {
|
|
183
|
+
filesStreamed.push(assetMetadata.filename);
|
|
184
|
+
})
|
|
185
|
+
.catch((error) => {
|
|
186
|
+
log(self.config, error, 'error');
|
|
187
|
+
});
|
|
188
|
+
}, { concurrency: self.assetConfig.uploadAssetsConcurrency })
|
|
189
|
+
.then(function () {
|
|
190
|
+
self.uidMapping[uid] = uidContainer[uid];
|
|
191
|
+
for (let url in urlContainer) {
|
|
192
|
+
self.urlMapping[url] = urlContainer[url];
|
|
193
|
+
}
|
|
194
|
+
// completed uploading all the versions of the asset
|
|
195
|
+
return resolve();
|
|
196
|
+
})
|
|
197
|
+
.catch(function (error) {
|
|
198
|
+
// failed to upload asset
|
|
199
|
+
// write it on fail logs, but do not stop the process
|
|
200
|
+
log(self.config, 'Failed to upload asset\n' + error, 'error');
|
|
201
|
+
return resolve();
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
updateAsset(assetPath, metadata, filesStreamed, _uidContainer, urlContainer) {
|
|
206
|
+
const self = this;
|
|
207
|
+
return new Promise(function (resolve, reject) {
|
|
208
|
+
let requestOption = {};
|
|
209
|
+
if (filesStreamed && filesStreamed.indexOf(metadata.filename) !== -1) {
|
|
210
|
+
log(self.config, 'Skipping re-upload/streaming of ' + metadata.uid + '/' + metadata.filename, 'success');
|
|
211
|
+
requestOption.formData = {};
|
|
212
|
+
return resolve();
|
|
213
|
+
}
|
|
214
|
+
log(self.config, 'Streaming: ' + metadata.uid + '/' + metadata.filename, 'success');
|
|
215
|
+
requestOption.formData = {};
|
|
216
|
+
if (metadata.hasOwnProperty('parent_uid') && typeof metadata.parent_uid === 'string') {
|
|
217
|
+
requestOption.formData['asset[parent_uid]'] = metadata.parent_uid;
|
|
218
|
+
}
|
|
219
|
+
if (metadata.hasOwnProperty('description') && typeof metadata.description === 'string') {
|
|
220
|
+
requestOption.formData['asset[description]'] = metadata.description;
|
|
221
|
+
}
|
|
222
|
+
if (metadata.hasOwnProperty('tags') && Array.isArray(metadata.tags)) {
|
|
223
|
+
requestOption.formData['asset[tags]'] = metadata.tags;
|
|
224
|
+
}
|
|
225
|
+
if (metadata.hasOwnProperty('title') && typeof metadata.title === 'string') {
|
|
226
|
+
requestOption.formData['asset[title]'] = metadata.title;
|
|
227
|
+
}
|
|
228
|
+
return uploadAssetHelper(self.config, requestOption, assetPath)
|
|
229
|
+
.then(function (response) {
|
|
230
|
+
urlContainer[metadata.url] = response.url;
|
|
231
|
+
return resolve();
|
|
232
|
+
})
|
|
233
|
+
.catch(function (error) {
|
|
234
|
+
log(self.config, error, 'error');
|
|
235
|
+
return reject(error);
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
uploadAsset(assetPath, metadata, uidContainer, urlContainer) {
|
|
240
|
+
const self = this;
|
|
241
|
+
return new Promise(function (resolve, reject) {
|
|
242
|
+
let requestOption = {};
|
|
243
|
+
if (metadata.hasOwnProperty('parent_uid') && typeof metadata.parent_uid === 'string') {
|
|
244
|
+
requestOption.parent_uid = metadata.parent_uid;
|
|
245
|
+
}
|
|
246
|
+
if (metadata.hasOwnProperty('description') && typeof metadata.description === 'string') {
|
|
247
|
+
requestOption.description = metadata.description;
|
|
248
|
+
}
|
|
249
|
+
// eslint-disable-next-line no-prototype-builtins
|
|
250
|
+
if (metadata.hasOwnProperty('tags') && Array.isArray(metadata.tags)) {
|
|
251
|
+
requestOption.tags = metadata.tags;
|
|
252
|
+
}
|
|
253
|
+
if (metadata.hasOwnProperty('title') && typeof metadata.title === 'string') {
|
|
254
|
+
requestOption.title = metadata.title;
|
|
255
|
+
}
|
|
256
|
+
return uploadAssetHelper(self.config, requestOption, assetPath)
|
|
257
|
+
.then(function (response) {
|
|
258
|
+
uidContainer[metadata.uid] = response.uid;
|
|
259
|
+
urlContainer[metadata.url] = response.url;
|
|
260
|
+
return resolve();
|
|
261
|
+
})
|
|
262
|
+
.catch(function (error) {
|
|
263
|
+
log(self.config, error, 'error');
|
|
264
|
+
return reject(error);
|
|
265
|
+
});
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
importFolders() {
|
|
269
|
+
let self = this;
|
|
270
|
+
return new Promise(function (resolve, reject) {
|
|
271
|
+
let mappedFolderPath = path.resolve(self.config.data, 'mapper', 'assets', 'folder-mapping.json');
|
|
272
|
+
self.folderDetails = fileHelper.readFileSync(path.resolve(self.assetsFolderPath, 'folders.json'));
|
|
273
|
+
if (_.isEmpty(self.folderDetails)) {
|
|
274
|
+
log(self.config, 'No folders were found at: ' + path.join(self.assetsFolderPath, 'folders.json'), 'success');
|
|
275
|
+
return resolve();
|
|
276
|
+
}
|
|
277
|
+
let tree = self.buildTree(_.cloneDeep(self.folderDetails));
|
|
278
|
+
let createdFolders = {};
|
|
279
|
+
let createdFolderUids = [];
|
|
280
|
+
// if a few folders have already been created, skip re-creating them
|
|
281
|
+
if (fs.existsSync(mappedFolderPath)) {
|
|
282
|
+
createdFolders = fileHelper.readFileSync(mappedFolderPath);
|
|
283
|
+
// check if the read file has mapped objects
|
|
284
|
+
if (_.isPlainObject(createdFolders)) {
|
|
285
|
+
createdFolderUids = Object.keys(createdFolders);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
self.buildFolderReqObjs(createdFolderUids, tree, null);
|
|
289
|
+
let idx = 0;
|
|
290
|
+
return Promise.map(self.folderBucket, function () {
|
|
291
|
+
let folder = self.folderBucket[idx];
|
|
292
|
+
if (createdFolders.hasOwnProperty(folder.json.asset.parent_uid)) {
|
|
293
|
+
// replace old uid with new
|
|
294
|
+
folder.json.asset.parent_uid = createdFolders[folder.json.asset.parent_uid];
|
|
295
|
+
}
|
|
296
|
+
return self.stackAPIClient
|
|
297
|
+
.asset()
|
|
298
|
+
.folder()
|
|
299
|
+
.create(folder.json)
|
|
300
|
+
.then((response) => {
|
|
301
|
+
log(self.config, "Created folder: '" + folder.json.asset.name + "'", 'success');
|
|
302
|
+
// assigning newUid to oldUid
|
|
303
|
+
createdFolders[folder.oldUid] = response.uid;
|
|
304
|
+
fileHelper.writeFileSync(mappedFolderPath, createdFolders);
|
|
305
|
+
idx++;
|
|
306
|
+
})
|
|
307
|
+
.catch(function (err) {
|
|
308
|
+
let error = JSON.parse(err.message);
|
|
309
|
+
if (error.errors.authorization || error.errors.api_key) {
|
|
310
|
+
log(self.config, 'Api_key or management_token is not valid', 'error');
|
|
311
|
+
return reject(error);
|
|
312
|
+
}
|
|
313
|
+
log(self.config, err, 'error');
|
|
314
|
+
return error;
|
|
315
|
+
});
|
|
316
|
+
}, { concurrency: self.assetConfig.importFoldersConcurrency })
|
|
317
|
+
.then(function () {
|
|
318
|
+
self.mappedFolderUids = fileHelper.readFileSync(mappedFolderPath);
|
|
319
|
+
// completed creating folders
|
|
320
|
+
return resolve();
|
|
321
|
+
})
|
|
322
|
+
.catch(function (error) {
|
|
323
|
+
log(self.config, error, 'error');
|
|
324
|
+
return reject(error);
|
|
325
|
+
});
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
buildFolderReqObjs(createdFolderUids, tree, parent_uid) {
|
|
329
|
+
let self = this;
|
|
330
|
+
for (let leaf in tree) {
|
|
331
|
+
// if the folder is already created, skip
|
|
332
|
+
if (createdFolderUids.indexOf(leaf) !== -1) {
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
let folderObj = _.find(self.folderDetails, { uid: leaf });
|
|
336
|
+
let requestOption = {
|
|
337
|
+
json: {
|
|
338
|
+
asset: {
|
|
339
|
+
name: folderObj.name,
|
|
340
|
+
parent_uid: parent_uid || null,
|
|
341
|
+
},
|
|
342
|
+
},
|
|
343
|
+
oldUid: leaf,
|
|
344
|
+
};
|
|
345
|
+
self.folderBucket.push(requestOption);
|
|
346
|
+
if (Object.keys(tree[leaf]).length > 0) {
|
|
347
|
+
self.buildFolderReqObjs(createdFolderUids, tree[leaf], leaf);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
buildTree(coll) {
|
|
352
|
+
let tree = {};
|
|
353
|
+
for (let i = 0; i < coll.length; i++) {
|
|
354
|
+
if (coll[i].parent_uid === null || !coll[i].hasOwnProperty('parent_uid')) {
|
|
355
|
+
tree[coll[i].uid] = {};
|
|
356
|
+
coll.splice(i, 1);
|
|
357
|
+
i--;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
this.findBranches(tree, _.keys(tree), coll);
|
|
361
|
+
return tree;
|
|
362
|
+
}
|
|
363
|
+
findBranches(tree, branches, coll) {
|
|
364
|
+
let self = this;
|
|
365
|
+
_.forEach(branches, (branch) => {
|
|
366
|
+
for (const element of coll) {
|
|
367
|
+
if (branch === element.parent_uid) {
|
|
368
|
+
let childUid = element.uid;
|
|
369
|
+
tree[branch][childUid] = {};
|
|
370
|
+
self.findBranches(tree[branch], [childUid], coll);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
publish(assetUid, assetObject) {
|
|
376
|
+
let envId = [];
|
|
377
|
+
let self = this;
|
|
378
|
+
let locales = [];
|
|
379
|
+
let requestObject = { json: { asset: {} } };
|
|
380
|
+
return new Promise(function (resolve, reject) {
|
|
381
|
+
_.forEach(assetObject.publish_details, function (pubObject) {
|
|
382
|
+
if (self.environment.hasOwnProperty(pubObject.environment)) {
|
|
383
|
+
envId.push(self.environment[pubObject.environment].name);
|
|
384
|
+
let idx = _.indexOf(locales, pubObject.locale);
|
|
385
|
+
if (idx === -1) {
|
|
386
|
+
locales.push(pubObject.locale);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
requestObject.json.asset.environments = envId;
|
|
391
|
+
requestObject.json.asset.locales = locales;
|
|
392
|
+
return self.stackAPIClient
|
|
393
|
+
.asset(assetUid)
|
|
394
|
+
.publish({ publishDetails: requestObject.json.asset })
|
|
395
|
+
.then(function () {
|
|
396
|
+
log(self.config, 'Asset ' + assetUid + ' published successfully', 'success');
|
|
397
|
+
return resolve();
|
|
398
|
+
})
|
|
399
|
+
.catch(function (err) {
|
|
400
|
+
if (err && err.message) {
|
|
401
|
+
let error;
|
|
402
|
+
try {
|
|
403
|
+
error = JSON.parse(err.message);
|
|
404
|
+
}
|
|
405
|
+
catch (cError) {
|
|
406
|
+
error = { errorMessage: err.message };
|
|
407
|
+
}
|
|
408
|
+
log(self.config, 'Asset ' + assetUid + ' not published, ' + error.errorMessage, 'error');
|
|
409
|
+
return reject(err);
|
|
410
|
+
}
|
|
411
|
+
return reject(err);
|
|
412
|
+
});
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export = ContentTypesImport;
|
|
2
|
+
declare class ContentTypesImport {
|
|
3
|
+
constructor(importConfig: any, stackAPIClient: any);
|
|
4
|
+
stackAPIClient: any;
|
|
5
|
+
importConfig: any;
|
|
6
|
+
contentTypeConfig: any;
|
|
7
|
+
globalFieldConfig: any;
|
|
8
|
+
importConcurrency: any;
|
|
9
|
+
writeConcurrency: any;
|
|
10
|
+
contentTypesFolderPath: string;
|
|
11
|
+
mapperFolderPath: string;
|
|
12
|
+
existingContentTypesPath: string;
|
|
13
|
+
globalFieldsFolderPath: string;
|
|
14
|
+
globalFieldMapperFolderPath: string;
|
|
15
|
+
globalFieldPendingPath: string;
|
|
16
|
+
ignoredFilesInContentTypesFolder: Map<string, string>;
|
|
17
|
+
contentTypes: any[];
|
|
18
|
+
existingContentTypesUIds: any[];
|
|
19
|
+
titleToUIdMap: Map<any, any>;
|
|
20
|
+
requestOptions: {
|
|
21
|
+
json: {};
|
|
22
|
+
};
|
|
23
|
+
fieldRules: any[];
|
|
24
|
+
installedExtensions: any[];
|
|
25
|
+
globalFields: any[];
|
|
26
|
+
existingGlobalFields: any[];
|
|
27
|
+
pendingGlobalFields: any[];
|
|
28
|
+
start(): Promise<void>;
|
|
29
|
+
seedContentType(contentType: any): Promise<boolean>;
|
|
30
|
+
updateContentType(contentType: any): Promise<void>;
|
|
31
|
+
updateGlobalFields(uid: any): Promise<boolean>;
|
|
32
|
+
mapUidToTitle(): Promise<void>;
|
|
33
|
+
}
|