contentful-import 9.1.0 → 9.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +3 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +1315 -157
- package/dist/index.mjs +1308 -0
- package/package.json +26 -6
- package/dist/parseOptions.js +0 -97
- package/dist/tasks/get-destination-data.js +0 -122
- package/dist/tasks/init-client.js +0 -19
- package/dist/tasks/push-to-space/assets.js +0 -75
- package/dist/tasks/push-to-space/creation.js +0 -178
- package/dist/tasks/push-to-space/publishing.js +0 -100
- package/dist/tasks/push-to-space/push-to-space.js +0 -329
- package/dist/transform/transform-space.js +0 -56
- package/dist/transform/transformers.js +0 -70
- package/dist/usageParams.js +0 -115
- package/dist/utils/headers.js +0 -38
- package/dist/utils/schema.js +0 -87
- package/dist/utils/sort-entries.js +0 -116
- package/dist/utils/sort-locales.js +0 -37
- package/dist/utils/validations.js +0 -54
package/package.json
CHANGED
|
@@ -1,22 +1,30 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "contentful-import",
|
|
3
|
-
"version": "9.1
|
|
3
|
+
"version": "9.2.1",
|
|
4
4
|
"description": "this tool allows you to import JSON dump exported by contentful-export",
|
|
5
|
-
"main": "dist/index.
|
|
5
|
+
"main": "dist/index.mjs",
|
|
6
|
+
"typings": "dist/index.d.ts",
|
|
7
|
+
"module": "dist/index.mjs",
|
|
6
8
|
"engines": {
|
|
7
9
|
"node": ">=16"
|
|
8
10
|
},
|
|
11
|
+
"exports": {
|
|
12
|
+
"default": "./dist/index.js",
|
|
13
|
+
"require": "./dist/index.js",
|
|
14
|
+
"import": "./dist/index.mjs",
|
|
15
|
+
"node": "./dist/index.js"
|
|
16
|
+
},
|
|
9
17
|
"bin": {
|
|
10
18
|
"contentful-import": "./bin/contentful-import"
|
|
11
19
|
},
|
|
12
20
|
"scripts": {
|
|
13
|
-
"build": "
|
|
21
|
+
"build": "tsup",
|
|
14
22
|
"build:watch": "babel lib --out-dir dist --watch",
|
|
15
23
|
"clean": "rimraf dist && rimraf coverage",
|
|
16
24
|
"lint": "eslint lib bin/contentful-import test",
|
|
17
25
|
"pretest": "npm run lint && npm run build",
|
|
18
26
|
"test": "npm run test:unit && npm run test:integration",
|
|
19
|
-
"test:unit": "jest --testPathPattern=test/unit --coverage",
|
|
27
|
+
"test:unit": "jest --testPathPattern=test/unit --runInBand --coverage",
|
|
20
28
|
"test:unit:debug": "node --inspect-brk ./node_modules/.bin/jest --runInBand --watch --testPathPattern=test/unit",
|
|
21
29
|
"test:unit:watch": "npm run test:unit -- --watch",
|
|
22
30
|
"test:integration": "jest --testPathPattern=test/integration",
|
|
@@ -49,12 +57,23 @@
|
|
|
49
57
|
"bugs": {
|
|
50
58
|
"url": "https://github.com/contentful/contentful-import/issues"
|
|
51
59
|
},
|
|
60
|
+
"tsup": {
|
|
61
|
+
"entry": [
|
|
62
|
+
"lib/index.ts"
|
|
63
|
+
],
|
|
64
|
+
"format": [
|
|
65
|
+
"cjs",
|
|
66
|
+
"esm"
|
|
67
|
+
],
|
|
68
|
+
"clean": true,
|
|
69
|
+
"dts": true
|
|
70
|
+
},
|
|
52
71
|
"dependencies": {
|
|
53
72
|
"@discoveryjs/json-ext": "^0.5.7",
|
|
54
73
|
"bluebird": "^3.5.1",
|
|
55
74
|
"cli-table3": "^0.6.0",
|
|
56
75
|
"contentful-batch-libs": "^9.5.0",
|
|
57
|
-
"contentful-management": "^10.46.
|
|
76
|
+
"contentful-management": "^10.46.4",
|
|
58
77
|
"date-fns": "^2.30.0",
|
|
59
78
|
"eslint": "^8.50.0",
|
|
60
79
|
"eslint-config-standard": "^17.1.0",
|
|
@@ -73,7 +92,7 @@
|
|
|
73
92
|
"@babel/preset-env": "^7.22.20",
|
|
74
93
|
"@types/jest": "^29.5.5",
|
|
75
94
|
"@types/node": "^20.6.3",
|
|
76
|
-
"@typescript-eslint/eslint-plugin": "^6.7.
|
|
95
|
+
"@typescript-eslint/eslint-plugin": "^6.7.3",
|
|
77
96
|
"@typescript-eslint/parser": "^6.7.2",
|
|
78
97
|
"babel-jest": "^29.7.0",
|
|
79
98
|
"babel-plugin-add-module-exports": "^1.0.2",
|
|
@@ -89,6 +108,7 @@
|
|
|
89
108
|
"rimraf": "^5.0.0",
|
|
90
109
|
"semantic-release": "^19.0.5",
|
|
91
110
|
"ts-jest": "^29.1.1",
|
|
111
|
+
"tsup": "^7.2.0",
|
|
92
112
|
"typescript": "^5.2.2"
|
|
93
113
|
},
|
|
94
114
|
"files": [
|
package/dist/parseOptions.js
DELETED
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const fs_1 = __importDefault(require("fs"));
|
|
7
|
-
const path_1 = require("path");
|
|
8
|
-
const format_1 = __importDefault(require("date-fns/format"));
|
|
9
|
-
const package_1 = require("../package");
|
|
10
|
-
const headers_1 = require("./utils/headers");
|
|
11
|
-
const proxy_1 = require("contentful-batch-libs/dist/proxy");
|
|
12
|
-
const add_sequence_header_1 = __importDefault(require("contentful-batch-libs/dist/add-sequence-header"));
|
|
13
|
-
const json_ext_1 = require("@discoveryjs/json-ext");
|
|
14
|
-
const SUPPORTED_ENTITY_TYPES = [
|
|
15
|
-
'contentTypes',
|
|
16
|
-
'tags',
|
|
17
|
-
'entries',
|
|
18
|
-
'assets',
|
|
19
|
-
'locales',
|
|
20
|
-
'webhooks',
|
|
21
|
-
'editorInterfaces'
|
|
22
|
-
];
|
|
23
|
-
async function parseOptions(params) {
|
|
24
|
-
const defaultOptions = {
|
|
25
|
-
skipContentModel: false,
|
|
26
|
-
skipLocales: false,
|
|
27
|
-
skipContentPublishing: false,
|
|
28
|
-
useVerboseRenderer: false,
|
|
29
|
-
environmentId: 'master',
|
|
30
|
-
rawProxy: false,
|
|
31
|
-
uploadAssets: false,
|
|
32
|
-
rateLimit: 7
|
|
33
|
-
};
|
|
34
|
-
const configFile = params.config
|
|
35
|
-
? require((0, path_1.resolve)(process.cwd(), params.config))
|
|
36
|
-
: {};
|
|
37
|
-
const options = {
|
|
38
|
-
...defaultOptions,
|
|
39
|
-
...configFile,
|
|
40
|
-
...params,
|
|
41
|
-
headers: (0, add_sequence_header_1.default)(params.headers || (0, headers_1.getHeadersConfig)(params.header))
|
|
42
|
-
};
|
|
43
|
-
// Validation
|
|
44
|
-
if (!options.spaceId) {
|
|
45
|
-
throw new Error('The `spaceId` option is required.');
|
|
46
|
-
}
|
|
47
|
-
if (!options.managementToken) {
|
|
48
|
-
throw new Error('The `managementToken` option is required.');
|
|
49
|
-
}
|
|
50
|
-
if (!options.contentFile && !options.content) {
|
|
51
|
-
throw new Error('Either the `contentFile` or `content` option are required.');
|
|
52
|
-
}
|
|
53
|
-
if (options.contentModelOnly && options.skipContentModel) {
|
|
54
|
-
throw new Error('`contentModelOnly` and `skipContentModel` cannot be used together');
|
|
55
|
-
}
|
|
56
|
-
if (options.skipLocales && !options.contentModelOnly) {
|
|
57
|
-
throw new Error('`skipLocales` can only be used together with `contentModelOnly`');
|
|
58
|
-
}
|
|
59
|
-
const proxySimpleExp = /.+:\d+/;
|
|
60
|
-
const proxyAuthExp = /.+:.+@.+:\d+/;
|
|
61
|
-
if (typeof options.proxy === 'string' && options.proxy && !(proxySimpleExp.test(options.proxy) || proxyAuthExp.test(options.proxy))) {
|
|
62
|
-
throw new Error('Please provide the proxy config in the following format:\nhost:port or user:password@host:port');
|
|
63
|
-
}
|
|
64
|
-
options.startTime = new Date();
|
|
65
|
-
if (!options.errorLogFile) {
|
|
66
|
-
options.errorLogFile = (0, path_1.resolve)(process.cwd(), `contentful-import-error-log-${options.spaceId}-${(0, format_1.default)(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json`);
|
|
67
|
-
}
|
|
68
|
-
else {
|
|
69
|
-
options.errorLogFile = (0, path_1.resolve)(process.cwd(), options.errorLogFile);
|
|
70
|
-
}
|
|
71
|
-
options.accessToken = options.managementToken;
|
|
72
|
-
if (!options.content) {
|
|
73
|
-
// using a stream parser allows input files > 512 MB
|
|
74
|
-
const fileStream = fs_1.default.createReadStream(options.contentFile, { encoding: 'utf8' });
|
|
75
|
-
options.content = await (0, json_ext_1.parseChunked)(fileStream);
|
|
76
|
-
}
|
|
77
|
-
// Clean up content to only include supported entity types
|
|
78
|
-
Object.keys(options.content).forEach((type) => {
|
|
79
|
-
if (SUPPORTED_ENTITY_TYPES.indexOf(type) === -1) {
|
|
80
|
-
delete options.content[type];
|
|
81
|
-
}
|
|
82
|
-
});
|
|
83
|
-
SUPPORTED_ENTITY_TYPES.forEach((type) => {
|
|
84
|
-
options.content[type] = options.content[type] || [];
|
|
85
|
-
});
|
|
86
|
-
if (typeof options.proxy === 'string') {
|
|
87
|
-
options.proxy = (0, proxy_1.proxyStringToObject)(options.proxy);
|
|
88
|
-
}
|
|
89
|
-
if (!options.rawProxy && options.proxy) {
|
|
90
|
-
options.httpsAgent = (0, proxy_1.agentFromProxy)(options.proxy);
|
|
91
|
-
delete options.proxy;
|
|
92
|
-
}
|
|
93
|
-
options.application = options.managementApplication || `contentful.import/${package_1.version}`;
|
|
94
|
-
options.feature = options.managementFeature || 'library-import';
|
|
95
|
-
return options;
|
|
96
|
-
}
|
|
97
|
-
exports.default = parseOptions;
|
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const bluebird_1 = __importDefault(require("bluebird"));
|
|
7
|
-
const logging_1 = require("contentful-batch-libs/dist/logging");
|
|
8
|
-
const BATCH_CHAR_LIMIT = 1990;
|
|
9
|
-
const BATCH_SIZE_LIMIT = 100;
|
|
10
|
-
const METHODS = {
|
|
11
|
-
contentTypes: { name: 'content types', method: 'getContentTypes' },
|
|
12
|
-
locales: { name: 'locales', method: 'getLocales' },
|
|
13
|
-
entries: { name: 'entries', method: 'getEntries' },
|
|
14
|
-
assets: { name: 'assets', method: 'getAssets' }
|
|
15
|
-
};
|
|
16
|
-
async function batchedIdQuery({ environment, type, ids, requestQueue }) {
|
|
17
|
-
const method = METHODS[type].method;
|
|
18
|
-
const entityTypeName = METHODS[type].name;
|
|
19
|
-
const batches = getIdBatches(ids);
|
|
20
|
-
let totalFetched = 0;
|
|
21
|
-
const allPendingResponses = batches.map((idBatch) => {
|
|
22
|
-
// TODO: add batch count to indicate that it's running
|
|
23
|
-
return requestQueue.add(async () => {
|
|
24
|
-
const response = await environment[method]({
|
|
25
|
-
'sys.id[in]': idBatch,
|
|
26
|
-
limit: idBatch.split(',').length
|
|
27
|
-
});
|
|
28
|
-
totalFetched = totalFetched + response.items.length;
|
|
29
|
-
logging_1.logEmitter.emit('info', `Fetched ${totalFetched} of ${response.total} ${entityTypeName}`);
|
|
30
|
-
return response.items;
|
|
31
|
-
});
|
|
32
|
-
});
|
|
33
|
-
const responses = await bluebird_1.default.all(allPendingResponses);
|
|
34
|
-
return responses.flat();
|
|
35
|
-
}
|
|
36
|
-
function getIdBatches(ids) {
|
|
37
|
-
const batches = [];
|
|
38
|
-
let currentBatch = '';
|
|
39
|
-
let currentSize = 0;
|
|
40
|
-
while (ids.length > 0) {
|
|
41
|
-
const id = ids.splice(0, 1);
|
|
42
|
-
currentBatch += id;
|
|
43
|
-
currentSize = currentSize + 1;
|
|
44
|
-
if (currentSize === BATCH_SIZE_LIMIT || currentBatch.length > BATCH_CHAR_LIMIT || ids.length === 0) {
|
|
45
|
-
batches.push(currentBatch);
|
|
46
|
-
currentBatch = '';
|
|
47
|
-
currentSize = 0;
|
|
48
|
-
}
|
|
49
|
-
else {
|
|
50
|
-
currentBatch += ',';
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
return batches;
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Gets content from a space which will have content copied to it, based on a
|
|
57
|
-
* collection of existing content.
|
|
58
|
-
*
|
|
59
|
-
* Only the supplied entry/asset/contentType/locale IDs will be retrieved.
|
|
60
|
-
* All tags will be retrieved.
|
|
61
|
-
*
|
|
62
|
-
*/
|
|
63
|
-
async function getDestinationData({ client, spaceId, environmentId, sourceData, contentModelOnly, skipLocales, skipContentModel, requestQueue }) {
|
|
64
|
-
const space = await client.getSpace(spaceId);
|
|
65
|
-
const environment = await space.getEnvironment(environmentId);
|
|
66
|
-
const result = {
|
|
67
|
-
contentTypes: [],
|
|
68
|
-
tags: [],
|
|
69
|
-
locales: [],
|
|
70
|
-
entries: [],
|
|
71
|
-
assets: []
|
|
72
|
-
};
|
|
73
|
-
// Make sure all required properties are available and at least an empty array
|
|
74
|
-
sourceData = {
|
|
75
|
-
...result,
|
|
76
|
-
...sourceData
|
|
77
|
-
};
|
|
78
|
-
if (!skipContentModel) {
|
|
79
|
-
const contentTypeIds = sourceData.contentTypes.map((e) => e.sys.id);
|
|
80
|
-
result.contentTypes = batchedIdQuery({
|
|
81
|
-
environment,
|
|
82
|
-
type: 'contentTypes',
|
|
83
|
-
ids: contentTypeIds,
|
|
84
|
-
requestQueue
|
|
85
|
-
});
|
|
86
|
-
if (!skipLocales) {
|
|
87
|
-
const localeIds = sourceData.locales.map((e) => e.sys.id);
|
|
88
|
-
result.locales = batchedIdQuery({
|
|
89
|
-
environment,
|
|
90
|
-
type: 'locales',
|
|
91
|
-
ids: localeIds,
|
|
92
|
-
requestQueue
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
// include tags even if contentModelOnly = true
|
|
97
|
-
result.tags = environment.getTags().then(response => response.items).catch((e) => {
|
|
98
|
-
// users without access to Tags will get 404
|
|
99
|
-
// if they dont have access, remove tags array so they're not handled in future steps
|
|
100
|
-
delete result.tags;
|
|
101
|
-
});
|
|
102
|
-
if (contentModelOnly) {
|
|
103
|
-
return bluebird_1.default.props(result);
|
|
104
|
-
}
|
|
105
|
-
const entryIds = sourceData.entries.map((e) => e.sys.id);
|
|
106
|
-
const assetIds = sourceData.assets.map((e) => e.sys.id);
|
|
107
|
-
result.entries = batchedIdQuery({
|
|
108
|
-
environment,
|
|
109
|
-
type: 'entries',
|
|
110
|
-
ids: entryIds,
|
|
111
|
-
requestQueue
|
|
112
|
-
});
|
|
113
|
-
result.assets = batchedIdQuery({
|
|
114
|
-
environment,
|
|
115
|
-
type: 'assets',
|
|
116
|
-
ids: assetIds,
|
|
117
|
-
requestQueue
|
|
118
|
-
});
|
|
119
|
-
result.webhooks = [];
|
|
120
|
-
return bluebird_1.default.props(result);
|
|
121
|
-
}
|
|
122
|
-
exports.default = getDestinationData;
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const contentful_management_1 = require("contentful-management");
|
|
4
|
-
const logging_1 = require("contentful-batch-libs/dist/logging");
|
|
5
|
-
function logHandler(level, data) {
|
|
6
|
-
logging_1.logEmitter.emit(level, data);
|
|
7
|
-
}
|
|
8
|
-
function initClient(opts) {
|
|
9
|
-
const defaultOpts = {
|
|
10
|
-
timeout: 30000,
|
|
11
|
-
logHandler
|
|
12
|
-
};
|
|
13
|
-
const config = {
|
|
14
|
-
...defaultOpts,
|
|
15
|
-
...opts
|
|
16
|
-
};
|
|
17
|
-
return (0, contentful_management_1.createClient)(config);
|
|
18
|
-
}
|
|
19
|
-
exports.default = initClient;
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.processAssets = exports.getAssetStreamForURL = void 0;
|
|
7
|
-
const fs_1 = __importDefault(require("fs"));
|
|
8
|
-
const path_1 = require("path");
|
|
9
|
-
const util_1 = require("util");
|
|
10
|
-
const get_entity_name_1 = __importDefault(require("contentful-batch-libs/dist/get-entity-name"));
|
|
11
|
-
const logging_1 = require("contentful-batch-libs/dist/logging");
|
|
12
|
-
const stat = (0, util_1.promisify)(fs_1.default.stat);
|
|
13
|
-
async function getAssetStreamForURL(url, assetsDirectory) {
|
|
14
|
-
const [, assetPath] = url.split('//');
|
|
15
|
-
const filePath = (0, path_1.join)(assetsDirectory, assetPath);
|
|
16
|
-
try {
|
|
17
|
-
await stat(filePath);
|
|
18
|
-
return fs_1.default.createReadStream(filePath);
|
|
19
|
-
}
|
|
20
|
-
catch (err) {
|
|
21
|
-
const error = new Error('Cannot open asset from filesystem');
|
|
22
|
-
error.filePath = filePath;
|
|
23
|
-
throw error;
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
exports.getAssetStreamForURL = getAssetStreamForURL;
|
|
27
|
-
async function processAssetForLocale(locale, asset, processingOptions) {
|
|
28
|
-
try {
|
|
29
|
-
return await asset.processForLocale(locale, processingOptions);
|
|
30
|
-
}
|
|
31
|
-
catch (err) {
|
|
32
|
-
err.entity = asset;
|
|
33
|
-
logging_1.logEmitter.emit('error', err);
|
|
34
|
-
throw err;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
// From
|
|
38
|
-
// https://stackoverflow.com/questions/67339630/how-to-get-last-resolved-promise-from-a-list-of-resolved-promises-in-javascript
|
|
39
|
-
async function lastResult(promises) {
|
|
40
|
-
if (!promises.length)
|
|
41
|
-
throw new RangeError('No last result from no promises');
|
|
42
|
-
const results = [];
|
|
43
|
-
await Promise.all(promises.map((p) => p.then((v) => {
|
|
44
|
-
results.push(v);
|
|
45
|
-
})));
|
|
46
|
-
return results[results.length - 1];
|
|
47
|
-
}
|
|
48
|
-
async function processAssets({ assets, timeout, retryLimit, requestQueue }) {
|
|
49
|
-
const processingOptions = Object.assign({}, timeout && { processingCheckWait: timeout }, retryLimit && { processingCheckRetry: retryLimit });
|
|
50
|
-
const pendingProcessingAssets = assets.map(async (asset) => {
|
|
51
|
-
logging_1.logEmitter.emit('info', `Processing Asset ${(0, get_entity_name_1.default)(asset)}`);
|
|
52
|
-
// We want to do what processForAllLocale does, but as the rate
|
|
53
|
-
// limit is only enforced if we have a dedicated requestQueue item
|
|
54
|
-
// for every processForLocale call, we need to duplicate the logic
|
|
55
|
-
// here
|
|
56
|
-
const locales = Object.keys(asset.fields.file || {});
|
|
57
|
-
let latestAssetVersion = asset;
|
|
58
|
-
try {
|
|
59
|
-
// The last resolved promise will return the most up to date asset
|
|
60
|
-
// version which we need for next import steps (e.g. publishing)
|
|
61
|
-
latestAssetVersion = await lastResult(locales.map((locale) => {
|
|
62
|
-
return requestQueue.add(() => processAssetForLocale(locale, asset, processingOptions));
|
|
63
|
-
}));
|
|
64
|
-
}
|
|
65
|
-
catch (err) {
|
|
66
|
-
// Handle any error that arises during the processing of any locale
|
|
67
|
-
return null;
|
|
68
|
-
}
|
|
69
|
-
return latestAssetVersion;
|
|
70
|
-
});
|
|
71
|
-
const potentiallyProcessedAssets = await Promise.all(pendingProcessingAssets);
|
|
72
|
-
// This filters out all process attempts which failed
|
|
73
|
-
return potentiallyProcessedAssets.filter((asset) => asset);
|
|
74
|
-
}
|
|
75
|
-
exports.processAssets = processAssets;
|
|
@@ -1,178 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.createEntries = exports.createLocales = exports.createEntities = void 0;
|
|
7
|
-
const collection_1 = require("lodash/collection");
|
|
8
|
-
const object_1 = require("lodash/object");
|
|
9
|
-
const get_entity_name_1 = __importDefault(require("contentful-batch-libs/dist/get-entity-name"));
|
|
10
|
-
const logging_1 = require("contentful-batch-libs/dist/logging");
|
|
11
|
-
/**
|
|
12
|
-
* Creates a list of entities
|
|
13
|
-
* Applies to all entities except Entries, as the CMA API for those is slightly different
|
|
14
|
-
* See handleCreationErrors for details on what errors reject the promise or not.
|
|
15
|
-
*/
|
|
16
|
-
function createEntities({ context, entities, destinationEntitiesById, requestQueue }) {
|
|
17
|
-
return createEntitiesWithConcurrency({ context, entities, destinationEntitiesById, requestQueue });
|
|
18
|
-
}
|
|
19
|
-
exports.createEntities = createEntities;
|
|
20
|
-
// TODO
|
|
21
|
-
// Locales need to be created in series
|
|
22
|
-
function createLocales({ context, entities, destinationEntitiesById, requestQueue }) {
|
|
23
|
-
return createEntitiesInSequence({ context, entities, destinationEntitiesById, requestQueue });
|
|
24
|
-
}
|
|
25
|
-
exports.createLocales = createLocales;
|
|
26
|
-
async function createEntitiesWithConcurrency({ context, entities, destinationEntitiesById, requestQueue }) {
|
|
27
|
-
const pendingCreatedEntities = entities.map((entity) => {
|
|
28
|
-
const destinationEntity = getDestinationEntityForSourceEntity(destinationEntitiesById, entity.transformed);
|
|
29
|
-
const operation = destinationEntity ? 'update' : 'create';
|
|
30
|
-
return requestQueue.add(async () => {
|
|
31
|
-
try {
|
|
32
|
-
const createdEntity = await (destinationEntity
|
|
33
|
-
? updateDestinationWithSourceData(destinationEntity, entity.transformed)
|
|
34
|
-
: createInDestination(context, entity.transformed));
|
|
35
|
-
creationSuccessNotifier(operation, createdEntity);
|
|
36
|
-
return createdEntity;
|
|
37
|
-
}
|
|
38
|
-
catch (err) {
|
|
39
|
-
return handleCreationErrors(entity, err);
|
|
40
|
-
}
|
|
41
|
-
});
|
|
42
|
-
});
|
|
43
|
-
const createdEntities = await Promise.all(pendingCreatedEntities);
|
|
44
|
-
// Filter null values in case of errors
|
|
45
|
-
return createdEntities.filter((entity) => entity);
|
|
46
|
-
}
|
|
47
|
-
async function createEntitiesInSequence({ context, entities, destinationEntitiesById, requestQueue }) {
|
|
48
|
-
const createdEntities = [];
|
|
49
|
-
for (const entity of entities) {
|
|
50
|
-
const destinationEntity = getDestinationEntityForSourceEntity(destinationEntitiesById, entity.transformed);
|
|
51
|
-
const operation = destinationEntity ? 'update' : 'create';
|
|
52
|
-
try {
|
|
53
|
-
// Even though we run things in sequence here,
|
|
54
|
-
// we still want to go through the normal rate limiting queue
|
|
55
|
-
const createdEntity = await requestQueue.add(async () => {
|
|
56
|
-
const createdOrUpdatedEntity = await (destinationEntity
|
|
57
|
-
? updateDestinationWithSourceData(destinationEntity, entity.transformed)
|
|
58
|
-
: createInDestination(context, entity.transformed));
|
|
59
|
-
return createdOrUpdatedEntity;
|
|
60
|
-
});
|
|
61
|
-
creationSuccessNotifier(operation, createdEntity);
|
|
62
|
-
createdEntities.push(createdEntity);
|
|
63
|
-
}
|
|
64
|
-
catch (err) {
|
|
65
|
-
const maybeSubstituteEntity = handleCreationErrors(entity, err);
|
|
66
|
-
if (maybeSubstituteEntity) {
|
|
67
|
-
createdEntities.push(maybeSubstituteEntity);
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
return createdEntities;
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* Creates a list of entries
|
|
75
|
-
*/
|
|
76
|
-
async function createEntries({ context, entities, destinationEntitiesById, requestQueue }) {
|
|
77
|
-
const createdEntries = await Promise.all(entities.map((entry) => {
|
|
78
|
-
return createEntry({ entry, target: context.target, skipContentModel: context.skipContentModel, destinationEntitiesById, requestQueue });
|
|
79
|
-
}));
|
|
80
|
-
return createdEntries.filter((entry) => entry);
|
|
81
|
-
}
|
|
82
|
-
exports.createEntries = createEntries;
|
|
83
|
-
async function createEntry({ entry, target, skipContentModel, destinationEntitiesById, requestQueue }) {
|
|
84
|
-
const contentTypeId = entry.original.sys.contentType.sys.id;
|
|
85
|
-
const destinationEntry = getDestinationEntityForSourceEntity(destinationEntitiesById, entry.transformed);
|
|
86
|
-
const operation = destinationEntry ? 'update' : 'create';
|
|
87
|
-
try {
|
|
88
|
-
const createdOrUpdatedEntry = await requestQueue.add(() => {
|
|
89
|
-
return (destinationEntry
|
|
90
|
-
? updateDestinationWithSourceData(destinationEntry, entry.transformed)
|
|
91
|
-
: createEntryInDestination(target, contentTypeId, entry.transformed));
|
|
92
|
-
});
|
|
93
|
-
creationSuccessNotifier(operation, createdOrUpdatedEntry);
|
|
94
|
-
return createdOrUpdatedEntry;
|
|
95
|
-
}
|
|
96
|
-
catch (err) {
|
|
97
|
-
/* If a field doesn't exist, it means it has been removed from the content types
|
|
98
|
-
* In that case, the field is removed from the entry, and creation is attempted again.
|
|
99
|
-
*/
|
|
100
|
-
if (skipContentModel && err.name === 'UnknownField') {
|
|
101
|
-
const errors = (0, object_1.get)(JSON.parse(err.message), 'details.errors');
|
|
102
|
-
entry.transformed.fields = cleanupUnknownFields(entry.transformed.fields, errors);
|
|
103
|
-
return createEntry({ entry, target, skipContentModel, destinationEntitiesById, requestQueue });
|
|
104
|
-
}
|
|
105
|
-
err.entity = entry;
|
|
106
|
-
logging_1.logEmitter.emit('error', err);
|
|
107
|
-
// No need to pass this entry down to publishing if it wasn't created
|
|
108
|
-
return null;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
function updateDestinationWithSourceData(destinationEntity, sourceEntity) {
|
|
112
|
-
const plainData = getPlainData(sourceEntity);
|
|
113
|
-
(0, object_1.assign)(destinationEntity, plainData);
|
|
114
|
-
return destinationEntity.update();
|
|
115
|
-
}
|
|
116
|
-
function createInDestination(context, sourceEntity) {
|
|
117
|
-
const { type, target } = context;
|
|
118
|
-
if (type === 'Tag') {
|
|
119
|
-
// tags are created with a different signature
|
|
120
|
-
return createTagInDestination(context, sourceEntity);
|
|
121
|
-
}
|
|
122
|
-
const id = (0, object_1.get)(sourceEntity, 'sys.id');
|
|
123
|
-
const plainData = getPlainData(sourceEntity);
|
|
124
|
-
return id
|
|
125
|
-
? target[`create${type}WithId`](id, plainData)
|
|
126
|
-
: target[`create${type}`](plainData);
|
|
127
|
-
}
|
|
128
|
-
function createEntryInDestination(space, contentTypeId, sourceEntity) {
|
|
129
|
-
const id = sourceEntity.sys.id;
|
|
130
|
-
const plainData = getPlainData(sourceEntity);
|
|
131
|
-
return id
|
|
132
|
-
? space.createEntryWithId(contentTypeId, id, plainData)
|
|
133
|
-
: space.createEntry(contentTypeId, plainData);
|
|
134
|
-
}
|
|
135
|
-
function createTagInDestination(context, sourceEntity) {
|
|
136
|
-
const id = sourceEntity.sys.id;
|
|
137
|
-
const visibility = sourceEntity.sys.visibility || 'private';
|
|
138
|
-
const name = sourceEntity.name;
|
|
139
|
-
return context.target.createTag(id, name, visibility);
|
|
140
|
-
}
|
|
141
|
-
/**
|
|
142
|
-
* Handles entity creation errors.
|
|
143
|
-
* If the error is a VersionMismatch the error is thrown and a message is returned
|
|
144
|
-
* instructing the user on what this situation probably means.
|
|
145
|
-
*/
|
|
146
|
-
function handleCreationErrors(entity, err) {
|
|
147
|
-
// Handle the case where a locale already exists and skip it
|
|
148
|
-
if ((0, object_1.get)(err, 'error.sys.id') === 'ValidationFailed') {
|
|
149
|
-
const errors = (0, object_1.get)(err, 'error.details.errors');
|
|
150
|
-
if (errors && errors.length > 0 && errors[0].name === 'taken') {
|
|
151
|
-
return entity;
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
err.entity = entity.original;
|
|
155
|
-
logging_1.logEmitter.emit('error', err);
|
|
156
|
-
// No need to pass this entity down to publishing if it wasn't created
|
|
157
|
-
return null;
|
|
158
|
-
}
|
|
159
|
-
function cleanupUnknownFields(fields, errors) {
|
|
160
|
-
return (0, object_1.omitBy)(fields, (field, fieldId) => {
|
|
161
|
-
return (0, collection_1.find)(errors, (error) => {
|
|
162
|
-
const [, errorFieldId] = error.path;
|
|
163
|
-
return error.name === 'unknown' && errorFieldId === fieldId;
|
|
164
|
-
});
|
|
165
|
-
});
|
|
166
|
-
}
|
|
167
|
-
function getDestinationEntityForSourceEntity(destinationEntitiesById, sourceEntity) {
|
|
168
|
-
return destinationEntitiesById.get((0, object_1.get)(sourceEntity, 'sys.id')) || null;
|
|
169
|
-
}
|
|
170
|
-
function creationSuccessNotifier(method, createdEntity) {
|
|
171
|
-
const verb = method[0].toUpperCase() + method.substr(1, method.length) + 'd';
|
|
172
|
-
logging_1.logEmitter.emit('info', `${verb} ${createdEntity.sys.type} ${(0, get_entity_name_1.default)(createdEntity)}`);
|
|
173
|
-
return createdEntity;
|
|
174
|
-
}
|
|
175
|
-
function getPlainData(entity) {
|
|
176
|
-
const data = entity.toPlainObject ? entity.toPlainObject() : entity;
|
|
177
|
-
return (0, object_1.omit)(data, 'sys');
|
|
178
|
-
}
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.archiveEntities = exports.publishEntities = void 0;
|
|
7
|
-
const get_entity_name_1 = __importDefault(require("contentful-batch-libs/dist/get-entity-name"));
|
|
8
|
-
const logging_1 = require("contentful-batch-libs/dist/logging");
|
|
9
|
-
/**
|
|
10
|
-
* Publish a list of entities.
|
|
11
|
-
* Does not return a rejected promise in the case of an error, pushing it
|
|
12
|
-
* to an error buffer instead.
|
|
13
|
-
*/
|
|
14
|
-
async function publishEntities({ entities, requestQueue }) {
|
|
15
|
-
const entitiesToPublish = entities.filter((entity) => {
|
|
16
|
-
if (!entity || !entity.publish) {
|
|
17
|
-
logging_1.logEmitter.emit('warning', `Unable to publish ${(0, get_entity_name_1.default)(entity)}`);
|
|
18
|
-
return false;
|
|
19
|
-
}
|
|
20
|
-
return true;
|
|
21
|
-
});
|
|
22
|
-
if (entitiesToPublish.length === 0) {
|
|
23
|
-
logging_1.logEmitter.emit('info', 'Skipping publishing since zero valid entities passed');
|
|
24
|
-
return [];
|
|
25
|
-
}
|
|
26
|
-
const entity = entities[0].original || entities[0];
|
|
27
|
-
const type = entity.sys.type || 'unknown type';
|
|
28
|
-
logging_1.logEmitter.emit('info', `Publishing ${entities.length} ${type}s`);
|
|
29
|
-
const result = await runQueue(entitiesToPublish, [], requestQueue);
|
|
30
|
-
logging_1.logEmitter.emit('info', `Successfully published ${result.length} ${type}s`);
|
|
31
|
-
return result;
|
|
32
|
-
}
|
|
33
|
-
exports.publishEntities = publishEntities;
|
|
34
|
-
async function archiveEntities({ entities, requestQueue }) {
|
|
35
|
-
const entitiesToArchive = entities.filter((entity) => {
|
|
36
|
-
if (!entity || !entity.archive) {
|
|
37
|
-
logging_1.logEmitter.emit('warning', `Unable to archive ${(0, get_entity_name_1.default)(entity)}`);
|
|
38
|
-
return false;
|
|
39
|
-
}
|
|
40
|
-
return true;
|
|
41
|
-
});
|
|
42
|
-
if (entitiesToArchive.length === 0) {
|
|
43
|
-
logging_1.logEmitter.emit('info', 'Skipping archiving since zero valid entities passed');
|
|
44
|
-
return [];
|
|
45
|
-
}
|
|
46
|
-
const entity = entities[0].original || entities[0];
|
|
47
|
-
const type = entity.sys.type || 'unknown type';
|
|
48
|
-
logging_1.logEmitter.emit('info', `Archiving ${entities.length} ${type}s`);
|
|
49
|
-
const pendingArchivedEntities = entitiesToArchive.map((entity) => {
|
|
50
|
-
return requestQueue.add(async () => {
|
|
51
|
-
try {
|
|
52
|
-
const archivedEntity = await entity.archive();
|
|
53
|
-
return archivedEntity;
|
|
54
|
-
}
|
|
55
|
-
catch (err) {
|
|
56
|
-
err.entity = entity;
|
|
57
|
-
logging_1.logEmitter.emit('error', err);
|
|
58
|
-
return null;
|
|
59
|
-
}
|
|
60
|
-
});
|
|
61
|
-
});
|
|
62
|
-
const allPossiblyArchivedEntities = await Promise.all(pendingArchivedEntities);
|
|
63
|
-
const allArchivedEntities = allPossiblyArchivedEntities.filter((entity) => entity);
|
|
64
|
-
logging_1.logEmitter.emit('info', `Successfully archived ${allArchivedEntities.length} ${type}s`);
|
|
65
|
-
return allArchivedEntities;
|
|
66
|
-
}
|
|
67
|
-
exports.archiveEntities = archiveEntities;
|
|
68
|
-
async function runQueue(queue, result = [], requestQueue) {
|
|
69
|
-
const publishedEntities = [];
|
|
70
|
-
for (const entity of queue) {
|
|
71
|
-
logging_1.logEmitter.emit('info', `Publishing ${entity.sys.type} ${(0, get_entity_name_1.default)(entity)}`);
|
|
72
|
-
try {
|
|
73
|
-
const publishedEntity = await requestQueue.add(() => entity.publish());
|
|
74
|
-
publishedEntities.push(publishedEntity);
|
|
75
|
-
}
|
|
76
|
-
catch (err) {
|
|
77
|
-
err.entity = entity;
|
|
78
|
-
logging_1.logEmitter.emit('error', err);
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
result = [
|
|
82
|
-
...result,
|
|
83
|
-
...publishedEntities
|
|
84
|
-
];
|
|
85
|
-
const publishedEntityIds = new Set(publishedEntities.map((entity) => entity.sys.id));
|
|
86
|
-
const unpublishedEntities = queue.filter((entity) => !publishedEntityIds.has(entity.sys.id));
|
|
87
|
-
if (unpublishedEntities.length > 0) {
|
|
88
|
-
if (queue.length === unpublishedEntities.length) {
|
|
89
|
-
// Fail when queue could not publish at least one item
|
|
90
|
-
const unpublishedEntityNames = unpublishedEntities.map(get_entity_name_1.default).join(', ');
|
|
91
|
-
logging_1.logEmitter.emit('error', `Could not publish the following entities: ${unpublishedEntityNames}`);
|
|
92
|
-
}
|
|
93
|
-
else {
|
|
94
|
-
// Rerun queue with unpublished entities
|
|
95
|
-
return runQueue(unpublishedEntities, result, requestQueue);
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
// Return only published entities + last result
|
|
99
|
-
return result;
|
|
100
|
-
}
|