contentful-import 9.0.21 → 9.2.0

package/package.json

@@ -1,22 +1,30 @@
 {
   "name": "contentful-import",
-  "version": "9.0.21",
+  "version": "9.2.0",
   "description": "this tool allows you to import JSON dump exported by contentful-export",
-  "main": "dist/index.js",
+  "main": "dist/index.mjs",
+  "typings": "dist/index.d.ts",
+  "module": "dist/index.mjs",
   "engines": {
     "node": ">=16"
   },
+  "exports": {
+    "default": "./dist/index.js",
+    "require": "./dist/index.js",
+    "import": "./dist/index.mjs",
+    "node": "./dist/index.js"
+  },
   "bin": {
     "contentful-import": "./bin/contentful-import"
   },
   "scripts": {
-    "build": "npm run clean && babel lib --out-dir dist",
+    "build": "tsup",
     "build:watch": "babel lib --out-dir dist --watch",
     "clean": "rimraf dist && rimraf coverage",
     "lint": "eslint lib bin/contentful-import test",
     "pretest": "npm run lint && npm run build",
     "test": "npm run test:unit && npm run test:integration",
-    "test:unit": "jest --testPathPattern=test/unit --coverage",
+    "test:unit": "jest --testPathPattern=test/unit --runInBand --coverage",
     "test:unit:debug": "node --inspect-brk ./node_modules/.bin/jest --runInBand --watch --testPathPattern=test/unit",
     "test:unit:watch": "npm run test:unit -- --watch",
     "test:integration": "jest --testPathPattern=test/integration",
@@ -49,8 +57,18 @@
   "bugs": {
     "url": "https://github.com/contentful/contentful-import/issues"
   },
+  "tsup": {
+    "entry": [
+      "lib/index.ts"
+    ],
+    "format": [
+      "cjs",
+      "esm"
+    ],
+    "clean": true,
+    "dts": true
+  },
   "dependencies": {
-    "@babel/eslint-parser": "^7.22.15",
     "@discoveryjs/json-ext": "^0.5.7",
     "bluebird": "^3.5.1",
     "cli-table3": "^0.6.0",
@@ -68,10 +86,14 @@
     "yargs": "^17.7.2"
   },
   "devDependencies": {
-    "@babel/cli": "^7.22.15",
-    "@babel/core": "^7.22.20",
+    "@babel/cli": "^7.23.0",
+    "@babel/core": "^7.23.0",
     "@babel/plugin-proposal-object-rest-spread": "^7.20.7",
     "@babel/preset-env": "^7.22.20",
+    "@types/jest": "^29.5.5",
+    "@types/node": "^20.6.3",
+    "@typescript-eslint/eslint-plugin": "^6.7.2",
+    "@typescript-eslint/parser": "^6.7.2",
     "babel-jest": "^29.7.0",
     "babel-plugin-add-module-exports": "^1.0.2",
     "babel-preset-env": "^1.7.0",
@@ -80,10 +102,14 @@
     "eslint-plugin-jest": "^27.4.0",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-standard": "^5.0.0",
     "husky": "^8.0.3",
     "jest": "^29.7.0",
     "rimraf": "^5.0.0",
-    "semantic-release": "^19.0.5"
+    "semantic-release": "^19.0.5",
+    "ts-jest": "^29.1.1",
+    "tsup": "^7.2.0",
+    "typescript": "^5.2.2"
   },
   "files": [
     "bin",
@@ -113,17 +139,5 @@
       "@semantic-release/npm",
       "@semantic-release/github"
     ]
-  },
-  "jest": {
-    "testEnvironment": "node",
-    "collectCoverageFrom": [
-      "lib/**/*.js"
-    ],
-    "coveragePathIgnorePatterns": [
-      "usageParams.js"
-    ],
-    "transformIgnorePatterns": [
-      "node_modules/(?!(contentful-batch-libs)/)"
-    ]
   }
 }

package/dist/parseOptions.js

@@ -1,94 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = parseOptions;
-var _fs = _interopRequireDefault(require("fs"));
-var _path = require("path");
-var _format = _interopRequireDefault(require("date-fns/format"));
-var _package = require("../package");
-var _headers = require("./utils/headers");
-var _proxy = require("contentful-batch-libs/dist/proxy");
-var _addSequenceHeader = _interopRequireDefault(require("contentful-batch-libs/dist/add-sequence-header"));
-var _jsonExt = require("@discoveryjs/json-ext");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
-function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
-function _defineProperty(obj, key, value) { key = _toPropertyKey(key); if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-function _toPropertyKey(arg) { var key = _toPrimitive(arg, "string"); return typeof key === "symbol" ? key : String(key); }
-function _toPrimitive(input, hint) { if (typeof input !== "object" || input === null) return input; var prim = input[Symbol.toPrimitive]; if (prim !== undefined) { var res = prim.call(input, hint || "default"); if (typeof res !== "object") return res; throw new TypeError("@@toPrimitive must return a primitive value."); } return (hint === "string" ? String : Number)(input); }
-const SUPPORTED_ENTITY_TYPES = ['contentTypes', 'tags', 'entries', 'assets', 'locales', 'webhooks', 'editorInterfaces'];
-async function parseOptions(params) {
-  const defaultOptions = {
-    skipContentModel: false,
-    skipLocales: false,
-    skipContentPublishing: false,
-    useVerboseRenderer: false,
-    environmentId: 'master',
-    rawProxy: false,
-    uploadAssets: false,
-    rateLimit: 7
-  };
-  const configFile = params.config ? require((0, _path.resolve)(process.cwd(), params.config)) : {};
-  const options = _objectSpread(_objectSpread(_objectSpread(_objectSpread({}, defaultOptions), configFile), params), {}, {
-    headers: (0, _addSequenceHeader.default)(params.headers || (0, _headers.getHeadersConfig)(params.header))
-  });
-
-  // Validation
-  if (!options.spaceId) {
-    throw new Error('The `spaceId` option is required.');
-  }
-  if (!options.managementToken) {
-    throw new Error('The `managementToken` option is required.');
-  }
-  if (!options.contentFile && !options.content) {
-    throw new Error('Either the `contentFile` or `content` option are required.');
-  }
-  if (options.contentModelOnly && options.skipContentModel) {
-    throw new Error('`contentModelOnly` and `skipContentModel` cannot be used together');
-  }
-  if (options.skipLocales && !options.contentModelOnly) {
-    throw new Error('`skipLocales` can only be used together with `contentModelOnly`');
-  }
-  const proxySimpleExp = /.+:\d+/;
-  const proxyAuthExp = /.+:.+@.+:\d+/;
-  if (typeof options.proxy === 'string' && options.proxy && !(proxySimpleExp.test(options.proxy) || proxyAuthExp.test(options.proxy))) {
-    throw new Error('Please provide the proxy config in the following format:\nhost:port or user:password@host:port');
-  }
-  options.startTime = new Date();
-  if (!options.errorLogFile) {
-    options.errorLogFile = (0, _path.resolve)(process.cwd(), `contentful-import-error-log-${options.spaceId}-${(0, _format.default)(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json`);
-  } else {
-    options.errorLogFile = (0, _path.resolve)(process.cwd(), options.errorLogFile);
-  }
-  options.accessToken = options.managementToken;
-  if (!options.content) {
-    // using a stream parser allows input files > 512 MB
-    const fileStream = _fs.default.createReadStream(options.contentFile, {
-      encoding: 'utf8'
-    });
-    options.content = await (0, _jsonExt.parseChunked)(fileStream);
-  }
-
-  // Clean up content to only include supported entity types
-  Object.keys(options.content).forEach(type => {
-    if (SUPPORTED_ENTITY_TYPES.indexOf(type) === -1) {
-      delete options.content[type];
-    }
-  });
-  SUPPORTED_ENTITY_TYPES.forEach(type => {
-    options.content[type] = options.content[type] || [];
-  });
-  if (typeof options.proxy === 'string') {
-    options.proxy = (0, _proxy.proxyStringToObject)(options.proxy);
-  }
-  if (!options.rawProxy && options.proxy) {
-    options.httpsAgent = (0, _proxy.agentFromProxy)(options.proxy);
-    delete options.proxy;
-  }
-  options.application = options.managementApplication || `contentful.import/${_package.version}`;
-  options.feature = options.managementFeature || 'library-import';
-  return options;
-}
-module.exports = exports.default;

package/dist/tasks/get-destination-data.js

@@ -1,155 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = getDestinationData;
-var _bluebird = _interopRequireDefault(require("bluebird"));
-var _logging = require("contentful-batch-libs/dist/logging");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
-function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
-function _defineProperty(obj, key, value) { key = _toPropertyKey(key); if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-function _toPropertyKey(arg) { var key = _toPrimitive(arg, "string"); return typeof key === "symbol" ? key : String(key); }
-function _toPrimitive(input, hint) { if (typeof input !== "object" || input === null) return input; var prim = input[Symbol.toPrimitive]; if (prim !== undefined) { var res = prim.call(input, hint || "default"); if (typeof res !== "object") return res; throw new TypeError("@@toPrimitive must return a primitive value."); } return (hint === "string" ? String : Number)(input); }
-const BATCH_CHAR_LIMIT = 1990;
-const BATCH_SIZE_LIMIT = 100;
-const METHODS = {
-  contentTypes: {
-    name: 'content types',
-    method: 'getContentTypes'
-  },
-  locales: {
-    name: 'locales',
-    method: 'getLocales'
-  },
-  entries: {
-    name: 'entries',
-    method: 'getEntries'
-  },
-  assets: {
-    name: 'assets',
-    method: 'getAssets'
-  }
-};
-async function batchedIdQuery({
-  environment,
-  type,
-  ids,
-  requestQueue
-}) {
-  const method = METHODS[type].method;
-  const entityTypeName = METHODS[type].name;
-  const batches = getIdBatches(ids);
-  let totalFetched = 0;
-  const allPendingResponses = batches.map(idBatch => {
-    // TODO: add batch count to indicate that it's running
-    return requestQueue.add(async () => {
-      const response = await environment[method]({
-        'sys.id[in]': idBatch,
-        limit: idBatch.split(',').length
-      });
-      totalFetched = totalFetched + response.items.length;
-      _logging.logEmitter.emit('info', `Fetched ${totalFetched} of ${response.total} ${entityTypeName}`);
-      return response.items;
-    });
-  });
-  const responses = await _bluebird.default.all(allPendingResponses);
-  return responses.flat();
-}
-function getIdBatches(ids) {
-  const batches = [];
-  let currentBatch = '';
-  let currentSize = 0;
-  while (ids.length > 0) {
-    const id = ids.splice(0, 1);
-    currentBatch += id;
-    currentSize = currentSize + 1;
-    if (currentSize === BATCH_SIZE_LIMIT || currentBatch.length > BATCH_CHAR_LIMIT || ids.length === 0) {
-      batches.push(currentBatch);
-      currentBatch = '';
-      currentSize = 0;
-    } else {
-      currentBatch += ',';
-    }
-  }
-  return batches;
-}
-
-/**
- * Gets content from a space which will have content copied to it, based on a
- * collection of existing content.
- *
- * Only the supplied entry/asset/contentType/locale IDs will be retrieved.
- * All tags will be retrieved.
- *
- */
-
-async function getDestinationData({
-  client,
-  spaceId,
-  environmentId,
-  sourceData,
-  contentModelOnly,
-  skipLocales,
-  skipContentModel,
-  requestQueue
-}) {
-  const space = await client.getSpace(spaceId);
-  const environment = await space.getEnvironment(environmentId);
-  const result = {
-    contentTypes: [],
-    tags: [],
-    locales: [],
-    entries: [],
-    assets: []
-  };
-
-  // Make sure all required properties are available and at least an empty array
-  sourceData = _objectSpread(_objectSpread({}, result), sourceData);
-  if (!skipContentModel) {
-    const contentTypeIds = sourceData.contentTypes.map(e => e.sys.id);
-    result.contentTypes = batchedIdQuery({
-      environment,
-      type: 'contentTypes',
-      ids: contentTypeIds,
-      requestQueue
-    });
-    if (!skipLocales) {
-      const localeIds = sourceData.locales.map(e => e.sys.id);
-      result.locales = batchedIdQuery({
-        environment,
-        type: 'locales',
-        ids: localeIds,
-        requestQueue
-      });
-    }
-  }
-
-  // include tags even if contentModelOnly = true
-  result.tags = environment.getTags().then(response => response.items).catch(e => {
-    // users without access to Tags will get 404
-    // if they dont have access, remove tags array so they're not handled in future steps
-    delete result.tags;
-  });
-  if (contentModelOnly) {
-    return _bluebird.default.props(result);
-  }
-  const entryIds = sourceData.entries.map(e => e.sys.id);
-  const assetIds = sourceData.assets.map(e => e.sys.id);
-  result.entries = batchedIdQuery({
-    environment,
-    type: 'entries',
-    ids: entryIds,
-    requestQueue
-  });
-  result.assets = batchedIdQuery({
-    environment,
-    type: 'assets',
-    ids: assetIds,
-    requestQueue
-  });
-  result.webhooks = [];
-  return _bluebird.default.props(result);
-}
-module.exports = exports.default;

package/dist/tasks/init-client.js

@@ -1,25 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = initClient;
-var _contentfulManagement = require("contentful-management");
-var _logging = require("contentful-batch-libs/dist/logging");
-function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
-function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
-function _defineProperty(obj, key, value) { key = _toPropertyKey(key); if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-function _toPropertyKey(arg) { var key = _toPrimitive(arg, "string"); return typeof key === "symbol" ? key : String(key); }
-function _toPrimitive(input, hint) { if (typeof input !== "object" || input === null) return input; var prim = input[Symbol.toPrimitive]; if (prim !== undefined) { var res = prim.call(input, hint || "default"); if (typeof res !== "object") return res; throw new TypeError("@@toPrimitive must return a primitive value."); } return (hint === "string" ? String : Number)(input); }
-function logHandler(level, data) {
-  _logging.logEmitter.emit(level, data);
-}
-function initClient(opts) {
-  const defaultOpts = {
-    timeout: 30000,
-    logHandler
-  };
-  const config = _objectSpread(_objectSpread({}, defaultOpts), opts);
-  return (0, _contentfulManagement.createClient)(config);
-}
-module.exports = exports.default;

package/dist/tasks/push-to-space/assets.js

@@ -1,83 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.getAssetStreamForURL = getAssetStreamForURL;
-exports.processAssets = processAssets;
-var _fs = _interopRequireDefault(require("fs"));
-var _path = require("path");
-var _util = require("util");
-var _getEntityName = _interopRequireDefault(require("contentful-batch-libs/dist/get-entity-name"));
-var _logging = require("contentful-batch-libs/dist/logging");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-const stat = (0, _util.promisify)(_fs.default.stat);
-async function getAssetStreamForURL(url, assetsDirectory) {
-  const [, assetPath] = url.split('//');
-  const filePath = (0, _path.join)(assetsDirectory, assetPath);
-  try {
-    await stat(filePath);
-    return _fs.default.createReadStream(filePath);
-  } catch (err) {
-    const error = new Error('Cannot open asset from filesystem');
-    error.filePath = filePath;
-    throw error;
-  }
-}
-async function processAssetForLocale(locale, asset, processingOptions) {
-  try {
-    return await asset.processForLocale(locale, processingOptions);
-  } catch (err) {
-    err.entity = asset;
-    _logging.logEmitter.emit('error', err);
-    throw err;
-  }
-}
-
-// From
-// https://stackoverflow.com/questions/67339630/how-to-get-last-resolved-promise-from-a-list-of-resolved-promises-in-javascript
-async function lastResult(promises) {
-  if (!promises.length) throw new RangeError('No last result from no promises');
-  const results = [];
-  await Promise.all(promises.map(p => p.then(v => {
-    results.push(v);
-  })));
-  return results[results.length - 1];
-}
-async function processAssets({
-  assets,
-  timeout,
-  retryLimit,
-  requestQueue
-}) {
-  const processingOptions = Object.assign({}, timeout && {
-    processingCheckWait: timeout
-  }, retryLimit && {
-    processingCheckRetry: retryLimit
-  });
-  const pendingProcessingAssets = assets.map(async asset => {
-    _logging.logEmitter.emit('info', `Processing Asset ${(0, _getEntityName.default)(asset)}`);
-
-    // We want to do what processForAllLocale does, but as the rate
-    // limit is only enforced if we have a dedicated requestQueue item
-    // for every processForLocale call, we need to duplicate the logic
-    // here
-    const locales = Object.keys(asset.fields.file || {});
-    let latestAssetVersion = asset;
-    try {
-      // The last resolved promise will return the most up to date asset
-      // version which we need for next import steps (e.g. publishing)
-      latestAssetVersion = await lastResult(locales.map(locale => {
-        return requestQueue.add(() => processAssetForLocale(locale, asset, processingOptions));
-      }));
-    } catch (err) {
-      // Handle any error that arises during the processing of any locale
-      return null;
-    }
-    return latestAssetVersion;
-  });
-  const potentiallyProcessedAssets = await Promise.all(pendingProcessingAssets);
-
-  // This filters out all process attempts which failed
-  return potentiallyProcessedAssets.filter(asset => asset);
-}