@contentstack/datasync-manager 1.2.3 → 2.0.0

package/.github/workflows/jira.yml

@@ -25,3 +25,4 @@ jobs:
           description: |
             PR: ${{ github.event.pull_request.html_url }}
 
+          fields: "${{ secrets.JIRA_FIELDS }}"

package/dist/api.js

@@ -8,36 +8,40 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.get = exports.init = void 0;
 const debug_1 = __importDefault(require("debug"));
 const https_1 = require("https");
 const path_1 = require("path");
 const querystring_1 = require("querystring");
+const sanitize_url_1 = require("@braintree/sanitize-url");
 const fs_1 = require("./util/fs");
-const debug = debug_1.default('api');
+const debug = (0, debug_1.default)('api');
 let MAX_RETRY_LIMIT;
 let Contentstack;
 /**
  * @description Initialize sync utilities API requests
  * @param {Object} contentstack - Contentstack configuration details
  */
-exports.init = (contentstack) => {
-    const packageInfo = JSON.parse(fs_1.readFileSync(path_1.join(__dirname, '..', 'package.json')));
+const init = (contentstack) => {
+    const packageInfo = JSON.parse((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '..', 'package.json')));
     Contentstack = contentstack;
     Contentstack.headers = {
         'X-User-Agent': `datasync-manager/v${packageInfo.version}`,
         'access_token': Contentstack.deliveryToken,
         'api_key': Contentstack.apiKey,
+        branch: Contentstack.branch,
     };
     if (Contentstack.MAX_RETRY_LIMIT) {
         MAX_RETRY_LIMIT = Contentstack.MAX_RETRY_LIMIT;
     }
 };
+exports.init = init;
 /**
  * @description Make API requests to Contentstack
  * @param {Object} req - API request object
  * @param {Number} RETRY - API request retry counter
  */
-exports.get = (req, RETRY = 1) => {
+const get = (req, RETRY = 1) => {
     return new Promise((resolve, reject) => {
         if (RETRY > MAX_RETRY_LIMIT) {
             return reject(new Error('Max retry limit exceeded!'));
@@ -45,13 +49,13 @@ exports.get = (req, RETRY = 1) => {
         req.method = Contentstack.verbs.get;
         req.path = req.path || Contentstack.apis.sync;
         if (req.qs) {
-            req.path += `?${querystring_1.stringify(req.qs)}`;
+            req.path += `?${(0, querystring_1.stringify)(req.qs)}`;
         }
         const options = {
             headers: Contentstack.headers,
             hostname: Contentstack.host,
             method: Contentstack.verbs.get,
-            path: req.path,
+            path: (0, sanitize_url_1.sanitizeUrl)(req.path),
             port: Contentstack.port,
             protocol: Contentstack.protocol,
         };
@@ -59,7 +63,7 @@ exports.get = (req, RETRY = 1) => {
             debug(`${options.method.toUpperCase()}: ${options.path}`);
             let timeDelay;
             let body = '';
-            https_1.request(options, (response) => {
+            (0, https_1.request)(options, (response) => {
                 response
                     .setEncoding('utf-8')
                     .on('data', (chunk) => body += chunk)
@@ -72,7 +76,7 @@ exports.get = (req, RETRY = 1) => {
                         timeDelay = Math.pow(Math.SQRT2, RETRY) * 200;
                         debug(`API rate limit exceeded. Retrying ${options.path} with ${timeDelay} sec delay`);
                         return setTimeout(() => {
-                            return exports.get(req, RETRY)
+                            return (0, exports.get)(req, RETRY)
                                 .then(resolve)
                                 .catch(reject);
                         }, timeDelay);
@@ -83,7 +87,7 @@ exports.get = (req, RETRY = 1) => {
                         debug(`Retrying ${options.path} with ${timeDelay} sec delay`);
                         RETRY++;
                         return setTimeout(() => {
-                            return exports.get(req, RETRY)
+                            return (0, exports.get)(req, RETRY)
                                 .then(resolve)
                                 .catch(reject);
                         }, timeDelay);
@@ -102,3 +106,4 @@ exports.get = (req, RETRY = 1) => {
         }
     });
 };
+exports.get = get;

package/dist/config.js

@@ -5,6 +5,7 @@
 * MIT Licensed
 */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.config = void 0;
 /**
  * @description Default application's internal config
  */
@@ -85,6 +86,7 @@ exports.config = {
             options: {
                 logAssetPaths: true,
                 logReferencePaths: true,
+                // other overrides...
             },
         },
         {

package/dist/core/index.js

@@ -4,10 +4,20 @@
  * Copyright (c) 2019 Contentstack LLC
  * MIT Licensed
  */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.unlock = exports.lock = exports.poke = exports.pop = exports.unshift = exports.push = exports.init = void 0;
 const debug_1 = __importDefault(require("debug"));
 const events_1 = require("events");
 const lodash_1 = require("lodash");
@@ -20,7 +30,7 @@ const promise_map_1 = require("../util/promise.map");
 const inet_1 = require("./inet");
 const q_1 = require("./q");
 const token_management_1 = require("./token-management");
-const debug = debug_1.default('sync-core');
+const debug = (0, debug_1.default)('sync-core');
 const emitter = new events_1.EventEmitter();
 const formattedAssetType = '_assets';
 const formattedContentType = '_content_types';
@@ -37,10 +47,10 @@ let Q;
  * @param {Object} connector - Content connector instance
  * @param {Object} config - Application config
  */
-exports.init = (contentStore, assetStore) => {
-    config = __1.getConfig();
+const init = (contentStore, assetStore) => {
+    config = (0, __1.getConfig)();
     Q = new q_1.Q(contentStore, assetStore, config);
-    api_1.init(config.contentstack);
+    (0, api_1.init)(config.contentstack);
     debug('Sync core:start invoked');
     return new Promise((resolve, reject) => {
         try {
@@ -60,8 +70,8 @@ exports.init = (contentStore, assetStore) => {
             else if (typeof Contentstack.pagination_token === 'string' && Contentstack.pagination_token.length !== 0) {
                 request.qs.pagination_token = Contentstack.pagination_token;
             }
-            else if (fs_1.existsSync(paths.token)) {
-                const token = JSON.parse(fs_1.readFileSync(paths.token));
+            else if ((0, fs_1.existsSync)(paths.token)) {
+                const token = JSON.parse((0, fs_1.readFileSync)(paths.token));
                 request.qs[token.name] = token.token;
             }
             else {
@@ -83,78 +93,102 @@ exports.init = (contentStore, assetStore) => {
         }
     });
 };
-exports.push = (data) => {
+exports.init = init;
+const push = (data) => {
     Q.emit('push', data);
 };
-exports.unshift = (data) => {
+exports.push = push;
+const unshift = (data) => {
     Q.emit('push', data);
 };
-exports.pop = () => {
+exports.unshift = unshift;
+const pop = () => {
     Q.emit('pop');
 };
+exports.pop = pop;
 /**
  * @description Notifies the sync manager utility to wake up and start syncing..
  */
-exports.poke = () => {
-    logger_1.logger.info('Received \'contentstack sync\' notification');
-    if (!flag.lockdown) {
-        flag.WQ = true;
-        check();
+const poke = () => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        debug('Invoked poke');
+        logger_1.logger.info('Received \'contentstack sync\' notification');
+        if (!flag.lockdown) {
+            flag.WQ = true;
+            return yield check();
+        }
+        return null;
     }
-};
+    catch (error) {
+        debug('Error [poke]', error);
+        throw error;
+    }
+});
+exports.poke = poke;
 /**
  * @description Check's if the status of the app when a new incoming notification is fired
  * @description Starts processing if the 'SQ: false'
  */
-const check = () => {
-    debug(`Check called. SQ status is ${flag.SQ} and WQ status is ${flag.WQ}`);
-    if (!flag.SQ && flag.WQ) {
-        flag.WQ = false;
-        flag.SQ = true;
-        sync().then(() => {
+const check = () => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        debug(`Check called. SQ status is ${flag.SQ} and WQ status is ${flag.WQ}`);
+        if (!flag.SQ && flag.WQ) {
+            flag.WQ = false;
+            flag.SQ = true;
+            yield sync();
             debug(`Sync completed and SQ flag updated. Cooloff duration is ${config.syncManager.cooloff}`);
             setTimeout(() => {
                 flag.SQ = false;
                 emitter.emit('check');
             }, config.syncManager.cooloff);
+        }
+    }
+    catch (error) {
+        logger_1.logger.error(error);
+        debug('Error [check]', error);
+        check().then(() => {
+            debug('passed [check] error');
         }).catch((error) => {
-            logger_1.logger.error(error);
-            check();
+            debug('failed [check] error', error);
         });
+        throw error;
     }
-};
+});
 /**
  * @description Gets saved token, builds request object and fires the sync process
  */
-const sync = () => {
-    return new Promise((resolve, reject) => {
-        return token_management_1.getToken().then((tokenObject) => {
-            const token = tokenObject;
-            const request = {
-                qs: {
-                    environment: process.env.SYNC_ENV || Contentstack.environment || 'development',
-                    limit: config.syncManager.limit,
-                    [token.name]: token.token,
-                },
-            };
-            return fire(request)
-                .then(resolve);
-        }).catch((error) => {
-            return reject(error);
-        });
-    });
-};
+const sync = () => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        debug('started [sync]');
+        const tokenObject = yield (0, token_management_1.getToken)();
+        debug('tokenObject [sync]', tokenObject);
+        const token = tokenObject;
+        const request = {
+            qs: {
+                environment: process.env.SYNC_ENV || Contentstack.environment || 'development',
+                limit: config.syncManager.limit,
+                [token.name]: token.token,
+            },
+        };
+        return yield fire(request);
+    }
+    catch (error) {
+        debug('Error [sync]', error);
+        throw error;
+    }
+});
 /**
  * @description Used to lockdown the 'sync' process in case of exceptions
  */
-exports.lock = () => {
+const lock = () => {
     debug('Contentstack sync locked..');
     flag.lockdown = true;
 };
+exports.lock = lock;
 /**
  * @description Used to unlock the 'sync' process in case of errors/exceptions
  */
-exports.unlock = (refire) => {
+const unlock = (refire) => {
     debug('Contentstack sync unlocked..', refire);
     flag.lockdown = false;
     if (typeof refire === 'boolean' && refire) {
@@ -167,6 +201,7 @@ exports.unlock = (refire) => {
     }
     check();
 };
+exports.unlock = unlock;
 /**
  * @description Description required
  * @param {Object} req - Contentstack sync API request object
@@ -175,22 +210,23 @@ const fire = (req) => {
     debug(`Fire called with: ${JSON.stringify(req)}`);
     flag.SQ = true;
     return new Promise((resolve, reject) => {
-        return api_1.get(req).then((response) => {
+        return (0, api_1.get)(req).then((response) => {
             delete req.qs.init;
             delete req.qs.pagination_token;
             delete req.qs.sync_token;
             delete req.path;
             const syncResponse = response;
+            debug('Response [fire]', syncResponse.items.length);
             if (syncResponse.items.length) {
-                return index_1.filterItems(syncResponse, config).then(() => {
+                return (0, index_1.filterItems)(syncResponse, config).then(() => {
                     if (syncResponse.items.length === 0) {
                         return postProcess(req, syncResponse)
                             .then(resolve)
                             .catch(reject);
                     }
-                    syncResponse.items = index_1.formatItems(syncResponse.items, config);
-                    let groupedItems = index_1.groupItems(syncResponse.items);
-                    groupedItems = index_1.markCheckpoint(groupedItems, syncResponse);
+                    syncResponse.items = (0, index_1.formatItems)(syncResponse.items, config);
+                    let groupedItems = (0, index_1.groupItems)(syncResponse.items);
+                    groupedItems = (0, index_1.markCheckpoint)(groupedItems, syncResponse);
                     // send assets data for processing
                     if (groupedItems[formattedAssetType]) {
                         groupedItems[formattedAssetType].forEach((asset) => {
@@ -205,7 +241,7 @@ const fire = (req) => {
                         delete groupedItems[formattedContentType];
                     }
                     const contentTypeUids = Object.keys(groupedItems);
-                    lodash_1.remove(contentTypeUids, (contentTypeUid) => {
+                    (0, lodash_1.remove)(contentTypeUids, (contentTypeUid) => {
                         const contentType = groupedItems[contentTypeUid];
                         if (contentType.length === 1 && !contentType[0].publish_details) {
                             Q.push(contentType[0]);
@@ -213,9 +249,9 @@ const fire = (req) => {
                         }
                         return false;
                     });
-                    return promise_map_1.map(contentTypeUids, (uid) => {
+                    return (0, promise_map_1.map)(contentTypeUids, (uid) => {
                         return new Promise((mapResolve, mapReject) => {
-                            return api_1.get({
+                            return (0, api_1.get)({
                                 path: `${Contentstack.apis.content_types}${uid}`,
                                 qs: {
                                     include_global_field_schema: config.contentstack.query.include_global_field_schema,
@@ -225,17 +261,17 @@ const fire = (req) => {
                                 if (schemaResponse.content_type) {
                                     const items = groupedItems[uid];
                                     items.forEach((entry) => {
-                                        entry._content_type = lodash_1.cloneDeep(schemaResponse.content_type);
+                                        entry._content_type = (0, lodash_1.cloneDeep)(schemaResponse.content_type);
                                         Q.push(entry);
                                     });
-                                    return mapResolve();
+                                    return mapResolve('');
                                 }
                                 const err = new Error('Content type ${uid} schema not found!');
                                 // Illegal content type call
                                 err.code = 'ICTC';
                                 return mapReject(err);
                             }).catch((error) => {
-                                if (inet_1.netConnectivityIssues(error)) {
+                                if ((0, inet_1.netConnectivityIssues)(error)) {
                                     flag.SQ = false;
                                 }
                                 return mapReject(error);
@@ -246,7 +282,7 @@ const fire = (req) => {
                             .then(resolve)
                             .catch(reject);
                     }).catch((error) => {
-                        if (inet_1.netConnectivityIssues(error)) {
+                        if ((0, inet_1.netConnectivityIssues)(error)) {
                             flag.SQ = false;
                         }
                         // Errorred while fetching content type schema
@@ -259,7 +295,8 @@ const fire = (req) => {
             return postProcess(req, syncResponse)
                 .then(resolve);
         }).catch((error) => {
-            if (inet_1.netConnectivityIssues(error)) {
+            debug('Error [fire]', error);
+            if ((0, inet_1.netConnectivityIssues)(error)) {
                 flag.SQ = false;
             }
             // do something
@@ -281,7 +318,7 @@ const postProcess = (req, resp) => {
         else {
             name = 'sync_token';
         }
-        return token_management_1.saveCheckpoint(name, resp[name])
+        return (0, token_management_1.saveCheckpoint)(name, resp[name])
             .then(() => {
             // re-fire!
             req.qs[name] = resp[name];
@@ -296,7 +333,7 @@ const postProcess = (req, resp) => {
             else {
                 if (name === 'sync_token') {
                     flag.SQ = false;
-                    return resolve();
+                    return resolve('');
                 }
                 return fire(req)
                     .then(resolve)

package/dist/core/inet.js

@@ -8,6 +8,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.netConnectivityIssues = exports.checkNetConnectivity = exports.init = void 0;
 const debug_1 = __importDefault(require("debug"));
 const dns_socket_1 = __importDefault(require("dns-socket"));
 const events_1 = require("events");
@@ -15,15 +16,15 @@ const index_1 = require("../index");
 const logger_1 = require("../util/logger");
 const index_2 = require("./index");
 const emitter = new events_1.EventEmitter();
-const debug = debug_1.default('inet');
+const debug = (0, debug_1.default)('inet');
 let disconnected = false;
 let sm;
 let query;
 let port;
 let dns;
 let currentTimeout;
-exports.init = () => {
-    sm = index_1.getConfig().syncManager;
+const init = () => {
+    sm = (0, index_1.getConfig)().syncManager;
     query = {
         questions: [
             {
@@ -39,8 +40,9 @@ exports.init = () => {
     // start checking for net connectivity, 30 seconds after the app has started
     setTimeout(exports.checkNetConnectivity, currentTimeout);
 };
-exports.checkNetConnectivity = () => {
-    const socket = dns_socket_1.default({
+exports.init = init;
+const checkNetConnectivity = () => {
+    const socket = (0, dns_socket_1.default)({
         retries: sm.inet.retries,
         timeout: sm.inet.timeout,
     });
@@ -55,7 +57,7 @@ exports.checkNetConnectivity = () => {
             });
         }
         else if (disconnected) {
-            index_2.poke();
+            (0, index_2.poke)();
         }
         disconnected = false;
         return socket.destroy(() => {
@@ -64,12 +66,14 @@ exports.checkNetConnectivity = () => {
         });
     });
 };
-exports.netConnectivityIssues = (error) => {
+exports.checkNetConnectivity = checkNetConnectivity;
+const netConnectivityIssues = (error) => {
     if (error.code === 'ENOTFOUND' || error.code === 'ETIMEDOUT') {
         return true;
     }
     return false;
 };
+exports.netConnectivityIssues = netConnectivityIssues;
 emitter.on('ok', () => {
     currentTimeout = sm.inet.retryTimeout;
     debug(`pinging ${sm.inet.host} in ${sm.inet.timeout} ms`);

package/dist/core/plugins.js

@@ -8,18 +8,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.load = void 0;
 const debug_1 = __importDefault(require("debug"));
 const lodash_1 = require("lodash");
 const index_1 = require("../util/index");
 const validations_1 = require("../util/validations");
-const debug = debug_1.default('plugins');
+const debug = (0, debug_1.default)('plugins');
 const pluginMethods = ['beforeSync', 'afterSync'];
 /**
  * @description Load registered plugins
  * @param {Object} config - Application config
  * @returns {Object} pluginInstance - An instance of plugins, with valid registered methods
  */
-exports.load = (config) => {
+const load = (config) => {
     debug('Plugins load called');
     const pluginInstances = {
         external: {},
@@ -31,20 +32,20 @@ exports.load = (config) => {
         pluginInstances.internal[pluginMethod] = pluginInstances[pluginMethod] || [];
     });
     plugins.forEach((plugin) => {
-        validations_1.validatePlugin(plugin);
+        (0, validations_1.validatePlugin)(plugin);
         const pluginName = plugin.name;
         const slicedName = pluginName.slice(0, 13);
         let isInternal = false;
         if (slicedName === '_cs_internal_') {
             isInternal = true;
         }
-        const pluginPath = index_1.normalizePluginPath(config, plugin, isInternal);
+        const pluginPath = (0, index_1.normalizePluginPath)(config, plugin, isInternal);
         const Plugin = require(pluginPath);
         Plugin.options = plugin.options || {};
         // execute/initiate plugin
         Plugin();
         pluginMethods.forEach((pluginMethod) => {
-            if (lodash_1.hasIn(Plugin, pluginMethod)) {
+            if ((0, lodash_1.hasIn)(Plugin, pluginMethod)) {
                 if (plugin.disabled) {
                     // do nothing
                 }
@@ -64,3 +65,4 @@ exports.load = (config) => {
     debug('Plugins loaded successfully!');
     return pluginInstances;
 };
+exports.load = load;

package/dist/core/process.js

@@ -5,6 +5,7 @@
 * MIT Licensed
 */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.configure = void 0;
 /**
  * @note 'SIGKILL' cannot have a listener installed, it will unconditionally terminate Node.js on all platforms.
  * @note 'SIGSTOP' cannot have a listener installed.
@@ -17,8 +18,8 @@ const index_2 = require("./index");
  * @param {String} signal - Process signal
  */
 const handleExit = (signal) => {
-    index_2.lock();
-    const { syncManager } = index_1.getConfig();
+    (0, index_2.lock)();
+    const { syncManager } = (0, index_1.getConfig)();
     const killDuration = (process.env.KILLDURATION) ? calculateKillDuration() : syncManager.processTimeout;
     logger_1.logger.info(`Received ${signal}. This will shut down the process in ${killDuration}ms..`);
     setTimeout(abort, killDuration);
@@ -33,9 +34,9 @@ const handleExit = (signal) => {
 const unhandledErrors = (error) => {
     logger_1.logger.error('Unhandled exception caught. Locking down process for 10s to recover..');
     logger_1.logger.error(error);
-    index_2.lock();
+    (0, index_2.lock)();
     setTimeout(() => {
-        index_2.unlock();
+        (0, index_2.unlock)();
     }, 10000);
 };
 /**
@@ -44,7 +45,7 @@ const unhandledErrors = (error) => {
 const calculateKillDuration = () => {
     const killDuration = parseInt(process.env.KILLDURATION, 10);
     if (isNaN(killDuration)) {
-        const { syncManager } = index_1.getConfig();
+        const { syncManager } = (0, index_1.getConfig)();
         return syncManager.processTimeout;
     }
     return killDuration;
@@ -55,9 +56,10 @@ const calculateKillDuration = () => {
 const abort = () => {
     process.abort();
 };
-exports.configure = () => {
+const configure = () => {
     process.on('SIGTERM', handleExit);
     process.on('SIGINT', handleExit);
     process.on('uncaughtException', unhandledErrors);
     process.on('unhandledRejection', unhandledErrors);
 };
+exports.configure = configure;