@contentstack/datasync-manager 1.2.4 → 2.0.1

package/LICENSE

@@ -1,6 +1,6 @@
 The MIT License
 
-Copyright (c) 2022 Contentstack LLC <https://www.contentstack.com/>
+Copyright (c) 2023 Contentstack LLC <https://www.contentstack.com/>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/dist/api.js

@@ -13,32 +13,37 @@ const debug_1 = __importDefault(require("debug"));
 const https_1 = require("https");
 const path_1 = require("path");
 const querystring_1 = require("querystring");
+const sanitize_url_1 = require("@braintree/sanitize-url");
 const fs_1 = require("./util/fs");
-const debug = debug_1.default('api');
+const debug = (0, debug_1.default)('api');
 let MAX_RETRY_LIMIT;
 let Contentstack;
 /**
  * @description Initialize sync utilities API requests
  * @param {Object} contentstack - Contentstack configuration details
  */
-exports.init = (contentstack) => {
-    const packageInfo = JSON.parse(fs_1.readFileSync(path_1.join(__dirname, '..', 'package.json')));
+const init = (contentstack) => {
+    const packageInfo = JSON.parse((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '..', 'package.json')));
     Contentstack = contentstack;
     Contentstack.headers = {
         'X-User-Agent': `datasync-manager/v${packageInfo.version}`,
         'access_token': Contentstack.deliveryToken,
-        'api_key': Contentstack.apiKey,
+        'api_key': Contentstack.apiKey
     };
+    if (Contentstack.branch) {
+        Contentstack.headers['branch'] = Contentstack.branch;
+    }
     if (Contentstack.MAX_RETRY_LIMIT) {
         MAX_RETRY_LIMIT = Contentstack.MAX_RETRY_LIMIT;
     }
 };
+exports.init = init;
 /**
  * @description Make API requests to Contentstack
  * @param {Object} req - API request object
  * @param {Number} RETRY - API request retry counter
  */
-exports.get = (req, RETRY = 1) => {
+const get = (req, RETRY = 1) => {
     return new Promise((resolve, reject) => {
         if (RETRY > MAX_RETRY_LIMIT) {
             return reject(new Error('Max retry limit exceeded!'));
@@ -46,13 +51,13 @@ exports.get = (req, RETRY = 1) => {
         req.method = Contentstack.verbs.get;
         req.path = req.path || Contentstack.apis.sync;
         if (req.qs) {
-            req.path += `?${querystring_1.stringify(req.qs)}`;
+            req.path += `?${(0, querystring_1.stringify)(req.qs)}`;
         }
         const options = {
             headers: Contentstack.headers,
             hostname: Contentstack.host,
             method: Contentstack.verbs.get,
-            path: req.path,
+            path: (0, sanitize_url_1.sanitizeUrl)(req.path),
             port: Contentstack.port,
             protocol: Contentstack.protocol,
         };
@@ -60,7 +65,7 @@ exports.get = (req, RETRY = 1) => {
             debug(`${options.method.toUpperCase()}: ${options.path}`);
             let timeDelay;
             let body = '';
-            https_1.request(options, (response) => {
+            (0, https_1.request)(options, (response) => {
                 response
                     .setEncoding('utf-8')
                     .on('data', (chunk) => body += chunk)
@@ -73,7 +78,7 @@ exports.get = (req, RETRY = 1) => {
                         timeDelay = Math.pow(Math.SQRT2, RETRY) * 200;
                         debug(`API rate limit exceeded. Retrying ${options.path} with ${timeDelay} sec delay`);
                         return setTimeout(() => {
-                            return exports.get(req, RETRY)
+                            return (0, exports.get)(req, RETRY)
                                 .then(resolve)
                                 .catch(reject);
                         }, timeDelay);
@@ -84,7 +89,7 @@ exports.get = (req, RETRY = 1) => {
                         debug(`Retrying ${options.path} with ${timeDelay} sec delay`);
                         RETRY++;
                         return setTimeout(() => {
-                            return exports.get(req, RETRY)
+                            return (0, exports.get)(req, RETRY)
                                 .then(resolve)
                                 .catch(reject);
                         }, timeDelay);
@@ -103,3 +108,4 @@ exports.get = (req, RETRY = 1) => {
         }
     });
 };
+exports.get = get;

package/dist/config.js

@@ -86,6 +86,7 @@ exports.config = {
             options: {
                 logAssetPaths: true,
                 logReferencePaths: true,
+                // other overrides...
             },
         },
         {

package/dist/core/index.js

@@ -30,7 +30,7 @@ const promise_map_1 = require("../util/promise.map");
 const inet_1 = require("./inet");
 const q_1 = require("./q");
 const token_management_1 = require("./token-management");
-const debug = debug_1.default('sync-core');
+const debug = (0, debug_1.default)('sync-core');
 const emitter = new events_1.EventEmitter();
 const formattedAssetType = '_assets';
 const formattedContentType = '_content_types';
@@ -47,10 +47,10 @@ let Q;
  * @param {Object} connector - Content connector instance
  * @param {Object} config - Application config
  */
-exports.init = (contentStore, assetStore) => {
-    config = __1.getConfig();
+const init = (contentStore, assetStore) => {
+    config = (0, __1.getConfig)();
     Q = new q_1.Q(contentStore, assetStore, config);
-    api_1.init(config.contentstack);
+    (0, api_1.init)(config.contentstack);
     debug('Sync core:start invoked');
     return new Promise((resolve, reject) => {
         try {
@@ -70,8 +70,8 @@ exports.init = (contentStore, assetStore) => {
             else if (typeof Contentstack.pagination_token === 'string' && Contentstack.pagination_token.length !== 0) {
                 request.qs.pagination_token = Contentstack.pagination_token;
             }
-            else if (fs_1.existsSync(paths.token)) {
-                const token = JSON.parse(fs_1.readFileSync(paths.token));
+            else if ((0, fs_1.existsSync)(paths.token)) {
+                const token = JSON.parse((0, fs_1.readFileSync)(paths.token));
                 request.qs[token.name] = token.token;
             }
             else {
@@ -93,19 +93,23 @@ exports.init = (contentStore, assetStore) => {
         }
     });
 };
-exports.push = (data) => {
+exports.init = init;
+const push = (data) => {
     Q.emit('push', data);
 };
-exports.unshift = (data) => {
+exports.push = push;
+const unshift = (data) => {
     Q.emit('push', data);
 };
-exports.pop = () => {
+exports.unshift = unshift;
+const pop = () => {
     Q.emit('pop');
 };
+exports.pop = pop;
 /**
  * @description Notifies the sync manager utility to wake up and start syncing..
  */
-exports.poke = () => __awaiter(void 0, void 0, void 0, function* () {
+const poke = () => __awaiter(void 0, void 0, void 0, function* () {
     try {
         debug('Invoked poke');
         logger_1.logger.info('Received \'contentstack sync\' notification');
@@ -120,6 +124,7 @@ exports.poke = () => __awaiter(void 0, void 0, void 0, function* () {
         throw error;
     }
 });
+exports.poke = poke;
 /**
  * @description Check's if the status of the app when a new incoming notification is fired
  * @description Starts processing if the 'SQ: false'
@@ -155,7 +160,7 @@ const check = () => __awaiter(void 0, void 0, void 0, function* () {
 const sync = () => __awaiter(void 0, void 0, void 0, function* () {
     try {
         debug('started [sync]');
-        const tokenObject = yield token_management_1.getToken();
+        const tokenObject = yield (0, token_management_1.getToken)();
         debug('tokenObject [sync]', tokenObject);
         const token = tokenObject;
         const request = {
@@ -175,14 +180,15 @@ const sync = () => __awaiter(void 0, void 0, void 0, function* () {
 /**
  * @description Used to lockdown the 'sync' process in case of exceptions
  */
-exports.lock = () => {
+const lock = () => {
     debug('Contentstack sync locked..');
     flag.lockdown = true;
 };
+exports.lock = lock;
 /**
  * @description Used to unlock the 'sync' process in case of errors/exceptions
  */
-exports.unlock = (refire) => {
+const unlock = (refire) => {
     debug('Contentstack sync unlocked..', refire);
     flag.lockdown = false;
     if (typeof refire === 'boolean' && refire) {
@@ -195,6 +201,7 @@ exports.unlock = (refire) => {
     }
     check();
 };
+exports.unlock = unlock;
 /**
  * @description Description required
  * @param {Object} req - Contentstack sync API request object
@@ -203,7 +210,7 @@ const fire = (req) => {
     debug(`Fire called with: ${JSON.stringify(req)}`);
     flag.SQ = true;
     return new Promise((resolve, reject) => {
-        return api_1.get(req).then((response) => {
+        return (0, api_1.get)(req).then((response) => {
             delete req.qs.init;
             delete req.qs.pagination_token;
             delete req.qs.sync_token;
@@ -211,15 +218,15 @@ const fire = (req) => {
             const syncResponse = response;
             debug('Response [fire]', syncResponse.items.length);
             if (syncResponse.items.length) {
-                return index_1.filterItems(syncResponse, config).then(() => {
+                return (0, index_1.filterItems)(syncResponse, config).then(() => {
                     if (syncResponse.items.length === 0) {
                         return postProcess(req, syncResponse)
                             .then(resolve)
                             .catch(reject);
                     }
-                    syncResponse.items = index_1.formatItems(syncResponse.items, config);
-                    let groupedItems = index_1.groupItems(syncResponse.items);
-                    groupedItems = index_1.markCheckpoint(groupedItems, syncResponse);
+                    syncResponse.items = (0, index_1.formatItems)(syncResponse.items, config);
+                    let groupedItems = (0, index_1.groupItems)(syncResponse.items);
+                    groupedItems = (0, index_1.markCheckpoint)(groupedItems, syncResponse);
                     // send assets data for processing
                     if (groupedItems[formattedAssetType]) {
                         groupedItems[formattedAssetType].forEach((asset) => {
@@ -234,7 +241,7 @@ const fire = (req) => {
                         delete groupedItems[formattedContentType];
                     }
                     const contentTypeUids = Object.keys(groupedItems);
-                    lodash_1.remove(contentTypeUids, (contentTypeUid) => {
+                    (0, lodash_1.remove)(contentTypeUids, (contentTypeUid) => {
                         const contentType = groupedItems[contentTypeUid];
                         if (contentType.length === 1 && !contentType[0].publish_details) {
                             Q.push(contentType[0]);
@@ -242,9 +249,9 @@ const fire = (req) => {
                         }
                         return false;
                     });
-                    return promise_map_1.map(contentTypeUids, (uid) => {
+                    return (0, promise_map_1.map)(contentTypeUids, (uid) => {
                         return new Promise((mapResolve, mapReject) => {
-                            return api_1.get({
+                            return (0, api_1.get)({
                                 path: `${Contentstack.apis.content_types}${uid}`,
                                 qs: {
                                     include_global_field_schema: config.contentstack.query.include_global_field_schema,
@@ -254,7 +261,7 @@ const fire = (req) => {
                                 if (schemaResponse.content_type) {
                                     const items = groupedItems[uid];
                                     items.forEach((entry) => {
-                                        entry._content_type = lodash_1.cloneDeep(schemaResponse.content_type);
+                                        entry._content_type = (0, lodash_1.cloneDeep)(schemaResponse.content_type);
                                         Q.push(entry);
                                     });
                                     return mapResolve('');
@@ -264,7 +271,7 @@ const fire = (req) => {
                                 err.code = 'ICTC';
                                 return mapReject(err);
                             }).catch((error) => {
-                                if (inet_1.netConnectivityIssues(error)) {
+                                if ((0, inet_1.netConnectivityIssues)(error)) {
                                     flag.SQ = false;
                                 }
                                 return mapReject(error);
@@ -275,7 +282,7 @@ const fire = (req) => {
                             .then(resolve)
                             .catch(reject);
                     }).catch((error) => {
-                        if (inet_1.netConnectivityIssues(error)) {
+                        if ((0, inet_1.netConnectivityIssues)(error)) {
                             flag.SQ = false;
                         }
                         // Errorred while fetching content type schema
@@ -289,7 +296,7 @@ const fire = (req) => {
                 .then(resolve);
         }).catch((error) => {
             debug('Error [fire]', error);
-            if (inet_1.netConnectivityIssues(error)) {
+            if ((0, inet_1.netConnectivityIssues)(error)) {
                 flag.SQ = false;
             }
             // do something
@@ -311,7 +318,7 @@ const postProcess = (req, resp) => {
         else {
             name = 'sync_token';
         }
-        return token_management_1.saveCheckpoint(name, resp[name])
+        return (0, token_management_1.saveCheckpoint)(name, resp[name])
             .then(() => {
             // re-fire!
             req.qs[name] = resp[name];

package/dist/core/inet.js

@@ -16,15 +16,15 @@ const index_1 = require("../index");
 const logger_1 = require("../util/logger");
 const index_2 = require("./index");
 const emitter = new events_1.EventEmitter();
-const debug = debug_1.default('inet');
+const debug = (0, debug_1.default)('inet');
 let disconnected = false;
 let sm;
 let query;
 let port;
 let dns;
 let currentTimeout;
-exports.init = () => {
-    sm = index_1.getConfig().syncManager;
+const init = () => {
+    sm = (0, index_1.getConfig)().syncManager;
     query = {
         questions: [
             {
@@ -40,8 +40,9 @@ exports.init = () => {
     // start checking for net connectivity, 30 seconds after the app has started
     setTimeout(exports.checkNetConnectivity, currentTimeout);
 };
-exports.checkNetConnectivity = () => {
-    const socket = dns_socket_1.default({
+exports.init = init;
+const checkNetConnectivity = () => {
+    const socket = (0, dns_socket_1.default)({
         retries: sm.inet.retries,
         timeout: sm.inet.timeout,
     });
@@ -56,7 +57,7 @@ exports.checkNetConnectivity = () => {
             });
         }
         else if (disconnected) {
-            index_2.poke();
+            (0, index_2.poke)();
         }
         disconnected = false;
         return socket.destroy(() => {
@@ -65,12 +66,14 @@ exports.checkNetConnectivity = () => {
         });
     });
 };
-exports.netConnectivityIssues = (error) => {
+exports.checkNetConnectivity = checkNetConnectivity;
+const netConnectivityIssues = (error) => {
     if (error.code === 'ENOTFOUND' || error.code === 'ETIMEDOUT') {
         return true;
     }
     return false;
 };
+exports.netConnectivityIssues = netConnectivityIssues;
 emitter.on('ok', () => {
     currentTimeout = sm.inet.retryTimeout;
     debug(`pinging ${sm.inet.host} in ${sm.inet.timeout} ms`);

package/dist/core/plugins.js

@@ -13,14 +13,14 @@ const debug_1 = __importDefault(require("debug"));
 const lodash_1 = require("lodash");
 const index_1 = require("../util/index");
 const validations_1 = require("../util/validations");
-const debug = debug_1.default('plugins');
+const debug = (0, debug_1.default)('plugins');
 const pluginMethods = ['beforeSync', 'afterSync'];
 /**
  * @description Load registered plugins
  * @param {Object} config - Application config
  * @returns {Object} pluginInstance - An instance of plugins, with valid registered methods
  */
-exports.load = (config) => {
+const load = (config) => {
     debug('Plugins load called');
     const pluginInstances = {
         external: {},
@@ -32,20 +32,20 @@ exports.load = (config) => {
         pluginInstances.internal[pluginMethod] = pluginInstances[pluginMethod] || [];
     });
     plugins.forEach((plugin) => {
-        validations_1.validatePlugin(plugin);
+        (0, validations_1.validatePlugin)(plugin);
         const pluginName = plugin.name;
         const slicedName = pluginName.slice(0, 13);
         let isInternal = false;
         if (slicedName === '_cs_internal_') {
             isInternal = true;
         }
-        const pluginPath = index_1.normalizePluginPath(config, plugin, isInternal);
+        const pluginPath = (0, index_1.normalizePluginPath)(config, plugin, isInternal);
         const Plugin = require(pluginPath);
         Plugin.options = plugin.options || {};
         // execute/initiate plugin
         Plugin();
         pluginMethods.forEach((pluginMethod) => {
-            if (lodash_1.hasIn(Plugin, pluginMethod)) {
+            if ((0, lodash_1.hasIn)(Plugin, pluginMethod)) {
                 if (plugin.disabled) {
                     // do nothing
                 }
@@ -65,3 +65,4 @@ exports.load = (config) => {
     debug('Plugins loaded successfully!');
     return pluginInstances;
 };
+exports.load = load;

package/dist/core/process.js

@@ -18,8 +18,8 @@ const index_2 = require("./index");
  * @param {String} signal - Process signal
  */
 const handleExit = (signal) => {
-    index_2.lock();
-    const { syncManager } = index_1.getConfig();
+    (0, index_2.lock)();
+    const { syncManager } = (0, index_1.getConfig)();
     const killDuration = (process.env.KILLDURATION) ? calculateKillDuration() : syncManager.processTimeout;
     logger_1.logger.info(`Received ${signal}. This will shut down the process in ${killDuration}ms..`);
     setTimeout(abort, killDuration);
@@ -34,9 +34,9 @@ const handleExit = (signal) => {
 const unhandledErrors = (error) => {
     logger_1.logger.error('Unhandled exception caught. Locking down process for 10s to recover..');
     logger_1.logger.error(error);
-    index_2.lock();
+    (0, index_2.lock)();
     setTimeout(() => {
-        index_2.unlock();
+        (0, index_2.unlock)();
     }, 10000);
 };
 /**
@@ -45,7 +45,7 @@ const unhandledErrors = (error) => {
 const calculateKillDuration = () => {
     const killDuration = parseInt(process.env.KILLDURATION, 10);
     if (isNaN(killDuration)) {
-        const { syncManager } = index_1.getConfig();
+        const { syncManager } = (0, index_1.getConfig)();
         return syncManager.processTimeout;
     }
     return killDuration;
@@ -56,9 +56,10 @@ const calculateKillDuration = () => {
 const abort = () => {
     process.abort();
 };
-exports.configure = () => {
+const configure = () => {
     process.on('SIGTERM', handleExit);
     process.on('SIGINT', handleExit);
     process.on('uncaughtException', unhandledErrors);
     process.on('unhandledRejection', unhandledErrors);
 };
+exports.configure = configure;

package/dist/core/q.js

@@ -28,7 +28,8 @@ const series_1 = require("../util/series");
 const unprocessible_1 = require("../util/unprocessible");
 const plugins_1 = require("./plugins");
 const token_management_1 = require("./token-management");
-const debug = debug_1.default('q');
+const index_2 = require("../index");
+const debug = (0, debug_1.default)('q');
 const notifications = new events_1.EventEmitter();
 exports.notifications = notifications;
 let instance = null;
@@ -47,7 +48,7 @@ class Q extends events_1.EventEmitter {
     constructor(contentStore, assetStore, config) {
         if (!instance && contentStore && assetStore && config) {
             super();
-            this.pluginInstances = plugins_1.load(config);
+            this.pluginInstances = (0, plugins_1.load)(config);
             this.contentStore = contentStore;
             this.syncManager = config.syncManager;
             this.iLock = false;
@@ -66,7 +67,7 @@ class Q extends events_1.EventEmitter {
         this.q.unshift(data);
         if (this.q.length > this.syncManager.queue.pause_threshold) {
             this.iLock = true;
-            _1.lock();
+            (0, _1.lock)();
         }
         debug(`Content type '${data._content_type_uid}' received for '${data._type}'`);
         this.emit('next');
@@ -79,7 +80,7 @@ class Q extends events_1.EventEmitter {
         this.q.push(data);
         if (this.q.length > this.syncManager.queue.pause_threshold) {
             this.iLock = true;
-            _1.lock();
+            (0, _1.lock)();
         }
         debug(`Content type '${data._content_type_uid}' received for '${data._type}'`);
         this.emit('next');
@@ -96,9 +97,9 @@ class Q extends events_1.EventEmitter {
                 logger_1.logger.error(obj);
                 debug(`Error handler called with ${JSON.stringify(obj)}`);
                 if (typeof obj.checkpoint !== 'undefined') {
-                    yield token_management_1.saveToken(obj.checkpoint.name, obj.checkpoint.token);
+                    yield (0, token_management_1.saveToken)(obj.checkpoint.name, obj.checkpoint.token);
                 }
-                yield unprocessible_1.saveFailedItems(obj);
+                yield (0, unprocessible_1.saveFailedItems)(obj);
                 this.inProgress = false;
                 this.emit('next');
             }
@@ -121,7 +122,7 @@ class Q extends events_1.EventEmitter {
         return __awaiter(this, void 0, void 0, function* () {
             try {
                 if (this.iLock && this.q.length < this.syncManager.queue.resume_threshold) {
-                    _1.unlock(true);
+                    (0, _1.unlock)(true);
                     this.iLock = false;
                 }
                 debug(`Calling 'next'. In progress status is ${this.inProgress}, and Q length is ${this.q.length}`);
@@ -162,6 +163,9 @@ class Q extends events_1.EventEmitter {
                 const contentType = data._content_type_uid;
                 const locale = data.locale;
                 const uid = data.uid;
+                const branch = (0, index_2.getConfig)().contentstack.branch;
+                if (branch)
+                    data.branch = branch;
                 if (data.hasOwnProperty('_checkpoint')) {
                     checkpoint = data._checkpoint;
                     delete data._checkpoint;
@@ -174,29 +178,29 @@ class Q extends events_1.EventEmitter {
                 const afterSyncPlugins = [];
                 let transformedData;
                 let transformedSchema;
-                let { schema } = index_1.getSchema(action, data);
-                data = index_1.filterUnwantedKeys(action, data);
+                let { schema } = (0, index_1.getSchema)(action, data);
+                data = (0, index_1.filterUnwantedKeys)(action, data);
                 if (typeof schema !== 'undefined') {
-                    schema = index_1.filterUnwantedKeys(action, schema);
+                    schema = (0, index_1.filterUnwantedKeys)(action, schema);
                 }
                 logger_1.logger.log(`${type}: { content_type: '${contentType}', ${(locale) ? `locale: '${locale}',` : ''} uid: '${uid}'} is in progress`);
                 this.pluginInstances.internal.beforeSync.forEach((method) => {
                     beforeSyncInternalPlugins.push(() => method(action, data, schema));
                 });
-                yield series_1.series(beforeSyncInternalPlugins);
+                yield (0, series_1.series)(beforeSyncInternalPlugins);
                 if (this.syncManager.pluginTransformations) {
                     transformedData = data;
                     transformedSchema = schema;
                 }
                 else {
-                    transformedData = lodash_1.cloneDeep(data);
-                    transformedSchema = lodash_1.cloneDeep(schema);
+                    transformedData = (0, lodash_1.cloneDeep)(data);
+                    transformedSchema = (0, lodash_1.cloneDeep)(schema);
                 }
                 if (this.syncManager.serializePlugins) {
                     this.pluginInstances.external.beforeSync.forEach((method) => {
                         beforeSyncPlugins.push(() => method(action, transformedData, transformedSchema));
                     });
-                    yield series_1.series(beforeSyncPlugins);
+                    yield (0, series_1.series)(beforeSyncPlugins);
                 }
                 else {
                     this.pluginInstances.external.beforeSync.forEach((method) => {
@@ -208,6 +212,8 @@ class Q extends events_1.EventEmitter {
                 yield this.contentStore[action](data);
                 debug(`Completed '${action}' on connector successfully!`);
                 if (typeof schema !== 'undefined') {
+                    if (branch)
+                        schema.branch = branch;
                     yield this.contentStore.updateContentType(schema);
                 }
                 debug('Connector instance called successfully!');
@@ -215,7 +221,7 @@ class Q extends events_1.EventEmitter {
                     this.pluginInstances.external.afterSync.forEach((method) => {
                         afterSyncPlugins.push(() => method(action, transformedData, transformedSchema));
                     });
-                    yield series_1.series(afterSyncPlugins);
+                    yield (0, series_1.series)(afterSyncPlugins);
                 }
                 else {
                     this.pluginInstances.external.afterSync.forEach((method) => {
@@ -224,7 +230,7 @@ class Q extends events_1.EventEmitter {
                     yield Promise.all(afterSyncPlugins);
                 }
                 if (typeof checkpoint !== 'undefined') {
-                    yield token_management_1.saveToken(checkpoint.name, checkpoint.token);
+                    yield (0, token_management_1.saveToken)(checkpoint.name, checkpoint.token);
                 }
                 debug('After action plugins executed successfully!');
                 logger_1.logger.log(`${type}: { content_type: '${contentType}', ${(locale) ? `locale: '${locale}',` : ''} uid: '${uid}'} was completed successfully!`);