@contentstack/datasync-manager 2.0.10 → 2.1.1

package/README.md

@@ -154,6 +154,23 @@ And here's an example to get you started:
 }
 ```
 
+### Environment Variables
+
+The following environment variables can be used to customize the behavior of Contentstack DataSync Manager:
+
+| Variable      | Description                                                                 | Default                |
+|--------------|-----------------------------------------------------------------------------|------------------------|
+| `TOKEN_PATH` | Path to the directory where token/checkpoint/ledger files are stored.        | Project root directory |
+| `PLUGIN_PATH`| Path to the directory where plugins are loaded from.                         | Project root directory |
+| `NODE_ENV`   | Node.js environment (affects config/environment selection).                  | `development`          |
+| `SYNC_ENV`   | Overrides the environment used for sync operations.                         | Value of `NODE_ENV`    |
+| `KILLDURATION`| Time (in ms) before the process is forcefully killed (overrides config).    | Value from config      |
+
+**Note:**
+- `TOKEN_PATH` is especially useful for storing token data in a custom directory (e.g., for selective re-syncing based on timestamps).
+- If a relative path is provided, it is resolved from the project root.
+- These variables can be set in your shell or in your process manager configuration.
+
 ### Further Reading
 
 - [Getting started with Contentstack DataSync](https://www.contentstack.com/docs/guide/synchronization/contentstack-datasync)

package/dist/config.js

@@ -129,4 +129,9 @@ exports.config = {
         saveFailedItems: true,
         saveFilteredItems: true,
     },
+    checkpoint: {
+        enabled: false,
+        filePath: ".checkpoint",
+        preserve: false // Set to true if you want to preserve the checkpoint file during clean operation
+    },
 };

package/dist/core/index.js

@@ -1,9 +1,27 @@
 "use strict";
-/*!
- * Contentstack DataSync Manager
- * Copyright (c) 2019 Contentstack LLC
- * MIT Licensed
- */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -18,6 +36,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.unlock = exports.lock = exports.poke = exports.pop = exports.unshift = exports.push = exports.init = void 0;
+/*!
+ * Contentstack DataSync Manager
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
 const debug_1 = __importDefault(require("debug"));
 const events_1 = require("events");
 const lodash_1 = require("lodash");
@@ -30,6 +55,7 @@ const promise_map_1 = require("../util/promise.map");
 const inet_1 = require("./inet");
 const q_1 = require("./q");
 const token_management_1 = require("./token-management");
+const helper_1 = require("../plugins/helper");
 const debug = (0, debug_1.default)('sync-core');
 const emitter = new events_1.EventEmitter();
 const formattedAssetType = '_assets';
@@ -55,6 +81,7 @@ const init = (contentStore, assetStore) => {
     return new Promise((resolve, reject) => {
         try {
             Contentstack = config.contentstack;
+            const checkPointConfig = config.checkpoint;
             const paths = config.paths;
             const environment = Contentstack.environment || process.env.NODE_ENV || 'development';
             debug(`Environment: ${environment}`);
@@ -64,6 +91,7 @@ const init = (contentStore, assetStore) => {
                     limit: config.syncManager.limit,
                 },
             };
+            loadCheckpoint(checkPointConfig, paths);
             if (typeof Contentstack.sync_token === 'string' && Contentstack.sync_token.length !== 0) {
                 request.qs.sync_token = Contentstack.sync_token;
             }
@@ -94,6 +122,37 @@ const init = (contentStore, assetStore) => {
     });
 };
 exports.init = init;
+const loadCheckpoint = (checkPointConfig, paths) => {
+    if (!(checkPointConfig === null || checkPointConfig === void 0 ? void 0 : checkPointConfig.enabled))
+        return;
+    // Try reading checkpoint from primary path
+    let checkpoint = readHiddenFile(paths.checkpoint);
+    // Fallback to filePath in config if not found
+    if (!checkpoint) {
+        const fallbackPath = path.join((0, helper_1.sanitizePath)(__dirname), (0, helper_1.sanitizePath)(checkPointConfig.filePath || ".checkpoint"));
+        checkpoint = readHiddenFile(fallbackPath);
+    }
+    // Set sync token if checkpoint is found
+    if (checkpoint) {
+        debug("Found sync token in checkpoint file:", checkpoint);
+        Contentstack.sync_token = checkpoint.token;
+        debug("Using sync token:", Contentstack.sync_token);
+    }
+};
+function readHiddenFile(filePath) {
+    try {
+        if (!fs.existsSync(filePath)) {
+            logger_1.logger.error("File does not exist:", filePath);
+            return;
+        }
+        const data = fs.readFileSync(filePath, "utf8");
+        return JSON.parse(data);
+    }
+    catch (err) {
+        logger_1.logger.error("Error reading file:", err);
+        return undefined;
+    }
+}
 const push = (data) => {
     Q.emit('push', data);
 };
@@ -251,6 +310,7 @@ const fire = (req) => {
                     });
                     return (0, promise_map_1.map)(contentTypeUids, (uid) => {
                         return new Promise((mapResolve, mapReject) => {
+                            debug(`API called with for content type: ${uid}`);
                             return (0, api_1.get)({
                                 path: `${Contentstack.apis.content_types}${uid}`,
                                 qs: {
@@ -271,6 +331,7 @@ const fire = (req) => {
                                 err.code = 'ICTC';
                                 return mapReject(err);
                             }).catch((error) => {
+                                debug('Error [map] fetching content type schema:', error);
                                 if ((0, inet_1.netConnectivityIssues)(error)) {
                                     flag.SQ = false;
                                 }
@@ -286,14 +347,17 @@ const fire = (req) => {
                             flag.SQ = false;
                         }
                         // Errorred while fetching content type schema
+                        debug('Error [mapResolve]:', error);
                         return reject(error);
                     });
                 }).catch((processError) => {
+                    debug('Error [filterItems]:', processError);
                     return reject(processError);
                 });
             }
             return postProcess(req, syncResponse)
-                .then(resolve);
+                .then(resolve)
+                .catch(reject);
         }).catch((error) => {
             debug('Error [fire]', error);
             if ((0, inet_1.netConnectivityIssues)(error)) {
@@ -335,6 +399,7 @@ const postProcess = (req, resp) => {
                     flag.SQ = false;
                     return resolve('');
                 }
+                debug(`Re-Fire called with: ${JSON.stringify(req)}`);
                 return fire(req)
                     .then(resolve)
                     .catch(reject);

package/dist/core/plugins.js

@@ -22,47 +22,53 @@ const pluginMethods = ['beforeSync', 'afterSync'];
  */
 const load = (config) => {
     debug('Plugins load called');
-    const pluginInstances = {
-        external: {},
-        internal: {},
-    };
-    const plugins = config.plugins || [];
-    pluginMethods.forEach((pluginMethod) => {
-        pluginInstances.external[pluginMethod] = pluginInstances[pluginMethod] || [];
-        pluginInstances.internal[pluginMethod] = pluginInstances[pluginMethod] || [];
-    });
-    plugins.forEach((plugin) => {
-        (0, validations_1.validatePlugin)(plugin);
-        const pluginName = plugin.name;
-        const slicedName = pluginName.slice(0, 13);
-        let isInternal = false;
-        if (slicedName === '_cs_internal_') {
-            isInternal = true;
-        }
-        const pluginPath = (0, index_1.normalizePluginPath)(config, plugin, isInternal);
-        const Plugin = require(pluginPath);
-        Plugin.options = plugin.options || {};
-        // execute/initiate plugin
-        Plugin();
+    try {
+        const pluginInstances = {
+            external: {},
+            internal: {},
+        };
+        const plugins = config.plugins || [];
         pluginMethods.forEach((pluginMethod) => {
-            if ((0, lodash_1.hasIn)(Plugin, pluginMethod)) {
-                if (plugin.disabled) {
-                    // do nothing
-                }
-                else if (isInternal) {
-                    pluginInstances.internal[pluginMethod].push(Plugin[pluginMethod]);
+            pluginInstances.external[pluginMethod] = pluginInstances[pluginMethod] || [];
+            pluginInstances.internal[pluginMethod] = pluginInstances[pluginMethod] || [];
+        });
+        plugins.forEach((plugin) => {
+            (0, validations_1.validatePlugin)(plugin);
+            const pluginName = plugin.name;
+            const slicedName = pluginName.slice(0, 13);
+            let isInternal = false;
+            if (slicedName === '_cs_internal_') {
+                isInternal = true;
+            }
+            const pluginPath = (0, index_1.normalizePluginPath)(config, plugin, isInternal);
+            const Plugin = require(pluginPath);
+            Plugin.options = plugin.options || {};
+            // execute/initiate plugin
+            Plugin();
+            pluginMethods.forEach((pluginMethod) => {
+                if ((0, lodash_1.hasIn)(Plugin, pluginMethod)) {
+                    if (plugin.disabled) {
+                        // do nothing
+                    }
+                    else if (isInternal) {
+                        pluginInstances.internal[pluginMethod].push(Plugin[pluginMethod]);
+                    }
+                    else {
+                        pluginInstances.external[pluginMethod].push(Plugin[pluginMethod]);
+                    }
+                    debug(`${pluginMethod} loaded from ${pluginName} successfully!`);
                 }
                 else {
-                    pluginInstances.external[pluginMethod].push(Plugin[pluginMethod]);
+                    debug(`${pluginMethod} not found in ${pluginName}`);
                 }
-                debug(`${pluginMethod} loaded from ${pluginName} successfully!`);
-            }
-            else {
-                debug(`${pluginMethod} not found in ${pluginName}`);
-            }
+            });
         });
-    });
-    debug('Plugins loaded successfully!');
-    return pluginInstances;
+        debug('Plugins loaded successfully!');
+        return pluginInstances;
+    }
+    catch (error) {
+        debug('Error while loading plugins:', error);
+        throw new Error(`Failed to load plugins: ${error === null || error === void 0 ? void 0 : error.message}`);
+    }
 };
 exports.load = load;

package/dist/util/index.js

@@ -131,8 +131,9 @@ const formatItems = (items, config) => {
                 items[i]._type = config.contentstack.actions.publish;
                 // extra keys
                 items[i]._synced_at = time;
-                items[i].locale = items[i].data.publish_details.locale;
-                items[i] = (0, lodash_1.merge)(items[i], items[i].data);
+                const assetLocale = items[i].data.publish_details.locale;
+                items[i] = (0, lodash_1.merge)((0, lodash_1.cloneDeep)(items[i]), items[i].data);
+                items[i].locale = assetLocale;
                 break;
             case 'asset_unpublished':
                 delete items[i].type;
@@ -152,8 +153,9 @@ const formatItems = (items, config) => {
                 items[i]._content_type_uid = items[i].content_type_uid;
                 // extra keys
                 items[i]._synced_at = time;
-                items[i].locale = items[i].data.publish_details.locale;
-                items[i] = (0, lodash_1.merge)(items[i], items[i].data);
+                const entryLocale = items[i].data.publish_details.locale;
+                items[i] = (0, lodash_1.merge)((0, lodash_1.cloneDeep)(items[i]), items[i].data);
+                items[i].locale = entryLocale;
                 break;
             case 'entry_unpublished':
                 delete items[i].type;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@contentstack/datasync-manager",
   "author": "Contentstack LLC <support@contentstack.com>",
-  "version": "2.0.10",
+  "version": "2.1.1",
   "description": "The primary module of Contentstack DataSync. Syncs Contentstack data with your server using Contentstack Sync API",
   "main": "dist/index.js",
   "dependencies": {