@contentstack/datasync-manager 2.1.0 → 2.1.2

package/README.md

@@ -154,6 +154,23 @@ And here's an example to get you started:
 }
 ```
 
+### Environment Variables
+
+The following environment variables can be used to customize the behavior of Contentstack DataSync Manager:
+
+| Variable      | Description                                                                 | Default                |
+|--------------|-----------------------------------------------------------------------------|------------------------|
+| `TOKEN_PATH` | Path to the directory where token/checkpoint/ledger files are stored.        | Project root directory |
+| `PLUGIN_PATH`| Path to the directory where plugins are loaded from.                         | Project root directory |
+| `NODE_ENV`   | Node.js environment (affects config/environment selection).                  | `development`          |
+| `SYNC_ENV`   | Overrides the environment used for sync operations.                         | Value of `NODE_ENV`    |
+| `KILLDURATION`| Time (in ms) before the process is forcefully killed (overrides config).    | Value from config      |
+
+**Note:**
+- `TOKEN_PATH` is especially useful for storing token data in a custom directory (e.g., for selective re-syncing based on timestamps).
+- If a relative path is provided, it is resolved from the project root.
+- These variables can be set in your shell or in your process manager configuration.
+
 ### Further Reading
 
 - [Getting started with Contentstack DataSync](https://www.contentstack.com/docs/guide/synchronization/contentstack-datasync)

package/dist/api.js

@@ -17,11 +17,13 @@ const sanitize_url_1 = require("@braintree/sanitize-url");
 const fs_1 = require("./util/fs");
 const debug = (0, debug_1.default)('api');
 let MAX_RETRY_LIMIT;
+let RETRY_DELAY_BASE = 200; // Default base delay in milliseconds
+let TIMEOUT = 30000; // Default timeout in milliseconds
 let Contentstack;
 /**
  * @description Initialize sync utilities API requests
  * @param {Object} contentstack - Contentstack configuration details
- */
+*/
 const init = (contentstack) => {
     const packageInfo = JSON.parse((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '..', 'package.json')));
     Contentstack = contentstack;
@@ -36,6 +38,12 @@ const init = (contentstack) => {
     if (Contentstack.MAX_RETRY_LIMIT) {
         MAX_RETRY_LIMIT = Contentstack.MAX_RETRY_LIMIT;
     }
+    if (Contentstack.RETRY_DELAY_BASE) {
+        RETRY_DELAY_BASE = Contentstack.RETRY_DELAY_BASE;
+    }
+    if (Contentstack.TIMEOUT) {
+        TIMEOUT = Contentstack.TIMEOUT;
+    }
 };
 exports.init = init;
 /**
@@ -60,12 +68,13 @@ const get = (req, RETRY = 1) => {
             path: (0, sanitize_url_1.sanitizeUrl)(encodeURI(req.path)),
             port: Contentstack.port,
             protocol: Contentstack.protocol,
+            timeout: TIMEOUT, // Configurable timeout to prevent socket hang ups
         };
         try {
             debug(`${options.method.toUpperCase()}: ${options.path}`);
             let timeDelay;
             let body = '';
-            (0, https_1.request)(options, (response) => {
+            const httpRequest = (0, https_1.request)(options, (response) => {
                 response
                     .setEncoding('utf-8')
                     .on('data', (chunk) => body += chunk)
@@ -75,8 +84,8 @@ const get = (req, RETRY = 1) => {
                         return resolve(JSON.parse(body));
                     }
                     else if (response.statusCode === 429) {
-                        timeDelay = Math.pow(Math.SQRT2, RETRY) * 200;
-                        debug(`API rate limit exceeded. Retrying ${options.path} with ${timeDelay} sec delay`);
+                        timeDelay = Math.pow(Math.SQRT2, RETRY) * RETRY_DELAY_BASE;
+                        debug(`API rate limit exceeded. Retrying ${options.path} with ${timeDelay} ms delay`);
                         return setTimeout(() => {
                             return (0, exports.get)(req, RETRY)
                                 .then(resolve)
@@ -85,8 +94,8 @@ const get = (req, RETRY = 1) => {
                     }
                     else if (response.statusCode >= 500) {
                         // retry, with delay
-                        timeDelay = Math.pow(Math.SQRT2, RETRY) * 200;
-                        debug(`Retrying ${options.path} with ${timeDelay} sec delay`);
+                        timeDelay = Math.pow(Math.SQRT2, RETRY) * RETRY_DELAY_BASE;
+                        debug(`Retrying ${options.path} with ${timeDelay} ms delay`);
                         RETRY++;
                         return setTimeout(() => {
                             return (0, exports.get)(req, RETRY)
@@ -99,9 +108,31 @@ const get = (req, RETRY = 1) => {
                         return reject(body);
                     }
                 });
-            })
-                .on('error', reject)
-                .end();
+            });
+            // Set socket timeout to handle socket hang ups
+            httpRequest.setTimeout(options.timeout, () => {
+                debug(`Request timeout for ${options.path || 'unknown'}`);
+                httpRequest.destroy();
+                reject(new Error('Request timeout'));
+            });
+            // Enhanced error handling for socket hang ups and connection resets
+            httpRequest.on('error', (error) => {
+                var _a;
+                debug(`Request error for ${options.path || 'unknown'}: ${(error === null || error === void 0 ? void 0 : error.message) || 'Unknown error'} (${(error === null || error === void 0 ? void 0 : error.code) || 'NO_CODE'})`);
+                // Handle socket hang up and connection reset errors with retry
+                if (((error === null || error === void 0 ? void 0 : error.code) === 'ECONNRESET' || ((_a = error === null || error === void 0 ? void 0 : error.message) === null || _a === void 0 ? void 0 : _a.includes('socket hang up'))) && RETRY <= MAX_RETRY_LIMIT) {
+                    timeDelay = Math.pow(Math.SQRT2, RETRY) * RETRY_DELAY_BASE;
+                    debug(`Socket hang up detected. Retrying ${options.path || 'unknown'} with ${timeDelay} ms delay (attempt ${RETRY}/${MAX_RETRY_LIMIT})`);
+                    RETRY++;
+                    return setTimeout(() => {
+                        return (0, exports.get)(req, RETRY)
+                            .then(resolve)
+                            .catch(reject);
+                    }, timeDelay);
+                }
+                return reject(error);
+            });
+            httpRequest.end();
         }
         catch (error) {
             return reject(error);

package/dist/config.js

@@ -35,6 +35,8 @@ exports.config = {
     },
     contentstack: {
         MAX_RETRY_LIMIT: 6,
+        TIMEOUT: 30000,
+        RETRY_DELAY_BASE: 200,
         actions: {
             delete: 'delete',
             publish: 'publish',

package/dist/core/index.js

@@ -310,6 +310,7 @@ const fire = (req) => {
                     });
                     return (0, promise_map_1.map)(contentTypeUids, (uid) => {
                         return new Promise((mapResolve, mapReject) => {
+                            debug(`API called with for content type: ${uid}`);
                             return (0, api_1.get)({
                                 path: `${Contentstack.apis.content_types}${uid}`,
                                 qs: {
@@ -330,6 +331,7 @@ const fire = (req) => {
                                 err.code = 'ICTC';
                                 return mapReject(err);
                             }).catch((error) => {
+                                debug('Error [map] fetching content type schema:', error);
                                 if ((0, inet_1.netConnectivityIssues)(error)) {
                                     flag.SQ = false;
                                 }
@@ -345,14 +347,17 @@ const fire = (req) => {
                             flag.SQ = false;
                         }
                         // Errorred while fetching content type schema
+                        debug('Error [mapResolve]:', error);
                         return reject(error);
                     });
                 }).catch((processError) => {
+                    debug('Error [filterItems]:', processError);
                     return reject(processError);
                 });
             }
             return postProcess(req, syncResponse)
-                .then(resolve);
+                .then(resolve)
+                .catch(reject);
         }).catch((error) => {
             debug('Error [fire]', error);
             if ((0, inet_1.netConnectivityIssues)(error)) {
@@ -394,6 +399,7 @@ const postProcess = (req, resp) => {
                     flag.SQ = false;
                     return resolve('');
                 }
+                debug(`Re-Fire called with: ${JSON.stringify(req)}`);
                 return fire(req)
                     .then(resolve)
                     .catch(reject);

package/dist/core/inet.js

@@ -68,7 +68,10 @@ const checkNetConnectivity = () => {
 };
 exports.checkNetConnectivity = checkNetConnectivity;
 const netConnectivityIssues = (error) => {
-    if (error.code === 'ENOTFOUND' || error.code === 'ETIMEDOUT') {
+    var _a;
+    // Include socket hang up and connection reset errors as network connectivity issues
+    const networkErrorCodes = ['ENOTFOUND', 'ETIMEDOUT', 'ECONNRESET', 'EPIPE', 'EHOSTUNREACH'];
+    if (networkErrorCodes.includes(error.code) || ((_a = error.message) === null || _a === void 0 ? void 0 : _a.includes('socket hang up'))) {
         return true;
     }
     return false;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@contentstack/datasync-manager",
   "author": "Contentstack LLC <support@contentstack.com>",
-  "version": "2.1.0",
+  "version": "2.1.2",
   "description": "The primary module of Contentstack DataSync. Syncs Contentstack data with your server using Contentstack Sync API",
   "main": "dist/index.js",
   "dependencies": {