locize-cli 7.15.0 → 7.15.2

package/CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 Project versioning adheres to [Semantic Versioning](http://semver.org/).
 Change log format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## [7.15.2](https://github.com/locize/locize-cli/compare/v7.15.1...v7.15.2) - 2023-01-06
+
+- optimize --skip-empty false handling for download command
+
+## [7.15.1](https://github.com/locize/locize-cli/compare/v7.15.0...v7.15.1) - 2023-12-19
+
+- refactor retriable requests handling
+
 ## [7.15.0](https://github.com/locize/locize-cli/compare/v7.14.15...v7.15.0) - 2023-12-18
 
 - locize sync: show publish mode advice if necessary

package/download.js

@@ -151,6 +151,43 @@ const handleError = (err, cb) => {
   if (cb) cb(err);
 };
 
+const filterDownloads = (opt, downloads) => {
+  if (opt.skipEmpty) return downloads.filter((d) => d.size > 2);
+  if (downloads.length < 1) return downloads;
+
+  const allNamespaces = [];
+  const downloadMap = {};
+  downloads.forEach((d) => {
+    const splitted = d.key.split('/');
+    const p = splitted[d.isPrivate ? 1 : 0];
+    const v = splitted[d.isPrivate ? 2 : 1];
+    const l = splitted[d.isPrivate ? 3 : 2];
+    const n = splitted[d.isPrivate ? 4 : 3];
+    downloadMap[p] = downloadMap[p] || {};
+    downloadMap[p][v] = downloadMap[p][v] || {};
+    downloadMap[p][v][l] = downloadMap[p][v][l] || {};
+    downloadMap[p][v][l][n] = d;
+    if (allNamespaces.indexOf(n) < 0) allNamespaces.push(n);
+  });
+  Object.keys(downloadMap).forEach((projectId) => {
+    Object.keys(downloadMap[projectId]).forEach((version) => {
+      Object.keys(downloadMap[projectId][version]).forEach((language) => {
+        allNamespaces.forEach((namespace) => {
+          if (!downloadMap[projectId][version][language][namespace]) {
+            downloads.push({
+              url: `${opt.apiPath}/${projectId}/${version}/${language}/${namespace}`,
+              key: `${projectId}/${version}/${language}/${namespace}`,
+              lastModified: '1960-01-01T00:00:00.000Z',
+              size: 0
+            });
+          }
+        });
+      });
+    });
+  });
+  return downloads;
+};
+
 const download = (opt, cb) => {
   opt.format = opt.format || 'json';
   if (!reversedFileExtensionsMap[opt.format]) {
@@ -203,20 +240,18 @@ const download = (opt, cb) => {
           'Authorization': opt.apiKey
         } : undefined
       }, (err, res, obj) => {
-        obj = obj || [];
         if (res && res.status === 401) {
           opt.apiKey = null;
           request(url, {
             method: 'get',
           }, (err, res, obj) => {
-            obj = obj || [];
-            if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
+            obj = filterDownloads(opt, obj || []);
             handleDownload(opt, url, err, res, obj, cb);
           });
           return;
         }
 
-        if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
+        obj = filterDownloads(opt, obj || []);
         handleDownload(opt, url, err, res, obj, cb);
       });
       return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locize-cli",
-  "version": "7.15.0",
+  "version": "7.15.2",
   "description": "locize cli to import locales",
   "main": "index.js",
   "bin": {
@@ -34,7 +34,7 @@
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.0/xlsx-0.20.0.tgz"
   },
   "devDependencies": {
-    "eslint": "8.53.0",
+    "eslint": "8.56.0",
     "gh-release": "7.0.2",
     "pkg": "5.8.1"
   },

package/request.js

@@ -8,6 +8,28 @@ cacheable.install(https.globalAgent);
 
 const httpProxy = process.env.http_proxy || process.env.HTTP_PROXY || process.env.https_proxy || process.env.HTTPS_PROXY;
 
+const isRetriableError = (err) => {
+  return err && err.message && (
+    err.message.indexOf('ETIMEDOUT') > -1 || // on timeout retry
+    // on dns errors
+    err.message.indexOf('ENOTFOUND') > -1 ||
+    err.message.indexOf('ENODATA') > -1 ||
+    err.message.indexOf('ENOENT') > -1 // Windows: name exists, but not this record type
+  );
+};
+
+const isJSONResponse = (res) => res.headers.get('content-type') && res.headers.get('content-type').indexOf('json') > 0;
+
+const handleResponse = (res) => {
+  if (isJSONResponse(res)) {
+    return new Promise((resolve, reject) => res.json().then((obj) => resolve({ res, obj })).catch(reject));
+  } else {
+    return { res };
+  }
+};
+
+const handleSuccessful = (callback) => (ret) => callback(null, ret.res, ret.obj);
+
 module.exports = (url, options, callback) => {
   if (httpProxy) {
     const httpsProxyAgent = new HttpsProxyAgent(httpProxy);
@@ -25,31 +47,16 @@ module.exports = (url, options, callback) => {
     }
   }
   if (options.headers['Authorization'] === undefined) delete options.headers['Authorization'];
-  fetch(url, options).then((res) => {
-    if (res.headers.get('content-type') && res.headers.get('content-type').indexOf('json') > 0) {
-      return new Promise((resolve, reject) => res.json().then((obj) => resolve({ res, obj })).catch(reject));
-    } else {
-      return { res };
-    }
-  }).then((ret) => callback(null, ret.res, ret.obj)).catch((err) => {
-    if (err && err.message && (
-      err.message.indexOf('ETIMEDOUT') > -1 || // on timeout retry
-      // on dns errors
-      err.message.indexOf('ENOTFOUND') > -1 ||
-      err.message.indexOf('ENODATA') > -1 ||
-      err.message.indexOf('ENOENT') > -1 // Windows: name exists, but not this record type
-    )) {
+
+  function retriableFetch(maxRetries) {
+    fetch(url, options).then(handleResponse).then(handleSuccessful(callback)).catch((err) => {
+      if (maxRetries < 1) return callback(err);
+      if (!isRetriableError(err)) return callback(err);
       setTimeout(() => {
-        fetch(url, options).then((res) => {
-          if (res.headers.get('content-type') && res.headers.get('content-type').indexOf('json') > 0) {
-            return new Promise((resolve, reject) => res.json().then((obj) => resolve({ res, obj })).catch(reject));
-          } else {
-            return { res };
-          }
-        }).then((ret) => callback(null, ret.res, ret.obj)).catch(callback);
+        retriableFetch(--maxRetries);
       }, 5000);
-      return;
-    }
-    callback(err);
-  });
+    });
+  }
+
+  retriableFetch(3);
 };