locize-cli 7.6.14 → 7.7.0

package/.github/stale.yml

@@ -0,0 +1,23 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 7
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - "discussion"
+  - "feature request"
+  - "bug"
+  - "breaking change"
+  - "doc"
+  - "issue"
+  - "help wanted"
+  - "good first issue"
+# Label to use when marking an issue as stale
+staleLabel: stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

package/CHANGELOG.md

@@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file.
 Project versioning adheres to [Semantic Versioning](http://semver.org/).
 Change log format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## [7.7.0](https://github.com/locize/locize-cli/compare/v7.6.17...v7.7.0) - 2021-12-08
+
+- sync: optimize api requests on bigger projects
+- retry on very short dns resolution errors
+
+
+## [7.6.17](https://github.com/locize/locize-cli/compare/v7.6.16...v7.6.17) - 2021-09-21
+
+- check skipEmpty flag when downloading translations
+
+
+## [7.6.16](https://github.com/locize/locize-cli/compare/v7.6.15...v7.6.16) - 2021-09-20
+
+- by default set also languageFolderPrefix to empty string when using programmatically
+
+
+## [7.6.15](https://github.com/locize/locize-cli/compare/v7.6.14...v7.6.15) - 2021-07-23
+
+- update some dependencies
+
+
 ## [7.6.14](https://github.com/locize/locize-cli/compare/v7.6.13...v7.6.14) - 2021-03-13
 
 - update all dependencies

package/README.md

@@ -88,6 +88,11 @@ locize download --project-id my-project-id-93e1-442a-ab35-24331fa294ba --ver lat
 
 
 ## Synchronize locize with your repository (or any other local directory)
+By using the sync command, you can keep your existing code setup and synchronize the translations with locize.
+An example on how this could look like can be seen in [this tutorial](https://github.com/locize/react-tutorial#step-1---keep-existing-code-setup-but-synchronize-with-locize).
+
+**⚠️ Since the remote source are the published translations, make sure the desired version is set to standard publish mode, or alternatively [publish](#publish-version) that version before you execute the cli command. ⚠️**
+
 ### Step 1: Go near to your translation files
 
 ```sh
@@ -139,6 +144,13 @@ Navigate to your locize project and check the results => [www.locize.app](https:
 
 
 ## Push missing keys to locize from your repository (or any other local directory)
+This is useful, when i.e. using [i18next-scanner](https://github.com/i18next/i18next-scanner), like described [here](https://github.com/locize/i18next-locize-backend/issues/315#issuecomment-586967039).
+The save-missing command uses the [missing API](https://docs.locize.com/integration/api#missing-translations) and the sync command uses the [update API](https://docs.locize.com/integration/api#update-remove-translations)
+So, if you want to save new keys (that does not exist in locize), the save-missing command is the better choice.
+Doing so, you can then for example make use of the “created by missing API" filter in the locize UI.
+
+But if you need to update existing keys, the sync command is the correct choice.
+
 ### Step 1: Go near to your translation files
 
 ```sh

package/download.js

@@ -152,10 +152,14 @@ const download = (opt, cb) => {
         opt.apiKey = null;
         request(url, {
           method: 'get',
-        }, (err, res, obj) => handleDownload(opt, url, err, res, obj, cb));
+        }, (err, res, obj) => {
+          if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
+          handleDownload(opt, url, err, res, obj, cb);
+        });
         return;
       }
 
+      if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
       handleDownload(opt, url, err, res, obj, cb);
     });
   });

package/missing.js

@@ -139,6 +139,7 @@ const missing = (opt, cb) => {
   opt.pathMaskInterpolationPrefix = opt.pathMaskInterpolationPrefix || '{{';
   opt.pathMaskInterpolationSuffix = opt.pathMaskInterpolationSuffix || '}}';
   opt.pathMask = opt.pathMask || `${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}${path.sep}${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`;
+  opt.languageFolderPrefix = opt.languageFolderPrefix || '';
   opt.pathMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, `${opt.languageFolderPrefix}${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`);
 
   getRemoteLanguages(opt, (err, remoteLanguages) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locize-cli",
-  "version": "7.6.14",
+  "version": "7.7.0",
   "description": "locize cli to import locales",
   "main": "index.js",
   "bin": {
@@ -9,32 +9,32 @@
   "dependencies": {
     "@js.properties/properties": "0.5.4",
     "android-string-resource": "2.3.4",
-    "async": "3.2.0",
+    "async": "3.2.1",
     "colors": "1.4.0",
-    "commander": "7.1.0",
+    "commander": "7.2.0",
     "csvjson": "5.1.0",
     "diff": "5.0.0",
     "flat": "5.0.2",
     "fluent_conv": "3.1.0",
-    "gettext-converter": "1.0.7",
+    "gettext-converter": "1.1.0",
     "https-proxy-agent": "5.0.0",
     "ini": "2.0.0",
-    "js-yaml": "4.0.0",
+    "js-yaml": "4.1.0",
     "laravelphp": "2.0.3",
     "lodash.clonedeep": "4.5.0",
     "mkdirp": "1.0.4",
-    "node-fetch": "2.6.1",
+    "node-fetch": "2.6.2",
     "resx": "2.0.3",
     "rimraf": "3.0.2",
     "strings-file": "0.0.5",
     "tmexchange": "2.0.4",
-    "xliff": "5.5.1",
-    "xlsx": "0.16.9"
+    "xliff": "5.6.2",
+    "xlsx": "0.17.2"
   },
   "devDependencies": {
-    "eslint": "7.22.0",
-    "gh-release": "5.0.0",
-    "pkg": "4.4.9"
+    "eslint": "7.32.0",
+    "gh-release": "6.0.0",
+    "pkg": "5.3.2"
   },
   "scripts": {
     "lint": "eslint .",

package/request.js

@@ -22,5 +22,19 @@ module.exports = (url, options, callback) => {
     } else {
       return { res };
     }
-  }).then((ret) => callback(null, ret.res, ret.obj)).catch(callback);
+  }).then((ret) => callback(null, ret.res, ret.obj)).catch((err) => {
+    if (err && err.message && err.message.indexOf('ENOTFOUND') > -1) {
+      setTimeout(() => {
+        fetch(url, options).then((res) => {
+          if (res.headers.get('content-type') && res.headers.get('content-type').indexOf('json') > 0) {
+            return new Promise((resolve, reject) => res.json().then((obj) => resolve({ res, obj })).catch(reject));
+          } else {
+            return { res };
+          }
+        }).then((ret) => callback(null, ret.res, ret.obj)).catch(callback);
+      }, 3000);
+      return;
+    }
+    callback(err);
+  });
 };

package/sync.js

@@ -36,6 +36,7 @@ const getDownloads = (opt, cb) => {
       }
       return cb(new Error(res.statusText + ' (' + res.status + ')'));
     }
+    if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
     cb(null, obj);
   });
 };
@@ -197,7 +198,7 @@ const downloadAll = (opt, remoteLanguages, omitRef, manipulate, cb) => {
   });
 };
 
-const update = (opt, lng, ns, cb) => {
+const update = (opt, lng, ns, shouldOmit, cb) => {
   var data = {};
   if (!opt.skipDelete) {
     ns.diff.toRemove.forEach((k) => data[k] = null);
@@ -214,7 +215,7 @@ const update = (opt, lng, ns, cb) => {
   var payloadKeysLimit = 1000;
 
   function send(d, clb, isRetrying) {
-    request(opt.apiPath + '/update/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns.namespace, {
+    request(opt.apiPath + '/update/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns.namespace + (shouldOmit ? '?omitstatsgeneration=true' : ''), {
       method: 'post',
       body: d,
       headers: {
@@ -320,6 +321,20 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
     compareNamespaces(opt, localNamespaces, (err, compared) => {
       if (err) return handleError(err);
 
+      const onlyToUpdate = compared.filter((ns) => ns.diff.toAdd.concat(opt.skipDelete ? [] : ns.diff.toRemove).concat(ns.diff.toUpdate).length > 0);
+
+      const lngsInReqs = [];
+      const nsInReqs = [];
+      onlyToUpdate.forEach((n) => {
+        if (lngsInReqs.indexOf(n.language) < 0) {
+          lngsInReqs.push(n.language);
+        }
+        if (nsInReqs.indexOf(n.namespace) < 0) {
+          nsInReqs.push(n.namespace);
+        }
+      });
+      const shouldOmit = lngsInReqs.length > 5 || nsInReqs.length > 5;
+
       var wasThereSomethingToUpdate = false;
       async.eachLimit(compared, Math.round(require('os').cpus().length / 2), (ns, clb) => {
         if (!cb) {
@@ -363,40 +378,63 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
           if (!somethingToUpdate) console.log(colors.grey(`nothing to update for ${ns.language}/${ns.namespace}`));
           if (!wasThereSomethingToUpdate && somethingToUpdate) wasThereSomethingToUpdate = true;
         }
-        update(opt, ns.language, ns, (err) => {
+        update(opt, ns.language, ns, shouldOmit, (err) => {
           if (err) return clb(err);
           if (ns.diff.toRemove.length === 0 || ns.language !== opt.referenceLanguage) return clb();
           const nsOnlyRemove = cloneDeep(ns);
           nsOnlyRemove.diff.toAdd = [];
           nsOnlyRemove.diff.toUpdate = [];
-          async.eachLimit(remoteLanguages, Math.round(require('os').cpus().length / 2), (lng, clb) => update(opt, lng, nsOnlyRemove, clb), clb);
+          async.eachLimit(remoteLanguages, Math.round(require('os').cpus().length / 2), (lng, clb) => update(opt, lng, nsOnlyRemove, shouldOmit, clb), clb);
         });
       }, (err) => {
         if (err) return handleError(err);
-
         if (!cb) console.log(colors.grey('syncing...'));
-        setTimeout(() => {
-          downloadAll(opt, remoteLanguages, false, opt.skipDelete ? (lng, namespace, ns) => {
-            const found = compared.find((n) => n.namespace === namespace && n.language === lng);
-            if (found && found.diff) {
-              if (found.diff.toAddLocally && found.diff.toAddLocally.length > 0) {
-                found.diff.toAddLocally.forEach((k) => {
-                  delete ns[k];
-                });
-              }
-              if (found.diff.toRemove && found.diff.toRemove.length > 0) {
-                found.diff.toRemove.forEach((k) => {
-                  delete ns[k];
-                });
+
+        function down() {
+          setTimeout(() => { // wait a bit before downloading... just to have a chance to get the newly published files
+            downloadAll(opt, remoteLanguages, false, opt.skipDelete ? (lng, namespace, ns) => {
+              const found = compared.find((n) => n.namespace === namespace && n.language === lng);
+              if (found && found.diff) {
+                if (found.diff.toAddLocally && found.diff.toAddLocally.length > 0) {
+                  found.diff.toAddLocally.forEach((k) => {
+                    delete ns[k];
+                  });
+                }
+                if (found.diff.toRemove && found.diff.toRemove.length > 0) {
+                  found.diff.toRemove.forEach((k) => {
+                    delete ns[k];
+                  });
+                }
               }
+            } : undefined, (err) => {
+              if (err) return handleError(err);
+              if (!cb) console.log(colors.green('FINISHED'));
+              if (cb) cb(null);
+            });
+          }, wasThereSomethingToUpdate && !opt.dry ? 5000 : 0);
+        }
+
+        if (!shouldOmit) return down();
+        if (opt.dry) return down();
+
+        // optimize stats generation...
+        request(opt.apiPath + '/stats/project/regenerate/' + opt.projectId + '/' + opt.version + (lngsInReqs.length === 1 ? `/${lngsInReqs[0]}` : '') + (nsInReqs.length === 1 ? `?namespace=${nsInReqs[0]}` : ''), {
+          method: 'post',
+          body: {},
+          headers: {
+            'Authorization': opt.apiKey
+          }
+        }, (err, res, obj) => {
+          if (err) return handleError(err);
+          if (res.status >= 300 && res.status !== 412) {
+            if (obj && (obj.errorMessage || obj.message)) {
+              return handleError(new Error((obj.errorMessage || obj.message)));
             }
-          } : undefined, (err) => {
-            if (err) return handleError(err);
-            if (!cb) console.log(colors.green('FINISHED'));
-            if (cb) cb(null);
-          });
-        }, wasThereSomethingToUpdate && !opt.dry ? 5000 : 0);
-      }); // wait a bit before downloading... just to have a chance to get the newly published files
+            return handleError(new Error(res.statusText + ' (' + res.status + ')'));
+          }
+          down();
+        });
+      });
     });
   });
 };
@@ -421,6 +459,7 @@ const sync = (opt, cb) => {
   opt.pathMaskInterpolationPrefix = opt.pathMaskInterpolationPrefix || '{{';
   opt.pathMaskInterpolationSuffix = opt.pathMaskInterpolationSuffix || '}}';
   opt.pathMask = opt.pathMask || `${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}${path.sep}${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`;
+  opt.languageFolderPrefix = opt.languageFolderPrefix || '';
   opt.pathMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, `${opt.languageFolderPrefix}${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`);
 
   getRemoteLanguages(opt, (err, remoteLanguages) => {