locize-cli 7.6.17 → 7.8.0

package/CHANGELOG.md

@@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file.
 Project versioning adheres to [Semantic Versioning](http://semver.org/).
 Change log format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## [7.8.0](https://github.com/locize/locize-cli/compare/v7.7.2...v7.8.0) - 2022-01-13
+
+- sync: introduce --delete-remote-namespace option
+
+
+## [7.7.2](https://github.com/locize/locize-cli/compare/v7.7.1...v7.7.2) - 2021-12-10
+
+- update dependencies
+
+
+## [7.7.1](https://github.com/locize/locize-cli/compare/v7.7.0...v7.7.1) - 2021-12-10
+
+- new binary build (x64)
+
+
+## [7.7.0](https://github.com/locize/locize-cli/compare/v7.6.17...v7.7.0) - 2021-12-08
+
+- sync: optimize api requests on bigger projects
+- retry on very short dns resolution errors
+
+
 ## [7.6.17](https://github.com/locize/locize-cli/compare/v7.6.16...v7.6.17) - 2021-09-21
 
 - check skipEmpty flag when downloading translations

package/README.md

@@ -91,6 +91,8 @@ locize download --project-id my-project-id-93e1-442a-ab35-24331fa294ba --ver lat
 By using the sync command, you can keep your existing code setup and synchronize the translations with locize.
 An example on how this could look like can be seen in [this tutorial](https://github.com/locize/react-tutorial#step-1---keep-existing-code-setup-but-synchronize-with-locize).
 
+**⚠️ Since the remote source are the published translations, make sure the desired version is set to standard publish mode, or alternatively [publish](#publish-version) that version before you execute the cli command. ⚠️**
+
 ### Step 1: Go near to your translation files
 
 ```sh

package/bin/locize

@@ -368,6 +368,7 @@ program
   .option('-cf, --clean-local-files <true|false>', 'Removes all local files without removing any folder (default: false)', 'false')
   .option('-u, --update-values <true|false>', 'This will update values of existing translations. (default: false)', 'false')
   .option('-S, --skip-delete <true|false>', 'This will skip the removal of keys on locize. (default: false)', 'false')
+  .option('-D, --delete-remote-namespace <true|false>', 'This will delete a complete namespace on locize, if a local file in reference language was deleted. (default: false)', 'false')
   .option('-m, --path-mask <mask>', 'This will define the folder and file structure; do not add a file extension (default: {{language}}/{{namespace}})', `{{language}}${path.sep}{{namespace}}`)
   .option('-P, --language-folder-prefix <prefix>', 'This will be added as a local folder name prefix in front of the language.', '')
   .option('-d, --dry <true|false>', 'Dry run (default: false)', 'false')
@@ -417,6 +418,7 @@ program
     const dry = options.dry === 'true';
     const updateValues = options.updateValues === 'true';
     const skipDelete = options.skipDelete === 'true';
+    const deleteRemoteNamespace = options.deleteRemoteNamespace === 'true';
     const languageFolderPrefix = options.languageFolderPrefix || '';
     const skipEmpty = options.skipEmpty === 'true';
     const referenceLanguageOnly = options.referenceLanguageOnly === 'false' ? false : true;
@@ -432,6 +434,7 @@ program
       format: options.format,
       updateValues: updateValues,
       skipDelete: skipDelete,
+      deleteRemoteNamespace: deleteRemoteNamespace,
       languageFolderPrefix: languageFolderPrefix,
       clean: clean,
       cleanLocalFiles: cleanLocalFiles,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locize-cli",
-  "version": "7.6.17",
+  "version": "7.8.0",
   "description": "locize cli to import locales",
   "main": "index.js",
   "bin": {
@@ -9,37 +9,37 @@
   "dependencies": {
     "@js.properties/properties": "0.5.4",
     "android-string-resource": "2.3.4",
-    "async": "3.2.1",
+    "async": "3.2.2",
     "colors": "1.4.0",
     "commander": "7.2.0",
     "csvjson": "5.1.0",
     "diff": "5.0.0",
     "flat": "5.0.2",
     "fluent_conv": "3.1.0",
-    "gettext-converter": "1.1.0",
+    "gettext-converter": "1.2.0",
     "https-proxy-agent": "5.0.0",
     "ini": "2.0.0",
     "js-yaml": "4.1.0",
     "laravelphp": "2.0.3",
     "lodash.clonedeep": "4.5.0",
     "mkdirp": "1.0.4",
-    "node-fetch": "2.6.2",
+    "node-fetch": "2.6.6",
     "resx": "2.0.3",
     "rimraf": "3.0.2",
     "strings-file": "0.0.5",
     "tmexchange": "2.0.4",
-    "xliff": "5.6.2",
-    "xlsx": "0.17.2"
+    "xliff": "5.7.2",
+    "xlsx": "0.17.4"
   },
   "devDependencies": {
     "eslint": "7.32.0",
-    "gh-release": "6.0.0",
-    "pkg": "5.3.2"
+    "gh-release": "6.0.1",
+    "pkg": "5.5.1"
   },
   "scripts": {
     "lint": "eslint .",
     "test": "npm run lint",
-    "pkg": "mkdir -p ./bins && pkg ./bin/locize --out-path ./bins",
+    "pkg": "mkdir -p ./bins && pkg ./bin/locize --out-path ./bins --targets node16-linux-x64,node16-macos-x64,node16-win-x64",
     "release": "gh-release --assets ./bins/locize-linux,./bins/locize-macos,./bins/locize-win.exe",
     "version": "npm run pkg",
     "postversion": "git push && npm run release"

package/request.js

@@ -22,5 +22,19 @@ module.exports = (url, options, callback) => {
     } else {
       return { res };
     }
-  }).then((ret) => callback(null, ret.res, ret.obj)).catch(callback);
+  }).then((ret) => callback(null, ret.res, ret.obj)).catch((err) => {
+    if (err && err.message && err.message.indexOf('ENOTFOUND') > -1) {
+      setTimeout(() => {
+        fetch(url, options).then((res) => {
+          if (res.headers.get('content-type') && res.headers.get('content-type').indexOf('json') > 0) {
+            return new Promise((resolve, reject) => res.json().then((obj) => resolve({ res, obj })).catch(reject));
+          } else {
+            return { res };
+          }
+        }).then((ret) => callback(null, ret.res, ret.obj)).catch(callback);
+      }, 3000);
+      return;
+    }
+    callback(err);
+  });
 };

package/sync.js

@@ -14,6 +14,7 @@ const parseLocalLanguages = require('./parseLocalLanguages');
 const parseLocalReference = require('./parseLocalReference');
 const formats = require('./formats');
 const lngCodes = require('./lngs.json');
+const deleteNamespace = require('./deleteNamespace');
 const reversedFileExtensionsMap = formats.reversedFileExtensionsMap;
 
 const getDirectories = (srcpath) => {
@@ -198,7 +199,7 @@ const downloadAll = (opt, remoteLanguages, omitRef, manipulate, cb) => {
   });
 };
 
-const update = (opt, lng, ns, cb) => {
+const update = (opt, lng, ns, shouldOmit, cb) => {
   var data = {};
   if (!opt.skipDelete) {
     ns.diff.toRemove.forEach((k) => data[k] = null);
@@ -215,7 +216,7 @@ const update = (opt, lng, ns, cb) => {
   var payloadKeysLimit = 1000;
 
   function send(d, clb, isRetrying) {
-    request(opt.apiPath + '/update/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns.namespace, {
+    request(opt.apiPath + '/update/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns.namespace + (shouldOmit ? '?omitstatsgeneration=true' : ''), {
       method: 'post',
       body: d,
       headers: {
@@ -303,6 +304,26 @@ const handleError = (err, cb) => {
   if (cb) cb(err);
 };
 
+const checkForMissingLocalNamespaces = (downloads, localNamespaces, opt) => {
+  const localMissingNamespaces = [];
+  downloads.forEach((d) => {
+    const splitted = d.url.split('/');
+    const namespace = splitted.pop();
+    const language = splitted.pop();
+    // if (!opt.referenceLanguageOnly || (opt.referenceLanguageOnly && language === opt.referenceLanguage)) {
+    if (language === opt.referenceLanguage) {
+      const foundLocalNamespace = localNamespaces.find((n) => n.namespace === namespace && n.language === language);
+      if (!foundLocalNamespace) {
+        localMissingNamespaces.push({
+          language,
+          namespace
+        });
+      }
+    }
+  });
+  return localMissingNamespaces;
+};
+
 const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
   if (!localNamespaces || localNamespaces.length === 0) {
     downloadAll(opt, remoteLanguages, (err) => {
@@ -318,86 +339,151 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
 
     opt.isPrivate = downloads.length > 0 && downloads[0].isPrivate;
 
+    const localMissingNamespaces = checkForMissingLocalNamespaces(downloads, localNamespaces, opt);
+
     compareNamespaces(opt, localNamespaces, (err, compared) => {
       if (err) return handleError(err);
 
+      const onlyToUpdate = compared.filter((ns) => ns.diff.toAdd.concat(opt.skipDelete ? [] : ns.diff.toRemove).concat(ns.diff.toUpdate).length > 0);
+
+      const lngsInReqs = [];
+      const nsInReqs = [];
+      onlyToUpdate.forEach((n) => {
+        if (lngsInReqs.indexOf(n.language) < 0) {
+          lngsInReqs.push(n.language);
+        }
+        if (nsInReqs.indexOf(n.namespace) < 0) {
+          nsInReqs.push(n.namespace);
+        }
+      });
+      const shouldOmit = lngsInReqs.length > 5 || nsInReqs.length > 5;
+
       var wasThereSomethingToUpdate = false;
-      async.eachLimit(compared, Math.round(require('os').cpus().length / 2), (ns, clb) => {
-        if (!cb) {
-          if (ns.diff.toRemove.length > 0) {
-            if (opt.skipDelete) {
-              console.log(colors.bgRed(`skipping the removal of ${ns.diff.toRemove.length} keys in ${ns.language}/${ns.namespace}...`));
-              if (opt.dry) console.log(colors.bgRed(`skipped to remove ${ns.diff.toRemove.join(', ')} in ${ns.language}/${ns.namespace}...`));
-            } else {
-              console.log(colors.red(`removing ${ns.diff.toRemove.length} keys in ${ns.language}/${ns.namespace}...`));
-              if (opt.dry) console.log(colors.red(`would remove ${ns.diff.toRemove.join(', ')} in ${ns.language}/${ns.namespace}...`));
+
+      function updateComparedNamespaces() {
+        async.eachLimit(compared, Math.round(require('os').cpus().length / 2), (ns, clb) => {
+          if (!cb) {
+            if (ns.diff.toRemove.length > 0) {
+              if (opt.skipDelete) {
+                console.log(colors.bgRed(`skipping the removal of ${ns.diff.toRemove.length} keys in ${ns.language}/${ns.namespace}...`));
+                if (opt.dry) console.log(colors.bgRed(`skipped to remove ${ns.diff.toRemove.join(', ')} in ${ns.language}/${ns.namespace}...`));
+              } else {
+                console.log(colors.red(`removing ${ns.diff.toRemove.length} keys in ${ns.language}/${ns.namespace}...`));
+                if (opt.dry) console.log(colors.red(`would remove ${ns.diff.toRemove.join(', ')} in ${ns.language}/${ns.namespace}...`));
+              }
             }
-          }
-          if (ns.diff.toRemoveLocally.length > 0) {
-            console.log(colors.red(`removing ${ns.diff.toRemoveLocally.length} keys in ${ns.language}/${ns.namespace} locally...`));
-            if (opt.dry) console.log(colors.red(`would remove ${ns.diff.toRemoveLocally.join(', ')} in ${ns.language}/${ns.namespace} locally...`));
-          }
-          if (ns.diff.toAdd.length > 0) {
-            console.log(colors.green(`adding ${ns.diff.toAdd.length} keys in ${ns.language}/${ns.namespace}...`));
-            if (opt.dry) console.log(colors.green(`would add ${ns.diff.toAdd.join(', ')} in ${ns.language}/${ns.namespace}...`));
-          }
-          if (ns.diff.toAddLocally.length > 0) {
-            if (opt.skipDelete) {
-              console.log(colors.bgGreen(`skipping the addition of ${ns.diff.toAddLocally.length} keys in ${ns.language}/${ns.namespace} locally...`));
-              if (opt.dry) console.log(colors.bgGreen(`skipped the addition of ${ns.diff.toAddLocally.join(', ')} in ${ns.language}/${ns.namespace} locally...`));
-            } else {
-              console.log(colors.green(`adding ${ns.diff.toAddLocally.length} keys in ${ns.language}/${ns.namespace} locally...`));
-              if (opt.dry) console.log(colors.green(`would add ${ns.diff.toAddLocally.join(', ')} in ${ns.language}/${ns.namespace} locally...`));
+            if (ns.diff.toRemoveLocally.length > 0) {
+              console.log(colors.red(`removing ${ns.diff.toRemoveLocally.length} keys in ${ns.language}/${ns.namespace} locally...`));
+              if (opt.dry) console.log(colors.red(`would remove ${ns.diff.toRemoveLocally.join(', ')} in ${ns.language}/${ns.namespace} locally...`));
             }
-          }
-          if (opt.updateValues) {
-            if (ns.diff.toUpdate.length > 0) {
-              console.log(colors.yellow(`updating ${ns.diff.toUpdate.length} keys in ${ns.language}/${ns.namespace}...`));
-              if (opt.dry) console.log(colors.yellow(`would update ${ns.diff.toUpdate.join(', ')} in ${ns.language}/${ns.namespace}...`));
+            if (ns.diff.toAdd.length > 0) {
+              console.log(colors.green(`adding ${ns.diff.toAdd.length} keys in ${ns.language}/${ns.namespace}...`));
+              if (opt.dry) console.log(colors.green(`would add ${ns.diff.toAdd.join(', ')} in ${ns.language}/${ns.namespace}...`));
             }
-            if (ns.diff.toUpdateLocally.length > 0) {
-              console.log(colors.yellow(`updating ${ns.diff.toUpdateLocally.length} keys in ${ns.language}/${ns.namespace} locally...`));
-              if (opt.dry) console.log(colors.yellow(`would update ${ns.diff.toUpdateLocally.join(', ')} in ${ns.language}/${ns.namespace} locally...`));
+            if (ns.diff.toAddLocally.length > 0) {
+              if (opt.skipDelete) {
+                console.log(colors.bgGreen(`skipping the addition of ${ns.diff.toAddLocally.length} keys in ${ns.language}/${ns.namespace} locally...`));
+                if (opt.dry) console.log(colors.bgGreen(`skipped the addition of ${ns.diff.toAddLocally.join(', ')} in ${ns.language}/${ns.namespace} locally...`));
+              } else {
+                console.log(colors.green(`adding ${ns.diff.toAddLocally.length} keys in ${ns.language}/${ns.namespace} locally...`));
+                if (opt.dry) console.log(colors.green(`would add ${ns.diff.toAddLocally.join(', ')} in ${ns.language}/${ns.namespace} locally...`));
+              }
             }
-          }
-          const somethingToUpdate = ns.diff.toAdd.concat(opt.skipDelete ? [] : ns.diff.toRemove)/*.concat(ns.diff.toUpdate)*/.length > 0;
-          if (!somethingToUpdate) console.log(colors.grey(`nothing to update for ${ns.language}/${ns.namespace}`));
-          if (!wasThereSomethingToUpdate && somethingToUpdate) wasThereSomethingToUpdate = true;
-        }
-        update(opt, ns.language, ns, (err) => {
-          if (err) return clb(err);
-          if (ns.diff.toRemove.length === 0 || ns.language !== opt.referenceLanguage) return clb();
-          const nsOnlyRemove = cloneDeep(ns);
-          nsOnlyRemove.diff.toAdd = [];
-          nsOnlyRemove.diff.toUpdate = [];
-          async.eachLimit(remoteLanguages, Math.round(require('os').cpus().length / 2), (lng, clb) => update(opt, lng, nsOnlyRemove, clb), clb);
-        });
-      }, (err) => {
-        if (err) return handleError(err);
-
-        if (!cb) console.log(colors.grey('syncing...'));
-        setTimeout(() => {
-          downloadAll(opt, remoteLanguages, false, opt.skipDelete ? (lng, namespace, ns) => {
-            const found = compared.find((n) => n.namespace === namespace && n.language === lng);
-            if (found && found.diff) {
-              if (found.diff.toAddLocally && found.diff.toAddLocally.length > 0) {
-                found.diff.toAddLocally.forEach((k) => {
-                  delete ns[k];
-                });
+            if (opt.updateValues) {
+              if (ns.diff.toUpdate.length > 0) {
+                console.log(colors.yellow(`updating ${ns.diff.toUpdate.length} keys in ${ns.language}/${ns.namespace}...`));
+                if (opt.dry) console.log(colors.yellow(`would update ${ns.diff.toUpdate.join(', ')} in ${ns.language}/${ns.namespace}...`));
               }
-              if (found.diff.toRemove && found.diff.toRemove.length > 0) {
-                found.diff.toRemove.forEach((k) => {
-                  delete ns[k];
-                });
+              if (ns.diff.toUpdateLocally.length > 0) {
+                console.log(colors.yellow(`updating ${ns.diff.toUpdateLocally.length} keys in ${ns.language}/${ns.namespace} locally...`));
+                if (opt.dry) console.log(colors.yellow(`would update ${ns.diff.toUpdateLocally.join(', ')} in ${ns.language}/${ns.namespace} locally...`));
               }
             }
-          } : undefined, (err) => {
+            const somethingToUpdate = ns.diff.toAdd.concat(opt.skipDelete ? [] : ns.diff.toRemove)/*.concat(ns.diff.toUpdate)*/.length > 0;
+            if (!somethingToUpdate) console.log(colors.grey(`nothing to update for ${ns.language}/${ns.namespace}`));
+            if (!wasThereSomethingToUpdate && somethingToUpdate) wasThereSomethingToUpdate = true;
+          }
+          update(opt, ns.language, ns, shouldOmit, (err) => {
+            if (err) return clb(err);
+            if (ns.diff.toRemove.length === 0 || ns.language !== opt.referenceLanguage) return clb();
+            const nsOnlyRemove = cloneDeep(ns);
+            nsOnlyRemove.diff.toAdd = [];
+            nsOnlyRemove.diff.toUpdate = [];
+            async.eachLimit(remoteLanguages, Math.round(require('os').cpus().length / 2), (lng, clb) => update(opt, lng, nsOnlyRemove, shouldOmit, clb), clb);
+          });
+        }, (err) => {
+          if (err) return handleError(err);
+          if (!cb) console.log(colors.grey('syncing...'));
+
+          function down() {
+            setTimeout(() => { // wait a bit before downloading... just to have a chance to get the newly published files
+              downloadAll(opt, remoteLanguages, false, opt.skipDelete ? (lng, namespace, ns) => {
+                const found = compared.find((n) => n.namespace === namespace && n.language === lng);
+                if (found && found.diff) {
+                  if (found.diff.toAddLocally && found.diff.toAddLocally.length > 0) {
+                    found.diff.toAddLocally.forEach((k) => {
+                      delete ns[k];
+                    });
+                  }
+                  if (found.diff.toRemove && found.diff.toRemove.length > 0) {
+                    found.diff.toRemove.forEach((k) => {
+                      delete ns[k];
+                    });
+                  }
+                }
+              } : undefined, (err) => {
+                if (err) return handleError(err);
+                if (!cb) console.log(colors.green('FINISHED'));
+                if (cb) cb(null);
+              });
+            }, wasThereSomethingToUpdate && !opt.dry ? 5000 : 0);
+          }
+
+          if (!shouldOmit) return down();
+          if (opt.dry) return down();
+
+          // optimize stats generation...
+          request(opt.apiPath + '/stats/project/regenerate/' + opt.projectId + '/' + opt.version + (lngsInReqs.length === 1 ? `/${lngsInReqs[0]}` : '') + (nsInReqs.length === 1 ? `?namespace=${nsInReqs[0]}` : ''), {
+            method: 'post',
+            body: {},
+            headers: {
+              'Authorization': opt.apiKey
+            }
+          }, (err, res, obj) => {
             if (err) return handleError(err);
-            if (!cb) console.log(colors.green('FINISHED'));
-            if (cb) cb(null);
+            if (res.status >= 300 && res.status !== 412) {
+              if (obj && (obj.errorMessage || obj.message)) {
+                return handleError(new Error((obj.errorMessage || obj.message)));
+              }
+              return handleError(new Error(res.statusText + ' (' + res.status + ')'));
+            }
+            down();
           });
-        }, wasThereSomethingToUpdate && !opt.dry ? 5000 : 0);
-      }); // wait a bit before downloading... just to have a chance to get the newly published files
+        });
+      }
+
+      if (opt.deleteRemoteNamespace && localMissingNamespaces.length > 0) {
+        wasThereSomethingToUpdate = true;
+        async.each(localMissingNamespaces, (n, clb) => {
+          if (opt.dry) {
+            console.log(colors.red(`would delete complete namespace ${n.namespace}...`));
+            return clb();
+          }
+          console.log(colors.red(`deleting complete namespace ${n.namespace}...`));
+          deleteNamespace({
+            apiPath: opt.apiPath,
+            apiKey: opt.apiKey,
+            projectId: opt.projectId,
+            version: opt.version,
+            namespace: n.namespace
+          }, clb);
+        }, (err) => {
+          if (err) return handleError(err);
+          updateComparedNamespaces();
+        });
+        return;
+      }
+      updateComparedNamespaces();
     });
   });
 };