locize-cli 7.11.0 → 7.12.2

package/CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 Project versioning adheres to [Semantic Versioning](http://semver.org/).
 Change log format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## [7.12.2](https://github.com/locize/locize-cli/compare/v7.12.1...v7.12.2) - 2022-07-12
+
+- limit parallel downloads
+
+## [7.12.1](https://github.com/locize/locize-cli/compare/v7.12.0...v7.12.1) - 2022-07-12
+
+- limit parallel downloads
+
+## [7.12.0](https://github.com/locize/locize-cli/compare/v7.11.0...v7.12.0) - 2022-06-16
+
+- sync and download: introduce --unpublished flag
+
 ## [7.11.0](https://github.com/locize/locize-cli/compare/v7.10.1...v7.11.0) - 2022-06-13
 
 - alpine build

package/README.md

@@ -91,7 +91,7 @@ locize download --project-id my-project-id-93e1-442a-ab35-24331fa294ba --ver lat
 By using the sync command, you can keep your existing code setup and synchronize the translations with locize.
 An example on how this could look like can be seen in [this tutorial](https://github.com/locize/react-tutorial#step-1---keep-existing-code-setup-but-synchronize-with-locize).
 
-**⚠️ Since the remote source are the published translations, make sure the desired version is set to standard/auto publish mode, or alternatively [publish](#publish-version) that version before you execute the cli command. ⚠️**
+**⚠️ Since the remote source are the published translations, make sure the desired version is set to auto publish mode. Alternatively use the `--unpublished true` argument (this will generate [private downloads costs](https://docs.locize.com/integration/api#fetch-filter-the-unpublished-namespace-resources)). ⚠️**
 
 ### Step 1: Go near to your translation files
 

package/bin/locize

@@ -238,6 +238,7 @@ program
   .option('-P, --language-folder-prefix <prefix>', 'This will be added as a local folder name prefix in front of the language.', '')
   .option('-m, --path-mask <mask>', 'This will define the folder and file structure; do not add a file extension (default: {{language}}/{{namespace}})', `{{language}}${path.sep}{{namespace}}`)
   .option('-c, --clean <true|false>', 'Removes all local files by removing the whole folder (default: false)', 'false')
+  .option('-up, --unpublished <true|false>', 'Downloads the current (unpublished) translations. This will generate private download costs (default: false)', 'false')
   .option('-C, --config-path <configPath>', `Specify the path to the optional locize config file (default: ${workingDirectory} or ${home})`)
   .action((options) => {
     try {
@@ -271,6 +272,8 @@ program
 
     const clean = options.clean === 'true';
 
+    const unpublished = options.unpublished === 'true';
+
     const languageFolderPrefix = options.languageFolderPrefix || '';
 
     const pathMask = options.pathMask;
@@ -287,7 +290,8 @@ program
       skipEmpty: skipEmpty,
       clean: clean,
       languageFolderPrefix: languageFolderPrefix,
-      pathMask: pathMask
+      pathMask: pathMask,
+      unpublished: unpublished
     });
   })
   .on('--help', () => {
@@ -377,6 +381,7 @@ program
   .option('-l, --language <lng>', 'The language that should be targeted')
   .option('-n, --namespace <ns>', 'The namespace that should be targeted')
   .option('-g, --get-path <url>', `Specify the get-path url that should be used (default: ${getPathUrl})`)
+  .option('-up, --unpublished <true|false>', 'Downloads the current (unpublished) translations. This will generate private download costs (default: false)', 'false')
   .option('-C, --config-path <configPath>', `Specify the path to the optional locize config file (default: ${workingDirectory} or ${home})`)
   .action((options) => {
     try {
@@ -424,6 +429,7 @@ program
     const referenceLanguageOnly = options.referenceLanguageOnly === 'false' ? false : true;
     const compareModificationTime = options.compareModificationTime === 'true';
     const pathMask = options.pathMask;
+    const unpublished = options.unpublished === 'true';
 
     sync({
       apiPath: url.parse(getPath).protocol + '//' + url.parse(getPath).host,
@@ -444,7 +450,8 @@ program
       language: language,
       namespace: namespace,
       dry: dry,
-      pathMask: pathMask
+      pathMask: pathMask,
+      unpublished: unpublished
     });
   })
   .on('--help', () => {

package/download.js

@@ -10,6 +10,7 @@ const getRemoteNamespace = require('./getRemoteNamespace');
 const getRemoteLanguages = require('./getRemoteLanguages');
 const convertToDesiredFormat = require('./convertToDesiredFormat');
 const formats = require('./formats');
+const getProjectStats = require('./getProjectStats');
 const reversedFileExtensionsMap = formats.reversedFileExtensionsMap;
 
 function handleDownload(opt, url, err, res, downloads, cb) {
@@ -33,7 +34,7 @@ function handleDownload(opt, url, err, res, downloads, cb) {
     return;
   }
 
-  async.each(downloads, (download, clb) => {
+  async.eachLimit(downloads, 20, (download, clb) => {
     const splitted = download.key.split('/');
     const version = splitted[download.isPrivate ? 2 : 1];
     const lng = splitted[download.isPrivate ? 3 : 2];
@@ -92,6 +93,56 @@ function handleDownload(opt, url, err, res, downloads, cb) {
   });
 }
 
+function handlePull(opt, toDownload, cb) {
+  const url = opt.apiPath + '/pull/' + opt.projectId + '/' + opt.version;
+  async.eachLimit(toDownload, 20, (download, clb) => {
+    const lng = download.language;
+    const namespace = download.namespace;
+
+    getRemoteNamespace(opt, lng, namespace, (err, ns, lastModified) => {
+      if (err) return clb(err);
+
+      if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
+        return clb(null);
+      }
+
+      convertToDesiredFormat(opt, namespace, lng, ns, lastModified, (err, converted) => {
+        if (err) {
+          err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
+          return clb(err);
+        }
+        var filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, lng).replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format];
+        var mkdirPath;
+        if (filledMask.lastIndexOf(path.sep) > 0) {
+          mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep));
+        }
+        if (!opt.language) {
+          if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath));
+          fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+          return;
+        }
+
+        if (filledMask.indexOf(path.sep) > 0) filledMask = filledMask.replace(opt.languageFolderPrefix + lng, '');
+        const parentDir = path.dirname(path.join(opt.path, filledMask));
+        mkdirp.sync(parentDir);
+        fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+      });
+    });
+  }, (err) => {
+    if (err) {
+      if (!cb) {
+        console.error(colors.red(err.message));
+        process.exit(1);
+      }
+      if (cb) cb(err);
+      return;
+    }
+
+    if (!cb) console.log(colors.green(`downloaded ${url} to ${opt.path}...`));
+    if (cb) cb(null);
+  });
+}
+
 const handleError = (err, cb) => {
   if (!cb && err) {
     console.error(colors.red(err.message));
@@ -103,7 +154,7 @@ const handleError = (err, cb) => {
 const download = (opt, cb) => {
   opt.format = opt.format || 'json';
   if (!reversedFileExtensionsMap[opt.format]) {
-    return handleError(new Error(`${opt.format} is not a valid format!`));
+    return handleError(new Error(`${opt.format} is not a valid format!`), cb);
   }
 
   if (opt.skipEmpty === undefined) opt.skipEmpty = true;
@@ -115,6 +166,9 @@ const download = (opt, cb) => {
   opt.pathMaskInterpolationSuffix = opt.pathMaskInterpolationSuffix || '}}';
   opt.pathMask = opt.pathMask || `${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}${path.sep}${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`;
   opt.pathMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, `${opt.languageFolderPrefix}${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`);
+  if (opt.unpublished && !opt.apiKey) {
+    return handleError(new Error('Please provide also an api-key!'), cb);
+  }
 
   var url = opt.apiPath + '/download/' + opt.projectId;
 
@@ -140,27 +194,59 @@ const download = (opt, cb) => {
   if (!cb) console.log(colors.yellow(`downloading ${url} to ${opt.path}...`));
 
   getRemoteLanguages(opt, (err) => {
-    if (err) return handleError(err);
-
-    request(url, {
-      method: 'get',
-      headers: opt.apiKey ? {
-        'Authorization': opt.apiKey
-      } : undefined
-    }, (err, res, obj) => {
-      if (res && res.status === 401) {
-        opt.apiKey = null;
-        request(url, {
-          method: 'get',
-        }, (err, res, obj) => {
-          if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
-          handleDownload(opt, url, err, res, obj, cb);
-        });
-        return;
-      }
+    if (err) return handleError(err, cb);
+
+    if (!opt.unpublished) {
+      request(url, {
+        method: 'get',
+        headers: opt.apiKey ? {
+          'Authorization': opt.apiKey
+        } : undefined
+      }, (err, res, obj) => {
+        obj = obj || [];
+        if (res && res.status === 401) {
+          opt.apiKey = null;
+          request(url, {
+            method: 'get',
+          }, (err, res, obj) => {
+            obj = obj || [];
+            if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
+            handleDownload(opt, url, err, res, obj, cb);
+          });
+          return;
+        }
 
-      if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
-      handleDownload(opt, url, err, res, obj, cb);
+        if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
+        handleDownload(opt, url, err, res, obj, cb);
+      });
+      return;
+    }
+
+    getProjectStats(opt, (err, res) => {
+      if (err) return handleError(err, cb);
+      if (!res || !res[opt.version]) return handleError(new Error('Nothing found!'), cb);
+
+      const toDownload = [];
+      const lngsToCheck = opt.language ? [opt.language] : Object.keys(res[opt.version]);
+      lngsToCheck.forEach((l) => {
+        if (opt.namespaces) {
+          opt.namespaces.forEach((n) => {
+            if (!res[opt.version][l][n]) return;
+            if (opt.skipEmpty && res[opt.version][l][n].segmentsTranslated === 0) return;
+            toDownload.push({ language: l, namespace: n });
+          });
+        } else if (opt.namespace) {
+          if (!res[opt.version][l][opt.namespace]) return;
+          if (opt.skipEmpty && res[opt.version][l][opt.namespace].segmentsTranslated === 0) return;
+          toDownload.push({ language: l, namespace: opt.namespace });
+        } else {
+          Object.keys(res[opt.version][l]).forEach((n) => {
+            if (opt.skipEmpty && res[opt.version][l][n].segmentsTranslated === 0) return;
+            toDownload.push({ language: l, namespace: n });
+          });
+        }
+      });
+      handlePull(opt, toDownload, cb);
     });
   });
 };

package/getProjectStats.js

@@ -0,0 +1,40 @@
+const colors = require('colors');
+const request = require('./request');
+
+const getProjectStats = (opt, cb) => {
+  request(opt.apiPath + '/stats/project/' + opt.projectId, {
+    method: 'get',
+    headers: {
+      'Authorization': opt.apiKey
+    }
+  }, (err, res, obj) => {
+    if (err || (obj && (obj.errorMessage || obj.message))) {
+      if (!cb) console.log(colors.red('getting job failed...'));
+
+      if (err) {
+        if (!cb) { console.error(colors.red(err.message)); process.exit(1); }
+        if (cb) cb(err);
+        return;
+      }
+      if (obj && (obj.errorMessage || obj.message)) {
+        if (!cb) { console.error(colors.red((obj.errorMessage || obj.message))); process.exit(1); }
+        if (cb) cb(new Error((obj.errorMessage || obj.message)));
+        return;
+      }
+    }
+    if (res.status === 404) {
+      if (!cb) { console.error(colors.yellow(res.statusText + ' (' + res.status + ')')); process.exit(1); }
+      if (cb) cb(null, null);
+      return;
+    }
+    if (res.status >= 300) {
+      if (!cb) { console.error(colors.red(res.statusText + ' (' + res.status + ')')); process.exit(1); }
+      if (cb) cb(new Error(res.statusText + ' (' + res.status + ')'));
+      return;
+    }
+    if (!cb) console.log(colors.green('getting project stats succesfull'));
+    if (cb) cb(null, obj);
+  });
+};
+
+module.exports = getProjectStats;

package/getRemoteNamespace.js

@@ -3,9 +3,9 @@ const flatten = require('flat');
 const sortFlatResources = require('./sortFlatResources');
 
 const getRemoteNamespace = (opt, lng, ns, cb) => {
-  request(opt.apiPath + (opt.isPrivate ? '/private' : '') + '/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns + '?ts=' + Date.now(), {
+  request(opt.apiPath + (opt.isPrivate ? '/private' : opt.unpublished ? '/pull' : '') + '/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns + '?ts=' + Date.now(), {
     method: 'get',
-    headers: opt.isPrivate ? {
+    headers: (opt.isPrivate || opt.unpublished) ? {
       'Authorization': opt.apiKey
     } : undefined
   }, (err, res, obj) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locize-cli",
-  "version": "7.11.0",
+  "version": "7.12.2",
   "description": "locize cli to import locales",
   "main": "index.js",
   "bin": {

package/sync.js

@@ -15,6 +15,7 @@ const parseLocalReference = require('./parseLocalReference');
 const formats = require('./formats');
 const lngCodes = require('./lngs.json');
 const deleteNamespace = require('./deleteNamespace');
+const getProjectStats = require('./getProjectStats');
 const reversedFileExtensionsMap = formats.reversedFileExtensionsMap;
 
 const getDirectories = (srcpath) => {
@@ -24,22 +25,66 @@ const getDirectories = (srcpath) => {
 };
 
 const getDownloads = (opt, cb) => {
-  request(opt.apiPath + '/download/' + opt.projectId + '/' + opt.version, {
-    method: 'get',
-    headers: opt.apiKey ? {
-      'Authorization': opt.apiKey
-    } : undefined
-  }, (err, res, obj) => {
-    if (err) return cb(err);
-    if (res.status >= 300) {
-      if (obj && (obj.errorMessage || obj.message)) {
-        return cb(new Error((obj.errorMessage || obj.message)));
+  if (!opt.unpublished) {
+    request(opt.apiPath + '/download/' + opt.projectId + '/' + opt.version, {
+      method: 'get',
+      headers: opt.apiKey ? {
+        'Authorization': opt.apiKey
+      } : undefined
+    }, (err, res, obj) => {
+      if (err) return cb(err);
+      if (res.status >= 300) {
+        if (obj && (obj.errorMessage || obj.message)) {
+          return cb(new Error((obj.errorMessage || obj.message)));
+        }
+        return cb(new Error(res.statusText + ' (' + res.status + ')'));
       }
-      return cb(new Error(res.statusText + ' (' + res.status + ')'));
-    }
-    if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
-    cb(null, obj);
-  });
+      if (opt.skipEmpty) obj = obj.filter((d) => d.size > 2);
+      cb(null, obj);
+    });
+  } else {
+    getProjectStats(opt, (err, res) => {
+      if (err) return handleError(err, cb);
+      if (!res || !res[opt.version]) return handleError(new Error('Nothing found!'), cb);
+
+      const toDownload = [];
+      const lngsToCheck = opt.language ? [opt.language] : Object.keys(res[opt.version]);
+      lngsToCheck.forEach((l) => {
+        if (opt.namespaces) {
+          opt.namespaces.forEach((n) => {
+            if (!res[opt.version][l][n]) return;
+            if (opt.skipEmpty && res[opt.version][l][n].segmentsTranslated === 0) return;
+            toDownload.push({
+              url: `${opt.apiPath}/${opt.projectId}/${opt.version}/${l}/${n}`,
+              key: `${opt.projectId}/${opt.version}/${l}/${n}`,
+              lastModified: '1960-01-01T00:00:00.000Z',
+              size: 0
+            });
+          });
+        } else if (opt.namespace) {
+          if (!res[opt.version][l][opt.namespace]) return;
+          if (opt.skipEmpty && res[opt.version][l][opt.namespace].segmentsTranslated === 0) return;
+          toDownload.push({
+            url: `${opt.apiPath}/${opt.projectId}/${opt.version}/${l}/${opt.namespace}`,
+            key: `${opt.projectId}/${opt.version}/${l}/${opt.namespace}`,
+            lastModified: '1960-01-01T00:00:00.000Z',
+            size: 0
+          });
+        } else {
+          Object.keys(res[opt.version][l]).forEach((n) => {
+            if (opt.skipEmpty && res[opt.version][l][n].segmentsTranslated === 0) return;
+            toDownload.push({
+              url: `${opt.apiPath}/${opt.projectId}/${opt.version}/${l}/${n}`,
+              key: `${opt.projectId}/${opt.version}/${l}/${n}`,
+              lastModified: '1960-01-01T00:00:00.000Z',
+              size: 0
+            });
+          });
+        }
+      });
+      cb(null, toDownload);
+    });
+  }
 };
 
 const compareNamespace = (local, remote, lastModifiedLocal, lastModifiedRemote) => {
@@ -160,7 +205,7 @@ const downloadAll = (opt, remoteLanguages, omitRef, manipulate, cb) => {
         return lng !== opt.referenceLanguage;
       });
     }
-    async.each(downloads, (download, clb) => {
+    async.eachLimit(downloads, 20, (download, clb) => {
       const splitted = download.key.split('/');
       const lng = splitted[download.isPrivate ? 3 : 2];
       const namespace = splitted[download.isPrivate ? 4 : 3];
@@ -327,7 +372,7 @@ const checkForMissingLocalNamespaces = (downloads, localNamespaces, opt) => {
 const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
   if (!localNamespaces || localNamespaces.length === 0) {
     downloadAll(opt, remoteLanguages, (err) => {
-      if (err) return handleError(err);
+      if (err) return handleError(err, cb);
       if (!cb) console.log(colors.green('FINISHED'));
       if (cb) cb(null);
     });
@@ -335,14 +380,14 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
   }
 
   getDownloads(opt, (err, downloads) => {
-    if (err) return handleError(err);
+    if (err) return handleError(err, cb);
 
     opt.isPrivate = downloads.length > 0 && downloads[0].isPrivate;
 
     const localMissingNamespaces = checkForMissingLocalNamespaces(downloads, localNamespaces, opt);
 
     compareNamespaces(opt, localNamespaces, (err, compared) => {
-      if (err) return handleError(err);
+      if (err) return handleError(err, cb);
 
       const onlyToUpdate = compared.filter((ns) => ns.diff.toAdd.concat(opt.skipDelete ? [] : ns.diff.toRemove).concat(ns.diff.toUpdate).length > 0);
 
@@ -412,7 +457,7 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
             async.eachLimit(remoteLanguages, Math.round(require('os').cpus().length / 2), (lng, clb) => update(opt, lng, nsOnlyRemove, shouldOmit, clb), clb);
           });
         }, (err) => {
-          if (err) return handleError(err);
+          if (err) return handleError(err, cb);
           if (!cb) console.log(colors.grey('syncing...'));
 
           function down() {
@@ -432,7 +477,7 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
                   }
                 }
               } : undefined, (err) => {
-                if (err) return handleError(err);
+                if (err) return handleError(err, cb);
                 if (!cb) console.log(colors.green('FINISHED'));
                 if (cb) cb(null);
               });
@@ -450,12 +495,12 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
               'Authorization': opt.apiKey
             }
           }, (err, res, obj) => {
-            if (err) return handleError(err);
+            if (err) return handleError(err, cb);
             if (res.status >= 300 && res.status !== 412) {
               if (obj && (obj.errorMessage || obj.message)) {
-                return handleError(new Error((obj.errorMessage || obj.message)));
+                return handleError(new Error((obj.errorMessage || obj.message)), cb);
               }
-              return handleError(new Error(res.statusText + ' (' + res.status + ')'));
+              return handleError(new Error(res.statusText + ' (' + res.status + ')'), cb);
             }
             down();
           });
@@ -464,7 +509,7 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
 
       if (opt.deleteRemoteNamespace && localMissingNamespaces.length > 0) {
         wasThereSomethingToUpdate = true;
-        async.each(localMissingNamespaces, (n, clb) => {
+        async.eachLimit(localMissingNamespaces, 20, (n, clb) => {
           if (opt.dry) {
             console.log(colors.red(`would delete complete namespace ${n.namespace}...`));
             return clb();
@@ -478,7 +523,7 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
             namespace: n.namespace
           }, clb);
         }, (err) => {
-          if (err) return handleError(err);
+          if (err) return handleError(err, cb);
           updateComparedNamespaces();
         });
         return;
@@ -491,7 +536,7 @@ const handleSync = (opt, remoteLanguages, localNamespaces, cb) => {
 const sync = (opt, cb) => {
   opt.format = opt.format || 'json';
   if (!reversedFileExtensionsMap[opt.format]) {
-    return handleError(new Error(`${opt.format} is not a valid format!`));
+    return handleError(new Error(`${opt.format} is not a valid format!`), cb);
   }
 
   opt.version = opt.version || 'latest';
@@ -510,9 +555,12 @@ const sync = (opt, cb) => {
   opt.pathMask = opt.pathMask || `${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}${path.sep}${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`;
   opt.languageFolderPrefix = opt.languageFolderPrefix || '';
   opt.pathMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, `${opt.languageFolderPrefix}${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`);
+  if (opt.unpublished && !opt.apiKey) {
+    return handleError(new Error('Please provide also an api-key!'), cb);
+  }
 
   getRemoteLanguages(opt, (err, remoteLanguages) => {
-    if (err) return handleError(err);
+    if (err) return handleError(err, cb);
 
     if (opt.referenceLanguageOnly && opt.language && opt.referenceLanguage !== opt.language) {
       opt.referenceLanguage = opt.language;
@@ -520,7 +568,7 @@ const sync = (opt, cb) => {
 
     if (opt.referenceLanguageOnly) {
       parseLocalReference(opt, (err, localNamespaces) => {
-        if (err) return handleError(err);
+        if (err) return handleError(err, cb);
 
         if (!opt.dry && opt.cleanLocalFiles) {
           localNamespaces.forEach((ln) => fs.unlinkSync(ln.path));
@@ -533,7 +581,7 @@ const sync = (opt, cb) => {
     }
 
     parseLocalLanguages(opt, remoteLanguages, (err, localNamespaces) => {
-      if (err) return handleError(err);
+      if (err) return handleError(err, cb);
 
       if (!opt.dry && opt.cleanLocalFiles) {
         localNamespaces.forEach((ln) => fs.unlinkSync(ln.path));