locize-cli 7.12.3 → 7.12.6

package/CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 Project versioning adheres to [Semantic Versioning](http://semver.org/).
 Change log format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## [7.12.6](https://github.com/locize/locize-cli/compare/v7.12.5...v7.12.6) - 2022-08-26
+
+- internal optimization of unpublished download workflow
+
+## [7.12.5](https://github.com/locize/locize-cli/compare/v7.12.4...v7.12.5) - 2022-08-25
+
+- update dependencies
+
+## [7.12.4](https://github.com/locize/locize-cli/compare/v7.12.3...v7.12.4) - 2022-07-20
+
+- optimize update handling
+- update dependencies
+
 ## [7.12.3](https://github.com/locize/locize-cli/compare/v7.12.2...v7.12.3) - 2022-07-20
 
 - update dependencies

package/getRemoteNamespace.js

@@ -2,10 +2,68 @@ const request = require('./request');
 const flatten = require('flat');
 const sortFlatResources = require('./sortFlatResources');
 
+const getRandomDelay = (delayFrom, delayTo) => Math.floor(Math.random() * delayTo) + delayFrom;
+
+const pullNamespacePaged = (opt, lng, ns, cb, next, retry) => {
+  next = next || '';
+  request(opt.apiPath + '/pull/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns + '?' + 'next=' + next + '&ts=' + Date.now(), {
+    method: 'get',
+    headers: {
+      'Authorization': opt.apiKey
+    }
+  }, (err, res, obj) => {
+    if (err) return cb(err);
+    if (res.status >= 300) {
+      retry = retry || 0;
+      if (retry < 3 && res.status !== 401) {
+        setTimeout(() => {
+          pullNamespacePaged(opt, lng, ns, cb, next, retry + 1);
+        }, getRandomDelay(3000, 10000));
+        return;
+      }
+      if (obj && (obj.errorMessage || obj.message)) {
+        return cb(new Error((obj.errorMessage || obj.message)));
+      }
+      return cb(new Error(res.statusText + ' (' + res.status + ')'));
+    }
+
+    cb(null, {
+      result: sortFlatResources(flatten(obj)),
+      next: res.headers.get('x-next-page'),
+      lastModified: res.headers.get('last-modified') ? new Date(res.headers.get('last-modified')) : undefined
+    });
+  });
+};
+
+const pullNamespace = (opt, lng, ns, cb) => {
+  var ret = {};
+  var lastModified = new Date(2000, 0, 1);
+  (function nextPage(next) {
+    pullNamespacePaged(opt, lng, ns, (err, info) => {
+      if (err) return cb(err);
+
+      Object.keys(info.result).forEach((k) => {
+        ret[k] = info.result[k];
+      });
+
+      if (info.lastModified.getTime() > lastModified.getTime()) {
+        lastModified = info.lastModified;
+      }
+
+      if (info.next) {
+        return nextPage(info.next);
+      }
+      cb(null, ret, lastModified);
+    }, next);
+  })();
+};
+
 const getRemoteNamespace = (opt, lng, ns, cb) => {
-  request(opt.apiPath + (opt.isPrivate ? '/private' : opt.unpublished ? '/pull' : '') + '/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns + '?ts=' + Date.now(), {
+  if (opt.unpublished) return pullNamespace(opt, lng, ns, cb);
+
+  request(opt.apiPath + (opt.isPrivate ? '/private' : '') + '/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns + '?ts=' + Date.now(), {
     method: 'get',
-    headers: (opt.isPrivate || opt.unpublished) ? {
+    headers: opt.isPrivate ? {
       'Authorization': opt.apiKey
     } : undefined
   }, (err, res, obj) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locize-cli",
-  "version": "7.12.3",
+  "version": "7.12.6",
   "description": "locize cli to import locales",
   "main": "index.js",
   "bin": {
@@ -10,7 +10,7 @@
     "@js.properties/properties": "0.5.4",
     "android-string-resource": "2.3.4",
     "async": "3.2.4",
-    "cacheable-lookup": "6.0.4",
+    "cacheable-lookup": "6.1.0",
     "colors": "1.4.0",
     "commander": "9.4.0",
     "csvjson": "5.1.0",
@@ -19,7 +19,7 @@
     "fluent_conv": "3.1.0",
     "gettext-converter": "1.2.3",
     "https-proxy-agent": "5.0.1",
-    "ini": "3.0.0",
+    "ini": "3.0.1",
     "js-yaml": "4.1.0",
     "laravelphp": "2.0.3",
     "lodash.clonedeep": "4.5.0",
@@ -33,7 +33,7 @@
     "xlsx": "0.18.5"
   },
   "devDependencies": {
-    "eslint": "8.20.0",
+    "eslint": "8.22.0",
     "gh-release": "6.0.4",
     "pkg": "5.8.0"
   },

package/sync.js

@@ -254,14 +254,15 @@ const update = (opt, lng, ns, shouldOmit, cb) => {
     ns.diff.toUpdate.forEach((k) => data[k] = ns.content[k]);
   }
 
-  if (Object.keys(data).length === 0) return cb(null);
+  var keysToSend = Object.keys(data).length;
+  if (keysToSend === 0) return cb(null);
 
   if (opt.dry) return cb(null);
 
   var payloadKeysLimit = 1000;
 
-  function send(d, clb, isRetrying) {
-    request(opt.apiPath + '/update/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns.namespace + (shouldOmit ? '?omitstatsgeneration=true' : ''), {
+  function send(d, so, clb, isRetrying) {
+    request(opt.apiPath + '/update/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns.namespace + (so ? '?omitstatsgeneration=true' : ''), {
       method: 'post',
       body: d,
       headers: {
@@ -270,7 +271,7 @@ const update = (opt, lng, ns, shouldOmit, cb) => {
     }, (err, res, obj) => {
       if (err) return clb(err);
       if (res.status === 504 && !isRetrying) {
-        return setTimeout(() => send(d, clb, true), 3000);
+        return setTimeout(() => send(d, so, clb, true), 3000);
       }
       if (res.status >= 300 && res.status !== 412) {
         if (obj && (obj.errorMessage || obj.message)) {
@@ -282,7 +283,7 @@ const update = (opt, lng, ns, shouldOmit, cb) => {
     });
   }
 
-  if (Object.keys(data).length > payloadKeysLimit) {
+  if (keysToSend > payloadKeysLimit) {
     var tasks = [];
     var keysInObj = Object.keys(data);
 
@@ -290,7 +291,8 @@ const update = (opt, lng, ns, shouldOmit, cb) => {
       (function() {
         var pagedData = {};
         keysInObj.splice(0, payloadKeysLimit).forEach((k) => pagedData[k] = data[k]);
-        tasks.push((c) => send(pagedData, c));
+        var hasMoreKeys = keysInObj.length > 0;
+        tasks.push((c) => send(pagedData, hasMoreKeys ? true : shouldOmit, c));
       })();
     }
 
@@ -298,13 +300,13 @@ const update = (opt, lng, ns, shouldOmit, cb) => {
 
     var finalPagedData = {};
     keysInObj.splice(0, keysInObj.length).forEach((k) => finalPagedData[k] = data[k]);
-    tasks.push((c) => send(finalPagedData, c));
+    tasks.push((c) => send(finalPagedData, shouldOmit, c));
 
     async.series(tasks, cb);
     return;
   }
 
-  send(data, cb);
+  send(data, shouldOmit, cb);
 };
 
 const cleanupLanguages = (opt, remoteLanguages) => {