locize-cli 8.6.1 → 8.7.0

package/CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 Project versioning adheres to [Semantic Versioning](http://semver.org/).
 Change log format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## [8.7.0](https://github.com/locize/locize-cli/compare/v8.6.2...v8.7.0) - 2025-03-20
+
+- intruduce xcstrings format [106](https://github.com/locize/locize-cli/issues/106)
+
+## [8.6.2](https://github.com/locize/locize-cli/compare/v8.6.1...v8.6.2) - 2025-03-18
+
+- fix migrate command to handle paged update
+
 ## [8.6.1](https://github.com/locize/locize-cli/compare/v8.6.0...v8.6.1) - 2025-03-11
 
 - fix .yml yaml format variats to address [104](https://github.com/locize/locize-cli/issues/104)

package/README.md

@@ -84,7 +84,7 @@ or
 locize download
 ```
 
-or add a format like (json, nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties)
+or add a format like (json, nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties, xcstrings)
 
 ```sh
 locize download --project-id my-project-id-93e1-442a-ab35-24331fa294ba --ver latest --language en --namespace namespace1 --path ./backup --format android
@@ -133,7 +133,7 @@ Add your api-key and your project-id and let's go...
 locize sync --api-key my-api-key-d9de-4f55-9855-a9ef0ed44672 --project-id my-project-id-93e1-442a-ab35-24331fa294ba
 ```
 
-or add a format like (json, nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties)
+or add a format like (json, nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties, xcstrings)
 
 ```sh
 locize sync --api-key my-api-key-d9de-4f55-9855-a9ef0ed44672 --project-id my-project-id-93e1-442a-ab35-24331fa294ba --format android
@@ -192,7 +192,7 @@ Add your api-key and your project-id and let's go...
 locize save-missing --api-key my-api-key-d9de-4f55-9855-a9ef0ed44672 --project-id my-project-id-93e1-442a-ab35-24331fa294ba
 ```
 
-or add a format like (json, nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties)
+or add a format like (json, nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties, xcstrings)
 
 ```sh
 locize save-missing --api-key my-api-key-d9de-4f55-9855-a9ef0ed44672 --project-id my-project-id-93e1-442a-ab35-24331fa294ba --format android

package/bin/locize

@@ -232,7 +232,7 @@ program
   .option('-p, --path <path>', `Specify the path that should be used (default: ${process.cwd()})`, process.cwd())
   .option('-g, --get-path <url>', `Specify the get-path url that should be used (default: ${getPathUrl})`)
   .option('-k, --api-key <apiKey>', 'The api-key that should be used')
-  .option('-f, --format <json>', 'File format of namespaces (default: json; [nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties])', 'json')
+  .option('-f, --format <json>', 'File format of namespaces (default: json; [nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties, xcstrings])', 'json')
   .option('-s, --skip-empty <true|false>', 'Skips to download empty files (default: true)', 'true')
   .option('-P, --language-folder-prefix <prefix>', 'This will be added as a local folder name prefix in front of the language.', '')
   .option('-m, --path-mask <mask>', 'This will define the folder and file structure; do not add a file extension (default: {{language}}/{{namespace}})', `{{language}}${path.sep}{{namespace}}`)
@@ -361,7 +361,7 @@ program
   .option('-p, --path <path>', `Specify the path that should be used (default: ${process.cwd()})`, process.cwd())
   .option('-B, --backup-deleted-path <path>', 'Saves the segments that will be deleted in this path')
   .option('-A, --auto-create-path <true|false>', 'This will automatically make sure the --path is created. (default: true)', 'true')
-  .option('-f, --format <json>', 'File format of namespaces (default: json; [nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties])', 'json')
+  .option('-f, --format <json>', 'File format of namespaces (default: json; [nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties, xcstrings])', 'json')
   .option('-s, --skip-empty <true|false>', 'Skips to download empty files (default: false)', 'false')
   .option('-c, --clean <true|false>', 'Removes all local files by removing the whole folder (default: false)', 'false')
   .option('-cf, --clean-local-files <true|false>', 'Removes all local files without removing any folder (default: false)', 'false')
@@ -472,7 +472,7 @@ program
   .option('-i, --project-id <projectId>', 'The project-id that should be used')
   .option('-v, --ver <version>', 'Found namespaces will be matched to this version (default: latest)')
   .option('-p, --path <path>', `Specify the path that should be used (default: ${process.cwd()})`, process.cwd())
-  .option('-f, --format <json>', 'File format of namespaces (default: json; [nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties])', 'json')
+  .option('-f, --format <json>', 'File format of namespaces (default: json; [nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties, xcstrings])', 'json')
   .option('-m, --path-mask <mask>', 'This will define the folder and file structure; do not add a file extension (default: {{language}}/{{namespace}})', `{{language}}${path.sep}{{namespace}}`)
   .option('-P, --language-folder-prefix <prefix>', 'This will be added as a local folder name prefix in front of the language.', '')
   .option('-d, --dry <true|false>', 'Dry run (default: false)', 'false')
@@ -695,7 +695,7 @@ program
   .command('format [fileOrDirectory]')
   .alias('ft')
   .description('format local files')
-  .option('-f, --format <json>', 'File format of namespaces (default: json; [nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties])', 'json')
+  .option('-f, --format <json>', 'File format of namespaces (default: json; [nested, flat, xliff2, xliff12, xlf2, xlf12, android, yaml, yaml-rails, yaml-nested, yml, yml-rails, yml-nested, csv, xlsx, po, strings, resx, fluent, tmx, laravel, properties, xcstrings])', 'json')
   .option('-l, --reference-language <lng>', 'Some format conversions need to know the reference language.', 'en')
   .option('-d, --dry <true|false>', 'Dry run (default: false)', 'false')
   .option('-C, --config-path <configPath>', `Specify the path to the optional locize config file (default: ${configInWorkingDirectory} or ${configInHome})`)

package/convertToFlatFormat.js

@@ -13,6 +13,7 @@ const ftl2js = require('fluent_conv/cjs/ftl2js');
 const tmx2js = require('tmexchange/cjs/tmx2js');
 const laravel2js = require('laravelphp/cjs/laravel2js');
 const javaProperties = require('@js.properties/properties');
+const xcstrings2locize = require('locize-xcstrings/cjs/xcstrings2locize');
 const flatten = require('flat');
 const prepareCombinedImport = require('./combineSubkeyPreprocessor').prepareImport;
 
@@ -293,6 +294,10 @@ const convertToFlatFormat = (opt, data, lng, cb) => {
       cb(null, javaProperties.parseToProperties(data.toString()));
       return;
     }
+    if (opt.format === 'xcstrings') {
+      cb(null, xcstrings2locize(data.toString()));
+      return;
+    }
     cb(new Error(`${opt.format} is not a valid format!`));
   } catch (err) {
     cb(err);

package/download.js

@@ -12,6 +12,15 @@ const convertToDesiredFormat = require('./convertToDesiredFormat');
 const formats = require('./formats');
 const getProjectStats = require('./getProjectStats');
 const reversedFileExtensionsMap = formats.reversedFileExtensionsMap;
+const locize2xcstrings = require('locize-xcstrings/cjs/locize2xcstrings');
+
+function getInfosInUrl(download) {
+  const splitted = download.key.split('/');
+  const version = splitted[download.isPrivate ? 2 : 1];
+  const language = splitted[download.isPrivate ? 3 : 2];
+  const namespace = splitted[download.isPrivate ? 4 : 3];
+  return { version, language, namespace };
+}
 
 function handleDownload(opt, url, err, res, downloads, cb) {
   if (err || (downloads && (downloads.errorMessage || downloads.message))) {
@@ -34,113 +43,263 @@ function handleDownload(opt, url, err, res, downloads, cb) {
     return;
   }
 
-  async.eachLimit(downloads, 20, (download, clb) => {
-    const splitted = download.key.split('/');
-    const version = splitted[download.isPrivate ? 2 : 1];
-    const lng = splitted[download.isPrivate ? 3 : 2];
-    const namespace = splitted[download.isPrivate ? 4 : 3];
-    opt.isPrivate = download.isPrivate;
-
-    if (opt.namespace && opt.namespace !== namespace) return clb(null);
-    if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null);
+  if (opt.format === 'xcstrings') { // 1 file per namespace including all languages
+    const downloadsByNamespace = {};
+    downloads.forEach((download) => {
+      const { version, namespace } = getInfosInUrl(download);
+      opt.isPrivate = download.isPrivate;
 
-    getRemoteNamespace(opt, lng, namespace, (err, ns, lastModified) => {
-      if (err) return clb(err);
+      downloadsByNamespace[version] = downloadsByNamespace[version] || {};
+      downloadsByNamespace[version][namespace] = downloadsByNamespace[version][namespace] || [];
+      downloadsByNamespace[version][namespace].push(download);
+    });
 
-      if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
-        return clb(null);
+    async.eachSeries(Object.keys(downloadsByNamespace), (version, clb) => {
+      async.eachLimit(Object.keys(downloadsByNamespace[version]), 20, (ns, clb) => {
+        if (opt.namespace && opt.namespace !== ns) return clb(null);
+        if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(ns) < 0) return clb(null);
+
+        const locizeData = {
+          sourceLng: opt.referenceLanguage,
+          resources: {}
+        };
+        async.eachLimit(downloadsByNamespace[version][ns], 20, (download, clb2) => {
+          const { language } = getInfosInUrl(download);
+          getRemoteNamespace(opt, language, ns, (err, ns, lastModified) => {
+            if (err) return clb2(err);
+
+            if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
+              return clb2(null);
+            }
+
+            locizeData.resources[language] = ns;
+            clb2();
+          });
+        }, (err) => {
+          if (err) return clb(err);
+
+          try {
+            const result = locize2xcstrings(locizeData);
+            const converted = JSON.stringify(result, null, 2);
+
+            var filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, '').replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, ns) + reversedFileExtensionsMap[opt.format];
+            var mkdirPath;
+            if (filledMask.lastIndexOf(path.sep) > 0) {
+              mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep));
+            }
+
+            function logAndClb(err) {
+              if (err) return clb(err);
+              if (!cb) console.log(colors.green(`downloaded ${version}/${ns} to ${opt.path}...`));
+              if (clb) clb(null);
+            }
+
+            if (!opt.version) {
+              if (mkdirPath) mkdirp.sync(path.join(opt.path, version, mkdirPath));
+              fs.writeFile(path.join(opt.path, version, filledMask), converted, logAndClb);
+              return;
+            }
+
+            if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath));
+            fs.writeFile(path.join(opt.path, filledMask), converted, logAndClb);
+          } catch (e) {
+            err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
+            return clb(err);
+          }
+        });
+      }, clb);
+    }, (err) => {
+      if (err) {
+        if (!cb) {
+          console.error(colors.red(err.message));
+          process.exit(1);
+        }
+        if (cb) cb(err);
+        return;
       }
 
-      convertToDesiredFormat(opt, namespace, lng, ns, lastModified, (err, converted) => {
-        if (err) {
-          err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
-          return clb(err);
-        }
-        var filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, lng).replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format];
-        var mkdirPath;
-        if (filledMask.lastIndexOf(path.sep) > 0) {
-          mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep));
-        }
-        if (!opt.version) {
-          if (mkdirPath) mkdirp.sync(path.join(opt.path, version, mkdirPath));
-          fs.writeFile(path.join(opt.path, version, filledMask), converted, clb);
-          return;
-        }
-        if (!opt.language) {
-          if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath));
-          fs.writeFile(path.join(opt.path, filledMask), converted, clb);
-          return;
+      if (cb) cb(null);
+    });
+  } else { // 1 file per namespace/lng
+    async.eachLimit(downloads, 20, (download, clb) => {
+      const { version, language, namespace } = getInfosInUrl(download);
+      opt.isPrivate = download.isPrivate;
+
+      if (opt.namespace && opt.namespace !== namespace) return clb(null);
+      if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null);
+
+      getRemoteNamespace(opt, language, namespace, (err, ns, lastModified) => {
+        if (err) return clb(err);
+
+        if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
+          return clb(null);
         }
 
-        if (filledMask.indexOf(path.sep) > 0) filledMask = filledMask.replace(opt.languageFolderPrefix + lng, '');
-        const parentDir = path.dirname(path.join(opt.path, filledMask));
-        mkdirp.sync(parentDir);
-        fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+        convertToDesiredFormat(opt, namespace, language, ns, lastModified, (err, converted) => {
+          if (err) {
+            err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
+            return clb(err);
+          }
+          var filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, language).replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format];
+          var mkdirPath;
+          if (filledMask.lastIndexOf(path.sep) > 0) {
+            mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep));
+          }
+          if (!opt.version) {
+            if (mkdirPath) mkdirp.sync(path.join(opt.path, version, mkdirPath));
+            fs.writeFile(path.join(opt.path, version, filledMask), converted, clb);
+            return;
+          }
+          if (!opt.language) {
+            if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath));
+            fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+            return;
+          }
+
+          if (filledMask.indexOf(path.sep) > 0) filledMask = filledMask.replace(opt.languageFolderPrefix + language, '');
+          const parentDir = path.dirname(path.join(opt.path, filledMask));
+          mkdirp.sync(parentDir);
+          fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+        });
       });
-    });
-  }, (err) => {
-    if (err) {
-      if (!cb) {
-        console.error(colors.red(err.message));
-        process.exit(1);
+    }, (err) => {
+      if (err) {
+        if (!cb) {
+          console.error(colors.red(err.message));
+          process.exit(1);
+        }
+        if (cb) cb(err);
+        return;
       }
-      if (cb) cb(err);
-      return;
-    }
 
-    if (!cb) console.log(colors.green(`downloaded ${url} to ${opt.path}...`));
-    if (cb) cb(null);
-  });
+      if (!cb) console.log(colors.green(`downloaded ${url} to ${opt.path}...`));
+      if (cb) cb(null);
+    });
+  }
 }
 
 function handlePull(opt, toDownload, cb) {
   const url = opt.apiPath + '/pull/' + opt.projectId + '/' + opt.version;
-  async.eachLimit(toDownload, 5, (download, clb) => {
-    const lng = download.language;
-    const namespace = download.namespace;
 
-    getRemoteNamespace(opt, lng, namespace, (err, ns, lastModified) => {
-      if (err) return clb(err);
+  if (opt.format === 'xcstrings') { // 1 file per namespace including all languages
+    const downloadsByNamespace = {};
+    toDownload.forEach((download) => {
+      const { namespace } = download;
+      downloadsByNamespace[namespace] = downloadsByNamespace[namespace] || [];
+      downloadsByNamespace[namespace].push(download);
+    });
 
-      if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
-        return clb(null);
-      }
+    async.eachLimit(Object.keys(downloadsByNamespace), 5, (namespace, clb) => {
+      if (opt.namespace && opt.namespace !== namespace) return clb(null);
+      if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null);
 
-      convertToDesiredFormat(opt, namespace, lng, ns, lastModified, (err, converted) => {
-        if (err) {
+      const locizeData = {
+        sourceLng: opt.referenceLanguage,
+        resources: {}
+      };
+
+      async.eachLimit(downloadsByNamespace[namespace], 5, (download, clb2) => {
+        const { language } = download;
+        opt.raw = true;
+        getRemoteNamespace(opt, language, namespace, (err, ns, lastModified) => {
+          if (err) return clb2(err);
+
+          if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
+            return clb2(null);
+          }
+
+          locizeData.resources[language] = ns;
+          clb2();
+        });
+      }, (err) => {
+        if (err) return clb(err);
+
+        try {
+          const result = locize2xcstrings(locizeData);
+          const converted = JSON.stringify(result, null, 2);
+
+          var filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, '').replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format];
+          var mkdirPath;
+          if (filledMask.lastIndexOf(path.sep) > 0) {
+            mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep));
+          }
+
+          function logAndClb(err) {
+            if (err) return clb(err);
+            if (!cb) console.log(colors.green(`downloaded ${opt.version}/${namespace} to ${opt.path}...`));
+            if (clb) clb(null);
+          }
+
+          if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath));
+          fs.writeFile(path.join(opt.path, filledMask), converted, logAndClb);
+        } catch (e) {
           err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
           return clb(err);
         }
-        var filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, lng).replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format];
-        var mkdirPath;
-        if (filledMask.lastIndexOf(path.sep) > 0) {
-          mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep));
+      });
+    }, (err) => {
+      if (err) {
+        if (!cb) {
+          console.error(colors.red(err.message));
+          process.exit(1);
         }
-        if (!opt.language) {
-          if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath));
-          fs.writeFile(path.join(opt.path, filledMask), converted, clb);
-          return;
+        if (cb) cb(err);
+        return;
+      }
+
+      if (cb) cb(null);
+    });
+  } else { // 1 file per namespace/lng
+    async.eachLimit(toDownload, 5, (download, clb) => {
+      const lng = download.language;
+      const namespace = download.namespace;
+
+      if (opt.namespace && opt.namespace !== namespace) return clb(null);
+      if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null);
+
+      getRemoteNamespace(opt, lng, namespace, (err, ns, lastModified) => {
+        if (err) return clb(err);
+
+        if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
+          return clb(null);
         }
 
-        if (filledMask.indexOf(path.sep) > 0) filledMask = filledMask.replace(opt.languageFolderPrefix + lng, '');
-        const parentDir = path.dirname(path.join(opt.path, filledMask));
-        mkdirp.sync(parentDir);
-        fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+        convertToDesiredFormat(opt, namespace, lng, ns, lastModified, (err, converted) => {
+          if (err) {
+            err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
+            return clb(err);
+          }
+          var filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, lng).replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format];
+          var mkdirPath;
+          if (filledMask.lastIndexOf(path.sep) > 0) {
+            mkdirPath = filledMask.substring(0, filledMask.lastIndexOf(path.sep));
+          }
+          if (!opt.language) {
+            if (mkdirPath) mkdirp.sync(path.join(opt.path, mkdirPath));
+            fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+            return;
+          }
+
+          if (filledMask.indexOf(path.sep) > 0) filledMask = filledMask.replace(opt.languageFolderPrefix + lng, '');
+          const parentDir = path.dirname(path.join(opt.path, filledMask));
+          mkdirp.sync(parentDir);
+          fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+        });
       });
-    });
-  }, (err) => {
-    if (err) {
-      if (!cb) {
-        console.error(colors.red(err.message));
-        process.exit(1);
+    }, (err) => {
+      if (err) {
+        if (!cb) {
+          console.error(colors.red(err.message));
+          process.exit(1);
+        }
+        if (cb) cb(err);
+        return;
       }
-      if (cb) cb(err);
-      return;
-    }
 
-    if (!cb) console.log(colors.green(`downloaded ${url} to ${opt.path}...`));
-    if (cb) cb(null);
-  });
+      if (!cb) console.log(colors.green(`downloaded ${url} to ${opt.path}...`));
+      if (cb) cb(null);
+    });
+  }
 }
 
 const handleError = (err, cb) => {

package/formats.js

@@ -13,7 +13,8 @@ const fileExtensionsMap = {
   '.ftl': ['fluent'],
   '.tmx': ['tmx'],
   '.php': ['laravel'],
-  '.properties': ['properties']
+  '.properties': ['properties'],
+  '.xcstrings': ['xcstrings']
 };
 
 const acceptedFileExtensions = Object.keys(fileExtensionsMap);

package/getRemoteNamespace.js

@@ -4,9 +4,26 @@ const sortFlatResources = require('./sortFlatResources');
 
 const getRandomDelay = (delayFrom, delayTo) => Math.floor(Math.random() * delayTo) + delayFrom;
 
+function onlyKeysFlat(resources, prefix, ret) {
+  if (!resources) resources;
+  ret = ret || {};
+  Object.keys(resources).forEach((k) => {
+    if (typeof resources[k] === 'string' || !resources[k] || typeof resources[k].value === 'string') {
+      if (prefix) {
+        ret[prefix + '.' + k] = resources[k];
+      } else {
+        ret[k] = resources[k];
+      }
+    } else {
+      onlyKeysFlat(resources[k], prefix ? prefix + '.' + k : k, ret);
+    }
+  });
+  return ret;
+}
+
 const pullNamespacePaged = (opt, lng, ns, cb, next, retry) => {
   next = next || '';
-  request(opt.apiPath + '/pull/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns + '?' + 'next=' + next + '&ts=' + Date.now(), {
+  request(opt.apiPath + '/pull/' + opt.projectId + '/' + opt.version + '/' + lng + '/' + ns + '?' + 'next=' + next + (opt.raw ? '&raw=true' : '') + '&ts=' + Date.now(), {
     method: 'get',
     headers: {
       'Authorization': opt.apiKey
@@ -31,7 +48,7 @@ const pullNamespacePaged = (opt, lng, ns, cb, next, retry) => {
     }
 
     cb(null, {
-      result: sortFlatResources(flatten(obj)),
+      result: opt.raw ? sortFlatResources(onlyKeysFlat(obj)) : sortFlatResources(flatten(obj)),
       next: res.headers.get('x-next-page'),
       lastModified: res.headers.get('last-modified') ? new Date(res.headers.get('last-modified')) : undefined
     });

package/lngs.json

@@ -111,6 +111,7 @@
   "kg",
   "ko",
   "ku",
+  "ckb",
   "kj",
   "la",
   "lb",

package/migrate.js

@@ -69,28 +69,79 @@ const transfer = (opt, ns, cb) => {
 
   if (!opt.replace) url = url.replace('/update/', '/missing/');
 
-  request(url + `?replace=${!!opt.replace}`, {
-    method: 'post',
-    body: ns.value,
-    headers: {
-      'Authorization': opt.apiKey
+  var data = ns.value;
+  var keysToSend = Object.keys(data).length;
+  if (keysToSend === 0) return cb(null);
+
+  var payloadKeysLimit = 1000;
+
+  function send(d, so, isFirst, clb, isRetrying) {
+    const queryParams = new URLSearchParams();
+    if (so) {
+      queryParams.append('omitstatsgeneration', 'true');
     }
-  }, (err, res, obj) => {
-    if (err || (obj && (obj.errorMessage || obj.message))) {
-      if (url.indexOf('/missing/') > -1 && res.status === 412) {
-        console.log(colors.green(`transfered ${opt.version}/${ns.language}/${ns.namespace} (but all keys already existed)...`));
-        cb(null);
-        return;
+    if (isFirst && opt.replace) {
+      queryParams.append('replace', 'true');
+    }
+
+    const queryString = queryParams.size > 0 ? '?' + queryParams.toString() : '';
+
+    request(url + queryString, {
+      method: 'post',
+      body: d,
+      headers: {
+        'Authorization': opt.apiKey
       }
-      console.log(colors.red(`transfer failed for ${opt.version}/${ns.language}/${ns.namespace}...`));
+    }, (err, res, obj) => {
+      if (err || (obj && (obj.errorMessage || obj.message))) {
+        if (url.indexOf('/missing/') > -1 && res.status === 412) {
+          console.log(colors.green(`transfered ${Object.keys(d).length} keys ${opt.version}/${ns.language}/${ns.namespace} (but all keys already existed)...`));
+          clb(null);
+          return;
+        }
+        if (res.status === 504 && !isRetrying) {
+          return setTimeout(() => send(d, so, isFirst, clb, true), 3000);
+        }
+        console.log(colors.red(`transfer failed for ${Object.keys(d).length} keys ${opt.version}/${ns.language}/${ns.namespace}...`));
 
-      if (err) return cb(err);
-      if (obj && (obj.errorMessage || obj.message)) return cb(new Error((obj.errorMessage || obj.message)));
+        if (err) return clb(err);
+        if (obj && (obj.errorMessage || obj.message)) return clb(new Error((obj.errorMessage || obj.message)));
+      }
+      if (res.status >= 300 && res.status !== 412) {
+        if (obj && (obj.errorMessage || obj.message)) {
+          return clb(new Error((obj.errorMessage || obj.message)));
+        }
+        return clb(new Error(res.statusText + ' (' + res.status + ')'));
+      }
+      console.log(colors.green(`transfered ${Object.keys(d).length} keys ${opt.version}/${ns.language}/${ns.namespace}...`));
+      clb(null);
+    });
+  }
+
+  if (keysToSend > payloadKeysLimit) {
+    var tasks = [];
+    var keysInObj = Object.keys(data);
+
+    while (keysInObj.length > payloadKeysLimit) {
+      (function() {
+        var pagedData = {};
+        keysInObj.splice(0, payloadKeysLimit).forEach((k) => pagedData[k] = data[k]);
+        var hasMoreKeys = keysInObj.length > 0;
+        tasks.push((c) => send(pagedData, hasMoreKeys, false, c));
+      })();
     }
-    if (res.status >= 300 && res.status !== 412) return cb(new Error(res.statusText + ' (' + res.status + ')'));
-    console.log(colors.green(`transfered ${opt.version}/${ns.language}/${ns.namespace}...`));
-    cb(null);
-  });
+
+    if (keysInObj.length === 0) return cb(null);
+
+    var finalPagedData = {};
+    keysInObj.splice(0, keysInObj.length).forEach((k) => finalPagedData[k] = data[k]);
+    tasks.push((c) => send(finalPagedData, false, false, c));
+
+    async.series(tasks, cb);
+    return;
+  }
+
+  send(data, false, true, cb);
 };
 
 const upload = (opt, nss, cb) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locize-cli",
-  "version": "8.6.1",
+  "version": "8.7.0",
   "description": "locize cli to import locales",
   "main": "index.js",
   "bin": {
@@ -23,6 +23,7 @@
     "ini": "4.1.3",
     "js-yaml": "4.1.0",
     "laravelphp": "2.0.4",
+    "locize-xcstrings": "1.0.0",
     "lodash.clonedeep": "4.5.0",
     "mkdirp": "3.0.1",
     "node-fetch": "2.7.0",
@@ -34,9 +35,9 @@
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
   },
   "devDependencies": {
+    "@yao-pkg/pkg": "6.3.1",
     "eslint": "8.56.0",
-    "gh-release": "7.0.2",
-    "@yao-pkg/pkg": "6.3.1"
+    "gh-release": "7.0.2"
   },
   "scripts": {
     "lint": "eslint .",

package/parseLocalLanguage.js

@@ -6,6 +6,7 @@ const convertToFlatFormat = require('./convertToFlatFormat');
 const formats = require('./formats');
 const fileExtensionsMap = formats.fileExtensionsMap;
 const acceptedFileExtensions = formats.acceptedFileExtensions;
+const xcstrings2locize = require('locize-xcstrings/cjs/xcstrings2locize');
 
 const getFiles = (srcpath) => {
   return fs.readdirSync(srcpath).filter((file) => {
@@ -21,7 +22,7 @@ const getDirectories = (srcpath) => {
 
 const parseLocalLanguage = (opt, lng, cb) => {
   const hasNamespaceInPath = opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`) > -1;
-  const filledLngMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, lng);
+  const filledLngMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, opt.format === 'xcstrings' ? '' : lng);
   var firstPartLngMask, lastPartLngMask;
   if (opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`) > opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`)) {
     const secondPartMask = opt.pathMask.substring(opt.pathMask.lastIndexOf(path.sep) + 1);
@@ -138,28 +139,63 @@ const parseLocalLanguage = (opt, lng, cb) => {
         }
       }
 
-      convertToFlatFormat(opt, data, lng, (err, content) => {
-        if (err) {
+      if (opt.format === 'xcstrings') { // 1 file per namespace including all languages
+        try {
+          const content = xcstrings2locize(JSON.parse(data));
+
+          fs.stat(fPath, (err, stat) => {
+            if (err) return clb(err);
+
+            clb(null, Object.keys(content.resources).map((l) => ({
+              namespace: namespace,
+              path: fPath,
+              extension: fExt,
+              content: content.resources[l],
+              language: l,
+              mtime: stat.mtime
+            })));
+          });
+        } catch (e) {
           err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
           err.message += '\n' + fPath;
           return clb(err);
         }
+      } else { // 1 file per namespace/lng
+        convertToFlatFormat(opt, data, lng, (err, content) => {
+          if (err) {
+            err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
+            err.message += '\n' + fPath;
+            return clb(err);
+          }
 
-        fs.stat(fPath, (err, stat) => {
-          if (err) return clb(err);
+          fs.stat(fPath, (err, stat) => {
+            if (err) return clb(err);
 
-          clb(null, {
-            namespace: namespace,
-            path: fPath,
-            extension: fExt,
-            content: content,
-            language: lng,
-            mtime: stat.mtime
+            clb(null, {
+              namespace: namespace,
+              path: fPath,
+              extension: fExt,
+              content: content,
+              language: lng,
+              mtime: stat.mtime
+            });
           });
         });
-      });
+      }
     });
-  }, cb);
+  }, (err, ret) => {
+    if (err) return cb(err);
+    // xcstrings, returns array in array
+    const r = ret.reduce((prev, cur) => {
+      if (Array.isArray(cur)) {
+        prev = prev.concat(cur);
+      } else {
+        prev.push(cur);
+      }
+      return prev;
+    }, []);
+    cb(null, r);
+  });
 };
 
 module.exports = parseLocalLanguage;

package/parseLocalReference.js

@@ -4,7 +4,7 @@ const filterNamespaces = require('./filterNamespaces');
 const parseLocalReference = (opt, cb) => parseLocalLanguage(opt, opt.referenceLanguage, (err, nss) => {
   if (err) return cb(err);
 
-  cb(err, filterNamespaces(opt, nss));
+  cb(err, filterNamespaces(opt, nss).filter((n) => n.language === opt.referenceLanguage));
 });
 
 module.exports = parseLocalReference;

package/sync.js

@@ -17,6 +17,7 @@ const lngCodes = require('./lngs.json');
 const deleteNamespace = require('./deleteNamespace');
 const getProjectStats = require('./getProjectStats');
 const reversedFileExtensionsMap = formats.reversedFileExtensionsMap;
+const locize2xcstrings = require('locize-xcstrings/cjs/locize2xcstrings');
 
 const getDirectories = (srcpath) => {
   return fs.readdirSync(srcpath).filter((file) => {
@@ -24,6 +25,14 @@ const getDirectories = (srcpath) => {
   });
 };
 
+function getInfosInUrl(download) {
+  const splitted = download.key.split('/');
+  const version = splitted[download.isPrivate ? 2 : 1];
+  const language = splitted[download.isPrivate ? 3 : 2];
+  const namespace = splitted[download.isPrivate ? 4 : 3];
+  return { version, language, namespace };
+}
+
 const getDownloads = (opt, cb) => {
   if (!opt.unpublished) {
     request(opt.apiPath + '/download/' + opt.projectId + '/' + opt.version, {
@@ -194,7 +203,7 @@ const downloadAll = (opt, remoteLanguages, omitRef, manipulate, cb) => {
     }
   }
 
-  if (!opt.dry) cleanupLanguages(opt, remoteLanguages);
+  if (!opt.dry && opt.format !== 'xcstrings') cleanupLanguages(opt, remoteLanguages);
 
   getDownloads(opt, (err, downloads) => {
     if (err) return cb(err);
@@ -208,43 +217,101 @@ const downloadAll = (opt, remoteLanguages, omitRef, manipulate, cb) => {
         return lng !== opt.referenceLanguage;
       });
     }
-    async.eachLimit(downloads, opt.unpublished ? 5 : 20, (download, clb) => {
-      const splitted = download.key.split('/');
-      const lng = splitted[download.isPrivate ? 3 : 2];
-      const namespace = splitted[download.isPrivate ? 4 : 3];
-      opt.isPrivate = download.isPrivate;
-
-      if (opt.language && opt.language !== lng && lng !== opt.referenceLanguage) return clb(null);
-      if (opt.languages && opt.languages.length > 0 && opt.languages.indexOf(lng) < 0 && lng !== opt.referenceLanguage) return clb(null);
-      if (opt.namespace && opt.namespace !== namespace) return clb(null);
-      if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null);
-
-      getRemoteNamespace(opt, lng, namespace, (err, ns, lastModified) => {
-        if (err) return clb(err);
-
-        if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
-          return clb(null);
-        }
 
-        if (manipulate && typeof manipulate == 'function') manipulate(lng, namespace, ns);
+    if (opt.format === 'xcstrings') { // 1 file per namespace including all languages
+      const downloadsByNamespace = {};
+      downloads.forEach((download) => {
+        const { namespace } = getInfosInUrl(download);
+        downloadsByNamespace[namespace] = downloadsByNamespace[namespace] || [];
+        downloadsByNamespace[namespace].push(download);
+      });
+
+      async.eachLimit(Object.keys(downloadsByNamespace), opt.unpublished ? 5 : 20, (namespace, clb) => {
+        const locizeData = {
+          sourceLng: opt.referenceLanguage,
+          resources: {}
+        };
+
+        async.eachLimit(downloadsByNamespace[namespace], opt.unpublished ? 5 : 20, (download, clb) => {
+          const { language } = getInfosInUrl(download);
+          opt.isPrivate = download.isPrivate;
+
+          if (opt.language && opt.language !== language && language !== opt.referenceLanguage) return clb(null);
+          if (opt.languages && opt.languages.length > 0 && opt.languages.indexOf(language) < 0 && language !== opt.referenceLanguage) return clb(null);
+          if (opt.namespace && opt.namespace !== namespace) return clb(null);
+          if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null);
+
+          if (opt.unpublished) opt.raw = true;
+          getRemoteNamespace(opt, language, namespace, (err, ns, lastModified) => {
+            if (err) return clb(err);
+
+            if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
+              return clb(null);
+            }
+
+            if (manipulate && typeof manipulate == 'function') manipulate(language, namespace, ns);
+
+            locizeData.resources[language] = ns;
+            clb();
+          });
+        }, (err) => {
+          if (err) return clb(err);
+
+          try {
+            const result = locize2xcstrings(locizeData);
+            const converted = JSON.stringify(result, null, 2);
 
-        convertToDesiredFormat(opt, namespace, lng, ns, lastModified, (err, converted) => {
-          if (err) {
+            const filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, '').replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format];
+            if (opt.dry) return clb(null);
+            if (opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`) > opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`) && filledMask.lastIndexOf(path.sep) > 0) {
+              mkdirp.sync(path.join(opt.path, filledMask.substring(0, filledMask.lastIndexOf(path.sep))));
+            }
+            const parentDir = path.dirname(path.join(opt.path, filledMask));
+            mkdirp.sync(parentDir);
+            fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+          } catch (e) {
             err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
             return clb(err);
           }
-
-          const filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, lng).replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format];
-          if (opt.dry) return clb(null);
-          if (opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`) > opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`) && filledMask.lastIndexOf(path.sep) > 0) {
-            mkdirp.sync(path.join(opt.path, filledMask.substring(0, filledMask.lastIndexOf(path.sep))));
+        });
+      }, cb);
+    } else { // 1 file per namespace/lng
+      async.eachLimit(downloads, opt.unpublished ? 5 : 20, (download, clb) => {
+        const { language, namespace } = getInfosInUrl(download);
+        opt.isPrivate = download.isPrivate;
+
+        if (opt.language && opt.language !== language && language !== opt.referenceLanguage) return clb(null);
+        if (opt.languages && opt.languages.length > 0 && opt.languages.indexOf(language) < 0 && language !== opt.referenceLanguage) return clb(null);
+        if (opt.namespace && opt.namespace !== namespace) return clb(null);
+        if (opt.namespaces && opt.namespaces.length > 0 && opt.namespaces.indexOf(namespace) < 0) return clb(null);
+
+        getRemoteNamespace(opt, language, namespace, (err, ns, lastModified) => {
+          if (err) return clb(err);
+
+          if (opt.skipEmpty && Object.keys(flatten(ns)).length === 0) {
+            return clb(null);
           }
-          const parentDir = path.dirname(path.join(opt.path, filledMask));
-          mkdirp.sync(parentDir);
-          fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+
+          if (manipulate && typeof manipulate == 'function') manipulate(language, namespace, ns);
+
+          convertToDesiredFormat(opt, namespace, language, ns, lastModified, (err, converted) => {
+            if (err) {
+              err.message = 'Invalid content for "' + opt.format + '" format!\n' + (err.message || '');
+              return clb(err);
+            }
+
+            const filledMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, language).replace(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`, namespace) + reversedFileExtensionsMap[opt.format];
+            if (opt.dry) return clb(null);
+            if (opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`) > opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`) && filledMask.lastIndexOf(path.sep) > 0) {
+              mkdirp.sync(path.join(opt.path, filledMask.substring(0, filledMask.lastIndexOf(path.sep))));
+            }
+            const parentDir = path.dirname(path.join(opt.path, filledMask));
+            mkdirp.sync(parentDir);
+            fs.writeFile(path.join(opt.path, filledMask), converted, clb);
+          });
         });
-      });
-    }, cb);
+      }, cb);
+    }
   });
 };
 
@@ -272,7 +339,6 @@ const update = (opt, lng, ns, shouldOmit, cb) => {
       queryParams.append('autotranslate', 'true');
     }
     if (so) {
-      // no API docs for this
       queryParams.append('omitstatsgeneration', 'true');
     }