locize-cli 7.8.0 → 7.9.0

package/CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 Project versioning adheres to [Semantic Versioning](http://semver.org/).
 Change log format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## [7.9.0](https://github.com/locize/locize-cli/compare/v7.8.2...v7.9.0) - 2022-03-01
+
+- xliff: detect i18n format and merge pluralforms if necessary
+
+
+## [7.8.2](https://github.com/locize/locize-cli/compare/v7.8.1...v7.8.2) - 2022-02-28
+
+- update dependencies
+
+
+## [7.8.1](https://github.com/locize/locize-cli/compare/v7.8.0...v7.8.1) - 2022-02-10
+
+- sync: optimize handling if pathmask does not include namespace
+
+
 ## [7.8.0](https://github.com/locize/locize-cli/compare/v7.7.2...v7.8.0) - 2022-01-13
 
 - sync: introduce --delete-remote-namespace option

package/README.md

@@ -91,7 +91,7 @@ locize download --project-id my-project-id-93e1-442a-ab35-24331fa294ba --ver lat
 By using the sync command, you can keep your existing code setup and synchronize the translations with locize.
 An example on how this could look like can be seen in [this tutorial](https://github.com/locize/react-tutorial#step-1---keep-existing-code-setup-but-synchronize-with-locize).
 
-**⚠️ Since the remote source are the published translations, make sure the desired version is set to standard publish mode, or alternatively [publish](#publish-version) that version before you execute the cli command. ⚠️**
+**⚠️ Since the remote source are the published translations, make sure the desired version is set to standard/auto publish mode, or alternatively [publish](#publish-version) that version before you execute the cli command. ⚠️**
 
 ### Step 1: Go near to your translation files
 

package/combineSubkeyPreprocessor.js

@@ -0,0 +1,136 @@
+const jsyaml = require('js-yaml');
+
+const delimiter = {
+  i18next: '_',
+  i18njs: '.'
+};
+
+const detectFormat = (keys) => {
+  const i18nextMatches = keys.filter((k) => k.indexOf(delimiter.i18next) > 0).length;
+  const i18njsMatches = keys.filter((k) => k.indexOf(delimiter.i18njs) > 0).length;
+  if (i18nextMatches > i18njsMatches) {
+    return 'i18next';
+  }
+  if (i18nextMatches < i18njsMatches) {
+    return 'i18njs';
+  }
+};
+
+const getBaseKey = (delimiter) => (k) => {
+  const parts = k.split(delimiter);
+  parts.pop();
+  const baseKey = parts.join(delimiter);
+  return baseKey;
+};
+
+const uniq = (value, index, self) => self.indexOf(value) === index;
+
+const stringify = (o) => {
+  let str = jsyaml.dump(o);
+  const subKeys = Object.keys(o);
+  subKeys.forEach((sk, i) => {
+    str = str.replace(new RegExp(`^(?:${sk}: )+`, 'm'), `{${sk}}: `);
+  });
+  return str;
+};
+
+const transformKeys = (segments, baseKeys, toMerge, deli) => {
+  baseKeys.forEach((bk) => {
+    const asObj = toMerge[bk].reduce((mem, k) => {
+      const subKey = k.substring((bk + deli).length);
+      mem[subKey] = segments[k];
+      return mem;
+    }, {});
+    if (Object.keys(asObj).length > 0) {
+      const value = stringify(asObj);
+      segments[`${bk}__#locize.com/combinedSubkey`] = value;
+      toMerge[bk].forEach((k) => {
+        delete segments[k];
+      });
+    }
+  });
+  return segments;
+};
+
+// CLDR
+const pluralForms = [
+  'zero',
+  'one',
+  'two',
+  'few',
+  'many',
+  'other'
+];
+
+const endsWithPluralForm = (k) => !!pluralForms.find((f) => k.endsWith(`.${f}`)) || !!pluralForms.find((f) => k.endsWith(`_${f}`)) || /_\d+$/.test(k) || k.endsWith('_plural');
+
+const prepareExport = (refRes, trgRes) => {
+  const refLngKeys = Object.keys(refRes);
+  const trgLngKeys = Object.keys(trgRes);
+
+  const nonMatchInRef = refLngKeys.filter((k) => trgLngKeys.indexOf(k) < 0 && endsWithPluralForm(k));
+  const nonMatchInTrg = trgLngKeys.filter((k) => refLngKeys.indexOf(k) < 0 && endsWithPluralForm(k));
+
+  const allMatches = nonMatchInRef.concat(nonMatchInTrg);
+
+  const format = detectFormat(allMatches);
+  if (!format) return { ref: refRes, trg: trgRes };
+
+  const nonMatchBaseKeysInRef = nonMatchInRef.map(getBaseKey(delimiter[format])).filter(uniq);
+  const nonMatchBaseKeysInTrg = nonMatchInTrg.map(getBaseKey(delimiter[format])).filter(uniq);
+  const nonMatchBaseKeys = nonMatchBaseKeysInRef.concat(nonMatchBaseKeysInTrg).filter(uniq);
+
+  const toMergeInRef = nonMatchBaseKeys.reduce((mem, bk) => {
+    mem[bk] = refLngKeys.filter((k) => k.indexOf(bk + delimiter[format]) === 0);
+    return mem;
+  }, {});
+  const toMergeInTrg = nonMatchBaseKeys.reduce((mem, bk) => {
+    mem[bk] = trgLngKeys.filter((k) => k.indexOf(bk + delimiter[format]) === 0);
+    return mem;
+  }, {});
+
+  let falseFlags = nonMatchBaseKeysInRef.filter((k) => toMergeInRef[k].length < 2 && (!toMergeInTrg[k] || toMergeInTrg[k].length < 2));
+  falseFlags = falseFlags.concat(nonMatchBaseKeysInTrg.filter((k) => toMergeInTrg[k].length < 2 && (!toMergeInRef[k] || toMergeInRef[k].length < 2)));
+  falseFlags.forEach((k) => {
+    delete toMergeInRef[k];
+    delete toMergeInTrg[k];
+    nonMatchBaseKeys.splice(nonMatchBaseKeys.indexOf(k), 1);
+  });
+
+  const transformedRef = transformKeys(refRes, nonMatchBaseKeys, toMergeInRef, delimiter[format]);
+  const transformedTrg = transformKeys(trgRes, nonMatchBaseKeys, toMergeInTrg, delimiter[format]);
+  return { ref: transformedRef, trg: transformedTrg };
+};
+
+const skRegex = new RegExp('^(?:{(.+)})+', 'gm');
+const parse = (s) => {
+  let matchArray;
+  while ((matchArray = skRegex.exec(s)) !== null) {
+    const [match, sk] = matchArray;
+    s = s.replace(new RegExp(`^(?:${match}: )+`, 'm'), `${sk}: `);
+  }
+  return jsyaml.load(s);
+};
+
+const prepareImport = (resources) => {
+  const keys = Object.keys(resources);
+  keys.forEach((k) => {
+    if (k.indexOf('__#locize.com/combinedSubkey') > -1) {
+      const baseKey = k.substring(0, k.indexOf('__#locize.com/combinedSubkey'));
+      if (resources[k]) {
+        const parsed = parse(resources[k]);
+        Object.keys(parsed).map((sk) => {
+          const skVal = parsed[sk];
+          resources[`${baseKey}_${sk}`] = skVal;
+        });
+        delete resources[k];
+      }
+    }
+  });
+  return resources;
+};
+
+module.exports = {
+  prepareExport,
+  prepareImport
+};

package/convertToDesiredFormat.js

@@ -16,6 +16,7 @@ const unflatten = require('./unflatten');
 const getRemoteNamespace = require('./getRemoteNamespace');
 const removeUndefinedFromArrays = require('./removeUndefinedFromArrays');
 const shouldUnflatten = require('./shouldUnflatten');
+const prepareCombinedExport = require('./combineSubkeyPreprocessor').prepareExport;
 
 const convertToDesiredFormat = (
   opt,
@@ -172,7 +173,8 @@ const convertToDesiredFormat = (
       opt.getNamespace(opt, opt.referenceLanguage, namespace, (err, refNs) => {
         if (err) return cb(err);
 
-        fn(opt.referenceLanguage, lng, refNs, flatten(data), namespace, cb);
+        const prepared = prepareCombinedExport(refNs, flatten(data));
+        fn(opt.referenceLanguage, lng, prepared.ref, prepared.trg, namespace, cb);
       });
       return;
     }

package/convertToFlatFormat.js

@@ -14,6 +14,7 @@ const tmx2js = require('tmexchange/cjs/tmx2js');
 const laravel2js = require('laravelphp/cjs/laravel2js');
 const javaProperties = require('@js.properties/properties');
 const flatten = require('flat');
+const prepareCombinedImport = require('./combineSubkeyPreprocessor').prepareImport;
 
 const convertToFlatFormat = (opt, data, lng, cb) => {
   if (!cb) {
@@ -171,34 +172,34 @@ const convertToFlatFormat = (opt, data, lng, cb) => {
         res.resources = res.resources || {};
         const ns = Object.keys(res.resources)[0];
         const orgRes = res.resources[ns] || res.resources;
-        function checkForContext(nsRes) {
+        function checkForPostProcessing(nsRes) {
           Object.keys(nsRes).forEach((k) => {
             if (orgRes[k].note && (typeof nsRes[k] === 'string' || !nsRes[k])) {
               nsRes[k] = {
                 value: nsRes[k],
                 context: {
                   text: orgRes[k].note,
-                },
+                }
               };
             }
           });
-          return nsRes;
+          return prepareCombinedImport(nsRes);
         }
         if (!res.targetLanguage) {
           sourceOfjs(res, (err, ret) => {
             if (err) return cb(err);
-            cb(null, checkForContext(ret));
+            cb(null, checkForPostProcessing(ret));
           });
         } else {
           let ret = targetOfjs(res);
-          if (lng !== opt.referenceLanguage) return cb(null, checkForContext(ret));
+          if (lng !== opt.referenceLanguage) return cb(null, checkForPostProcessing(ret));
           ret = ret || {};
           const keys = Object.keys(ret);
-          if (keys.length === 0) return cb(null, checkForContext(ret));
+          if (keys.length === 0) return cb(null, checkForPostProcessing(ret));
           const allEmpty = keys.filter((k) => ret[k] !== '').length === 0;
-          if (!allEmpty) return cb(null, checkForContext(ret));
+          if (!allEmpty) return cb(null, checkForPostProcessing(ret));
           ret = sourceOfjs(res);
-          cb(null, checkForContext(ret));
+          cb(null, checkForPostProcessing(ret));
         }
       });
       return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locize-cli",
-  "version": "7.8.0",
+  "version": "7.9.0",
   "description": "locize cli to import locales",
   "main": "index.js",
   "bin": {
@@ -9,9 +9,9 @@
   "dependencies": {
     "@js.properties/properties": "0.5.4",
     "android-string-resource": "2.3.4",
-    "async": "3.2.2",
+    "async": "3.2.3",
     "colors": "1.4.0",
-    "commander": "7.2.0",
+    "commander": "9.0.0",
     "csvjson": "5.1.0",
     "diff": "5.0.0",
     "flat": "5.0.2",
@@ -23,18 +23,18 @@
     "laravelphp": "2.0.3",
     "lodash.clonedeep": "4.5.0",
     "mkdirp": "1.0.4",
-    "node-fetch": "2.6.6",
+    "node-fetch": "2.6.7",
     "resx": "2.0.3",
     "rimraf": "3.0.2",
     "strings-file": "0.0.5",
     "tmexchange": "2.0.4",
-    "xliff": "5.7.2",
-    "xlsx": "0.17.4"
+    "xliff": "6.0.3",
+    "xlsx": "0.18.2"
   },
   "devDependencies": {
     "eslint": "7.32.0",
     "gh-release": "6.0.1",
-    "pkg": "5.5.1"
+    "pkg": "5.5.2"
   },
   "scripts": {
     "lint": "eslint .",

package/parseLocalLanguage.js

@@ -20,6 +20,7 @@ const getDirectories = (srcpath) => {
 };
 
 const parseLocalLanguage = (opt, lng, cb) => {
+  const hasNamespaceInPath = opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`) > -1;
   const filledLngMask = opt.pathMask.replace(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`, lng);
   var firstPartLngMask, lastPartLngMask;
   if (opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}language${opt.pathMaskInterpolationSuffix}`) > opt.pathMask.indexOf(`${opt.pathMaskInterpolationPrefix}namespace${opt.pathMaskInterpolationSuffix}`)) {
@@ -114,6 +115,8 @@ const parseLocalLanguage = (opt, lng, cb) => {
       } else {
         namespace = dirPath.substring(filledNsMask.indexOf(nsMask));
       }
+    } else if (!hasNamespaceInPath && startNsIndex < 0) {
+      namespace = opt.namespace;
     }
     var fPath = path.join(opt.path, lngPath || '', file);
     if (dirPath && lngPath.indexOf(nsMask) > -1) {