locize-cli 7.8.2 → 7.10.0

package/CHANGELOG.md

@@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file.
 Project versioning adheres to [Semantic Versioning](http://semver.org/).
 Change log format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## [7.10.0](https://github.com/locize/locize-cli/compare/v7.9.1...v7.10.0) - 2022-03-15
+
+- gettext_i18next: try to detect v4 format
+
+## [7.9.1](https://github.com/locize/locize-cli/compare/v7.9.0...v7.9.1) - 2022-03-02
+
+- xliff: fix combined plural keys
+
+
+## [7.9.0](https://github.com/locize/locize-cli/compare/v7.8.2...v7.9.0) - 2022-03-01
+
+- xliff: detect i18n format and merge pluralforms if necessary
+
+
 ## [7.8.2](https://github.com/locize/locize-cli/compare/v7.8.1...v7.8.2) - 2022-02-28
 
 - update dependencies

package/combineSubkeyPreprocessor.js

@@ -0,0 +1,154 @@
+const jsyaml = require('js-yaml');
+
+const delimiter = {
+  i18next: '_',
+  i18njs: '.'
+};
+
+const detectFormat = (keys) => {
+  const i18nextMatches = keys.filter((k) => k.indexOf(delimiter.i18next) > 0).length;
+  const i18njsMatches = keys.filter((k) => k.indexOf(delimiter.i18njs) > 0).length;
+  if (i18nextMatches > i18njsMatches) {
+    return 'i18next';
+  }
+  if (i18nextMatches < i18njsMatches) {
+    return 'i18njs';
+  }
+};
+
+const getBaseKey = (delimiter) => (k) => {
+  const parts = k.split(delimiter);
+  parts.pop();
+  const baseKey = parts.join(delimiter);
+  return baseKey;
+};
+
+const uniq = (value, index, self) => self.indexOf(value) === index;
+
+const stringify = (o) => {
+  let str = jsyaml.dump(o);
+  const subKeys = Object.keys(o);
+  subKeys.forEach((sk) => {
+    if (isNaN(sk)) {
+      str = str.replace(new RegExp(`^(?:${sk}: )+`, 'm'), `{${sk}}: `);
+    } else {
+      str = str.replace(new RegExp(`^(?:'${sk}': )+`, 'm'), `{${sk}}: `);
+    }
+  });
+  return str;
+};
+
+const transformKeys = (segments, baseKeys, toMerge, deli) => {
+  baseKeys.forEach((bk) => {
+    const asObj = toMerge[bk].reduce((mem, k) => {
+      const subKey = k.substring((bk + deli).length);
+      // special handling for i18next v3
+      if (deli === delimiter.i18next && subKey === 'plural' && segments[bk]) {
+        mem['__'] = segments[bk];
+        delete segments[bk];
+      }
+      mem[subKey] = segments[k];
+      return mem;
+    }, {});
+    if (Object.keys(asObj).length > 0) {
+      const value = stringify(asObj);
+      segments[`${bk}__#locize.com/combinedSubkey`] = value;
+      toMerge[bk].forEach((k) => {
+        delete segments[k];
+      });
+    }
+  });
+  return segments;
+};
+
+// CLDR
+const pluralForms = [
+  'zero',
+  'one',
+  'two',
+  'few',
+  'many',
+  'other'
+];
+
+const endsWithPluralForm = (k) => !!pluralForms.find((f) => k.endsWith(`.${f}`)) || !!pluralForms.find((f) => k.endsWith(`_${f}`)) || /_\d+$/.test(k) || k.endsWith('_plural');
+
+const prepareExport = (refRes, trgRes) => {
+  const refLngKeys = Object.keys(refRes);
+  const trgLngKeys = Object.keys(trgRes);
+
+  const nonMatchInRef = refLngKeys.filter((k) => trgLngKeys.indexOf(k) < 0 && endsWithPluralForm(k));
+  const nonMatchInTrg = trgLngKeys.filter((k) => refLngKeys.indexOf(k) < 0 && endsWithPluralForm(k));
+
+  const allMatches = nonMatchInRef.concat(nonMatchInTrg);
+
+  const format = detectFormat(allMatches);
+  if (!format) return { ref: refRes, trg: trgRes };
+
+  const nonMatchBaseKeysInRef = nonMatchInRef.map(getBaseKey(delimiter[format])).filter(uniq);
+  const nonMatchBaseKeysInTrg = nonMatchInTrg.map(getBaseKey(delimiter[format])).filter(uniq);
+  const nonMatchBaseKeys = nonMatchBaseKeysInRef.concat(nonMatchBaseKeysInTrg).filter(uniq);
+
+  const toMergeInRef = nonMatchBaseKeys.reduce((mem, bk) => {
+    mem[bk] = refLngKeys.filter((k) => k.indexOf(bk + delimiter[format]) === 0);
+    return mem;
+  }, {});
+  const toMergeInTrg = nonMatchBaseKeys.reduce((mem, bk) => {
+    mem[bk] = trgLngKeys.filter((k) => k.indexOf(bk + delimiter[format]) === 0);
+    return mem;
+  }, {});
+
+  let falseFlags = nonMatchBaseKeysInRef.filter((k) => toMergeInRef[k].length < 2 && (!toMergeInTrg[k] || toMergeInTrg[k].length < 2));
+  falseFlags = falseFlags.concat(nonMatchBaseKeysInTrg.filter((k) => toMergeInTrg[k].length < 2 && (!toMergeInRef[k] || toMergeInRef[k].length < 2)));
+  falseFlags.forEach((k) => {
+    delete toMergeInRef[k];
+    delete toMergeInTrg[k];
+    nonMatchBaseKeys.splice(nonMatchBaseKeys.indexOf(k), 1);
+  });
+
+  const transformedRef = transformKeys(refRes, nonMatchBaseKeys, toMergeInRef, delimiter[format]);
+  const transformedTrg = transformKeys(trgRes, nonMatchBaseKeys, toMergeInTrg, delimiter[format]);
+  return { ref: transformedRef, trg: transformedTrg };
+};
+
+const skRegex = new RegExp('^(?:{(.+)})+', 'gm');
+const parse = (s) => {
+  let matchArray;
+  while ((matchArray = skRegex.exec(s)) !== null) {
+    const [match, sk] = matchArray;
+    if (isNaN(sk)) {
+      s = s.replace(new RegExp(`^(?:${match}: )+`, 'm'), `${sk}: `);
+    } else {
+      const escapedMatch = match.replace('{', '\\{').replace('}', '\\}');
+      s = s.replace(new RegExp(`^(?:${escapedMatch}: )+`, 'm'), `${sk}: `);
+    }
+  }
+  return jsyaml.load(s);
+};
+
+const prepareImport = (resources) => {
+  const keys = Object.keys(resources);
+  keys.forEach((k) => {
+    if (k.indexOf('__#locize.com/combinedSubkey') > -1) {
+      const baseKey = k.substring(0, k.indexOf('__#locize.com/combinedSubkey'));
+      if (resources[k]) {
+        const parsed = parse(resources[k]);
+        Object.keys(parsed).map((sk) => {
+          const skVal = parsed[sk];
+          resources[`${baseKey}_${sk}`] = skVal;
+          if (sk === '__') {
+            resources[baseKey] = resources[`${baseKey}_${sk}`];
+            delete resources[`${baseKey}_${sk}`];
+          }
+        });
+        delete resources[k];
+      }
+    }
+  });
+  return resources;
+};
+
+module.exports = {
+  prepareExport,
+  prepareImport
+};

package/convertToDesiredFormat.js

@@ -16,6 +16,7 @@ const unflatten = require('./unflatten');
 const getRemoteNamespace = require('./getRemoteNamespace');
 const removeUndefinedFromArrays = require('./removeUndefinedFromArrays');
 const shouldUnflatten = require('./shouldUnflatten');
+const prepareCombinedExport = require('./combineSubkeyPreprocessor').prepareExport;
 
 const convertToDesiredFormat = (
   opt,
@@ -55,11 +56,13 @@ const convertToDesiredFormat = (
     }
     if (opt.format === 'po_i18next' || opt.format === 'gettext_i18next') {
       const flatData = flatten(data);
+      const compatibilityJSON = !!Object.keys(flatData).find((k) => /_(zero|one|two|few|many|other)/.test(k)) && 'v4';
       const gettextOpt = {
         project: 'locize',
         language: lng,
         potCreationDate: lastModified,
-        poRevisionDate: lastModified
+        poRevisionDate: lastModified,
+        compatibilityJSON
       };
       cb(null, i18next2po(lng, flatData, gettextOpt));
       return;
@@ -172,7 +175,8 @@ const convertToDesiredFormat = (
       opt.getNamespace(opt, opt.referenceLanguage, namespace, (err, refNs) => {
         if (err) return cb(err);
 
-        fn(opt.referenceLanguage, lng, refNs, flatten(data), namespace, cb);
+        const prepared = prepareCombinedExport(refNs, flatten(data));
+        fn(opt.referenceLanguage, lng, prepared.ref, prepared.trg, namespace, cb);
       });
       return;
     }

package/convertToFlatFormat.js

@@ -14,6 +14,7 @@ const tmx2js = require('tmexchange/cjs/tmx2js');
 const laravel2js = require('laravelphp/cjs/laravel2js');
 const javaProperties = require('@js.properties/properties');
 const flatten = require('flat');
+const prepareCombinedImport = require('./combineSubkeyPreprocessor').prepareImport;
 
 const convertToFlatFormat = (opt, data, lng, cb) => {
   if (!cb) {
@@ -39,7 +40,9 @@ const convertToFlatFormat = (opt, data, lng, cb) => {
     }
     if (opt.format === 'po_i18next' || opt.format === 'gettext_i18next') {
       try {
-        const ret = po2i18next(data.toString());
+        const potxt = data.toString();
+        const compatibilityJSON = /msgctxt "(zero|one|two|few|many|other)"/.test(potxt) && 'v4';
+        const ret = po2i18next(potxt, { compatibilityJSON });
         cb(null, flatten(ret));
       } catch (err) {
         cb(err);
@@ -171,34 +174,34 @@ const convertToFlatFormat = (opt, data, lng, cb) => {
         res.resources = res.resources || {};
         const ns = Object.keys(res.resources)[0];
         const orgRes = res.resources[ns] || res.resources;
-        function checkForContext(nsRes) {
+        function checkForPostProcessing(nsRes) {
           Object.keys(nsRes).forEach((k) => {
             if (orgRes[k].note && (typeof nsRes[k] === 'string' || !nsRes[k])) {
               nsRes[k] = {
                 value: nsRes[k],
                 context: {
                   text: orgRes[k].note,
-                },
+                }
               };
             }
           });
-          return nsRes;
+          return prepareCombinedImport(nsRes);
         }
         if (!res.targetLanguage) {
           sourceOfjs(res, (err, ret) => {
             if (err) return cb(err);
-            cb(null, checkForContext(ret));
+            cb(null, checkForPostProcessing(ret));
           });
         } else {
           let ret = targetOfjs(res);
-          if (lng !== opt.referenceLanguage) return cb(null, checkForContext(ret));
+          if (lng !== opt.referenceLanguage) return cb(null, checkForPostProcessing(ret));
           ret = ret || {};
           const keys = Object.keys(ret);
-          if (keys.length === 0) return cb(null, checkForContext(ret));
+          if (keys.length === 0) return cb(null, checkForPostProcessing(ret));
           const allEmpty = keys.filter((k) => ret[k] !== '').length === 0;
-          if (!allEmpty) return cb(null, checkForContext(ret));
+          if (!allEmpty) return cb(null, checkForPostProcessing(ret));
           ret = sourceOfjs(res);
-          cb(null, checkForContext(ret));
+          cb(null, checkForPostProcessing(ret));
         }
       });
       return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locize-cli",
-  "version": "7.8.2",
+  "version": "7.10.0",
   "description": "locize cli to import locales",
   "main": "index.js",
   "bin": {
@@ -16,7 +16,7 @@
     "diff": "5.0.0",
     "flat": "5.0.2",
     "fluent_conv": "3.1.0",
-    "gettext-converter": "1.2.0",
+    "gettext-converter": "1.2.2",
     "https-proxy-agent": "5.0.0",
     "ini": "2.0.0",
     "js-yaml": "4.1.0",
@@ -29,11 +29,11 @@
     "strings-file": "0.0.5",
     "tmexchange": "2.0.4",
     "xliff": "6.0.3",
-    "xlsx": "0.18.2"
+    "xlsx": "0.18.3"
   },
   "devDependencies": {
     "eslint": "7.32.0",
-    "gh-release": "6.0.1",
+    "gh-release": "6.0.2",
     "pkg": "5.5.2"
   },
   "scripts": {