@lingui/format-po-gettext 4.12.0 → 4.14.0

package/README.md

@@ -13,7 +13,7 @@
 
 > **Warning**
 > This formatter is made for compatibility with translation management systems, which do not support ICU expressions in PO files.
-> 
+>
 > It does not support all features of LinguiJS and should be carefully considered over other formats.
 >
 > Not supported features (native gettext doesn't support this):
@@ -72,10 +72,17 @@ export type PoGettextFormatterOptions = {
 
   /**
    * Disable warning about unsupported `Select` feature encountered in catalogs
-   * 
+   *
    * @default false
    */
   disableSelectWarning?: boolean
+
+  /**
+   * Overrides the default prefix for icu and plural comments in the final PO catalog.
+   *
+   * @default "js-lingui:"
+   */
+  customICUPrefix?: string
 }
 ```
 

package/dist/po-gettext.cjs

@@ -6,13 +6,72 @@ const PO = require('pofile');
 const gettextPlurals = require('node-gettext/lib/plurals');
 const generateMessageId = require('@lingui/message-utils/generateMessageId');
 const formatPo = require('@lingui/format-po');
+const cardinals = require('cldr-core/supplemental/plurals.json');
 
 function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
 
 const pluralsCldr__default = /*#__PURE__*/_interopDefaultCompat(pluralsCldr);
 const PO__default = /*#__PURE__*/_interopDefaultCompat(PO);
 const gettextPlurals__default = /*#__PURE__*/_interopDefaultCompat(gettextPlurals);
+const cardinals__default = /*#__PURE__*/_interopDefaultCompat(cardinals);
 
+function renameKeys(rules) {
+  const result = {};
+  Object.keys(rules).forEach((k) => {
+    const newKey = k.match(/[^-]+$/)[0];
+    result[newKey] = rules[k];
+  });
+  return result;
+}
+function fillRange(value) {
+  let [start, end] = value.split("~");
+  const decimals = (start.split(".")[1] || "").length;
+  let mult = Math.pow(10, decimals);
+  const startNum = Number(start);
+  const endNum = Number(end);
+  let range = Array(Math.ceil(endNum * mult - startNum * mult + 1)).fill(0).map((v, idx) => (idx + startNum * mult) / mult);
+  let last = range[range.length - 1];
+  if (endNum !== last) {
+    throw new Error(`Range create error for ${value}: last value is ${last}`);
+  }
+  return range.map((v) => Number(v));
+}
+function createSamples(src) {
+  let result = [];
+  src.replace(/…/, "").trim().replace(/,$/, "").split(",").map(function(val) {
+    return val.trim();
+  }).forEach((val) => {
+    if (val.indexOf("~") !== -1) {
+      result = result.concat(fillRange(val));
+    } else {
+      result.push(Number(val));
+    }
+  });
+  return result;
+}
+function createLocaleTest(rules) {
+  let result = {};
+  Object.keys(rules).forEach((form) => {
+    let samples = rules[form].split(/@integer|@decimal/).slice(1);
+    result[form] = [];
+    samples.forEach((sample) => {
+      result[form] = result[form].concat(createSamples(sample));
+    });
+  });
+  return result;
+}
+function getCldrPluralSamples() {
+  const pluralRules = {};
+  Object.entries(cardinals__default.supplemental["plurals-type-cardinal"]).forEach(
+    ([loc, ruleset]) => {
+      let rules = renameKeys(ruleset);
+      pluralRules[loc.toLowerCase()] = createLocaleTest(rules);
+    }
+  );
+  return pluralRules;
+}
+
+const cldrSamples = getCldrPluralSamples();
 function stringifyICUCase(icuCase) {
   return icuCase.tokens.map((token) => {
     if (token.type === "content") {
@@ -32,9 +91,10 @@ function stringifyICUCase(icuCase) {
 const ICU_PLURAL_REGEX = /^{.*, plural, .*}$/;
 const ICU_SELECT_REGEX = /^{.*, select(Ordinal)?, .*}$/;
 const LINE_ENDINGS = /\r?\n/g;
-const CTX_PREFIX = "js-lingui:";
+const DEFAULT_CTX_PREFIX = "js-lingui:";
 function serializePlurals(item, message, id, isGeneratedId, options) {
   const icuMessage = message.message;
+  const ctxPrefix = options.customICUPrefix || DEFAULT_CTX_PREFIX;
   if (!icuMessage) {
     return item;
   }
@@ -63,7 +123,7 @@ function serializePlurals(item, message, id, isGeneratedId, options) {
         item.msgid_plural = id + "_plural";
       }
       ctx.sort();
-      item.extractedComments.push(CTX_PREFIX + ctx.toString());
+      item.extractedComments.push(ctxPrefix + ctx.toString());
       if (message.translation?.length > 0) {
         const ast = parser.parse(message.translation)[0];
         if (ast.cases == null) {
@@ -89,14 +149,53 @@ function serializePlurals(item, message, id, isGeneratedId, options) {
   }
   return item;
 }
-const getPluralCases = (lang) => {
+const getPluralCases = (lang, pluralFormsHeader) => {
+  let gettextPluralsInfo;
+  if (pluralFormsHeader) {
+    gettextPluralsInfo = parsePluralFormsFn(pluralFormsHeader);
+  }
   const [correctLang] = lang.split(/[-_]/g);
-  const gettextPluralsInfo = gettextPlurals__default[correctLang];
-  return gettextPluralsInfo?.examples.map(
-    (pluralCase) => pluralsCldr__default(correctLang, pluralCase.sample)
-  );
+  if (!gettextPluralsInfo) {
+    gettextPluralsInfo = gettextPlurals__default[correctLang];
+  }
+  if (!gettextPluralsInfo) {
+    if (lang !== "pseudo") {
+      console.warn(
+        `No plural rules found for language "${lang}". Please add a Plural-Forms header.`
+      );
+    }
+    return void 0;
+  }
+  const cases = [...Array(pluralsCldr__default.forms(correctLang).length)];
+  for (let form of pluralsCldr__default.forms(correctLang)) {
+    const samples = cldrSamples[correctLang][form];
+    const pluralForm = Number(
+      gettextPluralsInfo.pluralsFunc(Number(samples[0]))
+    );
+    cases[pluralForm] = form;
+  }
+  return cases;
 };
-const convertPluralsToICU = (item, pluralForms, lang) => {
+function parsePluralFormsFn(pluralFormsHeader) {
+  const [npluralsExpr, expr] = pluralFormsHeader.split(";");
+  try {
+    const nplurals = new Function(npluralsExpr + "; return nplurals;")();
+    const pluralsFunc = new Function(
+      "n",
+      expr + "; return plural;"
+    );
+    return {
+      nplurals,
+      pluralsFunc
+    };
+  } catch (e) {
+    console.warn(
+      `Plural-Forms header has incorrect value: ${pluralFormsHeader}`
+    );
+    return void 0;
+  }
+}
+const convertPluralsToICU = (item, pluralForms, lang, ctxPrefix = DEFAULT_CTX_PREFIX) => {
   const translationCount = item.msgstr.length;
   const messageKey = item.msgid;
   if (translationCount <= 1 && !item.msgid_plural) {
@@ -109,11 +208,11 @@ const convertPluralsToICU = (item, pluralForms, lang) => {
     );
     return;
   }
-  const contextComment = item.extractedComments.find((comment) => comment.startsWith(CTX_PREFIX))?.substr(CTX_PREFIX.length);
+  const contextComment = item.extractedComments.find((comment) => comment.startsWith(ctxPrefix))?.substring(ctxPrefix.length);
   const ctx = new URLSearchParams(contextComment);
   if (contextComment != null) {
     item.extractedComments = item.extractedComments.filter(
-      (comment) => !comment.startsWith(CTX_PREFIX)
+      (comment) => !comment.startsWith(ctxPrefix)
     );
   }
   const storedICU = ctx.get("icu");
@@ -137,11 +236,13 @@ const convertPluralsToICU = (item, pluralForms, lang) => {
       messageKey
     );
   }
-  const pluralClauses = item.msgstr.map((str, index) => pluralForms[index] + " {" + str + "}").join(" ");
+  const pluralClauses = item.msgstr.map(
+    (str, index) => pluralForms[index] ? pluralForms[index] + " {" + str + "}" : ""
+  ).join(" ");
   let pluralizeOn = ctx.get("pluralize_on");
   if (!pluralizeOn) {
     console.warn(
-      `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${CTX_PREFIX}"), assuming "count".`,
+      `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${ctxPrefix}"), assuming "count".`,
       messageKey
     );
     pluralizeOn = "count";
@@ -160,9 +261,17 @@ function formatter(options = {}) {
     templateExtension: ".pot",
     parse(content, ctx) {
       const po = PO__default.parse(content);
-      let pluralForms = getPluralCases(po.headers.Language);
+      let pluralForms = getPluralCases(
+        po.headers.Language,
+        po.headers["Plural-Forms"]
+      );
       po.items.forEach((item) => {
-        convertPluralsToICU(item, pluralForms, po.headers.Language);
+        convertPluralsToICU(
+          item,
+          pluralForms,
+          po.headers.Language,
+          options.customICUPrefix
+        );
       });
       return formatter2.parse(po.toString(), ctx);
     },

package/dist/po-gettext.d.cts

@@ -3,6 +3,7 @@ import { PoFormatterOptions } from '@lingui/format-po';
 
 type PoGettextFormatterOptions = PoFormatterOptions & {
     disableSelectWarning?: boolean;
+    customICUPrefix?: string;
 };
 declare function formatter(options?: PoGettextFormatterOptions): CatalogFormatter;
 

package/dist/po-gettext.d.mts

@@ -3,6 +3,7 @@ import { PoFormatterOptions } from '@lingui/format-po';
 
 type PoGettextFormatterOptions = PoFormatterOptions & {
     disableSelectWarning?: boolean;
+    customICUPrefix?: string;
 };
 declare function formatter(options?: PoGettextFormatterOptions): CatalogFormatter;
 

package/dist/po-gettext.d.ts

@@ -3,6 +3,7 @@ import { PoFormatterOptions } from '@lingui/format-po';
 
 type PoGettextFormatterOptions = PoFormatterOptions & {
     disableSelectWarning?: boolean;
+    customICUPrefix?: string;
 };
 declare function formatter(options?: PoGettextFormatterOptions): CatalogFormatter;
 

package/dist/po-gettext.mjs

@@ -4,7 +4,65 @@ import PO from 'pofile';
 import gettextPlurals from 'node-gettext/lib/plurals';
 import { generateMessageId } from '@lingui/message-utils/generateMessageId';
 import { formatter as formatter$1 } from '@lingui/format-po';
+import cardinals from 'cldr-core/supplemental/plurals.json';
 
+function renameKeys(rules) {
+  const result = {};
+  Object.keys(rules).forEach((k) => {
+    const newKey = k.match(/[^-]+$/)[0];
+    result[newKey] = rules[k];
+  });
+  return result;
+}
+function fillRange(value) {
+  let [start, end] = value.split("~");
+  const decimals = (start.split(".")[1] || "").length;
+  let mult = Math.pow(10, decimals);
+  const startNum = Number(start);
+  const endNum = Number(end);
+  let range = Array(Math.ceil(endNum * mult - startNum * mult + 1)).fill(0).map((v, idx) => (idx + startNum * mult) / mult);
+  let last = range[range.length - 1];
+  if (endNum !== last) {
+    throw new Error(`Range create error for ${value}: last value is ${last}`);
+  }
+  return range.map((v) => Number(v));
+}
+function createSamples(src) {
+  let result = [];
+  src.replace(/…/, "").trim().replace(/,$/, "").split(",").map(function(val) {
+    return val.trim();
+  }).forEach((val) => {
+    if (val.indexOf("~") !== -1) {
+      result = result.concat(fillRange(val));
+    } else {
+      result.push(Number(val));
+    }
+  });
+  return result;
+}
+function createLocaleTest(rules) {
+  let result = {};
+  Object.keys(rules).forEach((form) => {
+    let samples = rules[form].split(/@integer|@decimal/).slice(1);
+    result[form] = [];
+    samples.forEach((sample) => {
+      result[form] = result[form].concat(createSamples(sample));
+    });
+  });
+  return result;
+}
+function getCldrPluralSamples() {
+  const pluralRules = {};
+  Object.entries(cardinals.supplemental["plurals-type-cardinal"]).forEach(
+    ([loc, ruleset]) => {
+      let rules = renameKeys(ruleset);
+      pluralRules[loc.toLowerCase()] = createLocaleTest(rules);
+    }
+  );
+  return pluralRules;
+}
+
+const cldrSamples = getCldrPluralSamples();
 function stringifyICUCase(icuCase) {
   return icuCase.tokens.map((token) => {
     if (token.type === "content") {
@@ -24,9 +82,10 @@ function stringifyICUCase(icuCase) {
 const ICU_PLURAL_REGEX = /^{.*, plural, .*}$/;
 const ICU_SELECT_REGEX = /^{.*, select(Ordinal)?, .*}$/;
 const LINE_ENDINGS = /\r?\n/g;
-const CTX_PREFIX = "js-lingui:";
+const DEFAULT_CTX_PREFIX = "js-lingui:";
 function serializePlurals(item, message, id, isGeneratedId, options) {
   const icuMessage = message.message;
+  const ctxPrefix = options.customICUPrefix || DEFAULT_CTX_PREFIX;
   if (!icuMessage) {
     return item;
   }
@@ -55,7 +114,7 @@ function serializePlurals(item, message, id, isGeneratedId, options) {
         item.msgid_plural = id + "_plural";
       }
       ctx.sort();
-      item.extractedComments.push(CTX_PREFIX + ctx.toString());
+      item.extractedComments.push(ctxPrefix + ctx.toString());
       if (message.translation?.length > 0) {
         const ast = parse(message.translation)[0];
         if (ast.cases == null) {
@@ -81,14 +140,53 @@ function serializePlurals(item, message, id, isGeneratedId, options) {
   }
   return item;
 }
-const getPluralCases = (lang) => {
+const getPluralCases = (lang, pluralFormsHeader) => {
+  let gettextPluralsInfo;
+  if (pluralFormsHeader) {
+    gettextPluralsInfo = parsePluralFormsFn(pluralFormsHeader);
+  }
   const [correctLang] = lang.split(/[-_]/g);
-  const gettextPluralsInfo = gettextPlurals[correctLang];
-  return gettextPluralsInfo?.examples.map(
-    (pluralCase) => pluralsCldr(correctLang, pluralCase.sample)
-  );
+  if (!gettextPluralsInfo) {
+    gettextPluralsInfo = gettextPlurals[correctLang];
+  }
+  if (!gettextPluralsInfo) {
+    if (lang !== "pseudo") {
+      console.warn(
+        `No plural rules found for language "${lang}". Please add a Plural-Forms header.`
+      );
+    }
+    return void 0;
+  }
+  const cases = [...Array(pluralsCldr.forms(correctLang).length)];
+  for (let form of pluralsCldr.forms(correctLang)) {
+    const samples = cldrSamples[correctLang][form];
+    const pluralForm = Number(
+      gettextPluralsInfo.pluralsFunc(Number(samples[0]))
+    );
+    cases[pluralForm] = form;
+  }
+  return cases;
 };
-const convertPluralsToICU = (item, pluralForms, lang) => {
+function parsePluralFormsFn(pluralFormsHeader) {
+  const [npluralsExpr, expr] = pluralFormsHeader.split(";");
+  try {
+    const nplurals = new Function(npluralsExpr + "; return nplurals;")();
+    const pluralsFunc = new Function(
+      "n",
+      expr + "; return plural;"
+    );
+    return {
+      nplurals,
+      pluralsFunc
+    };
+  } catch (e) {
+    console.warn(
+      `Plural-Forms header has incorrect value: ${pluralFormsHeader}`
+    );
+    return void 0;
+  }
+}
+const convertPluralsToICU = (item, pluralForms, lang, ctxPrefix = DEFAULT_CTX_PREFIX) => {
   const translationCount = item.msgstr.length;
   const messageKey = item.msgid;
   if (translationCount <= 1 && !item.msgid_plural) {
@@ -101,11 +199,11 @@ const convertPluralsToICU = (item, pluralForms, lang) => {
     );
     return;
   }
-  const contextComment = item.extractedComments.find((comment) => comment.startsWith(CTX_PREFIX))?.substr(CTX_PREFIX.length);
+  const contextComment = item.extractedComments.find((comment) => comment.startsWith(ctxPrefix))?.substring(ctxPrefix.length);
   const ctx = new URLSearchParams(contextComment);
   if (contextComment != null) {
     item.extractedComments = item.extractedComments.filter(
-      (comment) => !comment.startsWith(CTX_PREFIX)
+      (comment) => !comment.startsWith(ctxPrefix)
     );
   }
   const storedICU = ctx.get("icu");
@@ -129,11 +227,13 @@ const convertPluralsToICU = (item, pluralForms, lang) => {
       messageKey
     );
   }
-  const pluralClauses = item.msgstr.map((str, index) => pluralForms[index] + " {" + str + "}").join(" ");
+  const pluralClauses = item.msgstr.map(
+    (str, index) => pluralForms[index] ? pluralForms[index] + " {" + str + "}" : ""
+  ).join(" ");
   let pluralizeOn = ctx.get("pluralize_on");
   if (!pluralizeOn) {
     console.warn(
-      `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${CTX_PREFIX}"), assuming "count".`,
+      `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${ctxPrefix}"), assuming "count".`,
       messageKey
     );
     pluralizeOn = "count";
@@ -152,9 +252,17 @@ function formatter(options = {}) {
     templateExtension: ".pot",
     parse(content, ctx) {
       const po = PO.parse(content);
-      let pluralForms = getPluralCases(po.headers.Language);
+      let pluralForms = getPluralCases(
+        po.headers.Language,
+        po.headers["Plural-Forms"]
+      );
       po.items.forEach((item) => {
-        convertPluralsToICU(item, pluralForms, po.headers.Language);
+        convertPluralsToICU(
+          item,
+          pluralForms,
+          po.headers.Language,
+          options.customICUPrefix
+        );
       });
       return formatter2.parse(po.toString(), ctx);
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lingui/format-po-gettext",
-  "version": "4.12.0",
+  "version": "4.14.0",
   "description": "Gettext PO format with gettext-style plurals for Lingui Catalogs",
   "main": "./dist/po-gettext.cjs",
   "module": "./dist/po-gettext.mjs",
@@ -41,10 +41,11 @@
     "dist/"
   ],
   "dependencies": {
-    "@lingui/conf": "4.12.0",
-    "@lingui/format-po": "4.12.0",
-    "@lingui/message-utils": "4.12.0",
+    "@lingui/conf": "4.14.0",
+    "@lingui/format-po": "4.14.0",
+    "@lingui/message-utils": "4.14.0",
     "@messageformat/parser": "^5.0.0",
+    "cldr-core": "^45.0.0",
     "node-gettext": "^3.0.0",
     "plurals-cldr": "^2.0.1",
     "pofile": "^1.1.4"
@@ -55,5 +56,5 @@
     "tsd": "^0.28.0",
     "unbuild": "2.0.0"
   },
-  "gitHead": "83a6ea03f22d6284bdcf067bde7b406c4e3520bc"
+  "gitHead": "8dcb749688e2f452f85fc0eaad4d033a44975b42"
 }