@lingui/format-po-gettext 4.0.0-next.2

package/README.md

@@ -0,0 +1,92 @@
+[![License][badge-license]][license]
+[![Version][badge-version]][package]
+[![Downloads][badge-downloads]][package]
+
+# @lingui/format-po-gettext
+
+> Read and write message catalogs in Gettext PO format with gettext-style plurals
+>
+> Converts ICU Plural expressions into native gettext plurals
+
+`@lingui/format-po-gettext` is part of [LinguiJS][linguijs]. See the
+[documentation][documentation] for all information, tutorials and examples.
+
+> **Warning**
+> This formatter is made for compatibility with TMS, which does not support ICU expressions in PO files.
+> 
+> It does not support all features of LinguiJS and should be carefully considered over other formats.
+>
+> Not supported features (native gettext doesn't support this):
+> - SelectOrdinal
+> - Select
+> - Nested ICU Expressions
+> - Signed digits and fractions (-5, and 0.15) in plurals
+
+## Catalog example
+
+```po
+#. js-lingui-id: WGI12K
+#. js-lingui:icu=%7BanotherCount%2C+plural%2C+one+%7BSingular+case%7D+other+%7BCase+number+%7BanotherCount%7D%7D%7D&pluralize_on=anotherCount
+msgid "Singular case"
+msgid_plural "Case number {anotherCount}"
+msgstr[0] "Singular case"
+msgstr[1] "Case number {anotherCount}"
+```
+
+## Installation
+
+```sh
+npm install --save-dev @lingui/format-po-gettext
+# yarn add --dev @lingui/format-po-gettext
+```
+
+## Usage
+
+```js
+// lingui.config.{js,ts}
+import {formatter} from "@lingui/format-po-gettext"
+
+export default {
+  [...]
+  format: formatter({lineNumbers: false}),
+}
+```
+
+Possible options:
+
+```ts
+export type PoGettextFormatterOptions = {
+  /**
+   * Print places where message is used
+   *
+   * @default true
+   */
+  origins?: boolean
+
+  /**
+   * Print line numbers in origins
+   *
+   * @default true
+   */
+  lineNumbers?: boolean
+
+  /**
+   * Disable warning about unsupported `Select` feature encountered in catalogs
+   * 
+   * @default false
+   */
+  disableSelectWarning?: boolean
+}
+```
+
+## License
+
+This package is licensed under [MIT][license] license.
+
+[license]: https://github.com/lingui/js-lingui/blob/main/LICENSE
+[linguijs]: https://github.com/lingui/js-lingui
+[documentation]: https://lingui.dev
+[package]: https://www.npmjs.com/package/@lingui/format-po-gettext
+[badge-downloads]: https://img.shields.io/npm/dw/@lingui/format-po-gettext.svg
+[badge-version]: https://img.shields.io/npm/v/@lingui/format-po-gettext.svg
+[badge-license]: https://img.shields.io/npm/l/@lingui/format-po-gettext.svg

package/dist/po-gettext.cjs

@@ -0,0 +1,176 @@
+'use strict';
+
+const parser = require('@messageformat/parser');
+const pluralsCldr = require('plurals-cldr');
+const PO = require('pofile');
+const gettextPlurals = require('node-gettext/lib/plurals');
+const api = require('@lingui/cli/api');
+const formatPo = require('@lingui/format-po');
+
+function stringifyICUCase(icuCase) {
+  return icuCase.tokens.map((token) => {
+    if (token.type === "content") {
+      return token.value;
+    } else if (token.type === "octothorpe") {
+      return "#";
+    } else if (token.type === "argument") {
+      return "{" + token.arg + "}";
+    } else {
+      console.warn(
+        `Unexpected token "${token}" while stringifying plural case "${icuCase}". Token will be ignored.`
+      );
+      return "";
+    }
+  }).join("");
+}
+const ICU_PLURAL_REGEX = /^{.*, plural, .*}$/;
+const ICU_SELECT_REGEX = /^{.*, select(Ordinal)?, .*}$/;
+const LINE_ENDINGS = /\r?\n/g;
+const CTX_PREFIX = "js-lingui:";
+function serializePlurals(item, message, id, isGeneratedId, options) {
+  const icuMessage = message.message;
+  if (!icuMessage) {
+    return;
+  }
+  const _simplifiedMessage = icuMessage.replace(LINE_ENDINGS, " ");
+  if (ICU_PLURAL_REGEX.test(_simplifiedMessage)) {
+    try {
+      const messageAst = parser.parse(icuMessage)[0];
+      if (messageAst.cases.some(
+        (icuCase) => icuCase.tokens.some((token) => token.type === "plural")
+      )) {
+        console.warn(
+          `Nested plurals cannot be expressed with gettext plurals. Message with key "%s" will not be saved correctly.`,
+          id
+        );
+      }
+      const ctx = new URLSearchParams({
+        pluralize_on: messageAst.arg
+      });
+      if (isGeneratedId) {
+        item.msgid = stringifyICUCase(messageAst.cases[0]);
+        item.msgid_plural = stringifyICUCase(
+          messageAst.cases[messageAst.cases.length - 1]
+        );
+        ctx.set("icu", icuMessage);
+      } else {
+        item.msgid_plural = id + "_plural";
+      }
+      ctx.sort();
+      item.extractedComments.push(CTX_PREFIX + ctx.toString());
+      if (message.translation?.length > 0) {
+        const ast = parser.parse(message.translation)[0];
+        if (ast.cases == null) {
+          console.warn(
+            `Found translation without plural cases for key "${id}". This likely means that a translated .po file misses multiple msgstr[] entries for the key. Translation found: "${message.translation}"`
+          );
+          item.msgstr = [message.translation];
+        } else {
+          item.msgstr = ast.cases.map(stringifyICUCase);
+        }
+      }
+    } catch (e) {
+      console.error(`Error parsing message ICU for key "${id}":`, e);
+    }
+  } else {
+    if (!options.disableSelectWarning && ICU_SELECT_REGEX.test(_simplifiedMessage)) {
+      console.warn(
+        `ICU 'select' and 'selectOrdinal' formats cannot be expressed natively in gettext format. Item with key "%s" will be included in the catalog as raw ICU message. To disable this warning, include '{ disableSelectWarning: true }' in the config's 'formatOptions'`,
+        id
+      );
+    }
+    item.msgstr = [message.translation];
+  }
+  return item;
+}
+const getPluralCases = (lang) => {
+  const [correctLang] = lang.split(/[-_]/g);
+  const gettextPluralsInfo = gettextPlurals[correctLang];
+  return gettextPluralsInfo?.examples.map(
+    (pluralCase) => pluralsCldr(correctLang, pluralCase.sample)
+  );
+};
+const convertPluralsToICU = (item, pluralForms, lang) => {
+  const translationCount = item.msgstr.length;
+  const messageKey = item.msgid;
+  if (translationCount <= 1 && !item.msgid_plural) {
+    return;
+  }
+  if (!item.msgid_plural) {
+    console.warn(
+      `Multiple translations for item with key "%s" but missing 'msgid_plural' in catalog "${lang}". This is not supported and the plural cases will be ignored.`,
+      messageKey
+    );
+    return;
+  }
+  const contextComment = item.extractedComments.find((comment) => comment.startsWith(CTX_PREFIX))?.substr(CTX_PREFIX.length);
+  const ctx = new URLSearchParams(contextComment);
+  if (contextComment != null) {
+    item.extractedComments = item.extractedComments.filter(
+      (comment) => !comment.startsWith(CTX_PREFIX)
+    );
+  }
+  const storedICU = ctx.get("icu");
+  if (storedICU != null) {
+    item.msgid = storedICU;
+  }
+  if (item.msgstr.every((str) => str.length === 0)) {
+    return;
+  }
+  if (pluralForms == null) {
+    console.warn(
+      `Multiple translations for item with key "%s" in language "${lang}", but no plural cases were found. This prohibits the translation of .po plurals into ICU plurals. Pluralization will not work for this key.`,
+      messageKey
+    );
+    return;
+  }
+  const pluralCount = pluralForms.length;
+  if (translationCount > pluralCount) {
+    console.warn(
+      `More translations provided (${translationCount}) for item with key "%s" in language "${lang}" than there are plural cases available (${pluralCount}). This will result in not all translations getting picked up.`,
+      messageKey
+    );
+  }
+  const pluralClauses = item.msgstr.map((str, index) => pluralForms[index] + " {" + str + "}").join(" ");
+  let pluralizeOn = ctx.get("pluralize_on");
+  if (!pluralizeOn) {
+    console.warn(
+      `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${CTX_PREFIX}"), assuming "count".`,
+      messageKey
+    );
+    pluralizeOn = "count";
+  }
+  item.msgstr = ["{" + pluralizeOn + ", plural, " + pluralClauses + "}"];
+};
+function formatter(options = {}) {
+  options = {
+    origins: true,
+    lineNumbers: true,
+    ...options
+  };
+  const formatter2 = formatPo.formatter(options);
+  return {
+    catalogExtension: ".po",
+    templateExtension: ".pot",
+    parse(content) {
+      const po = PO.parse(content);
+      let pluralForms = getPluralCases(po.headers.Language);
+      po.items.forEach((item) => {
+        convertPluralsToICU(item, pluralForms, po.headers.Language);
+      });
+      return formatter2.parse(po.toString());
+    },
+    serialize(catalog, ctx) {
+      const po = PO.parse(formatter2.serialize(catalog, ctx));
+      po.items = po.items.map((item) => {
+        const isGeneratedId = !item.flags["explicit-id"];
+        const id = isGeneratedId ? api.generateMessageId(item.msgid, item.msgctxt) : item.msgid;
+        const message = catalog[id];
+        return serializePlurals(item, message, id, isGeneratedId, options);
+      });
+      return po.toString();
+    }
+  };
+}
+
+exports.formatter = formatter;

package/dist/po-gettext.d.ts

@@ -0,0 +1,17 @@
+import { CatalogType } from '@lingui/conf';
+import { PoFormatterOptions } from '@lingui/format-po';
+
+type PoGettextFormatterOptions = PoFormatterOptions & {
+    disableSelectWarning?: boolean;
+};
+declare function formatter(options?: PoGettextFormatterOptions): {
+    catalogExtension: string;
+    templateExtension: string;
+    parse(content: string): CatalogType;
+    serialize(catalog: CatalogType, ctx: {
+        locale: string;
+        existing: string;
+    }): string;
+};
+
+export { PoGettextFormatterOptions, formatter };

package/dist/po-gettext.mjs

@@ -0,0 +1,174 @@
+import { parse } from '@messageformat/parser';
+import pluralsCldr from 'plurals-cldr';
+import PO from 'pofile';
+import gettextPlurals from 'node-gettext/lib/plurals';
+import { generateMessageId } from '@lingui/cli/api';
+import { formatter as formatter$1 } from '@lingui/format-po';
+
+function stringifyICUCase(icuCase) {
+  return icuCase.tokens.map((token) => {
+    if (token.type === "content") {
+      return token.value;
+    } else if (token.type === "octothorpe") {
+      return "#";
+    } else if (token.type === "argument") {
+      return "{" + token.arg + "}";
+    } else {
+      console.warn(
+        `Unexpected token "${token}" while stringifying plural case "${icuCase}". Token will be ignored.`
+      );
+      return "";
+    }
+  }).join("");
+}
+const ICU_PLURAL_REGEX = /^{.*, plural, .*}$/;
+const ICU_SELECT_REGEX = /^{.*, select(Ordinal)?, .*}$/;
+const LINE_ENDINGS = /\r?\n/g;
+const CTX_PREFIX = "js-lingui:";
+function serializePlurals(item, message, id, isGeneratedId, options) {
+  const icuMessage = message.message;
+  if (!icuMessage) {
+    return;
+  }
+  const _simplifiedMessage = icuMessage.replace(LINE_ENDINGS, " ");
+  if (ICU_PLURAL_REGEX.test(_simplifiedMessage)) {
+    try {
+      const messageAst = parse(icuMessage)[0];
+      if (messageAst.cases.some(
+        (icuCase) => icuCase.tokens.some((token) => token.type === "plural")
+      )) {
+        console.warn(
+          `Nested plurals cannot be expressed with gettext plurals. Message with key "%s" will not be saved correctly.`,
+          id
+        );
+      }
+      const ctx = new URLSearchParams({
+        pluralize_on: messageAst.arg
+      });
+      if (isGeneratedId) {
+        item.msgid = stringifyICUCase(messageAst.cases[0]);
+        item.msgid_plural = stringifyICUCase(
+          messageAst.cases[messageAst.cases.length - 1]
+        );
+        ctx.set("icu", icuMessage);
+      } else {
+        item.msgid_plural = id + "_plural";
+      }
+      ctx.sort();
+      item.extractedComments.push(CTX_PREFIX + ctx.toString());
+      if (message.translation?.length > 0) {
+        const ast = parse(message.translation)[0];
+        if (ast.cases == null) {
+          console.warn(
+            `Found translation without plural cases for key "${id}". This likely means that a translated .po file misses multiple msgstr[] entries for the key. Translation found: "${message.translation}"`
+          );
+          item.msgstr = [message.translation];
+        } else {
+          item.msgstr = ast.cases.map(stringifyICUCase);
+        }
+      }
+    } catch (e) {
+      console.error(`Error parsing message ICU for key "${id}":`, e);
+    }
+  } else {
+    if (!options.disableSelectWarning && ICU_SELECT_REGEX.test(_simplifiedMessage)) {
+      console.warn(
+        `ICU 'select' and 'selectOrdinal' formats cannot be expressed natively in gettext format. Item with key "%s" will be included in the catalog as raw ICU message. To disable this warning, include '{ disableSelectWarning: true }' in the config's 'formatOptions'`,
+        id
+      );
+    }
+    item.msgstr = [message.translation];
+  }
+  return item;
+}
+const getPluralCases = (lang) => {
+  const [correctLang] = lang.split(/[-_]/g);
+  const gettextPluralsInfo = gettextPlurals[correctLang];
+  return gettextPluralsInfo?.examples.map(
+    (pluralCase) => pluralsCldr(correctLang, pluralCase.sample)
+  );
+};
+const convertPluralsToICU = (item, pluralForms, lang) => {
+  const translationCount = item.msgstr.length;
+  const messageKey = item.msgid;
+  if (translationCount <= 1 && !item.msgid_plural) {
+    return;
+  }
+  if (!item.msgid_plural) {
+    console.warn(
+      `Multiple translations for item with key "%s" but missing 'msgid_plural' in catalog "${lang}". This is not supported and the plural cases will be ignored.`,
+      messageKey
+    );
+    return;
+  }
+  const contextComment = item.extractedComments.find((comment) => comment.startsWith(CTX_PREFIX))?.substr(CTX_PREFIX.length);
+  const ctx = new URLSearchParams(contextComment);
+  if (contextComment != null) {
+    item.extractedComments = item.extractedComments.filter(
+      (comment) => !comment.startsWith(CTX_PREFIX)
+    );
+  }
+  const storedICU = ctx.get("icu");
+  if (storedICU != null) {
+    item.msgid = storedICU;
+  }
+  if (item.msgstr.every((str) => str.length === 0)) {
+    return;
+  }
+  if (pluralForms == null) {
+    console.warn(
+      `Multiple translations for item with key "%s" in language "${lang}", but no plural cases were found. This prohibits the translation of .po plurals into ICU plurals. Pluralization will not work for this key.`,
+      messageKey
+    );
+    return;
+  }
+  const pluralCount = pluralForms.length;
+  if (translationCount > pluralCount) {
+    console.warn(
+      `More translations provided (${translationCount}) for item with key "%s" in language "${lang}" than there are plural cases available (${pluralCount}). This will result in not all translations getting picked up.`,
+      messageKey
+    );
+  }
+  const pluralClauses = item.msgstr.map((str, index) => pluralForms[index] + " {" + str + "}").join(" ");
+  let pluralizeOn = ctx.get("pluralize_on");
+  if (!pluralizeOn) {
+    console.warn(
+      `Unable to determine plural placeholder name for item with key "%s" in language "${lang}" (should be stored in a comment starting with "#. ${CTX_PREFIX}"), assuming "count".`,
+      messageKey
+    );
+    pluralizeOn = "count";
+  }
+  item.msgstr = ["{" + pluralizeOn + ", plural, " + pluralClauses + "}"];
+};
+function formatter(options = {}) {
+  options = {
+    origins: true,
+    lineNumbers: true,
+    ...options
+  };
+  const formatter2 = formatter$1(options);
+  return {
+    catalogExtension: ".po",
+    templateExtension: ".pot",
+    parse(content) {
+      const po = PO.parse(content);
+      let pluralForms = getPluralCases(po.headers.Language);
+      po.items.forEach((item) => {
+        convertPluralsToICU(item, pluralForms, po.headers.Language);
+      });
+      return formatter2.parse(po.toString());
+    },
+    serialize(catalog, ctx) {
+      const po = PO.parse(formatter2.serialize(catalog, ctx));
+      po.items = po.items.map((item) => {
+        const isGeneratedId = !item.flags["explicit-id"];
+        const id = isGeneratedId ? generateMessageId(item.msgid, item.msgctxt) : item.msgid;
+        const message = catalog[id];
+        return serializePlurals(item, message, id, isGeneratedId, options);
+      });
+      return po.toString();
+    }
+  };
+}
+
+export { formatter };

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@lingui/format-po-gettext",
+  "version": "4.0.0-next.2",
+  "description": "Gettext PO format with gettext-style plurals for Lingui Catalogs",
+  "main": "./dist/po-gettext.cjs",
+  "module": "./dist/po-gettext.mjs",
+  "types": "./dist/po-gettext.d.ts",
+  "license": "MIT",
+  "keywords": [
+    "i18n",
+    "lingui-formatter",
+    "lingui-format",
+    "gettext",
+    "po",
+    "pot",
+    "internationalization",
+    "i10n",
+    "localization",
+    "i9n",
+    "translation"
+  ],
+  "scripts": {
+    "build": "rimraf ./dist && unbuild",
+    "stub": "unbuild --stub"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/lingui/js-lingui.git"
+  },
+  "bugs": {
+    "url": "https://github.com/lingui/js-lingui/issues"
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "files": [
+    "LICENSE",
+    "README.md",
+    "dist/"
+  ],
+  "dependencies": {
+    "@lingui/cli": "4.0.0-next.2",
+    "@lingui/conf": "4.0.0-next.2",
+    "@lingui/format-po": "4.0.0-next.2",
+    "@messageformat/parser": "^5.0.0",
+    "node-gettext": "^3.0.0",
+    "plurals-cldr": "^2.0.1",
+    "pofile": "^1.1.4"
+  },
+  "devDependencies": {
+    "@lingui/jest-mocks": "workspace:^",
+    "unbuild": "^1.1.2"
+  }
+}