@transifex/cli 3.2.2 → 4.1.0

package/README.md

@@ -113,6 +113,7 @@ OPTIONS
   --dry-run                              dry run, do not push to Transifex
   --key-generator=source|hash            [default: source] use hashed or source based keys
   --no-wait                              disable polling for upload results
+  --parser=auto|i18next                  [default: auto] file parser to use
   --purge                                purge content on Transifex
   --secret=secret                        native project secret
   --token=token                          native project public token
@@ -145,6 +146,7 @@ DESCRIPTION
   txjs-cli push --with-tags-only="home,error"
   txjs-cli push --without-tags-only="custom"
   txjs-cli push --token=mytoken --secret=mysecret
+  txjs-cli push en.json --parser=i18next
   TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push
 ```
 

package/oclif.manifest.json

@@ -1 +1 @@
-{"version":"3.2.2","commands":{"invalidate":{"id":"invalidate","description":"invalidate and refresh CDS cache\nContent for delivery is cached in CDS and refreshed automatically every hour.\nThis command triggers a refresh of cached content on the fly.\n\nBy default, invalidation does not remove existing cached content, but\nstarts the process of updating with latest translations from Transifex.\n\nPassing the --purge option, cached content will be forced to be deleted,\nhowever use that with caution, as it may introduce downtime of\ntranslation delivery to the apps until fresh content is cached in the CDS.\n\nTo invalidate translations some environment variables must be set:\nTRANSIFEX_TOKEN=<Transifex Native Project Token>\nTRANSIFEX_SECRET=<Transifex Native Project Secret>\n(optional) TRANSIFEX_CDS_HOST=<CDS HOST>\n\nor passed as --token=<TOKEN> --secret=<SECRET> parameters\n\nDefault CDS Host is https://cds.svc.transifex.net\n\nExamples:\ntxjs-cli invalidate\ntxjs-cli invalidate --purge\ntxjs-cli invalidate --token=mytoken --secret=mysecret\nTRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli invalidate\n","pluginName":"@transifex/cli","pluginType":"core","aliases":[],"flags":{"purge":{"name":"purge","type":"boolean","description":"force delete CDS cached content","allowNo":false},"token":{"name":"token","type":"option","description":"native project public token","default":""},"secret":{"name":"secret","type":"option","description":"native project secret","default":""},"cds-host":{"name":"cds-host","type":"option","description":"CDS host URL","default":""}},"args":[]},"push":{"id":"push","description":"detect and push source content to Transifex\nParse .js, .ts, .jsx, .tsx and .html files and detect phrases marked for\ntranslation by Transifex Native toolkit for Javascript and\nupload them to Transifex for translation.\n\nTo push content some environment variables must be set:\nTRANSIFEX_TOKEN=<Transifex Native Project Token>\nTRANSIFEX_SECRET=<Transifex Native Project Secret>\n(optional) TRANSIFEX_CDS_HOST=<CDS HOST>\n\nor passed as --token=<TOKEN> --secret=<SECRET> parameters\n\nDefault CDS Host is https://cds.svc.transifex.net\n\nExamples:\ntxjs-cli push -v\ntxjs-cli push src/\ntxjs-cli push /home/repo/src\ntxjs-cli push \"*.js\"\ntxjs-cli push --dry-run\ntxjs-cli push --no-wait\ntxjs-cli push --key-generator=hash\ntxjs-cli push --append-tags=\"master,release:2.5\"\ntxjs-cli push --with-tags-only=\"home,error\"\ntxjs-cli push --without-tags-only=\"custom\"\ntxjs-cli push --token=mytoken --secret=mysecret\nTRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push\n","pluginName":"@transifex/cli","pluginType":"core","aliases":[],"flags":{"dry-run":{"name":"dry-run","type":"boolean","description":"dry run, do not push to Transifex","allowNo":false},"verbose":{"name":"verbose","type":"boolean","char":"v","description":"verbose output","allowNo":false},"purge":{"name":"purge","type":"boolean","description":"purge content on Transifex","allowNo":false},"no-wait":{"name":"no-wait","type":"boolean","description":"disable polling for upload results","allowNo":false},"token":{"name":"token","type":"option","description":"native project public token","default":""},"secret":{"name":"secret","type":"option","description":"native project secret","default":""},"append-tags":{"name":"append-tags","type":"option","description":"append tags to strings","default":""},"with-tags-only":{"name":"with-tags-only","type":"option","description":"push strings with specific tags","default":""},"without-tags-only":{"name":"without-tags-only","type":"option","description":"push strings without specific tags","default":""},"cds-host":{"name":"cds-host","type":"option","description":"CDS host URL","default":""},"key-generator":{"name":"key-generator","type":"option","description":"use hashed or source based keys","options":["source","hash"],"default":"source"}},"args":[{"name":"pattern","description":"file pattern to scan for strings","required":false,"default":"**/*.{js,jsx,ts,tsx,html,vue,pug,ejs}"}]}}}
+{"version":"4.1.0","commands":{"invalidate":{"id":"invalidate","description":"invalidate and refresh CDS cache\nContent for delivery is cached in CDS and refreshed automatically every hour.\nThis command triggers a refresh of cached content on the fly.\n\nBy default, invalidation does not remove existing cached content, but\nstarts the process of updating with latest translations from Transifex.\n\nPassing the --purge option, cached content will be forced to be deleted,\nhowever use that with caution, as it may introduce downtime of\ntranslation delivery to the apps until fresh content is cached in the CDS.\n\nTo invalidate translations some environment variables must be set:\nTRANSIFEX_TOKEN=<Transifex Native Project Token>\nTRANSIFEX_SECRET=<Transifex Native Project Secret>\n(optional) TRANSIFEX_CDS_HOST=<CDS HOST>\n\nor passed as --token=<TOKEN> --secret=<SECRET> parameters\n\nDefault CDS Host is https://cds.svc.transifex.net\n\nExamples:\ntxjs-cli invalidate\ntxjs-cli invalidate --purge\ntxjs-cli invalidate --token=mytoken --secret=mysecret\nTRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli invalidate\n","pluginName":"@transifex/cli","pluginType":"core","aliases":[],"flags":{"purge":{"name":"purge","type":"boolean","description":"force delete CDS cached content","allowNo":false},"token":{"name":"token","type":"option","description":"native project public token","default":""},"secret":{"name":"secret","type":"option","description":"native project secret","default":""},"cds-host":{"name":"cds-host","type":"option","description":"CDS host URL","default":""}},"args":[]},"push":{"id":"push","description":"detect and push source content to Transifex\nParse .js, .ts, .jsx, .tsx and .html files and detect phrases marked for\ntranslation by Transifex Native toolkit for Javascript and\nupload them to Transifex for translation.\n\nTo push content some environment variables must be set:\nTRANSIFEX_TOKEN=<Transifex Native Project Token>\nTRANSIFEX_SECRET=<Transifex Native Project Secret>\n(optional) TRANSIFEX_CDS_HOST=<CDS HOST>\n\nor passed as --token=<TOKEN> --secret=<SECRET> parameters\n\nDefault CDS Host is https://cds.svc.transifex.net\n\nExamples:\ntxjs-cli push -v\ntxjs-cli push src/\ntxjs-cli push /home/repo/src\ntxjs-cli push \"*.js\"\ntxjs-cli push --dry-run\ntxjs-cli push --no-wait\ntxjs-cli push --key-generator=hash\ntxjs-cli push --append-tags=\"master,release:2.5\"\ntxjs-cli push --with-tags-only=\"home,error\"\ntxjs-cli push --without-tags-only=\"custom\"\ntxjs-cli push --token=mytoken --secret=mysecret\ntxjs-cli push en.json --parser=i18next\nTRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push\n","pluginName":"@transifex/cli","pluginType":"core","aliases":[],"flags":{"dry-run":{"name":"dry-run","type":"boolean","description":"dry run, do not push to Transifex","allowNo":false},"verbose":{"name":"verbose","type":"boolean","char":"v","description":"verbose output","allowNo":false},"purge":{"name":"purge","type":"boolean","description":"purge content on Transifex","allowNo":false},"no-wait":{"name":"no-wait","type":"boolean","description":"disable polling for upload results","allowNo":false},"token":{"name":"token","type":"option","description":"native project public token","default":""},"secret":{"name":"secret","type":"option","description":"native project secret","default":""},"append-tags":{"name":"append-tags","type":"option","description":"append tags to strings","default":""},"with-tags-only":{"name":"with-tags-only","type":"option","description":"push strings with specific tags","default":""},"without-tags-only":{"name":"without-tags-only","type":"option","description":"push strings without specific tags","default":""},"cds-host":{"name":"cds-host","type":"option","description":"CDS host URL","default":""},"parser":{"name":"parser","type":"option","description":"file parser to use","options":["auto","i18next"],"default":"auto"},"key-generator":{"name":"key-generator","type":"option","description":"use hashed or source based keys","options":["source","hash"],"default":"source"}},"args":[{"name":"pattern","description":"file pattern to scan for strings","required":false,"default":"**/*.{js,jsx,ts,tsx,html,vue,pug,ejs}"}]}}}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@transifex/cli",
   "description": "i18n CLI tool for Transifex Native",
-  "version": "3.2.2",
+  "version": "4.1.0",
   "author": "Transifex",
   "keywords": [
     "transifex",
@@ -24,19 +24,19 @@
     "access": "public"
   },
   "dependencies": {
-    "@babel/parser": "^7.17.8",
-    "@babel/traverse": "^7.13.0",
+    "@babel/parser": "^7.17.10",
+    "@babel/traverse": "^7.17.10",
     "@colors/colors": "^1.5.0",
     "@oclif/command": "^1.8.0",
     "@oclif/config": "^1.17.0",
     "@oclif/errors": "^1.3.4",
     "@oclif/plugin-help": "^5.1.12",
-    "@transifex/native": "^3.2.2",
+    "@transifex/native": "^4.1.0",
     "angular-html-parser": "^1.7.1",
-    "axios": "^0.26.1",
+    "axios": "^0.27.2",
     "cli-ux": "^6.0.9",
-    "ejs": "^3.1.6",
-    "glob": "^7.1.6",
+    "ejs": "^3.1.7",
+    "glob": "^8.0.1",
     "lodash": "^4.17.21",
     "pug": "^3.0.2",
     "shelljs": "^0.8.4",
@@ -48,14 +48,14 @@
     "chai": "^4.3.3",
     "eslint": "^7.21.0",
     "eslint-config-airbnb-base": "^14.2.1",
-    "eslint-plugin-import": "^2.22.1",
-    "globby": "^11.0.1",
-    "mocha": "^9.1.3",
+    "eslint-plugin-import": "^2.26.0",
+    "globby": "^11.1.0",
+    "mocha": "^10.0.0",
     "nock": "^13.0.11",
     "nyc": "^15.1.0"
   },
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=14.0.0"
   },
   "files": [
     "/bin",

package/src/api/extract.js

@@ -9,6 +9,7 @@ const pug = require('pug');
 const { parseHTMLTemplateFile } = require('./parsers/angularHTML');
 const { babelExtractPhrases } = require('./parsers/babel');
 const { extractVuePhrases } = require('./parsers/vue');
+const { extractI18NextPhrases } = require('./parsers/i18next');
 
 /**
  * Parse file and extract phrases using AST
@@ -20,29 +21,49 @@ const { extractVuePhrases } = require('./parsers/vue');
  * @param {String[]} options.filterWithTags
  * @param {String[]} options.filterWithoutTags
  * @param {Boolean} options.useHashedKeys
+ * @param {String} options.parser
  * @returns {Object}
  */
 function extractPhrases(file, relativeFile, options = {}) {
   const HASHES = {};
   let source = fs.readFileSync(file, 'utf8');
 
-  // Handle simple templates that compile to javascript
+  // i18next JSON
+  if (options.parser === 'i18next') {
+    extractI18NextPhrases(HASHES, source, relativeFile, options);
+    return HASHES;
+  }
+
+  // PUBG templates
   if (path.extname(file) === '.pug') {
     source = pug.compileClient(source);
-  } else if (path.extname(file) === '.ejs') {
+    babelExtractPhrases(HASHES, source, relativeFile, options);
+    return HASHES;
+  }
+
+  // EJS templates
+  if (path.extname(file) === '.ejs') {
     const template = new ejs.Template(source);
     template.generateSource();
     source = template.source;
+    babelExtractPhrases(HASHES, source, relativeFile, options);
+    return HASHES;
   }
 
+  // HTML templates
   if (path.extname(file) === '.html') {
     parseHTMLTemplateFile(HASHES, file, relativeFile, options);
-  } else if (path.extname(file) === '.vue') {
+    return HASHES;
+  }
+
+  // Vue templates
+  if (path.extname(file) === '.vue') {
     extractVuePhrases(HASHES, source, relativeFile, options);
-  } else if (path.extname(file) !== '.html') {
-    babelExtractPhrases(HASHES, source, relativeFile, options);
+    return HASHES;
   }
 
+  // default
+  babelExtractPhrases(HASHES, source, relativeFile, options);
   return HASHES;
 }
 

package/src/api/parsers/i18next.js

@@ -0,0 +1,99 @@
+const { implodePlurals } = require('@transifex/native');
+const _ = require('lodash');
+const { mergePayload } = require('../merge');
+const { createPayload, isPayloadValid } = require('./utils');
+
+const PLURAL_MAP = {
+  _1: 'one',
+  _2: 'two',
+  _3: 'few',
+  _4: 'many',
+  _5: 'other',
+  _one: 'one',
+  _two: 'two',
+  _few: 'few',
+  _many: 'many',
+  _other: 'other',
+};
+
+/**
+ * Parse i18next JSON v3 & v4 files
+ *
+ * @param {Object} HASHES A map of keys and content for phrases
+ * @param {String} source The content we want to parse
+ * @param {String} relativeFile occurence
+ * @param {Object} options
+ * @param {String[]} options.appendTags
+ * @param {String[]} options.filterWithTags
+ * @param {String[]} options.filterWithoutTags
+ * @param {Boolean} options.useHashedKeys
+ */
+function extractI18NextPhrases(HASHES, source, relativeFile, options) {
+  let json = {};
+  try {
+    json = JSON.parse(source);
+  } catch (err) {
+    return;
+  }
+
+  const plurals = {};
+
+  _.each(json, (string, key) => {
+    // nesting
+    if (!_.isString(string) && !_.isArray(string) && _.isObject(string)) {
+      _.each(string, (innerString, innerKey) => {
+        if (!_.isString(innerString)) return;
+        const partial = createPayload(innerString, {}, relativeFile, options);
+        if (!isPayloadValid(partial, options)) return;
+
+        mergePayload(HASHES, {
+          [`${key}.${innerKey}`]: {
+            string: partial.string,
+            meta: partial.meta,
+          },
+        });
+      });
+      return;
+    }
+
+    if (!_.isString(string)) return;
+
+    let isPlural = false;
+    _.each(_.keys(PLURAL_MAP), (suffix) => {
+      if (_.endsWith(key, suffix)) {
+        isPlural = true;
+        const pluralKey = key.slice(0, -suffix.length);
+        plurals[pluralKey] = plurals[pluralKey] || {};
+        plurals[pluralKey][PLURAL_MAP[suffix]] = string;
+      }
+    });
+    if (isPlural) return;
+
+    const partial = createPayload(string, {}, relativeFile, options);
+    if (!isPayloadValid(partial, options)) return;
+
+    mergePayload(HASHES, {
+      [key]: {
+        string: partial.string,
+        meta: partial.meta,
+      },
+    });
+  });
+
+  // add plurals
+  _.each(plurals, (plural, key) => {
+    const partial = createPayload(implodePlurals(plural, 'count'), {}, relativeFile, options);
+    if (!isPayloadValid(partial, options)) return;
+
+    mergePayload(HASHES, {
+      [`${key}_txplural`]: {
+        string: partial.string,
+        meta: partial.meta,
+      },
+    });
+  });
+}
+
+module.exports = {
+  extractI18NextPhrases,
+};

package/src/commands/push.js

@@ -73,6 +73,7 @@ class PushCommand extends Command {
       filterWithTags,
       filterWithoutTags,
       useHashedKeys,
+      parser: flags.parser,
     };
 
     _.each(allFiles, (file) => {
@@ -255,6 +256,7 @@ txjs-cli push --append-tags="master,release:2.5"
 txjs-cli push --with-tags-only="home,error"
 txjs-cli push --without-tags-only="custom"
 txjs-cli push --token=mytoken --secret=mysecret
+txjs-cli push en.json --parser=i18next
 TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push
 `;
 
@@ -307,6 +309,11 @@ PushCommand.flags = {
     description: 'CDS host URL',
     default: '',
   }),
+  parser: flags.string({
+    description: 'file parser to use',
+    default: 'auto',
+    options: ['auto', 'i18next'],
+  }),
   'key-generator': flags.string({
     description: 'use hashed or source based keys',
     default: 'source',