@transifex/cli 4.3.0 → 5.0.1

package/README.md

@@ -78,6 +78,7 @@ $ npm run push
 # Commands
 * [`txjs-cli help [COMMAND]`](#txjs-cli-help-command)
 * [`txjs-cli push [PATTERN]`](#txjs-cli-push-pattern)
+* [`txjs-cli pull [PATTERN]`](#txjs-cli-pull-pattern)
 * [`txjs-cli invalidate`](#txjs-cli-invalidate)
 
 ## `txjs-cli help [COMMAND]`
@@ -101,27 +102,32 @@ detect and push source content to Transifex
 
 ```
 USAGE
-  $ txjs-cli push [PATTERN]
+  $ txjs-cli push [PATTERN] [--dry-run] [--fake] [-v] [--purge] [--no-wait] [--token <value>] [--secret <value>] [--append-tags <value>] [--with-tags-only <value>] [--without-tags-only <value>]
+    [--cds-host <value>] [--parser auto|i18next|txnativejson] [--key-generator source|hash]
 
 ARGUMENTS
-  PATTERN  [default: **/*.{js,jsx,ts,tsx,vue}] file pattern to scan for strings
-
-OPTIONS
-  -v, --verbose                          verbose output
-  --append-tags=append-tags              append tags to strings
-  --cds-host=cds-host                    CDS host URL
-  --dry-run                              dry run, do not push to Transifex
-  --key-generator=source|hash            [default: source] use hashed or source based keys
-  --no-wait                              disable polling for upload results
-  --parser=auto|i18next                  [default: auto] file parser to use
-  --purge                                purge content on Transifex
-  --secret=secret                        native project secret
-  --token=token                          native project public token
-  --with-tags-only=with-tags-only        push strings with specific tags
-  --without-tags-only=without-tags-only  push strings without specific tags
+  PATTERN  [default: **/*.{js,jsx,ts,tsx,html,vue,pug,ejs}] file pattern to scan for strings
+
+FLAGS
+  -v, --verbose                verbose output
+  --append-tags=<value>        append tags to strings
+  --cds-host=<value>           CDS host URL
+  --dry-run                    dry run, do not apply changes in Transifex
+  --fake                       do not push content to remote server
+  --key-generator=<option>     [default: source] use hashed or source based keys
+                               <options: source|hash>
+  --no-wait                    disable polling for upload results
+  --parser=<option>            [default: auto] file parser to use
+                               <options: auto|i18next|txnativejson>
+  --purge                      purge content on Transifex
+  --secret=<value>             native project secret
+  --token=<value>              native project public token
+  --with-tags-only=<value>     push strings with specific tags
+  --without-tags-only=<value>  push strings without specific tags
 
 DESCRIPTION
-  Parse .js, .ts, .jsx, .tsx, .html and .vue files and detect phrases marked for
+  Detect and push source content to Transifex
+  Parse .js, .ts, .jsx, .tsx and .html files and detect phrases marked for
   translation by Transifex Native toolkit for Javascript and
   upload them to Transifex for translation.
 
@@ -140,6 +146,7 @@ DESCRIPTION
   txjs-cli push /home/repo/src
   txjs-cli push "*.js"
   txjs-cli push --dry-run
+  txjs-cli push --fake -v
   txjs-cli push --no-wait
   txjs-cli push --key-generator=hash
   txjs-cli push --append-tags="master,release:2.5"
@@ -147,29 +154,79 @@ DESCRIPTION
   txjs-cli push --without-tags-only="custom"
   txjs-cli push --token=mytoken --secret=mysecret
   txjs-cli push en.json --parser=i18next
+  txjs-cli push en.json --parser=txnativejson
   TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push
 ```
 
+## `txjs-cli pull [PATTERN]`
+
+Pull content from Transifex for offline caching
+
+```
+USAGE
+  $ txjs-cli pull [--token <value>] [--secret <value>] [-f <value>] [-l <value>] [--pretty] [--filter-tags <value>] [--filter-status reviewed|proofread|finalized] [--cds-host <value>]
+
+FLAGS
+  -f, --folder=<value>      output as files to folder
+  -l, --locale=<value>      pull specific language locale code
+  --cds-host=<value>        CDS host URL
+  --filter-status=<option>  filter over translation status
+                            <options: reviewed|proofread|finalized>
+  --filter-tags=<value>     filter over specific tags
+  --pretty                  beautify JSON output
+  --secret=<value>          native project secret
+  --token=<value>           native project public token
+
+DESCRIPTION
+  Pull content from Transifex for offline caching
+  Get content as JSON files, to be used by mobile Javascript SDKs for
+  offline support or warming up the cache with initial translations.
+
+  By default, JSON files are printed in the console,
+  unless the "-f foldername" parameter is provided. In that case
+  the JSON files will be downloaded to that folder with the <locale>.json format.
+
+  To pull content some environment variables must be set:
+  TRANSIFEX_TOKEN=<Transifex Native Project Token>
+  TRANSIFEX_SECRET=<Transifex Native Project Secret>
+  (optional) TRANSIFEX_CDS_HOST=<CDS HOST>
+
+  or passed as --token=<TOKEN> --secret=<SECRET> parameters
+
+  Default CDS Host is https://cds.svc.transifex.net
+
+  Examples:
+  txjs-cli pull
+  txjs-cli pull --pretty
+  txjs-cli pull -f languages/
+  txjs-cli pull --locale=fr -f .
+  txjs-cli pull --filter-tags="foo,bar"
+  txjs-cli pull --filter-status="reviewed"
+  txjs-cli pull --token=mytoken --secret=mysecret
+  TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli pull
+```
+
 ## `txjs-cli invalidate`
 
 invalidate and refresh CDS cache
 
 ```
 USAGE
-  $ txjs-cli invalidate
+  $ txjs-cli invalidate [--purge] [--token <value>] [--secret <value>] [--cds-host <value>]
 
-OPTIONS
-  --cds-host=cds-host  CDS host URL
-  --purge              force delete CDS cached content
-  --secret=secret      native project secret
-  --token=token        native project public token
+FLAGS
+  --cds-host=<value>  CDS host URL
+  --purge             force delete CDS cached content
+  --secret=<value>    native project secret
+  --token=<value>     native project public token
 
 DESCRIPTION
+  Invalidate and refresh CDS cache
   Content for delivery is cached in CDS and refreshed automatically every hour.
   This command triggers a refresh of cached content on the fly.
 
-  By default, invalidation does not remove existing cached content, but
-  starts the process of updating with latest translations from Transifex.
+  By default, invalidation does not remove existing cached content,
+  but starts the process of updating with latest translations from Transifex.
 
   Passing the --purge option, cached content will be forced to be deleted,
   however use that with caution, as it may introduce downtime of

package/bin/run

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
 
-require('@oclif/command').run()
-  .then(require('@oclif/command/flush'))
-  .catch(require('@oclif/errors/handle'));
+const oclif = require('@oclif/core');
+
+oclif.run().then(require('@oclif/core/flush')).catch(require('@oclif/core/handle'));

package/oclif.manifest.json

@@ -1 +1 @@
-{"version":"4.3.0","commands":{"invalidate":{"id":"invalidate","description":"invalidate and refresh CDS cache\nContent for delivery is cached in CDS and refreshed automatically every hour.\nThis command triggers a refresh of cached content on the fly.\n\nBy default, invalidation does not remove existing cached content, but\nstarts the process of updating with latest translations from Transifex.\n\nPassing the --purge option, cached content will be forced to be deleted,\nhowever use that with caution, as it may introduce downtime of\ntranslation delivery to the apps until fresh content is cached in the CDS.\n\nTo invalidate translations some environment variables must be set:\nTRANSIFEX_TOKEN=<Transifex Native Project Token>\nTRANSIFEX_SECRET=<Transifex Native Project Secret>\n(optional) TRANSIFEX_CDS_HOST=<CDS HOST>\n\nor passed as --token=<TOKEN> --secret=<SECRET> parameters\n\nDefault CDS Host is https://cds.svc.transifex.net\n\nExamples:\ntxjs-cli invalidate\ntxjs-cli invalidate --purge\ntxjs-cli invalidate --token=mytoken --secret=mysecret\nTRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli invalidate\n","pluginName":"@transifex/cli","pluginType":"core","aliases":[],"flags":{"purge":{"name":"purge","type":"boolean","description":"force delete CDS cached content","allowNo":false},"token":{"name":"token","type":"option","description":"native project public token","default":""},"secret":{"name":"secret","type":"option","description":"native project secret","default":""},"cds-host":{"name":"cds-host","type":"option","description":"CDS host URL","default":""}},"args":[]},"push":{"id":"push","description":"detect and push source content to Transifex\nParse .js, .ts, .jsx, .tsx and .html files and detect phrases marked for\ntranslation by Transifex Native toolkit for Javascript and\nupload them to Transifex for translation.\n\nTo push content some environment variables must be set:\nTRANSIFEX_TOKEN=<Transifex Native Project Token>\nTRANSIFEX_SECRET=<Transifex Native Project Secret>\n(optional) TRANSIFEX_CDS_HOST=<CDS HOST>\n\nor passed as --token=<TOKEN> --secret=<SECRET> parameters\n\nDefault CDS Host is https://cds.svc.transifex.net\n\nExamples:\ntxjs-cli push -v\ntxjs-cli push src/\ntxjs-cli push /home/repo/src\ntxjs-cli push \"*.js\"\ntxjs-cli push --dry-run\ntxjs-cli push --no-wait\ntxjs-cli push --key-generator=hash\ntxjs-cli push --append-tags=\"master,release:2.5\"\ntxjs-cli push --with-tags-only=\"home,error\"\ntxjs-cli push --without-tags-only=\"custom\"\ntxjs-cli push --token=mytoken --secret=mysecret\ntxjs-cli push en.json --parser=i18next\nTRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push\n","pluginName":"@transifex/cli","pluginType":"core","aliases":[],"flags":{"dry-run":{"name":"dry-run","type":"boolean","description":"dry run, do not push to Transifex","allowNo":false},"verbose":{"name":"verbose","type":"boolean","char":"v","description":"verbose output","allowNo":false},"purge":{"name":"purge","type":"boolean","description":"purge content on Transifex","allowNo":false},"no-wait":{"name":"no-wait","type":"boolean","description":"disable polling for upload results","allowNo":false},"token":{"name":"token","type":"option","description":"native project public token","default":""},"secret":{"name":"secret","type":"option","description":"native project secret","default":""},"append-tags":{"name":"append-tags","type":"option","description":"append tags to strings","default":""},"with-tags-only":{"name":"with-tags-only","type":"option","description":"push strings with specific tags","default":""},"without-tags-only":{"name":"without-tags-only","type":"option","description":"push strings without specific tags","default":""},"cds-host":{"name":"cds-host","type":"option","description":"CDS host URL","default":""},"parser":{"name":"parser","type":"option","description":"file parser to use","options":["auto","i18next"],"default":"auto"},"key-generator":{"name":"key-generator","type":"option","description":"use hashed or source based keys","options":["source","hash"],"default":"source"}},"args":[{"name":"pattern","description":"file pattern to scan for strings","required":false,"default":"**/*.{js,jsx,ts,tsx,html,vue,pug,ejs}"}]}}}
+{"version":"5.0.1","commands":{"invalidate":{"id":"invalidate","description":"Invalidate and refresh CDS cache\nContent for delivery is cached in CDS and refreshed automatically every hour.\nThis command triggers a refresh of cached content on the fly.\n\nBy default, invalidation does not remove existing cached content,\nbut starts the process of updating with latest translations from Transifex.\n\nPassing the --purge option, cached content will be forced to be deleted,\nhowever use that with caution, as it may introduce downtime of\ntranslation delivery to the apps until fresh content is cached in the CDS.\n\nTo invalidate translations some environment variables must be set:\nTRANSIFEX_TOKEN=<Transifex Native Project Token>\nTRANSIFEX_SECRET=<Transifex Native Project Secret>\n(optional) TRANSIFEX_CDS_HOST=<CDS HOST>\n\nor passed as --token=<TOKEN> --secret=<SECRET> parameters\n\nDefault CDS Host is https://cds.svc.transifex.net\n\nExamples:\ntxjs-cli invalidate\ntxjs-cli invalidate --purge\ntxjs-cli invalidate --token=mytoken --secret=mysecret\nTRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli invalidate\n","pluginName":"@transifex/cli","pluginType":"core","aliases":[],"flags":{"purge":{"name":"purge","type":"boolean","description":"force delete CDS cached content","allowNo":false},"token":{"name":"token","type":"option","description":"native project public token","default":""},"secret":{"name":"secret","type":"option","description":"native project secret","default":""},"cds-host":{"name":"cds-host","type":"option","description":"CDS host URL","default":""}},"args":[]},"pull":{"id":"pull","description":"Pull content from Transifex for offline caching\nGet content as JSON files, to be used by mobile Javascript SDKs for\noffline support or warming up the cache with initial translations.\n\nBy default, JSON files are printed in the console,\nunless the \"-f foldername\" parameter is provided. In that case\nthe JSON files will be downloaded to that folder with the <locale>.json format.\n\nTo pull content some environment variables must be set:\nTRANSIFEX_TOKEN=<Transifex Native Project Token>\nTRANSIFEX_SECRET=<Transifex Native Project Secret>\n(optional) TRANSIFEX_CDS_HOST=<CDS HOST>\n\nor passed as --token=<TOKEN> --secret=<SECRET> parameters\n\nDefault CDS Host is https://cds.svc.transifex.net\n\nExamples:\ntxjs-cli pull\ntxjs-cli pull --pretty\ntxjs-cli pull -f languages/\ntxjs-cli pull --locale=fr -f .\ntxjs-cli pull --filter-tags=\"foo,bar\"\ntxjs-cli pull --filter-status=\"reviewed\"\ntxjs-cli pull --token=mytoken --secret=mysecret\nTRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli pull\n","pluginName":"@transifex/cli","pluginType":"core","aliases":[],"flags":{"token":{"name":"token","type":"option","description":"native project public token","default":""},"secret":{"name":"secret","type":"option","description":"native project secret","default":""},"folder":{"name":"folder","type":"option","char":"f","description":"output as files to folder","default":""},"locale":{"name":"locale","type":"option","char":"l","description":"pull specific language locale code","default":""},"pretty":{"name":"pretty","type":"boolean","description":"beautify JSON output","allowNo":false},"filter-tags":{"name":"filter-tags","type":"option","description":"filter over specific tags","default":""},"filter-status":{"name":"filter-status","type":"option","description":"filter over translation status","options":["reviewed","proofread","finalized"],"default":""},"cds-host":{"name":"cds-host","type":"option","description":"CDS host URL","default":""}},"args":[]},"push":{"id":"push","description":"Detect and push source content to Transifex\nParse .js, .ts, .jsx, .tsx and .html files and detect phrases marked for\ntranslation by Transifex Native toolkit for Javascript and\nupload them to Transifex for translation.\n\nTo push content some environment variables must be set:\nTRANSIFEX_TOKEN=<Transifex Native Project Token>\nTRANSIFEX_SECRET=<Transifex Native Project Secret>\n(optional) TRANSIFEX_CDS_HOST=<CDS HOST>\n\nor passed as --token=<TOKEN> --secret=<SECRET> parameters\n\nDefault CDS Host is https://cds.svc.transifex.net\n\nExamples:\ntxjs-cli push -v\ntxjs-cli push src/\ntxjs-cli push /home/repo/src\ntxjs-cli push \"*.js\"\ntxjs-cli push --dry-run\ntxjs-cli push --fake -v\ntxjs-cli push --no-wait\ntxjs-cli push --key-generator=hash\ntxjs-cli push --append-tags=\"master,release:2.5\"\ntxjs-cli push --with-tags-only=\"home,error\"\ntxjs-cli push --without-tags-only=\"custom\"\ntxjs-cli push --token=mytoken --secret=mysecret\ntxjs-cli push en.json --parser=i18next\ntxjs-cli push en.json --parser=txnativejson\nTRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push\n","pluginName":"@transifex/cli","pluginType":"core","aliases":[],"flags":{"dry-run":{"name":"dry-run","type":"boolean","description":"dry run, do not apply changes in Transifex","allowNo":false},"fake":{"name":"fake","type":"boolean","description":"do not push content to remote server","allowNo":false},"verbose":{"name":"verbose","type":"boolean","char":"v","description":"verbose output","allowNo":false},"purge":{"name":"purge","type":"boolean","description":"purge content on Transifex","allowNo":false},"no-wait":{"name":"no-wait","type":"boolean","description":"disable polling for upload results","allowNo":false},"token":{"name":"token","type":"option","description":"native project public token","default":""},"secret":{"name":"secret","type":"option","description":"native project secret","default":""},"append-tags":{"name":"append-tags","type":"option","description":"append tags to strings","default":""},"with-tags-only":{"name":"with-tags-only","type":"option","description":"push strings with specific tags","default":""},"without-tags-only":{"name":"without-tags-only","type":"option","description":"push strings without specific tags","default":""},"cds-host":{"name":"cds-host","type":"option","description":"CDS host URL","default":""},"parser":{"name":"parser","type":"option","description":"file parser to use","options":["auto","i18next","txnativejson"],"default":"auto"},"key-generator":{"name":"key-generator","type":"option","description":"use hashed or source based keys","options":["source","hash"],"default":"source"}},"args":[{"name":"pattern","description":"file pattern to scan for strings","required":false,"default":"**/*.{js,jsx,ts,tsx,html,vue,pug,ejs}"}]}}}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@transifex/cli",
   "description": "i18n CLI tool for Transifex Native",
-  "version": "4.3.0",
+  "version": "5.0.1",
   "author": "Transifex",
   "keywords": [
     "transifex",
@@ -24,34 +24,29 @@
     "access": "public"
   },
   "dependencies": {
-    "@babel/parser": "^7.18.9",
-    "@babel/traverse": "^7.18.9",
+    "@babel/parser": "^7.20.3",
+    "@babel/traverse": "^7.20.1",
     "@colors/colors": "^1.5.0",
-    "@oclif/command": "^1.8.16",
-    "@oclif/config": "^1.18.3",
-    "@oclif/errors": "^1.3.5",
-    "@oclif/plugin-help": "^5.1.12",
-    "@transifex/native": "^4.3.0",
+    "@oclif/core": "^1.20.4",
+    "@transifex/native": "^5.0.1",
+    "@vue/compiler-sfc": "^3.2.45",
     "angular-html-parser": "^1.8.0",
-    "axios": "^0.27.2",
-    "cli-ux": "^6.0.9",
+    "axios": "^1.1.3",
     "ejs": "^3.1.8",
     "glob": "^8.0.3",
     "lodash": "^4.17.21",
     "pug": "^3.0.2",
-    "shelljs": "^0.8.5",
-    "vue-template-compiler": "^2.7.7"
+    "shelljs": "^0.8.5"
   },
   "devDependencies": {
     "@oclif/dev-cli": "^1.26.10",
-    "@oclif/test": "^2.1.0",
-    "chai": "^4.3.6",
-    "eslint": "^7.32.0",
-    "eslint-config-airbnb-base": "^14.2.1",
+    "@oclif/test": "^2.2.12",
+    "chai": "^4.3.7",
+    "eslint": "^8.28.0",
+    "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-import": "^2.26.0",
-    "globby": "^11.1.0",
-    "mocha": "^10.0.0",
-    "nock": "^13.2.8",
+    "mocha": "^10.1.0",
+    "nock": "^13.2.9",
     "nyc": "^15.1.0"
   },
   "engines": {
@@ -66,6 +61,9 @@
   "main": "src/index.js",
   "oclif": {
     "commands": "./src/commands",
+    "additionalHelpFlags": [
+      "-h"
+    ],
     "bin": "txjs-cli",
     "plugins": [
       "@oclif/plugin-help"

package/src/api/download.js

@@ -0,0 +1,83 @@
+const axios = require('axios');
+const { version } = require('../../package.json');
+const { sleep } = require('./utils');
+
+/**
+ * Download languages
+ *
+ * @param {*} { cdsHost, token, secret }
+ * @return {Promise}
+ */
+async function downloadLanguages({ cdsHost, token, secret }) {
+  let response;
+  let lastResponseStatus = 202;
+  while (lastResponseStatus === 202) {
+    /* eslint-disable no-await-in-loop */
+    response = await axios.get(`${cdsHost}/languages`, {
+      headers: {
+        Authorization: `Bearer ${token}:${secret}`,
+        'Accept-version': 'v2',
+        'Content-Type': 'application/json;charset=utf-8',
+        'X-NATIVE-SDK': `txjs/cli/${version}`,
+      },
+    });
+    lastResponseStatus = response.status;
+    if (lastResponseStatus === 202) {
+      await sleep(1000);
+    }
+    /* eslint-enable no-await-in-loop */
+  }
+
+  return response.data;
+}
+
+/**
+ * Download phrases
+ *
+ * @param {*} {
+ *   cdsHost, locale, filterTags, filterStatus, token, secret,
+ * }
+ * @return {Promise}
+ */
+async function downloadPhrases({
+  cdsHost, locale, filterTags, filterStatus, token, secret,
+}) {
+  let response;
+  let lastResponseStatus = 202;
+  while (lastResponseStatus === 202) {
+    let url = `${cdsHost}/content/${locale}`;
+
+    const getOptions = [];
+    if (filterTags) {
+      getOptions.push(`filter[tags]=${filterTags}`);
+    }
+    if (filterStatus) {
+      getOptions.push(`filter[status]=${filterStatus}`);
+    }
+    if (getOptions.length) {
+      url = `${url}?${getOptions.join('&')}`;
+    }
+
+    /* eslint-disable no-await-in-loop */
+    response = await axios.get(url, {
+      headers: {
+        Authorization: `Bearer ${token}:${secret}`,
+        'Accept-version': 'v2',
+        'Content-Type': 'application/json;charset=utf-8',
+        'X-NATIVE-SDK': `txjs/cli/${version}`,
+      },
+    });
+    lastResponseStatus = response.status;
+    if (lastResponseStatus === 202) {
+      await sleep(1000);
+    }
+    /* eslint-enable no-await-in-loop */
+  }
+
+  return response.data;
+}
+
+module.exports = {
+  downloadPhrases,
+  downloadLanguages,
+};

package/src/api/extract.js

@@ -10,6 +10,7 @@ const { parseHTMLTemplateFile } = require('./parsers/angularHTML');
 const { babelExtractPhrases } = require('./parsers/babel');
 const { extractVuePhrases } = require('./parsers/vue');
 const { extractI18NextPhrases } = require('./parsers/i18next');
+const { extractTXNativeJSONPhrases } = require('./parsers/txnativejson');
 
 /**
  * Parse file and extract phrases using AST
@@ -34,6 +35,12 @@ function extractPhrases(file, relativeFile, options = {}) {
     return HASHES;
   }
 
+  // Transifex Native JSON
+  if (options.parser === 'txnativejson') {
+    extractTXNativeJSONPhrases(HASHES, source, relativeFile, options);
+    return HASHES;
+  }
+
   // PUBG templates
   if (path.extname(file) === '.pug') {
     source = pug.compileClient(source);

package/src/api/parsers/txnativejson.js

@@ -0,0 +1,41 @@
+const _ = require('lodash');
+const { mergePayload } = require('../merge');
+
+/**
+ * Parse i18next JSON v3 & v4 files
+ *
+ * @param {Object} HASHES A map of keys and content for phrases
+ * @param {String} source The content we want to parse
+ * @param {String} relativeFile occurence
+ * @param {Object} options
+ * @param {String[]} options.appendTags
+ * @param {String[]} options.filterWithTags
+ * @param {String[]} options.filterWithoutTags
+ * @param {Boolean} options.useHashedKeys
+ */
+function extractTXNativeJSONPhrases(HASHES, source, relativeFile, options) {
+  let json = {};
+  try {
+    json = JSON.parse(source);
+  } catch (err) {
+    return;
+  }
+  _.each(json, (value, key) => {
+    mergePayload(HASHES, {
+      [key]: value,
+    });
+    if (options.appendTags) {
+      mergePayload(HASHES, {
+        [key]: {
+          meta: {
+            tags: options.appendTags,
+          },
+        },
+      });
+    }
+  });
+}
+
+module.exports = {
+  extractTXNativeJSONPhrases,
+};

package/src/api/parsers/vue.js

@@ -1,11 +1,11 @@
 const _ = require('lodash');
-const vueTemplateCompiler = require('vue-template-compiler');
+const vueTemplateCompiler = require('@vue/compiler-sfc');
 const { createPayload, isPayloadValid } = require('./utils');
 const { mergePayload } = require('../merge');
 const { babelExtractPhrases } = require('./babel');
 
 /**
- * A function to traverse the AST provided by the vue-template-compiler package
+ * A function to traverse the AST provided by the vue/compiler-sfc package
  *
  * @param {*} ast
  * @param {*} visitor
@@ -14,7 +14,7 @@ function traverseVueTemplateAst(ast, visitor = {}) {
   // Use this in order to identify expressions in the AST since there is no
   // Tag to identify it with when traversing
   const VISITORS = {
-    2: 'Expression',
+    5: 'Expression',
   };
 
   function traverseArray(array, parent) {
@@ -28,10 +28,13 @@ function traverseVueTemplateAst(ast, visitor = {}) {
     if (visitor.enter) visitor.enter(node, parent);
     if (visitor[node.tag]) visitor[node.tag](node, parent);
     // Take care of expressions
-    if (visitor[VISITORS[node.type]] && node.expression) {
-      visitor[VISITORS[node.type]](node, parent);
+    if (visitor[VISITORS[node.type]] && node.content) {
+      if (node.content.loc.source) visitor[VISITORS[node.type]](node, parent);
     }
     if (node.children) traverseArray(node.children, node);
+    if (node.content) {
+      if (node.content.children) traverseArray(node.content.children, node);
+    }
     if (visitor.exit) visitor.exit(node, parent);
   }
   traverseNode(ast, null);
@@ -54,11 +57,11 @@ function vueElementVisitor(HASHES, relativeFile, options) {
   return (node) => {
     let string;
     const params = {};
-    _.each(node.attrsList, (attr) => {
+    _.each(node.props, (attr) => {
       const property = attr.name;
       if (!property || !attr.value) return;
 
-      const attrValue = attr.value;
+      const attrValue = attr.value.content;
 
       if (!attrValue) return;
 
@@ -102,25 +105,25 @@ function vueElementVisitor(HASHES, relativeFile, options) {
  */
 function extractVuePhrases(HASHES, source, relativeFile, options) {
   // Use the vue-template-compiler API to parse content
-  const vueContent = vueTemplateCompiler.parseComponent(source);
-
+  const vueContent = vueTemplateCompiler.parse(source);
   // Get the JS Content from the file and extract hashes/phrases with Babel
-  if (vueContent.script && vueContent.script.content) {
-    const script = vueContent.script.content;
+  if (vueContent.descriptor.script && vueContent.descriptor.script.content) {
+    const script = vueContent.descriptor.script.content;
     babelExtractPhrases(HASHES, script, relativeFile, options);
   }
 
   // Get the template content from the file and extract hashes/phrases with
   // custom traverse function
-  if (vueContent.template && vueContent.template.content) {
+  if (vueContent.descriptor.template && vueContent.descriptor.template.content) {
     // Compile to get the AST
-    const template = vueTemplateCompiler.compile(vueContent.template.content, {
-      preserveWhitespace: false,
+    const template = vueTemplateCompiler.compileTemplate({
+      id: 'sfc-compiler',
+      source: vueContent.descriptor.template.content,
     });
 
     traverseVueTemplateAst(template.ast, {
       Expression(node) {
-        babelExtractPhrases(HASHES, node.expression, relativeFile, options);
+        babelExtractPhrases(HASHES, node.content.loc.source, relativeFile, options);
       },
       T: vueElementVisitor(HASHES, relativeFile, options),
       UT: vueElementVisitor(HASHES, relativeFile, options),

package/src/api/upload.js

@@ -18,6 +18,7 @@ async function uploadPhrases(payload, params) {
     data: payload,
     meta: {
       purge: !!params.purge,
+      dry_run: !!params.dry_run,
     },
   }, {
     headers: {

package/src/api/utils.js

@@ -23,7 +23,22 @@ function mergeArrays(array1, array2) {
   return _.uniq(_.concat(array1 || [], array2 || []));
 }
 
+/**
+ * Async/await sleep
+ *
+ * @param {Number} msec
+ * @return {Promise}
+ */
+function sleep(msec) {
+  return new Promise((resolve) => {
+    setTimeout(() => {
+      resolve();
+    }, msec);
+  });
+}
+
 module.exports = {
   stringToArray,
   mergeArrays,
+  sleep,
 };

package/src/commands/invalidate.js

@@ -1,13 +1,13 @@
 /* eslint no-shadow: 0 */
 
 require('@colors/colors');
-const { Command, flags } = require('@oclif/command');
-const { cli } = require('cli-ux');
+const { Command, Flags } = require('@oclif/core');
+const { CliUx } = require('@oclif/core');
 const { invalidateCDS } = require('../api/invalidate');
 
 class InvalidateCommand extends Command {
   async run() {
-    const { flags } = this.parse(InvalidateCommand);
+    const { flags } = await this.parse(InvalidateCommand);
 
     let cdsHost = process.env.TRANSIFEX_CDS_HOST || 'https://cds.svc.transifex.net';
     let projectToken = process.env.TRANSIFEX_TOKEN;
@@ -24,9 +24,9 @@ class InvalidateCommand extends Command {
     }
 
     if (flags.purge) {
-      cli.action.start('Invalidating and purging CDS cache', '', { stdout: true });
+      CliUx.ux.action.start('Invalidating and purging CDS cache', '', { stdout: true });
     } else {
-      cli.action.start('Invalidating CDS cache', '', { stdout: true });
+      CliUx.ux.action.start('Invalidating CDS cache', '', { stdout: true });
     }
 
     try {
@@ -36,22 +36,22 @@ class InvalidateCommand extends Command {
         secret: projectSecret,
         purge: flags.purge,
       });
-      cli.action.stop('Success'.green);
+      CliUx.ux.action.stop('Success'.green);
       this.log(`${(res.data.count || 0).toString().green} records invalidated`);
       this.log('Note: It might take a few minutes for fresh content to be available'.yellow);
     } catch (err) {
-      cli.action.stop('Failed'.red);
+      CliUx.ux.action.stop('Failed'.red);
       this.error(err);
     }
   }
 }
 
-InvalidateCommand.description = `invalidate and refresh CDS cache
+InvalidateCommand.description = `Invalidate and refresh CDS cache
 Content for delivery is cached in CDS and refreshed automatically every hour.
 This command triggers a refresh of cached content on the fly.
 
-By default, invalidation does not remove existing cached content, but
-starts the process of updating with latest translations from Transifex.
+By default, invalidation does not remove existing cached content,
+but starts the process of updating with latest translations from Transifex.
 
 Passing the --purge option, cached content will be forced to be deleted,
 however use that with caution, as it may introduce downtime of
@@ -76,19 +76,19 @@ TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli invalidate
 InvalidateCommand.args = [];
 
 InvalidateCommand.flags = {
-  purge: flags.boolean({
+  purge: Flags.boolean({
     description: 'force delete CDS cached content',
     default: false,
   }),
-  token: flags.string({
+  token: Flags.string({
     description: 'native project public token',
     default: '',
   }),
-  secret: flags.string({
+  secret: Flags.string({
     description: 'native project secret',
     default: '',
   }),
-  'cds-host': flags.string({
+  'cds-host': Flags.string({
     description: 'CDS host URL',
     default: '',
   }),

package/src/commands/pull.js

@@ -0,0 +1,142 @@
+/* eslint-disable no-await-in-loop */
+require('@colors/colors');
+const _ = require('lodash');
+const fs = require('fs');
+const path = require('path');
+const { Command, Flags } = require('@oclif/core');
+const { CliUx } = require('@oclif/core');
+const { downloadLanguages, downloadPhrases } = require('../api/download');
+
+class PullCommand extends Command {
+  async run() {
+    const { flags } = await this.parse(PullCommand);
+
+    let cdsHost = process.env.TRANSIFEX_CDS_HOST || 'https://cds.svc.transifex.net';
+    let projectToken = process.env.TRANSIFEX_TOKEN;
+    let projectSecret = process.env.TRANSIFEX_SECRET;
+
+    if (flags.token) projectToken = flags.token;
+    if (flags.secret) projectSecret = flags.secret;
+    if (flags['cds-host']) cdsHost = flags['cds-host'];
+
+    if (!projectToken || !projectSecret) {
+      this.log(`${'✘'.red} Cannot pull content, credentials are missing.`);
+      this.log('Tip: Set TRANSIFEX_TOKEN and TRANSIFEX_SECRET environment variables'.yellow);
+      process.exit();
+    }
+
+    try {
+      // get source lang
+      const params = {
+        cdsHost,
+        token: projectToken,
+        secret: projectSecret,
+        filterTags: flags['filter-tags'],
+        filterStatus: flags['filter-status'],
+        locale: '',
+      };
+      const locales = [];
+      if (flags.locale) {
+        locales.push(flags.locale);
+      } else {
+        CliUx.ux.action.start('Getting available languages', '', { stdout: true });
+        const languages = await downloadLanguages(params);
+        _.each(languages.data, (entry) => {
+          if (entry.code) {
+            locales.push(entry.code);
+          }
+        });
+      }
+      for (let i = 0; i < locales.length; i += 1) {
+        params.locale = locales[i];
+        CliUx.ux.action.start(
+          `Pulling content for ${params.locale.green} locale (${i + 1} of ${locales.length})`,
+          '',
+          { stdout: true },
+        );
+        const { data } = await downloadPhrases(params);
+        const json = flags.pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
+        if (flags.folder) {
+          fs.writeFileSync(path.join(flags.folder, `${params.locale}.json`), json);
+        }
+        if (!flags.folder) {
+          this.log(json);
+        }
+      }
+      CliUx.ux.action.stop('Done'.green);
+    } catch (err) {
+      CliUx.ux.action.stop('Failed'.red);
+      this.error(err);
+    }
+  }
+}
+
+PullCommand.description = `Pull content from Transifex for offline caching
+Get content as JSON files, to be used by mobile Javascript SDKs for
+offline support or warming up the cache with initial translations.
+
+By default, JSON files are printed in the console,
+unless the "-f foldername" parameter is provided. In that case
+the JSON files will be downloaded to that folder with the <locale>.json format.
+
+To pull content some environment variables must be set:
+TRANSIFEX_TOKEN=<Transifex Native Project Token>
+TRANSIFEX_SECRET=<Transifex Native Project Secret>
+(optional) TRANSIFEX_CDS_HOST=<CDS HOST>
+
+or passed as --token=<TOKEN> --secret=<SECRET> parameters
+
+Default CDS Host is https://cds.svc.transifex.net
+
+Examples:
+txjs-cli pull
+txjs-cli pull --pretty
+txjs-cli pull -f languages/
+txjs-cli pull --locale=fr -f .
+txjs-cli pull --filter-tags="foo,bar"
+txjs-cli pull --filter-status="reviewed"
+txjs-cli pull --token=mytoken --secret=mysecret
+TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli pull
+`;
+
+PullCommand.args = [];
+
+PullCommand.flags = {
+  token: Flags.string({
+    description: 'native project public token',
+    default: '',
+  }),
+  secret: Flags.string({
+    description: 'native project secret',
+    default: '',
+  }),
+  folder: Flags.string({
+    char: 'f',
+    description: 'output as files to folder',
+    default: '',
+  }),
+  locale: Flags.string({
+    char: 'l',
+    description: 'pull specific language locale code',
+    default: '',
+  }),
+  pretty: Flags.boolean({
+    description: 'beautify JSON output',
+    default: false,
+  }),
+  'filter-tags': Flags.string({
+    description: 'filter over specific tags',
+    default: '',
+  }),
+  'filter-status': Flags.string({
+    description: 'filter over translation status',
+    default: '',
+    options: ['reviewed', 'proofread', 'finalized'],
+  }),
+  'cds-host': Flags.string({
+    description: 'CDS host URL',
+    default: '',
+  }),
+};
+
+module.exports = PullCommand;

package/src/commands/push.js

@@ -5,10 +5,10 @@ require('@colors/colors');
 const fs = require('fs');
 const path = require('path');
 const _ = require('lodash');
-const { Command, flags } = require('@oclif/command');
+const { Command, Flags } = require('@oclif/core');
 const shelljs = require('shelljs');
 const { glob } = require('glob');
-const { cli } = require('cli-ux');
+const { CliUx } = require('@oclif/core');
 const { extractPhrases } = require('../api/extract');
 const { uploadPhrases, pollJob } = require('../api/upload');
 const { mergePayload } = require('../api/merge');
@@ -31,7 +31,7 @@ function isFolder(path) {
 
 class PushCommand extends Command {
   async run() {
-    const { args, flags } = this.parse(PushCommand);
+    const { args, flags } = await this.parse(PushCommand);
     const pwd = `${shelljs.pwd()}`;
     let filePattern = path.isAbsolute(args.pattern)
       ? args.pattern
@@ -61,7 +61,7 @@ class PushCommand extends Command {
     const payload = {};
     const tree = {};
 
-    const bar = cli.progress({
+    const bar = CliUx.ux.progress({
       format: '⸨{bar}⸩ {value}/{total} {file}',
       barCompleteChar: '\u2588',
       barIncompleteChar: '⠂',
@@ -137,7 +137,7 @@ class PushCommand extends Command {
       }
     }
 
-    if (!flags['dry-run']) {
+    if (!flags.fake) {
       if (_.isEmpty(payload)) {
         this.log('⚠ Nothing to upload.'.yellow);
         process.exit();
@@ -157,20 +157,23 @@ class PushCommand extends Command {
         process.exit();
       }
 
-      const uploadMessage = 'Uploading content to Transifex';
+      const uploadMessage = flags['dry-run']
+        ? 'Uploading content to Transifex (dry run, no changes will be applied)'
+        : 'Uploading content to Transifex';
 
       this.log('');
-      cli.action.start(uploadMessage, '', { stdout: true });
+      CliUx.ux.action.start(uploadMessage, '', { stdout: true });
       try {
         let res = await uploadPhrases(payload, {
           url: cdsHost,
           token: projectToken,
           secret: projectSecret,
           purge: flags.purge,
+          dry_run: flags['dry-run'],
         });
 
         if (flags['no-wait']) {
-          cli.action.stop('Queued'.green);
+          CliUx.ux.action.stop('Queued'.green);
           return;
         }
 
@@ -178,7 +181,7 @@ class PushCommand extends Command {
         const jobUrl = `${cdsHost}${res.jobUrl}`;
         let status = '';
         do {
-          await cli.wait(1500);
+          await CliUx.ux.wait(1500);
           res = await pollJob({
             url: jobUrl,
             token: projectToken,
@@ -188,10 +191,10 @@ class PushCommand extends Command {
             status = res.status;
             switch (status) {
               case 'pending':
-                cli.action.start(uploadMessage, 'In queue', { stdout: true });
+                CliUx.ux.action.start(uploadMessage, 'In queue', { stdout: true });
                 break;
               case 'processing':
-                cli.action.start(uploadMessage, 'Processing', { stdout: true });
+                CliUx.ux.action.start(uploadMessage, 'Processing', { stdout: true });
                 break;
               default:
                 break;
@@ -200,7 +203,7 @@ class PushCommand extends Command {
         } while (status === 'pending' || status === 'processing');
 
         if (status === 'completed') {
-          cli.action.stop('Success'.green);
+          CliUx.ux.action.stop('Success'.green);
           this.log(`${'✓'.green} Successfully pushed strings to Transifex:`);
           if (res.created > 0) {
             this.log(`  Created strings: ${res.created.toString().green}`);
@@ -218,20 +221,20 @@ class PushCommand extends Command {
             this.log(`  Failed strings: ${res.failed.toString().red}`);
           }
         } else {
-          cli.action.stop('Failed'.red);
+          CliUx.ux.action.stop('Failed'.red);
         }
         _.each(res.errors, (error) => {
           this.log(`Error: ${JSON.stringify(error).red}`);
         });
       } catch (err) {
-        cli.action.stop('Failed'.red);
+        CliUx.ux.action.stop('Failed'.red);
         this.error(err);
       }
     }
   }
 }
 
-PushCommand.description = `detect and push source content to Transifex
+PushCommand.description = `Detect and push source content to Transifex
 Parse .js, .ts, .jsx, .tsx and .html files and detect phrases marked for
 translation by Transifex Native toolkit for Javascript and
 upload them to Transifex for translation.
@@ -251,6 +254,7 @@ txjs-cli push src/
 txjs-cli push /home/repo/src
 txjs-cli push "*.js"
 txjs-cli push --dry-run
+txjs-cli push --fake -v
 txjs-cli push --no-wait
 txjs-cli push --key-generator=hash
 txjs-cli push --append-tags="master,release:2.5"
@@ -258,6 +262,7 @@ txjs-cli push --with-tags-only="home,error"
 txjs-cli push --without-tags-only="custom"
 txjs-cli push --token=mytoken --secret=mysecret
 txjs-cli push en.json --parser=i18next
+txjs-cli push en.json --parser=txnativejson
 TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push
 `;
 
@@ -269,53 +274,57 @@ PushCommand.args = [{
 }];
 
 PushCommand.flags = {
-  'dry-run': flags.boolean({
-    description: 'dry run, do not push to Transifex',
+  'dry-run': Flags.boolean({
+    description: 'dry run, do not apply changes in Transifex',
     default: false,
   }),
-  verbose: flags.boolean({
+  fake: Flags.boolean({
+    description: 'do not push content to remote server',
+    default: false,
+  }),
+  verbose: Flags.boolean({
     char: 'v',
     description: 'verbose output',
     default: false,
   }),
-  purge: flags.boolean({
+  purge: Flags.boolean({
     description: 'purge content on Transifex',
     default: false,
   }),
-  'no-wait': flags.boolean({
+  'no-wait': Flags.boolean({
     description: 'disable polling for upload results',
     default: false,
   }),
-  token: flags.string({
+  token: Flags.string({
     description: 'native project public token',
     default: '',
   }),
-  secret: flags.string({
+  secret: Flags.string({
     description: 'native project secret',
     default: '',
   }),
-  'append-tags': flags.string({
+  'append-tags': Flags.string({
     description: 'append tags to strings',
     default: '',
   }),
-  'with-tags-only': flags.string({
+  'with-tags-only': Flags.string({
     description: 'push strings with specific tags',
     default: '',
   }),
-  'without-tags-only': flags.string({
+  'without-tags-only': Flags.string({
     description: 'push strings without specific tags',
     default: '',
   }),
-  'cds-host': flags.string({
+  'cds-host': Flags.string({
     description: 'CDS host URL',
     default: '',
   }),
-  parser: flags.string({
+  parser: Flags.string({
     description: 'file parser to use',
     default: 'auto',
-    options: ['auto', 'i18next'],
+    options: ['auto', 'i18next', 'txnativejson'],
   }),
-  'key-generator': flags.string({
+  'key-generator': Flags.string({
     description: 'use hashed or source based keys',
     default: 'source',
     options: ['source', 'hash'],

package/src/index.js

@@ -1 +1,3 @@
-module.exports = require('@oclif/command');
+const { run } = require('@oclif/core');
+
+module.exports = run;