@formatjs/cli-lib 5.0.6 → 5.0.7

package/index.ts

@@ -0,0 +1,7 @@
+export {default as extractAndWrite, extract} from './src/extract'
+export type {ExtractCLIOptions, ExtractOpts} from './src/extract'
+export type {MessageDescriptor} from '@formatjs/ts-transformer'
+export type {FormatFn, CompileFn} from './src/formatters/default'
+export type {Element, Comparator} from 'json-stable-stringify'
+export {default as compileAndWrite, compile} from './src/compile'
+export type {CompileCLIOpts, Opts as CompileOpts} from './src/compile'

package/main.ts

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+
+'use strict'
+
+require('./src/cli').default(process.argv)

package/package.json

@@ -1,64 +1,64 @@
 {
-    "name": "@formatjs/cli-lib",
-    "version": "5.0.6",
-    "description": "Lib for CLI for formatjs.",
-    "keywords": [
-        "intl",
-        "i18n",
-        "internationalization",
-        "locale",
-        "localization",
-        "globalization",
-        "react",
-        "react-intl",
-        "reactjs",
-        "format",
-        "formatjs",
-        "formatting",
-        "translate",
-        "translation",
-        "cli"
-    ],
-    "author": "Linjie Ding <linjie@airtable.com>",
-    "homepage": "https://github.com/formatjs/formatjs",
-    "license": "MIT",
-    "main": "index.js",
-    "repository": {
-        "type": "git",
-        "url": "git+ssh://git@github.com/formatjs/formatjs.git"
+  "name": "@formatjs/cli-lib",
+  "version": "5.0.7",
+  "description": "Lib for CLI for formatjs.",
+  "keywords": [
+    "intl",
+    "i18n",
+    "internationalization",
+    "locale",
+    "localization",
+    "globalization",
+    "react",
+    "react-intl",
+    "reactjs",
+    "format",
+    "formatjs",
+    "formatting",
+    "translate",
+    "translation",
+    "cli"
+  ],
+  "author": "Linjie Ding <linjie@airtable.com>",
+  "homepage": "https://github.com/formatjs/formatjs",
+  "license": "MIT",
+  "main": "index.js",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/formatjs/formatjs.git"
+  },
+  "bugs": {
+    "url": "https://github.com/formatjs/formatjs/issues"
+  },
+  "dependencies": {
+    "@formatjs/icu-messageformat-parser": "2.1.5",
+    "@formatjs/ts-transformer": "3.9.10",
+    "@types/estree": "^0.0.50",
+    "@types/fs-extra": "^9.0.1",
+    "@types/json-stable-stringify": "^1.0.32",
+    "@types/node": "14 || 16 || 17",
+    "chalk": "^4.0.0",
+    "commander": "8",
+    "fast-glob": "^3.2.7",
+    "fs-extra": "10",
+    "json-stable-stringify": "^1.0.1",
+    "loud-rejection": "^2.2.0",
+    "tslib": "2.4.0",
+    "typescript": "^4.5"
+  },
+  "peerDependencies": {
+    "@vue/compiler-core": "^3.2.23",
+    "@vue/compiler-sfc": "^3.2.34"
+  },
+  "peerDependenciesMeta": {
+    "@vue/compiler-sfc": {
+      "optional": true
     },
-    "bugs": {
-        "url": "https://github.com/formatjs/formatjs/issues"
-    },
-    "dependencies": {
-        "@formatjs/icu-messageformat-parser": "2.1.4",
-        "@formatjs/ts-transformer": "3.9.9",
-        "@types/estree": "^0.0.50",
-        "@types/fs-extra": "^9.0.1",
-        "@types/json-stable-stringify": "^1.0.32",
-        "@types/node": "14 || 16 || 17",
-        "chalk": "^4.0.0",
-        "commander": "8",
-        "fast-glob": "^3.2.7",
-        "fs-extra": "10",
-        "json-stable-stringify": "^1.0.1",
-        "loud-rejection": "^2.2.0",
-        "tslib": "2.4.0",
-        "typescript": "^4.5"
-    },
-    "peerDependencies": {
-        "@vue/compiler-core": "^3.2.23",
-        "@vue/compiler-sfc": "^3.2.34"
-    },
-    "peerDependenciesMeta": {
-        "@vue/compiler-sfc": {
-            "optional": true
-        },
-        "@vue/compiler-core": {
-            "optional": true
-        }
-    },
-    "engines": {
-        "node": ">= 16.5.0"
+    "@vue/compiler-core": {
+      "optional": true
     }
-}
+  },
+  "engines": {
+    "node": ">= 16.5.0"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,240 @@
+import {program} from 'commander'
+import loudRejection from 'loud-rejection'
+import extract, {ExtractCLIOptions} from './extract'
+import compile, {CompileCLIOpts, Opts} from './compile'
+import compileFolder from './compile_folder'
+import {sync as globSync} from 'fast-glob'
+import {debug} from './console_utils'
+
+const KNOWN_COMMANDS = ['extract']
+
+async function main(argv: string[]) {
+  loudRejection()
+
+  program
+    // TODO: fix this
+    .version('5.0.6', '-v, --version')
+    .usage('<command> [flags]')
+    .action(command => {
+      if (!KNOWN_COMMANDS.includes(command)) {
+        program.help()
+      }
+    })
+
+  program
+    .command('help', {isDefault: true})
+    .description('Show this help message.')
+    .action(() => program.help())
+
+  // Long text wrapping to available terminal columns: https://github.com/tj/commander.js/pull/956
+  // NOTE: please keep the help text in sync with babel-plugin-formatjs documentation.
+  program
+    .command('extract [files...]')
+    .description(
+      `Extract string messages from React components that use react-intl.
+The input language is expected to be TypeScript or ES2017 with JSX.`
+    )
+    .option(
+      '--format <path>',
+      `Path to a formatter file that controls the shape of JSON file from \`--out-file\`.
+The formatter file must export a function called \`format\` with the signature
+\`\`\`
+type FormatFn = <T = Record<string, MessageDescriptor>>(
+  msgs: Record<string, MessageDescriptor>
+) => T
+\`\`\` 
+This is especially useful to convert from our extracted format to a TMS-specific format.
+`
+    )
+    .option(
+      '--out-file <path>',
+      `The target file path where the plugin will output an aggregated 
+\`.json\` file of all the translations from the \`files\` supplied.`
+    )
+    .option(
+      '--id-interpolation-pattern <pattern>',
+      `If certain message descriptors don't have id, this \`pattern\` will be used to automatically
+generate IDs for them. Default to \`[sha512:contenthash:base64:6]\` where \`contenthash\` is the hash of
+\`defaultMessage\` and \`description\`.
+See https://github.com/webpack/loader-utils#interpolatename for sample patterns`,
+      '[sha512:contenthash:base64:6]'
+    )
+    .option(
+      '--extract-source-location',
+      `Whether the metadata about the location of the message in the source file should be 
+extracted. If \`true\`, then \`file\`, \`start\`, and \`end\` fields will exist for each 
+extracted message descriptors.`,
+      false
+    )
+    .option(
+      '--remove-default-message',
+      'Remove `defaultMessage` field in generated js after extraction',
+      false
+    )
+    .option(
+      '--additional-component-names <comma-separated-names>',
+      `Additional component names to extract messages from, e.g: \`'FormattedFooBarMessage'\`. 
+**NOTE**: By default we check for the fact that \`FormattedMessage\` 
+is imported from \`moduleSourceName\` to make sure variable alias 
+works. This option does not do that so it's less safe.`,
+      (val: string) => val.split(',')
+    )
+    .option(
+      '--additional-function-names <comma-separated-names>',
+      `Additional function names to extract messages from, e.g: \`'$t'\`.`,
+      (val: string) => val.split(',')
+    )
+    .option(
+      '--ignore <files...>',
+      'List of glob paths to **not** extract translations from.'
+    )
+    .option(
+      '--throws',
+      'Whether to throw an exception when we fail to process any file in the batch.'
+    )
+    .option(
+      '--pragma <pragma>',
+      `parse specific additional custom pragma. This allows you to tag certain file with metadata such as \`project\`. For example with this file:
+
+      \`\`\`
+      // @intl-meta project:my-custom-project
+      import {FormattedMessage} from 'react-intl';
+
+      <FormattedMessage defaultMessage="foo" id="bar" />;
+      \`\`\`
+
+      and with option \`{pragma: "intl-meta"}\`, we'll parse out \`// @intl-meta project:my-custom-project\` into \`{project: 'my-custom-project'}\` in the result file.`
+    )
+    .option(
+      '--preserve-whitespace',
+      'Whether to preserve whitespace and newlines.'
+    )
+    .option(
+      '--flatten',
+      `Whether to hoist selectors & flatten sentences as much as possible. E.g:
+"I have {count, plural, one{a dog} other{many dogs}}"
+becomes "{count, plural, one{I have a dog} other{I have many dogs}}".
+The goal is to provide as many full sentences as possible since fragmented
+sentences are not translator-friendly.`
+    )
+    .action(async (filePatterns: string[], cmdObj: ExtractCLIOptions) => {
+      debug('File pattern:', filePatterns)
+      debug('Options:', cmdObj)
+      const files = globSync(filePatterns, {
+        ignore: cmdObj.ignore,
+      })
+
+      debug('Files to extract:', files)
+
+      await extract(files, {
+        outFile: cmdObj.outFile,
+        idInterpolationPattern:
+          cmdObj.idInterpolationPattern || '[sha1:contenthash:base64:6]',
+        extractSourceLocation: cmdObj.extractSourceLocation,
+        removeDefaultMessage: cmdObj.removeDefaultMessage,
+        additionalComponentNames: cmdObj.additionalComponentNames,
+        additionalFunctionNames: cmdObj.additionalFunctionNames,
+        throws: cmdObj.throws,
+        pragma: cmdObj.pragma,
+        format: cmdObj.format,
+        // It is possible that the glob pattern does NOT match anything.
+        // But so long as the glob pattern is provided, don't read from stdin.
+        readFromStdin: filePatterns.length === 0,
+        preserveWhitespace: cmdObj.preserveWhitespace,
+        flatten: cmdObj.flatten,
+      })
+      process.exit(0)
+    })
+
+  program
+    .command('compile [translation_files...]')
+    .description(
+      `Compile extracted translation file into react-intl consumable JSON
+We also verify that the messages are valid ICU and not malformed. 
+<translation_files> can be a glob like "foo/**/en.json"`
+    )
+    .option(
+      '--format <path>',
+      `Path to a formatter file that converts \`<translation_file>\` to \`Record<string, string>\`
+so we can compile. The file must export a function named \`compile\` with the signature:
+\`\`\`
+type CompileFn = <T = Record<string, MessageDescriptor>>(
+  msgs: T
+) => Record<string, string>;
+\`\`\`
+This is especially useful to convert from a TMS-specific format back to react-intl format
+`
+    )
+    .option(
+      '--out-file <path>',
+      `Compiled translation output file.
+If this is not provided, result will be printed to stdout`
+    )
+    .option(
+      '--ast',
+      `Whether to compile to AST. See https://formatjs.io/docs/guides/advanced-usage#pre-parsing-messages
+for more information`
+    )
+    .option(
+      '--skip-errors',
+      `Whether to continue compiling messages after encountering an error. Any keys with errors will not be included in the output file.`
+    )
+    .option(
+      '--pseudo-locale <pseudoLocale>',
+      `Whether to generate pseudo-locale files. See https://formatjs.io/docs/tooling/cli#--pseudo-locale-pseudolocale for possible values. 
+"--ast" is required for this to work.`
+    )
+    .action(async (filePatterns: string[], opts: CompileCLIOpts) => {
+      debug('File pattern:', filePatterns)
+      debug('Options:', opts)
+      const files = globSync(filePatterns)
+      if (!files.length) {
+        throw new Error(`No input file found with pattern ${filePatterns}`)
+      }
+      debug('Files to compile:', files)
+      await compile(files, opts)
+    })
+
+  program
+    .command('compile-folder <folder> <outFolder>')
+    .description(
+      `Batch compile all extracted translation JSON files in <folder> to <outFolder> containing
+react-intl consumable JSON. We also verify that the messages are 
+valid ICU and not malformed.`
+    )
+    .option(
+      '--format <path>',
+      `Path to a formatter file that converts JSON files in \`<folder>\` to \`Record<string, string>\`
+so we can compile. The file must export a function named \`compile\` with the signature:
+\`\`\`
+type CompileFn = <T = Record<string, MessageDescriptor>>(
+  msgs: T
+) => Record<string, string>;
+\`\`\`
+This is especially useful to convert from a TMS-specific format back to react-intl format
+`
+    )
+    .option(
+      '--ast',
+      `Whether to compile to AST. See https://formatjs.io/docs/guides/advanced-usage#pre-parsing-messages
+for more information`
+    )
+    .action(async (folder: string, outFolder: string, opts?: Opts) => {
+      debug('Folder:', folder)
+      debug('Options:', opts)
+      // fast-glob expect `/` in Windows as well
+      const files = globSync(`${folder}/*.json`)
+      if (!files.length) {
+        throw new Error(`No JSON file found in ${folder}`)
+      }
+      debug('Files to compile:', files)
+      await compileFolder(files, outFolder, opts)
+    })
+
+  if (argv.length < 3) {
+    program.help()
+  } else {
+    program.parse(argv)
+  }
+}
+export default main

package/src/compile.ts

@@ -0,0 +1,141 @@
+import {parse, MessageFormatElement} from '@formatjs/icu-messageformat-parser'
+import {outputFile, readJSON} from 'fs-extra'
+import stringify from 'json-stable-stringify'
+import {debug, warn, writeStdout} from './console_utils'
+import {resolveBuiltinFormatter, Formatter} from './formatters'
+import {
+  generateXXAC,
+  generateXXLS,
+  generateXXHA,
+  generateENXA,
+  generateENXB,
+} from './pseudo_locale'
+
+export type CompileFn = (msgs: any) => Record<string, string>
+
+export type PseudoLocale = 'xx-LS' | 'xx-AC' | 'xx-HA' | 'en-XA' | 'en-XB'
+
+export interface CompileCLIOpts extends Opts {
+  /**
+   * The target file that contains compiled messages.
+   */
+  outFile?: string
+}
+export interface Opts {
+  /**
+   * Whether to compile message into AST instead of just string
+   */
+  ast?: boolean
+  /**
+   * Whether to continue compiling messages after encountering an error.
+   * Any keys with errors will not be included in the output file.
+   */
+  skipErrors?: boolean
+  /**
+   * Path to a formatter file that converts <translation_files> to
+   * `Record<string, string>` so we can compile.
+   */
+  format?: string | Formatter
+  /**
+   * Whether to compile to pseudo locale
+   */
+  pseudoLocale?: PseudoLocale
+}
+
+/**
+ * Aggregate `inputFiles` into a single JSON blob and compile.
+ * Also checks for conflicting IDs.
+ * Then returns the serialized result as a `string` since key order
+ * makes a difference in some vendor.
+ * @param inputFiles Input files
+ * @param opts Options
+ * @returns serialized result in string format
+ */
+export async function compile(inputFiles: string[], opts: Opts = {}) {
+  debug('Compiling files:', inputFiles)
+  const {ast, format, pseudoLocale, skipErrors} = opts
+  const formatter = await resolveBuiltinFormatter(format)
+
+  const messages: Record<string, string> = {}
+  const messageAsts: Record<string, MessageFormatElement[]> = {}
+  const idsWithFileName: Record<string, string> = {}
+  const compiledFiles = await Promise.all(
+    inputFiles.map(f => readJSON(f).then(formatter.compile))
+  )
+  debug('Compiled files:', compiledFiles)
+  for (let i = 0; i < inputFiles.length; i++) {
+    const inputFile = inputFiles[i]
+    debug('Processing file:', inputFile)
+    const compiled = compiledFiles[i]
+    for (const id in compiled) {
+      if (messages[id] && messages[id] !== compiled[id]) {
+        throw new Error(`Conflicting ID "${id}" with different translation found in these 2 files:
+ID: ${id}
+Message from ${idsWithFileName[id]}: ${messages[id]}
+Message from ${compiled[id]}: ${inputFile}
+`)
+      }
+      try {
+        const msgAst = parse(compiled[id])
+        messages[id] = compiled[id]
+        switch (pseudoLocale) {
+          case 'xx-LS':
+            messageAsts[id] = generateXXLS(msgAst)
+            break
+          case 'xx-AC':
+            messageAsts[id] = generateXXAC(msgAst)
+            break
+          case 'xx-HA':
+            messageAsts[id] = generateXXHA(msgAst)
+            break
+          case 'en-XA':
+            messageAsts[id] = generateENXA(msgAst)
+            break
+          case 'en-XB':
+            messageAsts[id] = generateENXB(msgAst)
+            break
+          default:
+            messageAsts[id] = msgAst
+            break
+        }
+        idsWithFileName[id] = inputFile
+      } catch (e) {
+        warn(
+          'Error validating message "%s" with ID "%s" in file "%s"',
+          compiled[id],
+          id,
+          inputFile
+        )
+        if (!skipErrors) {
+          throw e
+        }
+      }
+    }
+  }
+
+  return stringify(ast ? messageAsts : messages, {
+    space: 2,
+    cmp: formatter.compareMessages || undefined,
+  })
+}
+
+/**
+ * Aggregate `inputFiles` into a single JSON blob and compile.
+ * Also checks for conflicting IDs and write output to `outFile`.
+ * @param inputFiles Input files
+ * @param compileOpts options
+ * @returns A `Promise` that resolves if file was written successfully
+ */
+export default async function compileAndWrite(
+  inputFiles: string[],
+  compileOpts: CompileCLIOpts = {}
+) {
+  const {outFile, ...opts} = compileOpts
+  const serializedResult = await compile(inputFiles, opts)
+  if (outFile) {
+    debug('Writing output file:', outFile)
+    return outputFile(outFile, serializedResult)
+  }
+  await writeStdout(serializedResult)
+  await writeStdout('\n')
+}

package/src/compile_folder.ts

@@ -0,0 +1,15 @@
+import {Opts, compile} from './compile'
+import {join, basename} from 'path'
+import {outputFile} from 'fs-extra'
+export default async function compileFolder(
+  files: string[],
+  outFolder: string,
+  opts: Opts = {}
+) {
+  const results = await Promise.all(files.map(f => compile([f], opts)))
+  const outFiles = files.map(f => join(outFolder, basename(f)))
+
+  return Promise.all(
+    outFiles.map((outFile, i) => outputFile(outFile, results[i]))
+  )
+}

package/src/console_utils.ts

@@ -0,0 +1,78 @@
+import {supportsColor, green, red, yellow} from 'chalk'
+import readline from 'readline'
+import {format, promisify} from 'util'
+
+const CLEAR_WHOLE_LINE = 0
+
+export const writeStderr = promisify(process.stderr.write).bind(process.stderr)
+export const writeStdout = promisify(process.stdout.write).bind(process.stdout)
+
+const nativeClearLine = promisify(readline.clearLine).bind(readline)
+const nativeCursorTo = promisify(readline.cursorTo).bind(readline)
+
+// From:
+// https://github.com/yarnpkg/yarn/blob/53d8004229f543f342833310d5af63a4b6e59c8a/src/reporters/console/util.js
+export async function clearLine(terminal: typeof process['stderr']) {
+  if (!supportsColor) {
+    if (terminal.isTTY) {
+      // terminal
+      if (terminal.columns > 0) {
+        await writeStderr(`\r${' '.repeat(terminal.columns - 1)}`)
+      }
+      await writeStderr(`\r`)
+    }
+    // ignore piping to file
+  } else {
+    await nativeClearLine(terminal, CLEAR_WHOLE_LINE)
+    await nativeCursorTo(terminal, 0, undefined)
+  }
+}
+
+const LEVEL_COLORS = {
+  debug: green,
+  warn: yellow,
+  error: red,
+}
+
+function label(level: keyof typeof LEVEL_COLORS, message: string) {
+  return `[@formatjs/cli] [${LEVEL_COLORS[level](
+    level.toUpperCase()
+  )}] ${message}`
+}
+
+export async function debug(message: string, ...args: any[]) {
+  if (process.env.LOG_LEVEL !== 'debug') {
+    return
+  }
+  await clearLine(process.stderr)
+  await writeStderr(format(label('debug', message), ...args))
+  await writeStderr('\n')
+}
+
+export async function warn(message: string, ...args: any[]) {
+  await clearLine(process.stderr)
+  await writeStderr(format(label('warn', message), ...args))
+  await writeStderr('\n')
+}
+
+export async function error(message: string, ...args: any[]) {
+  await clearLine(process.stderr)
+  await writeStderr(format(label('error', message), ...args))
+  await writeStderr('\n')
+}
+
+export function getStdinAsString(): Promise<string> {
+  let result = ''
+  return new Promise(resolve => {
+    process.stdin.setEncoding('utf-8')
+    process.stdin.on('readable', () => {
+      let chunk
+      while ((chunk = process.stdin.read())) {
+        result += chunk
+      }
+    })
+    process.stdin.on('end', () => {
+      resolve(result)
+    })
+  })
+}