@formatjs/cli-lib 6.6.6 → 7.0.1

package/lib_esnext/index.d.ts

@@ -0,0 +1,7 @@
+export { default as extractAndWrite, extract } from './src/extract';
+export type { ExtractCLIOptions, ExtractOpts } from './src/extract';
+export type { MessageDescriptor } from '@formatjs/ts-transformer';
+export type { FormatFn, CompileFn } from './src/formatters/default';
+export type { Element, Comparator } from 'json-stable-stringify';
+export { default as compileAndWrite, compile } from './src/compile';
+export type { CompileCLIOpts, Opts as CompileOpts } from './src/compile';

package/lib_esnext/index.js

@@ -0,0 +1,2 @@
+export { default as extractAndWrite, extract } from './src/extract';
+export { default as compileAndWrite, compile } from './src/compile';

package/lib_esnext/main.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/lib_esnext/main.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+'use strict';
+require('./src/cli').default(process.argv);

package/lib_esnext/src/cli.d.ts

@@ -0,0 +1,2 @@
+declare function main(argv: string[]): Promise<void>;
+export default main;

package/lib_esnext/src/cli.js

@@ -0,0 +1,153 @@
+import { program } from 'commander';
+import { sync as globSync } from 'fast-glob';
+import loudRejection from 'loud-rejection';
+import compile from './compile';
+import compileFolder from './compile_folder';
+import { debug } from './console_utils';
+import extract from './extract';
+const KNOWN_COMMANDS = ['extract'];
+async function main(argv) {
+    loudRejection();
+    program
+        // TODO: fix this
+        .version('5.0.6', '-v, --version')
+        .usage('<command> [flags]')
+        .action(command => {
+        if (!KNOWN_COMMANDS.includes(command)) {
+            program.help();
+        }
+    });
+    program
+        .command('help', { isDefault: true })
+        .description('Show this help message.')
+        .action(() => program.help());
+    // Long text wrapping to available terminal columns: https://github.com/tj/commander.js/pull/956
+    // NOTE: please keep the help text in sync with babel-plugin-formatjs documentation.
+    program
+        .command('extract [files...]')
+        .description(`Extract string messages from React components that use react-intl.
+The input language is expected to be TypeScript or ES2017 with JSX.`)
+        .option('--format <path>', `Path to a formatter file that controls the shape of JSON file from \`--out-file\`.
+The formatter file must export a function called \`format\` with the signature
+\`\`\`
+type FormatFn = <T = Record<string, MessageDescriptor>>(
+  msgs: Record<string, MessageDescriptor>
+) => T
+\`\`\` 
+This is especially useful to convert from our extracted format to a TMS-specific format.
+`)
+        .option('--out-file <path>', `The target file path where the plugin will output an aggregated 
+\`.json\` file of all the translations from the \`files\` supplied.`)
+        .option('--id-interpolation-pattern <pattern>', `If certain message descriptors don't have id, this \`pattern\` will be used to automatically
+generate IDs for them. Default to \`[sha512:contenthash:base64:6]\` where \`contenthash\` is the hash of
+\`defaultMessage\` and \`description\`.
+See https://github.com/webpack/loader-utils#interpolatename for sample patterns`, '[sha512:contenthash:base64:6]')
+        .option('--extract-source-location', `Whether the metadata about the location of the message in the source file should be 
+extracted. If \`true\`, then \`file\`, \`start\`, and \`end\` fields will exist for each 
+extracted message descriptors.`, false)
+        .option('--remove-default-message', 'Remove `defaultMessage` field in generated js after extraction', false)
+        .option('--additional-component-names <comma-separated-names>', `Additional component names to extract messages from, e.g: \`'FormattedFooBarMessage'\`. 
+**NOTE**: By default we check for the fact that \`FormattedMessage\` 
+is imported from \`moduleSourceName\` to make sure variable alias 
+works. This option does not do that so it's less safe.`, (val) => val.split(','))
+        .option('--additional-function-names <comma-separated-names>', `Additional function names to extract messages from, e.g: \`'$t'\`.`, (val) => val.split(','))
+        .option('--ignore <files...>', 'List of glob paths to **not** extract translations from.')
+        .option('--throws', 'Whether to throw an exception when we fail to process any file in the batch.')
+        .option('--pragma <pragma>', `parse specific additional custom pragma. This allows you to tag certain file with metadata such as \`project\`. For example with this file:
+
+      \`\`\`
+      // @intl-meta project:my-custom-project
+      import {FormattedMessage} from 'react-intl';
+
+      <FormattedMessage defaultMessage="foo" id="bar" />;
+      \`\`\`
+
+      and with option \`{pragma: "intl-meta"}\`, we'll parse out \`// @intl-meta project:my-custom-project\` into \`{project: 'my-custom-project'}\` in the result file.`)
+        .option('--preserve-whitespace', 'Whether to preserve whitespace and newlines.')
+        .option('--flatten', `Whether to hoist selectors & flatten sentences as much as possible. E.g:
+"I have {count, plural, one{a dog} other{many dogs}}"
+becomes "{count, plural, one{I have a dog} other{I have many dogs}}".
+The goal is to provide as many full sentences as possible since fragmented
+sentences are not translator-friendly.`)
+        .action(async (filePatterns, cmdObj) => {
+        debug('File pattern:', filePatterns);
+        debug('Options:', cmdObj);
+        const files = globSync(filePatterns, {
+            ignore: cmdObj.ignore,
+        });
+        debug('Files to extract:', files);
+        await extract(files, {
+            outFile: cmdObj.outFile,
+            idInterpolationPattern: cmdObj.idInterpolationPattern || '[sha1:contenthash:base64:6]',
+            extractSourceLocation: cmdObj.extractSourceLocation,
+            removeDefaultMessage: cmdObj.removeDefaultMessage,
+            additionalComponentNames: cmdObj.additionalComponentNames,
+            additionalFunctionNames: cmdObj.additionalFunctionNames,
+            throws: cmdObj.throws,
+            pragma: cmdObj.pragma,
+            format: cmdObj.format,
+            // It is possible that the glob pattern does NOT match anything.
+            // But so long as the glob pattern is provided, don't read from stdin.
+            readFromStdin: filePatterns.length === 0,
+            preserveWhitespace: cmdObj.preserveWhitespace,
+            flatten: cmdObj.flatten,
+        });
+        process.exit(0);
+    });
+    program
+        .command('compile [translation_files...]')
+        .description(`Compile extracted translation file into react-intl consumable JSON We also verify that the messages are valid ICU and not malformed. <translation_files> can be a glob like "foo/**/en.json"`)
+        .option('--format <path>', `Path to a formatter file that converts \`<translation_file>\` to \`Record<string, string>\` so we can compile. The file must export a function named \`compile\` with the signature:
+\`\`\`
+type CompileFn = <T = Record<string, MessageDescriptor>>(
+  msgs: T
+) => Record<string, string>;
+\`\`\`
+This is especially useful to convert from a TMS-specific format back to react-intl format
+`)
+        .option('--out-file <path>', `Compiled translation output file. If this is not provided, result will be printed to stdout`)
+        .option('--ast', `Whether to compile to AST. See https://formatjs.github.io/docs/guides/advanced-usage#pre-parsing-messages for more information`)
+        .option('--skip-errors', `Whether to continue compiling messages after encountering an error. Any keys with errors will not be included in the output file.`)
+        .option('--pseudo-locale <pseudoLocale>', `Whether to generate pseudo-locale files. See https://formatjs.github.io/docs/tooling/cli#--pseudo-locale-pseudolocale for possible values. "--ast" is required for this to work.`)
+        .option('--ignore-tag', `Whether the parser to treat HTML/XML tags as string literal instead of parsing them as tag token. When this is false we only allow simple tags without any attributes.`)
+        .action(async (filePatterns, opts) => {
+        debug('File pattern:', filePatterns);
+        debug('Options:', opts);
+        const files = globSync(filePatterns);
+        if (!files.length) {
+            throw new Error(`No input file found with pattern ${filePatterns}`);
+        }
+        debug('Files to compile:', files);
+        await compile(files, opts);
+    });
+    program
+        .command('compile-folder <folder> <outFolder>')
+        .description(`Batch compile all extracted translation JSON files in <folder> to <outFolder> containing react-intl consumable JSON. We also verify that the messages are valid ICU and not malformed.`)
+        .option('--format <path>', `Path to a formatter file that converts JSON files in \`<folder>\` to \`Record<string, string>\` so we can compile. The file must export a function named \`compile\` with the signature:
+\`\`\`
+type CompileFn = <T = Record<string, MessageDescriptor>>(
+  msgs: T
+) => Record<string, string>;
+\`\`\`
+This is especially useful to convert from a TMS-specific format back to react-intl format
+`)
+        .option('--ast', `Whether to compile to AST. See https://formatjs.github.io/docs/guides/advanced-usage#pre-parsing-messages for more information`)
+        .action(async (folder, outFolder, opts) => {
+        debug('Folder:', folder);
+        debug('Options:', opts);
+        // fast-glob expect `/` in Windows as well
+        const files = globSync(`${folder}/*.json`);
+        if (!files.length) {
+            throw new Error(`No JSON file found in ${folder}`);
+        }
+        debug('Files to compile:', files);
+        await compileFolder(files, outFolder, opts);
+    });
+    if (argv.length < 3) {
+        program.help();
+    }
+    else {
+        program.parse(argv);
+    }
+}
+export default main;

package/lib_esnext/src/compile.d.ts

@@ -0,0 +1,54 @@
+import { Formatter } from './formatters';
+export type CompileFn = (msgs: any) => Record<string, string>;
+export type PseudoLocale = 'xx-LS' | 'xx-AC' | 'xx-HA' | 'en-XA' | 'en-XB';
+export interface CompileCLIOpts extends Opts {
+    /**
+     * The target file that contains compiled messages.
+     */
+    outFile?: string;
+}
+export interface Opts {
+    /**
+     * Whether to compile message into AST instead of just string
+     */
+    ast?: boolean;
+    /**
+     * Whether to continue compiling messages after encountering an error.
+     * Any keys with errors will not be included in the output file.
+     */
+    skipErrors?: boolean;
+    /**
+     * Path to a formatter file that converts <translation_files> to
+     * `Record<string, string>` so we can compile.
+     */
+    format?: string | Formatter<unknown>;
+    /**
+     * Whether to compile to pseudo locale
+     */
+    pseudoLocale?: PseudoLocale;
+    /**
+     * Whether the parser to treat HTML/XML tags as string literal
+     * instead of parsing them as tag token.
+     * When this is false we only allow simple tags without
+     * any attributes
+     */
+    ignoreTag?: boolean;
+}
+/**
+ * Aggregate `inputFiles` into a single JSON blob and compile.
+ * Also checks for conflicting IDs.
+ * Then returns the serialized result as a `string` since key order
+ * makes a difference in some vendor.
+ * @param inputFiles Input files
+ * @param opts Options
+ * @returns serialized result in string format
+ */
+export declare function compile(inputFiles: string[], opts?: Opts): Promise<string>;
+/**
+ * Aggregate `inputFiles` into a single JSON blob and compile.
+ * Also checks for conflicting IDs and write output to `outFile`.
+ * @param inputFiles Input files
+ * @param compileOpts options
+ * @returns A `Promise` that resolves if file was written successfully
+ */
+export default function compileAndWrite(inputFiles: string[], compileOpts?: CompileCLIOpts): Promise<void>;

package/lib_esnext/src/compile.js

@@ -0,0 +1,90 @@
+import { parse } from '@formatjs/icu-messageformat-parser';
+import { outputFile, readJSON } from 'fs-extra';
+import stringify from 'json-stable-stringify';
+import { debug, warn, writeStdout } from './console_utils';
+import { resolveBuiltinFormatter } from './formatters';
+import { generateENXA, generateENXB, generateXXAC, generateXXHA, generateXXLS, } from './pseudo_locale';
+/**
+ * Aggregate `inputFiles` into a single JSON blob and compile.
+ * Also checks for conflicting IDs.
+ * Then returns the serialized result as a `string` since key order
+ * makes a difference in some vendor.
+ * @param inputFiles Input files
+ * @param opts Options
+ * @returns serialized result in string format
+ */
+export async function compile(inputFiles, opts = {}) {
+    debug('Compiling files:', inputFiles);
+    const { ast, format, pseudoLocale, skipErrors, ignoreTag } = opts;
+    const formatter = await resolveBuiltinFormatter(format);
+    const messages = {};
+    const messageAsts = {};
+    const idsWithFileName = {};
+    const compiledFiles = await Promise.all(inputFiles.map(f => readJSON(f).then(formatter.compile)));
+    debug('Compiled files:', compiledFiles);
+    for (let i = 0; i < inputFiles.length; i++) {
+        const inputFile = inputFiles[i];
+        debug('Processing file:', inputFile);
+        const compiled = compiledFiles[i];
+        for (const id in compiled) {
+            if (messages[id] && messages[id] !== compiled[id]) {
+                throw new Error(`Conflicting ID "${id}" with different translation found in these 2 files:
+ID: ${id}
+Message from ${idsWithFileName[id]}: ${messages[id]}
+Message from ${inputFile}: ${compiled[id]}
+`);
+            }
+            try {
+                const msgAst = parse(compiled[id], { ignoreTag });
+                messages[id] = compiled[id];
+                switch (pseudoLocale) {
+                    case 'xx-LS':
+                        messageAsts[id] = generateXXLS(msgAst);
+                        break;
+                    case 'xx-AC':
+                        messageAsts[id] = generateXXAC(msgAst);
+                        break;
+                    case 'xx-HA':
+                        messageAsts[id] = generateXXHA(msgAst);
+                        break;
+                    case 'en-XA':
+                        messageAsts[id] = generateENXA(msgAst);
+                        break;
+                    case 'en-XB':
+                        messageAsts[id] = generateENXB(msgAst);
+                        break;
+                    default:
+                        messageAsts[id] = msgAst;
+                        break;
+                }
+                idsWithFileName[id] = inputFile;
+            }
+            catch (e) {
+                warn('Error validating message "%s" with ID "%s" in file "%s"', compiled[id], id, inputFile);
+                if (!skipErrors) {
+                    throw e;
+                }
+            }
+        }
+    }
+    return stringify(ast ? messageAsts : messages, {
+        space: 2,
+        cmp: formatter.compareMessages || undefined,
+    });
+}
+/**
+ * Aggregate `inputFiles` into a single JSON blob and compile.
+ * Also checks for conflicting IDs and write output to `outFile`.
+ * @param inputFiles Input files
+ * @param compileOpts options
+ * @returns A `Promise` that resolves if file was written successfully
+ */
+export default async function compileAndWrite(inputFiles, compileOpts = {}) {
+    const { outFile, ...opts } = compileOpts;
+    const serializedResult = (await compile(inputFiles, opts)) + '\n';
+    if (outFile) {
+        debug('Writing output file:', outFile);
+        return outputFile(outFile, serializedResult);
+    }
+    await writeStdout(serializedResult);
+}

package/lib_esnext/src/compile_folder.d.ts

@@ -0,0 +1,2 @@
+import { Opts } from './compile';
+export default function compileFolder(files: string[], outFolder: string, opts?: Opts): Promise<void[]>;

package/lib_esnext/src/compile_folder.js

@@ -0,0 +1,8 @@
+import { outputFile } from 'fs-extra';
+import { basename, join } from 'path';
+import { compile } from './compile';
+export default async function compileFolder(files, outFolder, opts = {}) {
+    const results = await Promise.all(files.map(f => compile([f], opts)));
+    const outFiles = files.map(f => join(outFolder, basename(f)));
+    return Promise.all(outFiles.map((outFile, i) => outputFile(outFile, results[i])));
+}

package/lib_esnext/src/console_utils.d.ts

@@ -0,0 +1,7 @@
+export declare const writeStderr: (arg1: string | Uint8Array) => Promise<void>;
+export declare const writeStdout: (arg1: string | Uint8Array) => Promise<void>;
+export declare function clearLine(terminal: (typeof process)['stderr']): Promise<void>;
+export declare function debug(message: string, ...args: any[]): Promise<void>;
+export declare function warn(message: string, ...args: any[]): Promise<void>;
+export declare function error(message: string, ...args: any[]): Promise<void>;
+export declare function getStdinAsString(): Promise<string>;

package/lib_esnext/src/console_utils.js

@@ -0,0 +1,67 @@
+import { green, red, supportsColor, yellow } from 'chalk';
+import readline from 'readline';
+import { format, promisify } from 'util';
+const CLEAR_WHOLE_LINE = 0;
+export const writeStderr = promisify(process.stderr.write).bind(process.stderr);
+export const writeStdout = promisify(process.stdout.write).bind(process.stdout);
+const nativeClearLine = promisify(readline.clearLine).bind(readline);
+const nativeCursorTo = promisify(readline.cursorTo).bind(readline);
+// From:
+// https://github.com/yarnpkg/yarn/blob/53d8004229f543f342833310d5af63a4b6e59c8a/src/reporters/console/util.js
+export async function clearLine(terminal) {
+    if (!supportsColor) {
+        if (terminal.isTTY) {
+            // terminal
+            if (terminal.columns > 0) {
+                await writeStderr(`\r${' '.repeat(terminal.columns - 1)}`);
+            }
+            await writeStderr(`\r`);
+        }
+        // ignore piping to file
+    }
+    else {
+        await nativeClearLine(terminal, CLEAR_WHOLE_LINE);
+        await nativeCursorTo(terminal, 0, undefined);
+    }
+}
+const LEVEL_COLORS = {
+    debug: green,
+    warn: yellow,
+    error: red,
+};
+function label(level, message) {
+    return `[@formatjs/cli] [${LEVEL_COLORS[level](level.toUpperCase())}] ${message}`;
+}
+export async function debug(message, ...args) {
+    if (process.env.LOG_LEVEL !== 'debug') {
+        return;
+    }
+    await clearLine(process.stderr);
+    await writeStderr(format(label('debug', message), ...args));
+    await writeStderr('\n');
+}
+export async function warn(message, ...args) {
+    await clearLine(process.stderr);
+    await writeStderr(format(label('warn', message), ...args));
+    await writeStderr('\n');
+}
+export async function error(message, ...args) {
+    await clearLine(process.stderr);
+    await writeStderr(format(label('error', message), ...args));
+    await writeStderr('\n');
+}
+export function getStdinAsString() {
+    let result = '';
+    return new Promise(resolve => {
+        process.stdin.setEncoding('utf-8');
+        process.stdin.on('readable', () => {
+            let chunk;
+            while ((chunk = process.stdin.read())) {
+                result += chunk;
+            }
+        });
+        process.stdin.on('end', () => {
+            resolve(result);
+        });
+    });
+}

package/lib_esnext/src/extract.d.ts

@@ -0,0 +1,74 @@
+import { MessageDescriptor, Opts } from '@formatjs/ts-transformer';
+import { Formatter } from './formatters';
+export interface ExtractionResult<M = Record<string, string>> {
+    /**
+     * List of extracted messages
+     */
+    messages: MessageDescriptor[];
+    /**
+     * Metadata extracted w/ `pragma`
+     */
+    meta?: M;
+}
+export interface ExtractedMessageDescriptor extends MessageDescriptor {
+    /**
+     * Line number
+     */
+    line?: number;
+    /**
+     * Column number
+     */
+    col?: number;
+    /**
+     * Metadata extracted from pragma
+     */
+    meta?: Record<string, string>;
+}
+export type ExtractCLIOptions = Omit<ExtractOpts, 'overrideIdFn' | 'onMsgExtracted' | 'onMetaExtracted'> & {
+    /**
+     * Output File
+     */
+    outFile?: string;
+    /**
+     * Ignore file glob pattern
+     */
+    ignore?: string[];
+};
+export type ExtractOpts = Opts & {
+    /**
+     * Whether to throw an error if we had any issues with
+     * 1 of the source files
+     */
+    throws?: boolean;
+    /**
+     * Message ID interpolation pattern
+     */
+    idInterpolationPattern?: string;
+    /**
+     * Whether we read from stdin instead of a file
+     */
+    readFromStdin?: boolean;
+    /**
+     * Either path to a formatter file that controls the shape of JSON file from `outFile` or {@link Formatter} object.
+     */
+    format?: string | Formatter<any>;
+    /**
+     * Whether to hoist selectors & flatten sentences
+     */
+    flatten?: boolean;
+} & Pick<Opts, 'onMsgExtracted' | 'onMetaExtracted'>;
+/**
+ * Extract strings from source files
+ * @param files list of files
+ * @param extractOpts extract options
+ * @returns messages serialized as JSON string since key order
+ * matters for some `format`
+ */
+export declare function extract(files: readonly string[], extractOpts: ExtractOpts): Promise<string>;
+/**
+ * Extract strings from source files, also writes to a file.
+ * @param files list of files
+ * @param extractOpts extract options
+ * @returns A Promise that resolves if output file was written successfully
+ */
+export default function extractAndWrite(files: readonly string[], extractOpts: ExtractCLIOptions): Promise<void>;