@formatjs/cli-lib 6.6.6 → 7.0.0

package/lib_esnext/src/extract.js

@@ -0,0 +1,192 @@
+import { interpolateName, } from '@formatjs/ts-transformer';
+import { outputFile, readFile } from 'fs-extra';
+import { debug, getStdinAsString, warn, writeStdout } from './console_utils';
+import { parse } from '@formatjs/icu-messageformat-parser';
+import { hoistSelectors } from '@formatjs/icu-messageformat-parser/manipulator';
+import { printAST } from '@formatjs/icu-messageformat-parser/printer';
+import stringify from 'json-stable-stringify';
+import { resolveBuiltinFormatter } from './formatters';
+import { parseScript } from './parse_script';
+function calculateLineColFromOffset(text, start) {
+    if (!start) {
+        return { line: 1, col: 1 };
+    }
+    const chunk = text.slice(0, start);
+    const lines = chunk.split('\n');
+    const lastLine = lines[lines.length - 1];
+    return { line: lines.length, col: lastLine.length };
+}
+async function processFile(source, fn, { idInterpolationPattern, ...opts }) {
+    let messages = [];
+    let meta;
+    const onMsgExtracted = opts.onMsgExtracted;
+    const onMetaExtracted = opts.onMetaExtracted;
+    opts = {
+        ...opts,
+        additionalComponentNames: [
+            '$formatMessage',
+            ...(opts.additionalComponentNames || []),
+        ],
+        onMsgExtracted(filePath, msgs) {
+            if (opts.extractSourceLocation) {
+                msgs = msgs.map(msg => ({
+                    ...msg,
+                    ...calculateLineColFromOffset(source, msg.start),
+                }));
+            }
+            messages = messages.concat(msgs);
+            if (onMsgExtracted) {
+                onMsgExtracted(filePath, msgs);
+            }
+        },
+        onMetaExtracted(filePath, m) {
+            meta = m;
+            if (onMetaExtracted) {
+                onMetaExtracted(filePath, m);
+            }
+        },
+    };
+    if (!opts.overrideIdFn && idInterpolationPattern) {
+        opts = {
+            ...opts,
+            overrideIdFn: (id, defaultMessage, description, fileName) => id ||
+                interpolateName({
+                    resourcePath: fileName,
+                }, idInterpolationPattern, {
+                    content: description
+                        ? `${defaultMessage}#${typeof description === 'string'
+                            ? description
+                            : stringify(description)}`
+                        : defaultMessage,
+                }),
+        };
+    }
+    debug('Processing opts for %s: %s', fn, opts);
+    const scriptParseFn = parseScript(opts, fn);
+    if (fn.endsWith('.vue')) {
+        debug('Processing %s using vue extractor', fn);
+        const { parseFile } = await import('./vue_extractor.js');
+        parseFile(source, fn, scriptParseFn);
+    }
+    else if (fn.endsWith('.hbs')) {
+        debug('Processing %s using hbs extractor', fn);
+        const { parseFile } = await import('./hbs_extractor.js');
+        parseFile(source, fn, opts);
+    }
+    else if (fn.endsWith('.gts') || fn.endsWith('.gjs')) {
+        debug('Processing %s as gts/gjs file', fn);
+        const { parseFile } = await import('./gts_extractor.js');
+        parseFile(source, fn, opts);
+    }
+    else {
+        debug('Processing %s using typescript extractor', fn);
+        scriptParseFn(source);
+    }
+    debug('Done extracting %s messages: %s', fn, messages);
+    if (meta) {
+        debug('Extracted meta:', meta);
+        messages.forEach(m => (m.meta = meta));
+    }
+    return { messages, meta };
+}
+/**
+ * Extract strings from source files
+ * @param files list of files
+ * @param extractOpts extract options
+ * @returns messages serialized as JSON string since key order
+ * matters for some `format`
+ */
+export async function extract(files, extractOpts) {
+    const { throws, readFromStdin, flatten, ...opts } = extractOpts;
+    let rawResults = [];
+    try {
+        if (readFromStdin) {
+            debug(`Reading input from stdin`);
+            // Read from stdin
+            if (process.stdin.isTTY) {
+                warn('Reading source file from TTY.');
+            }
+            const stdinSource = await getStdinAsString();
+            rawResults = [await processFile(stdinSource, 'dummy', opts)];
+        }
+        else {
+            rawResults = await Promise.all(files.map(async (fn) => {
+                debug('Extracting file:', fn);
+                const source = await readFile(fn, 'utf8');
+                return processFile(source, fn, opts);
+            }));
+        }
+    }
+    catch (e) {
+        if (throws) {
+            throw e;
+        }
+        else {
+            warn(String(e));
+        }
+    }
+    const formatter = await resolveBuiltinFormatter(opts.format);
+    const extractionResults = rawResults.filter((r) => !!r);
+    const extractedMessages = new Map();
+    for (const { messages } of extractionResults) {
+        for (const message of messages) {
+            const { id, description, defaultMessage } = message;
+            if (!id) {
+                const error = new Error(`[FormatJS CLI] Missing message id for message:
+${JSON.stringify(message, undefined, 2)}`);
+                if (throws) {
+                    throw error;
+                }
+                else {
+                    warn(error.message);
+                }
+                continue;
+            }
+            if (extractedMessages.has(id)) {
+                const existing = extractedMessages.get(id);
+                if (stringify(description) !== stringify(existing.description) ||
+                    defaultMessage !== existing.defaultMessage) {
+                    const error = new Error(`[FormatJS CLI] Duplicate message id: "${id}", ` +
+                        'but the `description` and/or `defaultMessage` are different.');
+                    if (throws) {
+                        throw error;
+                    }
+                    else {
+                        warn(error.message);
+                    }
+                }
+            }
+            extractedMessages.set(id, message);
+        }
+    }
+    const results = {};
+    const messages = Array.from(extractedMessages.values());
+    for (const { id, ...msg } of messages) {
+        if (flatten && msg.defaultMessage) {
+            msg.defaultMessage = printAST(hoistSelectors(parse(msg.defaultMessage)));
+        }
+        results[id] = msg;
+    }
+    if (typeof formatter.serialize === 'function') {
+        return formatter.serialize(formatter.format(results));
+    }
+    return stringify(formatter.format(results), {
+        space: 2,
+        cmp: formatter.compareMessages || undefined,
+    });
+}
+/**
+ * Extract strings from source files, also writes to a file.
+ * @param files list of files
+ * @param extractOpts extract options
+ * @returns A Promise that resolves if output file was written successfully
+ */
+export default async function extractAndWrite(files, extractOpts) {
+    const { outFile, ...opts } = extractOpts;
+    const serializedResult = (await extract(files, opts)) + '\n';
+    if (outFile) {
+        debug('Writing output file:', outFile);
+        return outputFile(outFile, serializedResult);
+    }
+    await writeStdout(serializedResult);
+}

package/lib_esnext/src/formatters/crowdin.d.ts

@@ -0,0 +1,7 @@
+import { CompileFn, FormatFn } from './default';
+export type SmartlingJson = Record<string, {
+    message: string;
+    description?: string;
+}>;
+export declare const format: FormatFn<SmartlingJson>;
+export declare const compile: CompileFn<SmartlingJson>;

package/lib_esnext/src/formatters/crowdin.js

@@ -0,0 +1,22 @@
+export const format = msgs => {
+    const results = {};
+    for (const [id, msg] of Object.entries(msgs)) {
+        results[id] = {
+            message: msg.defaultMessage,
+            description: typeof msg.description === 'string'
+                ? msg.description
+                : JSON.stringify(msg.description),
+        };
+    }
+    return results;
+};
+export const compile = msgs => {
+    const results = {};
+    for (const [id, msg] of Object.entries(msgs)) {
+        if (id === 'smartling') {
+            continue;
+        }
+        results[id] = msg.message;
+    }
+    return results;
+};

package/lib_esnext/src/formatters/default.d.ts

@@ -0,0 +1,6 @@
+import { MessageDescriptor } from '@formatjs/ts-transformer';
+export type FormatFn<T = Record<string, MessageDescriptor>> = (msgs: Record<string, MessageDescriptor>) => T;
+export type CompileFn<T = Record<string, MessageDescriptor>> = (msgs: T) => Record<string, string>;
+export type SerializeFn<T = Record<string, MessageDescriptor>> = (msgs: T) => string;
+export declare const format: FormatFn;
+export declare const compile: CompileFn;

package/lib_esnext/src/formatters/default.js

@@ -0,0 +1,8 @@
+export const format = msgs => msgs;
+export const compile = msgs => {
+    const results = {};
+    for (const k in msgs) {
+        results[k] = msgs[k].defaultMessage;
+    }
+    return results;
+};

package/lib_esnext/src/formatters/index.d.ts

@@ -0,0 +1,9 @@
+import type { Comparator } from 'json-stable-stringify';
+import { CompileFn, FormatFn, SerializeFn } from './default';
+export interface Formatter<T> {
+    serialize?: SerializeFn<T>;
+    format: FormatFn<T>;
+    compile: CompileFn<T>;
+    compareMessages?: Comparator;
+}
+export declare function resolveBuiltinFormatter(format?: string | Formatter<unknown>): Promise<any>;

package/lib_esnext/src/formatters/index.js

@@ -0,0 +1,36 @@
+import { resolve } from 'path';
+import { pathToFileURL } from 'url';
+import * as crowdin from './crowdin';
+import * as defaultFormatter from './default';
+import * as lokalise from './lokalise';
+import * as simple from './simple';
+import * as smartling from './smartling';
+import * as transifex from './transifex';
+export async function resolveBuiltinFormatter(format) {
+    if (!format) {
+        return defaultFormatter;
+    }
+    if (typeof format !== 'string') {
+        return format;
+    }
+    switch (format) {
+        case 'transifex':
+            return transifex;
+        case 'smartling':
+            return smartling;
+        case 'simple':
+            return simple;
+        case 'lokalise':
+            return lokalise;
+        case 'crowdin':
+            return crowdin;
+    }
+    try {
+        // eslint-disable-next-line import/dynamic-import-chunkname
+        return import(pathToFileURL(resolve(process.cwd(), format)).href);
+    }
+    catch (e) {
+        console.error(`Cannot resolve formatter ${format}`);
+        throw e;
+    }
+}

package/lib_esnext/src/formatters/lokalise.d.ts

@@ -0,0 +1,9 @@
+import { CompileFn, FormatFn } from './default';
+export type StructuredJson = Record<string, {
+    translation: string;
+    notes?: string;
+    context?: string;
+    limit?: string;
+}>;
+export declare const format: FormatFn<StructuredJson>;
+export declare const compile: CompileFn<StructuredJson>;

package/lib_esnext/src/formatters/lokalise.js

@@ -0,0 +1,19 @@
+export const format = msgs => {
+    const results = {};
+    for (const [id, msg] of Object.entries(msgs)) {
+        results[id] = {
+            translation: msg.defaultMessage,
+            notes: typeof msg.description === 'string'
+                ? msg.description
+                : JSON.stringify(msg.description),
+        };
+    }
+    return results;
+};
+export const compile = msgs => {
+    const results = {};
+    for (const [id, msg] of Object.entries(msgs)) {
+        results[id] = msg.translation;
+    }
+    return results;
+};

package/lib_esnext/src/formatters/simple.d.ts

@@ -0,0 +1,4 @@
+import { CompileFn, FormatFn } from './default';
+export type PhraseJson = Record<string, string>;
+export declare const format: FormatFn<PhraseJson>;
+export declare const compile: CompileFn<PhraseJson>;

package/lib_esnext/src/formatters/simple.js

@@ -0,0 +1,7 @@
+export const format = msgs => {
+    return Object.keys(msgs).reduce((all, k) => {
+        all[k] = msgs[k].defaultMessage;
+        return all;
+    }, {});
+};
+export const compile = msgs => msgs;

package/lib_esnext/src/formatters/smartling.d.ts

@@ -0,0 +1,23 @@
+import type { Comparator } from 'json-stable-stringify';
+import { CompileFn, FormatFn } from './default';
+export interface SmartlingDirectives {
+    translate_paths: [
+        {
+            path: string;
+            key: string;
+            instruction: string;
+        }
+    ];
+    variants_enabled: boolean;
+    string_format: string;
+    [k: string]: any;
+}
+export type SmartlingJson = {
+    smartling: SmartlingDirectives;
+} & Record<string, {
+    message: string;
+    description?: string;
+}>;
+export declare const format: FormatFn<SmartlingJson>;
+export declare const compareMessages: Comparator;
+export declare const compile: CompileFn<SmartlingJson>;

package/lib_esnext/src/formatters/smartling.js

@@ -0,0 +1,44 @@
+export const format = msgs => {
+    const results = {
+        smartling: {
+            translate_paths: [
+                {
+                    path: '*/message',
+                    key: '{*}/message',
+                    instruction: '*/description',
+                },
+            ],
+            variants_enabled: true,
+            string_format: 'icu',
+        },
+    };
+    for (const [id, msg] of Object.entries(msgs)) {
+        results[id] = {
+            message: msg.defaultMessage,
+            description: typeof msg.description === 'string'
+                ? msg.description
+                : JSON.stringify(msg.description),
+        };
+    }
+    return results;
+};
+export const compareMessages = (el1, el2) => {
+    // `smartling` has to be the 1st key
+    if (el1.key === 'smartling') {
+        return -1;
+    }
+    if (el2.key === 'smartling') {
+        return 1;
+    }
+    return el1.key < el2.key ? -1 : el1.key === el2.key ? 0 : 1;
+};
+export const compile = msgs => {
+    const results = {};
+    for (const [id, msg] of Object.entries(msgs)) {
+        if (id === 'smartling') {
+            continue;
+        }
+        results[id] = msg.message;
+    }
+    return results;
+};

package/lib_esnext/src/formatters/transifex.d.ts

@@ -0,0 +1,9 @@
+import { CompileFn, FormatFn } from './default';
+export type StructuredJson = Record<string, {
+    string: string;
+    developer_comment?: string;
+    context?: string;
+    character_limit?: string;
+}>;
+export declare const format: FormatFn<StructuredJson>;
+export declare const compile: CompileFn<StructuredJson>;

package/lib_esnext/src/formatters/transifex.js

@@ -0,0 +1,19 @@
+export const format = msgs => {
+    const results = {};
+    for (const [id, msg] of Object.entries(msgs)) {
+        results[id] = {
+            string: msg.defaultMessage,
+            developer_comment: typeof msg.description === 'string'
+                ? msg.description
+                : JSON.stringify(msg.description),
+        };
+    }
+    return results;
+};
+export const compile = msgs => {
+    const results = {};
+    for (const [id, msg] of Object.entries(msgs)) {
+        results[id] = msg.string;
+    }
+    return results;
+};

package/lib_esnext/src/gts_extractor.d.ts

@@ -0,0 +1 @@
+export declare function parseFile(source: string, fileName: string, options: any): void;

package/lib_esnext/src/gts_extractor.js

@@ -0,0 +1,14 @@
+import { Preprocessor } from 'content-tag';
+import { parseFile as parseHbsFile } from './hbs_extractor';
+import { parseScript } from './parse_script';
+let p = new Preprocessor();
+export function parseFile(source, fileName, options) {
+    const scriptParseFn = parseScript(options, fileName);
+    const transformedSource = p.process(source, { filename: fileName });
+    scriptParseFn(transformedSource.code);
+    // extract template from transformed source to then run through hbs processor
+    const parseResult = p.parse(source, { filename: fileName });
+    for (let parsed of parseResult) {
+        parseHbsFile(parsed.contents, fileName, options);
+    }
+}

package/lib_esnext/src/hbs_extractor.d.ts

@@ -0,0 +1 @@
+export declare function parseFile(source: string, fileName: string, options: any): void;

package/lib_esnext/src/hbs_extractor.js

@@ -0,0 +1,45 @@
+import { transform } from 'ember-template-recast';
+function extractText(node, fileName, options) {
+    if (!options.onMsgExtracted)
+        return;
+    if (!options.overrideIdFn)
+        return;
+    if (node.path.type !== 'PathExpression')
+        return;
+    if (['format-message', 'formatMessage'].includes(node.path.original)) {
+        let [first, second] = node.params;
+        if (first.type !== 'StringLiteral')
+            return;
+        let message = first?.value;
+        let desc;
+        if (second?.type === 'StringLiteral') {
+            desc = second.value?.trim().replace(/\s+/gm, ' ');
+        }
+        let defaultMessage = message?.trim().replace(/\s+/gm, ' ');
+        let id = typeof options.overrideIdFn === 'string'
+            ? options.overrideIdFn
+            : options.overrideIdFn(undefined, defaultMessage, desc, fileName);
+        options.onMsgExtracted(fileName, [
+            {
+                id: id,
+                defaultMessage: defaultMessage,
+                description: desc,
+            },
+        ]);
+    }
+}
+export function parseFile(source, fileName, options) {
+    let visitor = function () {
+        return {
+            MustacheStatement(node) {
+                extractText(node, fileName, options);
+            },
+            SubExpression(node) {
+                extractText(node, fileName, options);
+            },
+        };
+    };
+    // SAFETY: ember-template-recast's types are out of date,
+    // but it does not affect runtime
+    transform(source, visitor);
+}

package/lib_esnext/src/parse_script.d.ts

@@ -0,0 +1,7 @@
+import { Opts } from '@formatjs/ts-transformer';
+/**
+ * Invoid TypeScript module transpilation with our TS transformer
+ * @param opts Formatjs TS Transformer opt
+ * @param fn filename
+ */
+export declare function parseScript(opts: Opts, fn?: string): (source: string) => void;

package/lib_esnext/src/parse_script.js

@@ -0,0 +1,46 @@
+import { transformWithTs } from '@formatjs/ts-transformer';
+import ts from 'typescript';
+import { debug } from './console_utils';
+/**
+ * Invoid TypeScript module transpilation with our TS transformer
+ * @param opts Formatjs TS Transformer opt
+ * @param fn filename
+ */
+export function parseScript(opts, fn) {
+    return (source) => {
+        let output;
+        try {
+            debug('Using TS compiler to process file', fn);
+            output = ts.transpileModule(source, {
+                compilerOptions: {
+                    allowJs: true,
+                    target: ts.ScriptTarget.ESNext,
+                    noEmit: true,
+                    experimentalDecorators: true,
+                },
+                reportDiagnostics: true,
+                fileName: fn,
+                transformers: {
+                    before: [transformWithTs(ts, opts)],
+                },
+            });
+        }
+        catch (e) {
+            if (e instanceof Error) {
+                e.message = `Error processing file ${fn} 
+${e.message || ''}`;
+            }
+            throw e;
+        }
+        if (output.diagnostics) {
+            const errs = output.diagnostics.filter(d => d.category === ts.DiagnosticCategory.Error);
+            if (errs.length) {
+                throw new Error(ts.formatDiagnosticsWithColorAndContext(errs, {
+                    getCanonicalFileName: fileName => fileName,
+                    getCurrentDirectory: () => process.cwd(),
+                    getNewLine: () => ts.sys.newLine,
+                }));
+            }
+        }
+    };
+}

package/lib_esnext/src/pseudo_locale.d.ts

@@ -0,0 +1,22 @@
+import { MessageFormatElement } from '@formatjs/icu-messageformat-parser';
+export declare function generateXXLS(msg: string | MessageFormatElement[]): MessageFormatElement[];
+export declare function generateXXAC(msg: string | MessageFormatElement[]): MessageFormatElement[];
+export declare function generateXXHA(msg: string | MessageFormatElement[]): MessageFormatElement[];
+/**
+ * accented - Ȧȧƈƈḗḗƞŧḗḗḓ Ḗḗƞɠŀīīşħ
+ * --------------------------------
+ *
+ * This locale replaces all Latin characters with their accented equivalents, and duplicates some
+ * vowels to create roughly 30% longer strings. Strings are wrapped in markers (square brackets),
+ * which help with detecting truncation.
+ */
+export declare function generateENXA(msg: string | MessageFormatElement[]): MessageFormatElement[];
+/**
+ * bidi - ɥsıʅƃuƎ ıpıԐ
+ * -------------------
+ *
+ * This strategy replaces all Latin characters with their 180 degree rotated versions and enforces
+ * right to left text flow using Unicode UAX#9 Explicit Directional Embeddings. In this mode, the UI
+ * directionality will also be set to right-to-left.
+ */
+export declare function generateENXB(msg: string | MessageFormatElement[]): MessageFormatElement[];