@formatjs/cli-lib 5.1.6 → 5.1.8

package/src/extract.ts

@@ -0,0 +1,273 @@
+import {warn, getStdinAsString, debug, writeStdout} from './console_utils'
+import {readFile, outputFile} from 'fs-extra'
+import {
+  interpolateName,
+  Opts,
+  MessageDescriptor,
+} from '@formatjs/ts-transformer'
+
+import {resolveBuiltinFormatter, Formatter} from './formatters'
+import stringify from 'json-stable-stringify'
+import {parseScript} from './parse_script'
+import {printAST} from '@formatjs/icu-messageformat-parser/printer'
+import {hoistSelectors} from '@formatjs/icu-messageformat-parser/manipulator'
+import {parse} from '@formatjs/icu-messageformat-parser'
+export interface ExtractionResult<M = Record<string, string>> {
+  /**
+   * List of extracted messages
+   */
+  messages: MessageDescriptor[]
+  /**
+   * Metadata extracted w/ `pragma`
+   */
+  meta?: M
+}
+
+export interface ExtractedMessageDescriptor extends MessageDescriptor {
+  /**
+   * Line number
+   */
+  line?: number
+  /**
+   * Column number
+   */
+  col?: number
+  /**
+   * Metadata extracted from pragma
+   */
+  meta?: Record<string, string>
+}
+
+export type ExtractCLIOptions = Omit<
+  ExtractOpts,
+  'overrideIdFn' | 'onMsgExtracted' | 'onMetaExtracted'
+> & {
+  /**
+   * Output File
+   */
+  outFile?: string
+  /**
+   * Ignore file glob pattern
+   */
+  ignore?: string[]
+}
+
+export type ExtractOpts = Opts & {
+  /**
+   * Whether to throw an error if we had any issues with
+   * 1 of the source files
+   */
+  throws?: boolean
+  /**
+   * Message ID interpolation pattern
+   */
+  idInterpolationPattern?: string
+  /**
+   * Whether we read from stdin instead of a file
+   */
+  readFromStdin?: boolean
+  /**
+   * Path to a formatter file that controls the shape of JSON file from `outFile`.
+   */
+  format?: string | Formatter
+  /**
+   * Whether to hoist selectors & flatten sentences
+   */
+  flatten?: boolean
+} & Pick<Opts, 'onMsgExtracted' | 'onMetaExtracted'>
+
+function calculateLineColFromOffset(
+  text: string,
+  start?: number
+): Pick<ExtractedMessageDescriptor, 'line' | 'col'> {
+  if (!start) {
+    return {line: 1, col: 1}
+  }
+  const chunk = text.slice(0, start)
+  const lines = chunk.split('\n')
+  const lastLine = lines[lines.length - 1]
+  return {line: lines.length, col: lastLine.length}
+}
+
+async function processFile(
+  source: string,
+  fn: string,
+  {idInterpolationPattern, ...opts}: Opts & {idInterpolationPattern?: string}
+) {
+  let messages: ExtractedMessageDescriptor[] = []
+  let meta: Record<string, string> | undefined
+
+  opts = {
+    ...opts,
+    additionalComponentNames: [
+      '$formatMessage',
+      ...(opts.additionalComponentNames || []),
+    ],
+    onMsgExtracted(_, msgs) {
+      if (opts.extractSourceLocation) {
+        msgs = msgs.map(msg => ({
+          ...msg,
+          ...calculateLineColFromOffset(source, msg.start),
+        }))
+      }
+      messages = messages.concat(msgs)
+    },
+    onMetaExtracted(_, m) {
+      meta = m
+    },
+  }
+
+  if (!opts.overrideIdFn && idInterpolationPattern) {
+    opts = {
+      ...opts,
+      overrideIdFn: (id, defaultMessage, description, fileName) =>
+        id ||
+        interpolateName(
+          {
+            resourcePath: fileName,
+          } as any,
+          idInterpolationPattern,
+          {
+            content: description
+              ? `${defaultMessage}#${
+                  typeof description === 'string'
+                    ? description
+                    : stringify(description)
+                }`
+              : defaultMessage,
+          }
+        ),
+    }
+  }
+
+  debug('Processing opts for %s: %s', fn, opts)
+
+  const scriptParseFn = parseScript(opts, fn)
+  if (fn.endsWith('.vue')) {
+    debug('Processing %s using vue extractor', fn)
+    const {parseFile} = await import('./vue_extractor')
+    parseFile(source, fn, scriptParseFn)
+  } else {
+    debug('Processing %s using typescript extractor', fn)
+    scriptParseFn(source)
+  }
+  debug('Done extracting %s messages: %s', fn, messages)
+  if (meta) {
+    debug('Extracted meta:', meta)
+    messages.forEach(m => (m.meta = meta))
+  }
+  return {messages, meta}
+}
+
+/**
+ * Extract strings from source files
+ * @param files list of files
+ * @param extractOpts extract options
+ * @returns messages serialized as JSON string since key order
+ * matters for some `format`
+ */
+export async function extract(
+  files: readonly string[],
+  extractOpts: ExtractOpts
+) {
+  const {throws, readFromStdin, flatten, ...opts} = extractOpts
+  let rawResults: Array<ExtractionResult | undefined>
+  if (readFromStdin) {
+    debug(`Reading input from stdin`)
+    // Read from stdin
+    if (process.stdin.isTTY) {
+      warn('Reading source file from TTY.')
+    }
+    const stdinSource = await getStdinAsString()
+    rawResults = [await processFile(stdinSource, 'dummy', opts)]
+  } else {
+    rawResults = await Promise.all(
+      files.map(async fn => {
+        debug('Extracting file:', fn)
+        try {
+          const source = await readFile(fn, 'utf8')
+          return processFile(source, fn, opts)
+        } catch (e) {
+          if (throws) {
+            throw e
+          } else {
+            warn(String(e))
+          }
+        }
+      })
+    )
+  }
+
+  const formatter = await resolveBuiltinFormatter(opts.format)
+  const extractionResults = rawResults.filter((r): r is ExtractionResult => !!r)
+
+  const extractedMessages = new Map<string, MessageDescriptor>()
+
+  for (const {messages} of extractionResults) {
+    for (const message of messages) {
+      const {id, description, defaultMessage} = message
+      if (!id) {
+        const error = new Error(
+          `[FormatJS CLI] Missing message id for message: 
+${JSON.stringify(message, undefined, 2)}`
+        )
+        if (throws) {
+          throw error
+        } else {
+          warn(error.message)
+        }
+        continue
+      }
+
+      if (extractedMessages.has(id)) {
+        const existing = extractedMessages.get(id)!
+        if (
+          stringify(description) !== stringify(existing.description) ||
+          defaultMessage !== existing.defaultMessage
+        ) {
+          const error = new Error(
+            `[FormatJS CLI] Duplicate message id: "${id}", ` +
+              'but the `description` and/or `defaultMessage` are different.'
+          )
+          if (throws) {
+            throw error
+          } else {
+            warn(error.message)
+          }
+        }
+      }
+      extractedMessages.set(id, message)
+    }
+  }
+  const results: Record<string, Omit<MessageDescriptor, 'id'>> = {}
+  const messages = Array.from(extractedMessages.values())
+  for (const {id, ...msg} of messages) {
+    if (flatten && msg.defaultMessage) {
+      msg.defaultMessage = printAST(hoistSelectors(parse(msg.defaultMessage)))
+    }
+    results[id] = msg
+  }
+  return stringify(formatter.format(results), {
+    space: 2,
+    cmp: formatter.compareMessages || undefined,
+  })
+}
+
+/**
+ * Extract strings from source files, also writes to a file.
+ * @param files list of files
+ * @param extractOpts extract options
+ * @returns A Promise that resolves if output file was written successfully
+ */
+export default async function extractAndWrite(
+  files: readonly string[],
+  extractOpts: ExtractCLIOptions
+) {
+  const {outFile, ...opts} = extractOpts
+  const serializedResult = (await extract(files, opts)) + '\n'
+  if (outFile) {
+    debug('Writing output file:', outFile)
+    return outputFile(outFile, serializedResult)
+  }
+  await writeStdout(serializedResult)
+}

package/src/formatters/crowdin.ts

@@ -0,0 +1,34 @@
+import {CompileFn, FormatFn} from './default'
+
+export type SmartlingJson = Record<
+  string,
+  {
+    message: string
+    description?: string
+  }
+>
+
+export const format: FormatFn<SmartlingJson> = msgs => {
+  const results: SmartlingJson = {}
+  for (const [id, msg] of Object.entries(msgs)) {
+    results[id] = {
+      message: msg.defaultMessage!,
+      description:
+        typeof msg.description === 'string'
+          ? msg.description
+          : JSON.stringify(msg.description),
+    }
+  }
+  return results
+}
+
+export const compile: CompileFn<SmartlingJson> = msgs => {
+  const results: Record<string, string> = {}
+  for (const [id, msg] of Object.entries(msgs)) {
+    if (id === 'smartling') {
+      continue
+    }
+    results[id] = msg.message
+  }
+  return results
+}

package/src/formatters/default.ts

@@ -0,0 +1,19 @@
+import {MessageDescriptor} from '@formatjs/ts-transformer'
+
+export type FormatFn<T = Record<string, MessageDescriptor>> = (
+  msgs: Record<string, MessageDescriptor>
+) => T
+
+export type CompileFn<T = Record<string, MessageDescriptor>> = (
+  msgs: T
+) => Record<string, string>
+
+export const format: FormatFn = msgs => msgs
+
+export const compile: CompileFn = msgs => {
+  const results: Record<string, string> = {}
+  for (const k in msgs) {
+    results[k] = msgs[k].defaultMessage!
+  }
+  return results
+}

package/src/formatters/index.ts

@@ -0,0 +1,46 @@
+import * as defaultFormatter from './default'
+import {FormatFn, CompileFn} from './default'
+import * as transifex from './transifex'
+import * as smartling from './smartling'
+import * as simple from './simple'
+import * as lokalise from './lokalise'
+import * as crowdin from './crowdin'
+import {Comparator} from 'json-stable-stringify'
+import {resolve} from 'path'
+import {pathToFileURL} from 'url'
+
+export interface Formatter {
+  format: FormatFn
+  compile: CompileFn
+  compareMessages?: Comparator
+}
+
+export async function resolveBuiltinFormatter(
+  format?: string | Formatter
+): Promise<any> {
+  if (!format) {
+    return defaultFormatter
+  }
+  if (typeof format !== 'string') {
+    return format
+  }
+  switch (format) {
+    case 'transifex':
+      return transifex
+    case 'smartling':
+      return smartling
+    case 'simple':
+      return simple
+    case 'lokalise':
+      return lokalise
+    case 'crowdin':
+      return crowdin
+  }
+  try {
+    // eslint-disable-next-line import/dynamic-import-chunkname
+    return import(pathToFileURL(resolve(process.cwd(), format)).href)
+  } catch (e) {
+    console.error(`Cannot resolve formatter ${format}`)
+    throw e
+  }
+}

package/src/formatters/lokalise.ts

@@ -0,0 +1,33 @@
+import {CompileFn, FormatFn} from './default'
+
+export type StructuredJson = Record<
+  string,
+  {
+    translation: string
+    notes?: string
+    context?: string
+    limit?: string
+  }
+>
+
+export const format: FormatFn<StructuredJson> = msgs => {
+  const results: StructuredJson = {}
+  for (const [id, msg] of Object.entries(msgs)) {
+    results[id] = {
+      translation: msg.defaultMessage!,
+      notes:
+        typeof msg.description === 'string'
+          ? msg.description
+          : JSON.stringify(msg.description),
+    }
+  }
+  return results
+}
+
+export const compile: CompileFn<StructuredJson> = msgs => {
+  const results: Record<string, string> = {}
+  for (const [id, msg] of Object.entries(msgs)) {
+    results[id] = msg.translation
+  }
+  return results
+}

package/src/formatters/simple.ts

@@ -0,0 +1,12 @@
+import {CompileFn, FormatFn} from './default'
+
+export type PhraseJson = Record<string, string>
+
+export const format: FormatFn<PhraseJson> = msgs => {
+  return Object.keys(msgs).reduce((all: PhraseJson, k) => {
+    all[k] = msgs[k].defaultMessage!
+    return all
+  }, {})
+}
+
+export const compile: CompileFn<PhraseJson> = msgs => msgs

package/src/formatters/smartling.ts

@@ -0,0 +1,73 @@
+import {Comparator} from 'json-stable-stringify'
+import {CompileFn, FormatFn} from './default'
+
+export interface SmartlingDirectives {
+  translate_paths: [
+    {
+      path: string
+      key: string
+      instruction: string
+    }
+  ]
+  variants_enabled: boolean
+  string_format: string
+  [k: string]: any
+}
+
+export type SmartlingJson = {
+  smartling: SmartlingDirectives
+} & Record<
+  string,
+  {
+    message: string
+    description?: string
+  }
+>
+
+export const format: FormatFn<SmartlingJson> = msgs => {
+  const results: SmartlingJson = {
+    smartling: {
+      translate_paths: [
+        {
+          path: '*/message',
+          key: '{*}/message',
+          instruction: '*/description',
+        },
+      ],
+      variants_enabled: true,
+      string_format: 'icu',
+    },
+  } as any
+  for (const [id, msg] of Object.entries(msgs)) {
+    results[id] = {
+      message: msg.defaultMessage!,
+      description:
+        typeof msg.description === 'string'
+          ? msg.description
+          : JSON.stringify(msg.description),
+    }
+  }
+  return results
+}
+
+export const compareMessages: Comparator = (el1, el2) => {
+  // `smartling` has to be the 1st key
+  if (el1.key === 'smartling') {
+    return -1
+  }
+  if (el2.key === 'smartling') {
+    return 1
+  }
+  return el1.key < el2.key ? -1 : el1.key === el2.key ? 0 : 1
+}
+
+export const compile: CompileFn<SmartlingJson> = msgs => {
+  const results: Record<string, string> = {}
+  for (const [id, msg] of Object.entries(msgs)) {
+    if (id === 'smartling') {
+      continue
+    }
+    results[id] = msg.message
+  }
+  return results
+}

package/src/formatters/transifex.ts

@@ -0,0 +1,33 @@
+import {CompileFn, FormatFn} from './default'
+
+export type StructuredJson = Record<
+  string,
+  {
+    string: string
+    developer_comment?: string
+    context?: string
+    character_limit?: string
+  }
+>
+
+export const format: FormatFn<StructuredJson> = msgs => {
+  const results: StructuredJson = {}
+  for (const [id, msg] of Object.entries(msgs)) {
+    results[id] = {
+      string: msg.defaultMessage!,
+      developer_comment:
+        typeof msg.description === 'string'
+          ? msg.description
+          : JSON.stringify(msg.description),
+    }
+  }
+  return results
+}
+
+export const compile: CompileFn<StructuredJson> = msgs => {
+  const results: Record<string, string> = {}
+  for (const [id, msg] of Object.entries(msgs)) {
+    results[id] = msg.string
+  }
+  return results
+}

package/src/parse_script.ts

@@ -0,0 +1,49 @@
+import {Opts, transformWithTs} from '@formatjs/ts-transformer'
+import ts from 'typescript'
+import {debug} from './console_utils'
+/**
+ * Invoid TypeScript module transpilation with our TS transformer
+ * @param opts Formatjs TS Transformer opt
+ * @param fn filename
+ */
+export function parseScript(opts: Opts, fn?: string) {
+  return (source: string) => {
+    let output
+    try {
+      debug('Using TS compiler to process file', fn)
+      output = ts.transpileModule(source, {
+        compilerOptions: {
+          allowJs: true,
+          target: ts.ScriptTarget.ESNext,
+          noEmit: true,
+          experimentalDecorators: true,
+        },
+        reportDiagnostics: true,
+        fileName: fn,
+        transformers: {
+          before: [transformWithTs(ts, opts)],
+        },
+      })
+    } catch (e) {
+      if (e instanceof Error) {
+        e.message = `Error processing file ${fn} 
+${e.message || ''}`
+      }
+      throw e
+    }
+    if (output.diagnostics) {
+      const errs = output.diagnostics.filter(
+        d => d.category === ts.DiagnosticCategory.Error
+      )
+      if (errs.length) {
+        throw new Error(
+          ts.formatDiagnosticsWithColorAndContext(errs, {
+            getCanonicalFileName: fileName => fileName,
+            getCurrentDirectory: () => process.cwd(),
+            getNewLine: () => ts.sys.newLine,
+          })
+        )
+      }
+    }
+  }
+}

package/src/pseudo_locale.ts

@@ -0,0 +1,113 @@
+import {
+  parse,
+  MessageFormatElement,
+  TYPE,
+  isLiteralElement,
+  isPluralElement,
+  isSelectElement,
+  isTagElement,
+} from '@formatjs/icu-messageformat-parser'
+
+export function generateXXLS(
+  msg: string | MessageFormatElement[]
+): MessageFormatElement[] {
+  const ast = typeof msg === 'string' ? parse(msg) : msg
+  const lastChunk = ast[ast.length - 1]
+  if (lastChunk && isLiteralElement(lastChunk)) {
+    lastChunk.value += 'SSSSSSSSSSSSSSSSSSSSSSSSS'
+    return ast
+  }
+  return [...ast, {type: TYPE.literal, value: 'SSSSSSSSSSSSSSSSSSSSSSSSS'}]
+}
+
+export function generateXXAC(
+  msg: string | MessageFormatElement[]
+): MessageFormatElement[] {
+  const ast = typeof msg === 'string' ? parse(msg) : msg
+  ast.forEach(el => {
+    if (isLiteralElement(el)) {
+      el.value = el.value.toUpperCase()
+    } else if (isPluralElement(el) || isSelectElement(el)) {
+      for (const opt of Object.values(el.options)) {
+        generateXXAC(opt.value)
+      }
+    } else if (isTagElement(el)) {
+      generateXXAC(el.children)
+    }
+  })
+  return ast
+}
+
+export function generateXXHA(
+  msg: string | MessageFormatElement[]
+): MessageFormatElement[] {
+  const ast = typeof msg === 'string' ? parse(msg) : msg
+  const firstChunk = ast.shift()
+  if (firstChunk && isLiteralElement(firstChunk)) {
+    firstChunk.value = '[javascript]' + firstChunk.value
+    return [firstChunk, ...ast]
+  }
+  return [{type: TYPE.literal, value: '[javascript]'}, ...ast]
+}
+
+const ASCII = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+const ACCENTED_ASCII = 'âḃćḋèḟĝḫíĵǩĺṁńŏṗɋŕśṭůṿẘẋẏẓḀḂḈḊḔḞḠḢḬĴḴĻḾŊÕṔɊŔṠṮŨṼẄẌŸƵ'
+
+export function generateENXA(
+  msg: string | MessageFormatElement[]
+): MessageFormatElement[] {
+  const ast = typeof msg === 'string' ? parse(msg) : msg
+  ast.forEach(el => {
+    if (isLiteralElement(el)) {
+      el.value = el.value
+        .split('')
+        .map(c => {
+          const i = ASCII.indexOf(c)
+          if (i < 0) {
+            return c
+          }
+          return ACCENTED_ASCII[i]
+        })
+        .join('')
+    } else if (isPluralElement(el) || isSelectElement(el)) {
+      for (const opt of Object.values(el.options)) {
+        generateENXA(opt.value)
+      }
+    } else if (isTagElement(el)) {
+      generateENXA(el.children)
+    }
+  })
+  return ast
+}
+
+export function generateENXB(
+  msg: string | MessageFormatElement[]
+): MessageFormatElement[] {
+  const ast = typeof msg === 'string' ? parse(msg) : msg
+  ast.forEach(el => {
+    if (isLiteralElement(el)) {
+      const pseudoString = el.value
+        .split('')
+        .map((c, index) => {
+          const i = ASCII.indexOf(c)
+          const canPad = (index + 1) % 3 === 0
+
+          if (i < 0) {
+            return c
+          }
+
+          return canPad ? ACCENTED_ASCII[i].repeat(3) : ACCENTED_ASCII[i]
+        })
+        .join('')
+
+      el.value = `[!! ${pseudoString} !!]`
+    } else if (isPluralElement(el) || isSelectElement(el)) {
+      for (const opt of Object.values(el.options)) {
+        generateENXB(opt.value)
+      }
+    } else if (isTagElement(el)) {
+      generateENXB(el.children)
+    }
+  })
+  return ast
+}