@freckle/i18n-scripts 1.0.0

package/src/check-i18n-variables

@@ -0,0 +1,133 @@
+#!/usr/bin/env node
+//
+// Checks all locize keys (in the codebase) with known variables against key
+// values in locize ensuring that variables in locize refer to known vars.
+//
+const {promises: fs} = require('fs')
+const p = require('path')
+const i18next = require('i18next')
+const ICU = require('i18next-icu')
+const HttpBackend = require('i18next-http-backend')
+const {IntlMessageFormat} = require('intl-messageformat')
+
+const {
+  extractTranslationKeysAndVariables,
+  KNOWN_VARIABLES_TAG
+} = require('./translation-extraction.js')
+const {getFiles, filterJsFiles, supportedLngs} = require('./helpers.js')
+
+const [_nodeBin, _scriptPath, ...projects] = process.argv
+
+if (projects.length < 1) {
+  console.log('Usage: check-i18n-variables <path>')
+  return
+}
+
+// TODO: Pass in via ENV?
+const ns = ['school', 'student', 'console', 'classroom', 'common']
+
+const translationKeysWithKnownVariablesInProject = async project => {
+  const files = await getFiles(project)
+  const jsFiles = files.filter(filterJsFiles)
+  const results = []
+
+  jsFiles.forEach(({path}) => {
+    extractTranslationKeysAndVariables(path)
+      .filter(({tag}) => tag === KNOWN_VARIABLES_TAG)
+      .forEach(keyWithVars => results.push(keyWithVars))
+  })
+
+  return results
+}
+
+// The AST is undocumented, however, its type definitions are here
+//    https://github.com/formatjs/formatjs/blob/82a2e5d07f9a999624ffb6387a58464f9a5f7399/packages/icu-messageformat-parser/types.ts#L122-L131
+const extractVariablesFromICUAst = ast => {
+  // Some AST field values are `null`
+  if (!ast) {
+    return []
+  }
+
+  if (ast.constructor === Array) {
+    return ast.flatMap(extractVariablesFromICUAst)
+  }
+  if (ast.constructor === Object) {
+    const subVariables = Object.values(ast).flatMap(extractVariablesFromICUAst)
+    // `type`s that aren't 0 (string literals) are variables when `.value` exists
+    const hasVariable = ast.type && typeof ast.type === 'number' && ast.type !== 0 && ast.value
+
+    return (hasVariable ? [ast.value] : []).concat(subVariables)
+  }
+
+  return []
+}
+
+const main = async () => {
+  const t = await i18next
+    .use(ICU)
+    .use(HttpBackend)
+    .init({
+      supportedLngs,
+      allowMultiLoading: true,
+      keySeparator: false,
+      ns,
+      backend: {
+        loadPath: 'https://translations.freckle.com/latest/{{lng}}/{{ns}}'
+      }
+    })
+
+  let anyErrors = false
+
+  // Project key extraction is not cheap so cache per project
+  const keysPerProject = {}
+
+  await Promise.all(
+    projects.map(
+      async project =>
+        (keysPerProject[project] = await translationKeysWithKnownVariablesInProject(project))
+    )
+  )
+
+  for (const lang of supportedLngs) {
+    await i18next.changeLanguage(lang)
+    const translations = i18next.getDataByLanguage(lang)
+
+    for (const [project, keys] of Object.entries(keysPerProject)) {
+      for (const {i18nKey, variables: codeVariables} of keys) {
+        const keyValue =
+          (translations[project] && translations[project][i18nKey]) || translations.common[i18nKey]
+
+        try {
+          if (keyValue) {
+            const locizeVariables = extractVariablesFromICUAst(
+              new IntlMessageFormat(keyValue, lang, {}, {ignoreTag: true}).getAst()
+            )
+
+            for (const locizeVariable of locizeVariables) {
+              if (!codeVariables.includes(locizeVariable)) {
+                anyErrors = true
+                console.error(
+                  `${project}:${lang}:${i18nKey} does not pass the variable ${locizeVariable}`
+                )
+              }
+            }
+          }
+        } catch (e) {
+          // We have another script that checks that `IntlMessageFormat` can
+          // successfully parse the ICU text so let's not fail for that reason.
+        }
+      }
+    }
+  }
+
+  if (anyErrors) {
+    console.error(
+      'The above keys have variables not passed by the code and need to be fixed in Locize.'
+    )
+    process.exit(1)
+  } else {
+    process.exit(0)
+  }
+}
+
+main()

package/src/check-i18n-variables.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+const {spawn} = require('node:child_process')
+const path = require('path')
+
+const [_nodeBin, scriptPath, ...args] = process.argv
+
+const proc = spawn(`${path.dirname(scriptPath)}/check-i18n-variables`, args)
+
+proc.stdout.on('data', data => {
+  process.stdout.write(data)
+})
+
+proc.stderr.on('data', data => {
+  process.stderr.write(data)
+})

package/src/check-i18next-icu-parser

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+// @flow
+//
+// Checks all keys in Locize to make sure that they can be parsed by the `i18next` `t` function
+//
+const i18next = require('i18next')
+const ICU = require('i18next-icu')
+const HttpBackend = require('i18next-http-backend')
+const {supportedLngs} = require('./helpers.js')
+
+const VALID_NAMESPACES = ['school', 'student', 'console', 'classroom', 'common']
+
+const ns = process.argv.slice(2)
+
+for (const arg of ns) {
+  if (!VALID_NAMESPACES.some(ns => ns === arg)) {
+    throw new Error(`Invalid arguments: ${arg} is not a namespace`)
+  }
+}
+
+
+const main = async () => {
+  const t = await i18next
+    .use(ICU)
+    .use(HttpBackend)
+    .init({
+      supportedLngs,
+      allowMultiLoading: true,
+      keySeparator: false,
+      ns,
+      backend: {
+        loadPath: 'https://translations.freckle.com/latest/{{lng}}/{{ns}}'
+      }
+    })
+
+  let anyErrors = false
+
+  for (const l of supportedLngs) {
+    await i18next.changeLanguage(l)
+    for (const n of ns) {
+      for (const key of Object.keys(i18next.getDataByLanguage(l)[n])) {
+        try {
+          t(`${n}:${key}`)
+        } catch (error) {
+          if (!isInterpolationError(error)) {
+            anyErrors = true
+            console.error(`${n}:${l}:${key}`)
+          }
+        }
+      }
+    }
+  }
+
+  if (anyErrors) {
+    console.error('The above keys cannot be parsed as ICU and need to be fixed in Locize.')
+    process.exit(1)
+  } else {
+    process.exit(0)
+  }
+}
+
+const isInterpolationError = error => /^The intl string context variable/.test(error.message)
+
+main()

package/src/check-i18next-icu-parser.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+const {spawn} = require('node:child_process')
+const path = require('path')
+
+const [_nodeBin, scriptPath, ...args] = process.argv
+
+const proc = spawn(`${path.dirname(scriptPath)}/check-i18next-icu-parser`, args)
+
+proc.stdout.on('data', data => {
+  process.stdout.write(data)
+})
+
+proc.stderr.on('data', data => {
+  process.stderr.write(data)
+})

package/src/check-missing-translations

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+set -eu
+
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+
+translation_keys_in_code() {
+  local project_path=$1
+
+  git ls-files "$project_path" |
+    grep -E '\.(js|ts|tsx)$' |
+    grep -v '\.min\.js$' |
+    "$script_dir/translation-keys"
+}
+
+translation_keys_in_locize() {
+  local namespace=$1
+
+  curl --silent --fail --show-error \
+    "https://translations.freckle.com/latest/en/$namespace" |
+    jq --raw-output 'keys[]'
+}
+
+empty_translation_keys_in_locize() {
+  local namespace=$1
+
+  curl --silent --fail --show-error \
+    "https://translations.freckle.com/latest/en/$namespace" |
+    jq --raw-output '[to_entries | .[] | select(.key | test("^[A-Z_]+$")) | select(.key == .value)] | .[] | .key' |
+    grep -xv 'OK' | grep -xv 'ELA' # Whitelist some common translation keys
+}
+
+translation_keys_only_in_code() {
+  local project_path=$1 namespace=$2
+
+  comm -23 \
+    <(translation_keys_in_code "$project_path" | sort -u) \
+    <(translation_keys_in_locize "$namespace" | sort -u)
+}
+
+empty_translation_keys_used_in_locize() {
+  local project_path=$1 namespace=$2
+
+  comm -12 \
+    <(translation_keys_in_code "$project_path" | sort -u) \
+    <(empty_translation_keys_in_locize "$namespace" | sort -u)
+}
+
+if (($# < 1)); then
+  echo "Usage: check-missing-translations namespace [project/path]" >&2
+  exit 64
+fi
+
+namespace=${1}
+project_path=${2-.}
+
+tmp=$(mktemp)
+trap 'rm -f -- "$tmp"' EXIT
+
+# Check which keys are present in the frontend but not Locize
+translation_keys_only_in_code "$project_path" "$namespace" | sed 's/$/ not present/' | tee "$tmp"
+
+# Check which keys are present in both the frontend and Locize but are untranslated
+empty_translation_keys_used_in_locize "$project_path" "$namespace" | sed 's/$/ not translated/' | tee -a "$tmp"
+
+if [ -s "$tmp" ]; then
+  echo
+  echo "The above keys either need to be added to Locize or translated" >&2
+  exit 1
+fi

package/src/check-missing-translations.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+const {spawn} = require('node:child_process')
+const path = require('path')
+
+const [_nodeBin, scriptPath, ...args] = process.argv
+
+const proc = spawn(`${path.dirname(scriptPath)}/check-missing-translations`, args)
+
+proc.stdout.on('data', data => {
+  process.stdout.write(data)
+})
+
+proc.stderr.on('data', data => {
+  process.stderr.write(data)
+})

package/src/helpers.js

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+const {promises: fs} = require('fs')
+const p = require('path')
+
+// Modified from https://dev.to/leonard/get-files-recursive-with-the-node-js-file-system-fs-2n7o
+async function getFiles(path) {
+  const allEntries = await fs.readdir(path, {withFileTypes: true})
+
+  const entries = allEntries.filter(file => file.name !== 'node_modules' && file.name !== '.cache')
+
+  const files = entries
+    .filter(file => !file.isDirectory())
+    .map(file => ({...file, path: p.join(path, file.name)}))
+
+  const folders = entries.filter(folder => folder.isDirectory())
+
+  for (const folder of folders) {
+    files.push(...(await getFiles(p.join(path, folder.name))))
+  }
+
+  return files
+}
+
+module.exports.getFiles = getFiles
+
+const filterJsFiles = f =>
+  (f.name.endsWith('.js') || f.name.endsWith('.ts') || f.name.endsWith('.tsx')) &&
+  !f.name.endsWith('min.js')
+
+module.exports.filterJsFiles = filterJsFiles
+
+const supportedLngs = ['es', 'en', 'en-GB']
+
+module.exports.supportedLngs = supportedLngs

package/src/translation-extraction.js

@@ -0,0 +1,141 @@
+const fs = require('fs')
+const parser = require('flow-parser')
+const walk = require('esprima-walk')
+
+const isIdentifierT = node => node && node.type === 'Identifier' && node.name === 't'
+
+const isTransTag = node =>
+  node.type === 'JSXElement' &&
+  node.openingElement.type === 'JSXOpeningElement' &&
+  node.openingElement.name.name === 'Trans'
+
+const isJsxAttribute = attr => node =>
+  node && node.type === 'JSXAttribute' && node.name.name === attr
+
+const naiveLooksLikeApplication = (content, node, previousNode) =>
+  content.slice(previousNode.range[1], node.range[0]).trim() === '('
+
+const looksLikeTApplication = (content, node, previousNode) =>
+  node.type === 'Literal' &&
+  isIdentifierT(previousNode) &&
+  naiveLooksLikeApplication(content, node, previousNode)
+
+const isObjectWithSimpleProperties = node =>
+  node.type === 'ObjectExpression' &&
+  node.properties.every(p => p.type === 'Property' && p.key.type === 'Identifier')
+
+const KNOWN_VARIABLES_TAG = 'i18n-key-with-known-variables'
+const UNKNOWN_VARIABLES_TAG = 'i18n-key-with-unknown-variables'
+
+const knownVariables = (source, i18nKey, objectWithSimpleProperties) => ({
+  tag: KNOWN_VARIABLES_TAG,
+  source,
+  i18nKey,
+  variables: objectWithSimpleProperties.properties.map(p => p.key.name)
+})
+
+const knownNoVariables = (source, i18nKey) => ({
+  tag: KNOWN_VARIABLES_TAG,
+  source,
+  i18nKey,
+  variables: []
+})
+
+const unknownVariables = (source, i18nKey) => ({
+  tag: UNKNOWN_VARIABLES_TAG,
+  source,
+  i18nKey
+})
+
+module.exports.KNOWN_VARIABLES_TAG = KNOWN_VARIABLES_TAG
+module.exports.UNKNOWN_VARIABLES_TAG = UNKNOWN_VARIABLES_TAG
+
+// Attempt to find all translations keys and their provided variables in use in JS files.
+//
+// Methodology:
+//
+//   Locate any
+//     - Literal nodes that immediately follow an Identifier node named t, and
+//       output their value, or
+//     - Trans component instances and output their i18nKey values
+//
+//   If possible, output any variables (if possible) used by the keys. This is
+//   done with no further analysis than looking for a variable object and
+//     - if one's not found, report no variables
+//     - if one's found and is a simple object, report its properties
+//     - if one's found and is a more complex object, report that the variables
+//       are not known
+//
+// Yields keys with variables as encountered without sorting or de-duplication.
+module.exports.extractTranslationKeysAndVariables = path => {
+  const content = fs.readFileSync(path, {encoding: 'utf8'})
+  const ast = parser.parse(content)
+
+  const results = []
+  let previousNode = null
+
+  // Possible `tag` values are
+  //  - 'finding-i18n-key' which means we're trying to find an i18n key
+  //  - 'found-t-application' which means we found a key like `t('KEY'` and we
+  //    ought to check the next token to see if it's a simple object literal
+  let state = {tag: 'finding-i18n-key'}
+
+  walk(ast, node => {
+    switch (state.tag) {
+      case 'finding-i18n-key': {
+        if (looksLikeTApplication(content, node, previousNode)) {
+          state = {tag: 'found-t-application', key: node.value}
+        } else if (isTransTag(node)) {
+          const i18nKey = node.openingElement.attributes.find(isJsxAttribute('i18nKey'))
+          const values = node.openingElement.attributes.find(isJsxAttribute('values'))
+          const hasCount = node.openingElement.attributes.some(isJsxAttribute('count'))
+
+          // `Trans` keys can have a `count` property which ends up being a
+          // variable. If we have known variables we've gotta account for it...
+          const accountForCount = i18nKeyVithVariables => ({
+            ...i18nKeyVithVariables,
+            variables: i18nKeyVithVariables.variables.concat(hasCount ? ['count'] : [])
+          })
+
+          if (
+            i18nKey.value.value &&
+            values &&
+            values.value.type === 'JSXExpressionContainer' &&
+            isObjectWithSimpleProperties(values.value.expression)
+          ) {
+            results.push(
+              accountForCount(
+                knownVariables('trans-component', i18nKey.value.value, values.value.expression)
+              )
+            )
+          } else if (i18nKey.value.value && !values) {
+            results.push(accountForCount(knownNoVariables('trans-component', i18nKey.value.value)))
+          } else if (i18nKey.value.value) {
+            results.push(unknownVariables('trans-component', i18nKey.value.value))
+          }
+        }
+        break
+      }
+      case 'found-t-application': {
+        if (isObjectWithSimpleProperties(node)) {
+          results.push(knownVariables('t-application', state.key, node))
+        } else if (typeof node === 'number') {
+          // No object after the key means no variables. For some reason this is an integer
+          results.push(knownNoVariables('t-application', state.key))
+        } else {
+          results.push(unknownVariables('t-application', state.key))
+        }
+        state = {tag: 'finding-i18n-key'}
+        break
+      }
+      case 'found-trans-component': {
+        state = {tag: 'finding-i18n-key'}
+        break
+      }
+    }
+
+    previousNode = node
+  })
+
+  return results
+}

package/src/translation-keys

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+//
+// Attempt to find all translations keys in use in JS files passed on stdin.
+//
+// Usage:
+//
+//   git ls-files |
+//      grep '\.js$' |
+//      grep -v '\.min\.js$' |
+//     ./scripts/translation-keys
+//
+// We stream out the keys as seen. We do no sorting or de-duplication.
+//
+const {extractTranslationKeysAndVariables} = require('./translation-extraction.js')
+const readline = require('readline')
+
+const interface = readline.createInterface({
+  input: process.stdin,
+  output: process.stdout,
+  terminal: false
+})
+
+interface.on('line', path => {
+  extractTranslationKeysAndVariables(path).forEach(({i18nKey}) =>
+    console.log(i18nKey)
+  )
+})