@shipi18n/core 2.6.1 → 2.8.0

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @shipi18n/core
 
+## 2.8.0
+
+- New: **ICU MessageFormat validation** (P6). When a source string is an ICU plural/select message,
+  it's now checked ICU-aware:
+  - `plural-category` (warning): the translation's ICU plural is missing a plural category the
+    target language needs for everyday counts under CLDR — e.g. a Russian plural with only
+    one/other, missing few/many. Categories come from the runtime's `Intl.PluralRules` (no data
+    dep); Spanish/French "many" (compact-notation only) is deliberately not flagged.
+  - `icu-invalid` (error): the source is valid ICU but the translation no longer parses.
+  - Fixes a real false positive: ICU select sub-messages (`{he}`/`{she}`) are no longer mistaken
+    for placeholders. Adds `@formatjs/icu-messageformat-parser`.
+
+## 2.7.0
+
+- New: **YAML locale files** (`.yaml` / `.yml`) are checked alongside JSON — flat and nested trees,
+  every existing check (placeholders, plurals, missing/orphan keys, glossary). The check logic was
+  already format-agnostic; this adds parsing + file discovery. Adds one dependency (`yaml`).
+
 ## 2.6.1
 
 - Fix: the OpenAI adapter sent `max_tokens`, which current OpenAI models (gpt-5.x) reject with a

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shipi18n/core",
-  "version": "2.6.1",
+  "version": "2.8.0",
   "description": "Translation QA for i18n locale files: placeholder and plural validation, key parity, coverage, and an LLM-as-judge semantic review. Also a structure-preserving translation engine — bring your own OpenAI or Anthropic key.",
   "type": "module",
   "main": "src/index.js",
@@ -58,7 +58,9 @@
     "jest": "^29.7.0"
   },
   "dependencies": {
-    "chalk": "^5.3.0"
+    "chalk": "^5.3.0",
+    "yaml": "^2.5.0",
+    "@formatjs/icu-messageformat-parser": "^2.11.0"
   },
   "scripts": {
     "test": "NODE_OPTIONS='--experimental-vm-modules' jest",

package/src/check.js

@@ -7,6 +7,7 @@
  */
 import { flatten } from './translate.js'
 import { validatePlaceholders } from './placeholders.js'
+import { isICUControl, checkICU } from './icu.js'
 
 /**
  * vue-i18n expresses plurals as one pipe-separated string
@@ -121,28 +122,34 @@ export function checkTranslations({ source, target, targetLang = 'target', gloss
       continue
     }
 
-    const { missing, added } = validatePlaceholders(s, t)
-    if (missing.length) {
-      findings.push({
-        type: 'placeholder-missing',
-        severity: 'error',
-        path,
-        missing,
-        message: `dropped ${missing.join(', ')}`,
-        source: s,
-        translation: t,
-      })
-    }
-    if (added.length) {
-      findings.push({
-        type: 'placeholder-added',
-        severity: 'warning',
-        path,
-        added,
-        message: `unexpected ${added.join(', ')}`,
-        source: s,
-        translation: t,
-      })
+    // ICU MessageFormat (plural/select) is validated ICU-aware — the regex
+    // placeholder check would read its sub-messages as bogus placeholders.
+    if (isICUControl(s)) {
+      findings.push(...checkICU(s, t, targetLang, path))
+    } else {
+      const { missing, added } = validatePlaceholders(s, t)
+      if (missing.length) {
+        findings.push({
+          type: 'placeholder-missing',
+          severity: 'error',
+          path,
+          missing,
+          message: `dropped ${missing.join(', ')}`,
+          source: s,
+          translation: t,
+        })
+      }
+      if (added.length) {
+        findings.push({
+          type: 'placeholder-added',
+          severity: 'warning',
+          path,
+          added,
+          message: `unexpected ${added.join(', ')}`,
+          source: s,
+          translation: t,
+        })
+      }
     }
 
     const srcForms = pluralFormCount(s)

package/src/icu.js

@@ -0,0 +1,135 @@
+/**
+ * ICU MessageFormat validation (P6).
+ *
+ * Strings like `{count, plural, one {# item} other {# items}}` are ICU
+ * MessageFormat, not plain placeholders — the sub-messages `{# item}` are NOT
+ * interpolation tokens, so the ordinary placeholder check mis-reads them. When
+ * the source is an ICU control message we validate it ICU-aware instead:
+ *
+ *   - the same ARGUMENTS survive translation (dropped/added → placeholder-*),
+ *   - the translation still PARSES as ICU (malformed → icu-invalid),
+ *   - each plural has the plural CATEGORIES its target locale requires per CLDR
+ *     (missing → plural-category). CLDR categories come from the runtime's
+ *     built-in Intl.PluralRules — no data dependency, always current.
+ *
+ * plural-category is a WARNING: some projects intentionally simplify plurals,
+ * and a QA gate that fails builds on that judgment call gets uninstalled.
+ */
+import { parse, TYPE } from '@formatjs/icu-messageformat-parser'
+
+/** Does this string use ICU plural/select control syntax (vs. a plain {name})? */
+export const isICUControl = (s) =>
+  typeof s === 'string' && /\{\s*[a-zA-Z0-9_]+\s*,\s*(?:plural|selectordinal|select)\s*,/.test(s)
+
+/** Walk an ICU AST collecting argument names and plural nodes. */
+function analyze(ast, acc) {
+  for (const n of ast) {
+    if (n.type === TYPE.argument) acc.args.add(n.value)
+    if (n.type === TYPE.select || n.type === TYPE.plural) {
+      acc.args.add(n.value)
+      if (n.type === TYPE.plural) {
+        acc.plurals.push({
+          arg: n.value,
+          ordinal: n.pluralType === 'ordinal',
+          categories: Object.keys(n.options).filter((k) => !k.startsWith('=')),
+        })
+      }
+      for (const opt of Object.values(n.options)) analyze(opt.value, acc)
+    }
+    if (n.type === TYPE.tag) analyze(n.children || [], acc)
+  }
+}
+
+const parseICU = (s) => {
+  const acc = { args: new Set(), plurals: [] }
+  analyze(parse(s), acc)
+  return acc
+}
+
+/**
+ * Plural categories a locale needs for everyday INTEGER quantities, plus the
+ * always-mandatory `other`. Sampling integers (not resolvedOptions) is
+ * deliberate: Spanish/French declare a `many` category that only fires for
+ * compact notation (millions), never plain counts — flagging its absence would
+ * be pedantic noise. Integer sampling keeps the high-value cases (Russian/
+ * Arabic/Polish few·many, reachable at small counts) and drops that noise.
+ */
+function requiredCategories(lang, ordinal) {
+  try {
+    const pr = new Intl.PluralRules(lang, { type: ordinal ? 'ordinal' : 'cardinal' })
+    const cats = new Set(['other']) // ICU always requires `other`
+    for (let n = 0; n <= 200; n++) cats.add(pr.select(n))
+    return [...cats]
+  } catch {
+    return null // unknown/invalid locale tag — skip the category check
+  }
+}
+
+/**
+ * Validate an ICU source/translation pair. Returns finding objects shaped like
+ * the rest of check.js (type/severity/path/message).
+ */
+export function checkICU(source, translation, targetLang, path) {
+  const findings = []
+  let src
+  try {
+    src = parseICU(source)
+  } catch {
+    return findings // malformed SOURCE is a usage error, not a translation defect
+  }
+
+  let tr
+  try {
+    tr = parseICU(translation)
+  } catch (err) {
+    findings.push({
+      type: 'icu-invalid',
+      severity: 'error',
+      path,
+      message: `translation is not valid ICU MessageFormat: ${err.message}`,
+      source,
+      translation,
+    })
+    return findings // can't compare further against a string we can't parse
+  }
+
+  const missing = [...src.args].filter((a) => !tr.args.has(a))
+  const added = [...tr.args].filter((a) => !src.args.has(a))
+  if (missing.length)
+    findings.push({
+      type: 'placeholder-missing',
+      severity: 'error',
+      path,
+      missing: missing.map((a) => `{${a}}`),
+      message: `dropped ${missing.map((a) => `{${a}}`).join(', ')}`,
+      source,
+      translation,
+    })
+  if (added.length)
+    findings.push({
+      type: 'placeholder-added',
+      severity: 'warning',
+      path,
+      added: added.map((a) => `{${a}}`),
+      message: `unexpected ${added.map((a) => `{${a}}`).join(', ')}`,
+      source,
+      translation,
+    })
+
+  for (const p of tr.plurals) {
+    const required = requiredCategories(targetLang, p.ordinal)
+    if (!required) continue
+    const missingCats = required.filter((c) => !p.categories.includes(c))
+    if (missingCats.length)
+      findings.push({
+        type: 'plural-category',
+        severity: 'warning',
+        path,
+        message: `plural {${p.arg}} is missing CLDR ${targetLang} categor${missingCats.length > 1 ? 'ies' : 'y'} ${missingCats.join(', ')} (has ${p.categories.join(', ') || 'none'})`,
+        source,
+        translation,
+      })
+  }
+
+  return findings
+}

package/src/index.js

@@ -12,6 +12,7 @@
  */
 export { translateJSON, translateStrings, flatten, unflatten } from './translate.js'
 export { extractPlaceholders, validatePlaceholders } from './placeholders.js'
+export { isICUControl, checkICU } from './icu.js'
 export { checkTranslations } from './check.js'
 export { runCheck, runSemantic, discoverLayout, compileIgnores, statsFrom, aggregateLanguage, SEP } from './tree.js'
 export { lockId, lockEntry, lockFinding, emptyLocks, normalizeLocks, LOCKS_VERSION } from './locks.js'

package/src/reporters.js

@@ -62,6 +62,8 @@ export const RULE_META = {
   'placeholder-missing': 'A placeholder from the source string was dropped in the translation.',
   'placeholder-added': 'The translation contains a placeholder the source does not have.',
   'plural-forms': 'A pipe-separated plural lost one or more of its forms in translation.',
+  'plural-category': 'An ICU plural is missing a plural category the target language requires under CLDR.',
+  'icu-invalid': 'The source is valid ICU MessageFormat but the translation does not parse as ICU.',
   'empty-value': 'The translation of a non-empty source string is empty.',
   'untranslated': 'The translation is identical to a multi-word source string.',
   'type-mismatch': 'Source and translation values have different JSON types.',

package/src/tree.js

@@ -11,8 +11,17 @@
 import { readFileSync, readdirSync, existsSync, statSync } from 'node:fs'
 import { join, basename, resolve, relative, dirname } from 'node:path'
 import { checkTranslations } from './check.js'
+import { parse as parseYaml } from 'yaml'
 import { parseArbBundle } from './formats/arb.js'
 import { parseXcstrings } from './formats/xcstrings.js'
+
+// Locale files are JSON or YAML. The check logic is format-agnostic once the
+// file is parsed to an object, so support is entirely a parse + discovery
+// concern. `.json` is matched first so it stays the default when both exist.
+const LOCALE_EXT = /\.(json|ya?ml)$/i
+const stripExt = (name) => name.replace(LOCALE_EXT, '')
+const readData = (file) =>
+  /\.ya?ml$/i.test(file) ? parseYaml(readFileSync(file, 'utf8')) : JSON.parse(readFileSync(file, 'utf8'))
 import { flatten } from './translate.js'
 import { lockId, lockFinding } from './locks.js'
 import { reviewTranslations } from './review.js'
@@ -37,19 +46,19 @@ export function discoverLayout(inputPath, sourceLang) {
 
   if (statSync(path).isFile()) {
     const dir = resolve(path, '..')
-    const lang = basename(path).replace(/\.json$/, '')
+    const lang = stripExt(basename(path))
     return flatLayout(dir, lang)
   }
 
   const entries = readdirSync(path, { withFileTypes: true })
-  if (entries.some((e) => e.isFile() && e.name === `${sourceLang}.json`)) {
+  if (entries.some((e) => e.isFile() && LOCALE_EXT.test(e.name) && stripExt(e.name) === sourceLang)) {
     return flatLayout(path, sourceLang)
   }
   if (entries.some((e) => e.isDirectory() && e.name === sourceLang)) {
     return nestedLayout(path, sourceLang)
   }
   throw new Error(
-    `no source locale found: expected ${join(inputPath, sourceLang + '.json')} or ${join(inputPath, sourceLang)}/`
+    `no source locale found: expected ${join(inputPath, sourceLang + '.{json,yaml}')} or ${join(inputPath, sourceLang)}/`
   )
 }
 
@@ -63,18 +72,26 @@ export function discoverLayout(inputPath, sourceLang) {
 const LOCALE_NAME = /^[a-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$/
 
 export function flatLayout(dir, sourceLang) {
-  const langs = readdirSync(dir)
-    .filter((f) => f.endsWith('.json') && !f.startsWith('.'))
-    .map((f) => f.replace(/\.json$/, ''))
-    .filter((name) => LOCALE_NAME.test(name) || name === sourceLang)
-  if (!langs.includes(sourceLang)) throw new Error(`source file not found: ${join(dir, sourceLang + '.json')}`)
-  const files = (lang) => ({ translation: join(dir, `${lang}.json`) })
+  // lang → actual file path (extension resolved, since a tree may be .json or
+  // .yaml — .json wins if both somehow exist for the same lang).
+  const pathByLang = {}
+  for (const f of readdirSync(dir).sort()) {
+    if (f.startsWith('.') || !LOCALE_EXT.test(f)) continue
+    const name = stripExt(f)
+    if (!(LOCALE_NAME.test(name) || name === sourceLang)) continue
+    if (!pathByLang[name] || f.endsWith('.json')) pathByLang[name] = join(dir, f)
+  }
+  const langs = Object.keys(pathByLang)
+  if (!langs.includes(sourceLang))
+    throw new Error(`source file not found: ${join(dir, sourceLang + '.{json,yaml}')}`)
   return {
     layout: 'flat',
     dir,
     sourceLang,
-    source: files(sourceLang),
-    targets: langs.filter((l) => l !== sourceLang).map((lang) => ({ lang, files: files(lang) })),
+    source: { translation: pathByLang[sourceLang] },
+    targets: langs
+      .filter((l) => l !== sourceLang)
+      .map((lang) => ({ lang, files: { translation: pathByLang[lang] } })),
   }
 }
 
@@ -91,12 +108,15 @@ export function nestedLayout(dir, sourceLang) {
         (LOCALE_NAME.test(e.name) || e.name === sourceLang)
     )
     .map((e) => e.name)
-  const nsFiles = (lang) =>
-    Object.fromEntries(
-      readdirSync(join(dir, lang))
-        .filter((f) => f.endsWith('.json'))
-        .map((f) => [f.replace(/\.json$/, ''), join(dir, lang, f)])
-    )
+  const nsFiles = (lang) => {
+    const out = {}
+    for (const f of readdirSync(join(dir, lang)).sort()) {
+      if (!LOCALE_EXT.test(f)) continue
+      const ns = stripExt(f)
+      if (!out[ns] || f.endsWith('.json')) out[ns] = join(dir, lang, f)
+    }
+    return out
+  }
   return {
     layout: 'nested',
     dir,
@@ -182,7 +202,7 @@ export function jsonMode({ input, source, isIgnored, glossary, locks }) {
   const layout = discoverLayout(input, source)
 
   const sourceData = {}
-  for (const [ns, file] of Object.entries(layout.source)) sourceData[ns] = readJson(file) // broken source = usage error
+  for (const [ns, file] of Object.entries(layout.source)) sourceData[ns] = readData(file) // broken source = usage error
 
   const perLang = {}
   const languages = []
@@ -200,9 +220,9 @@ export function jsonMode({ input, source, isIgnored, glossary, locks }) {
       }
       let data
       try {
-        data = readJson(file)
+        data = readData(file)
       } catch (err) {
-        const findings = [{ type: 'invalid-json', severity: 'error', path: ns, message: `invalid JSON: ${err.message}` }]
+        const findings = [{ type: 'invalid-json', severity: 'error', path: ns, message: `could not parse ${rel(file)}: ${err.message}` }]
         namespaces.push({ ns, file: rel(file), findings, stats: statsFrom(findings, srcKeys, 0) })
         continue
       }