@likolabs/i18nmd 0.1.1 → 0.3.0

package/bin/i18nmd.mjs

@@ -6,6 +6,7 @@ import { spawn } from 'node:child_process';
 import { parseCatalog, serializeCatalog, parseLanguageFiles, languageFromFilename, renameToken, textOf, divisionOf, namespaceFor, accessorFor } from '../lib/catalog.mjs';
 import { generateModule, generateModules, divisionFile, runtimeTypes } from '../lib/compiler.mjs';
 import { extractSource, missingDivisionImports } from '../lib/extractor.mjs';
+import { extractHtml, renderHtml } from '../lib/html.mjs';
 import { lockPathFor, findLock, namedRoot, readLock, serializeLock, report, syncLock, markCurrent } from '../lib/lock.mjs';
 import { resolveLanguage, topLanguages, RANKED } from '../lib/languages.mjs';
 import { llmConfig, translateLanguage } from '../lib/llm.mjs';
@@ -32,12 +33,16 @@ Keep files in shape
                                            One Markdown file with every language, and back.
 
 Build
+  render <site> [--out dist] [--url https://example.com]
+                                           Static HTML pages, one copy per language (/haw/…).
   compile [--out src/i18n] [--target ts|js|json|python] [--skip division,...] [--eager]
   extract <src...> [--out translations/<division>] [--in-place] [--runtime src/i18n/i18n]
           [--source en] [--locale-expr locale]
 
 Other tools
   import <files.json...> --from ${FORMATS.join('|')} [--out translations] [--source en]
+      --merge                              Add source-language messages to the source file in
+                                           --out, keeping its tokens, translations and the lock.
   export --to ${FORMATS.join('|')} --out <dir>
   import-python <module.py> --out <dir>
   languages                                The ranked language list used by --top.
@@ -60,14 +65,16 @@ function python(args) {
   });
 }
 
-async function filesAt(input) {
-  if (!(await stat(input)).isDirectory()) return /\.(?:[cm]?[jt]sx?)$/.test(input) ? [input] : [];
+// Source files under input; html: also .html pages (for extract).
+async function filesAt(input, { html = false } = {}) {
+  const wanted = html ? /\.(?:[cm]?[jt]sx?|html?)$/ : /\.(?:[cm]?[jt]sx?)$/;
+  if (!(await stat(input)).isDirectory()) return wanted.test(input) ? [input] : [];
   const files = [];
   for (const entry of (await readdir(input, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {
     if (entry.name.startsWith('.') || ['node_modules', 'dist', 'build', 'vendor', 'coverage'].includes(entry.name)) continue;
     const file = path.join(input, entry.name);
-    if (entry.isDirectory()) files.push(...await filesAt(file));
-    else if (entry.isFile() && /\.(?:[cm]?[jt]sx?)$/.test(file) && !/\.(test|spec)\.[jt]sx?$/.test(file) && !/\.d\.[cm]?ts$/.test(file)) files.push(file);
+    if (entry.isDirectory()) files.push(...await filesAt(file, { html }));
+    else if (entry.isFile() && wanted.test(file) && !/\.(test|spec)\.[jt]sx?$/.test(file) && !/\.d\.[cm]?ts$/.test(file)) files.push(file);
   }
   return files;
 }
@@ -117,8 +124,8 @@ async function open(input, options) {
   const strict = !!options.strict;
   const raw = directory ? await readLanguageFiles(input) : { [input]: await readFile(input, 'utf8') };
   const catalog = directory
-    ? parseLanguageFiles(raw, { source: options.source || lock.source, syntax: options.syntax || 'icu', strict, prefix: await divisionPrefix(input, lockFile) })
-    : parseCatalog(raw[input], { filename: input, syntax: options.syntax, allowIncomplete: !strict, lenient: !strict });
+    ? parseLanguageFiles(raw, { source: options.source || lock.source, syntax: options.syntax || 'icu', strict, prefix: await divisionPrefix(input, lockFile), formatters: lock.formatters })
+    : parseCatalog(raw[input], { filename: input, syntax: options.syntax, allowIncomplete: !strict, lenient: !strict, formatters: lock.formatters });
   catalog.orphans ||= {};
   catalog.divisions ||= [''];
   let queue = Promise.resolve();
@@ -218,7 +225,7 @@ async function main() {
   if (!command || ['help', '--help', '-h'].includes(command)) { console.log(help); return; }
   if (['--version', '-v', 'version'].includes(command)) { console.log(JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')).version); return; }
   const aliases = { locale: 'source', language: 'locale-expr' };
-  const valued = /^(skip|out|source|locale|language|locale-expr|dest|runtime|target|table|languages|syntax|in|from|to|model|base-url|provider|batch|only|dir)$/;
+  const valued = /^(url|skip|out|source|locale|language|locale-expr|dest|runtime|target|table|languages|syntax|in|from|to|model|base-url|provider|batch|only|dir)$/;
   const options = Object.create(null), positional = [];
   for (let i = 0; i < rest.length; i++) {
     const arg = rest[i];
@@ -358,6 +365,45 @@ async function main() {
     const divisions = Object.keys(files).filter(n => !/^(?:i18n|language|runtime)\.|^languages\//.test(n) && !n.endsWith('.d.mts')).length;
     console.log(`Compiled ${plural(catalog.messages.length, 'token')} into ${options.out}: ${plural(divisions, 'division module')}, ${plural(Object.keys(catalog.languages).length - 1, 'language chunk')}${options.eager ? ' (eager)' : ''}.`); return;
   }
+  if (command === 'render') {
+    // Static pages, one copy per language: the source language where the pages
+    // are, every other language under its code (/haw/), and every other file copied.
+    const site = one();
+    if (!site || !(await isDirectory(site))) throw new Error('Usage: i18nmd render <site directory> [--out dist] [--url https://example.com]');
+    const out = path.resolve(options.out || 'dist');
+    const root = path.resolve(site);
+    if (out === root) throw new Error('Render into a directory other than the site itself.');
+    const ws = await open(options.dir, options); ws.warn();
+    const { catalog } = ws;
+    const base = (options.url || '').replace(/\/+$/, '');
+    if (base && !/^https?:\/\//.test(base)) throw new Error('--url needs a full address, such as https://example.com.');
+    const codes = Object.keys(catalog.languages);
+    const tables = Object.fromEntries(codes.map(code => [code, Object.fromEntries(catalog.messages.filter(m => Object.hasOwn(m.translations, code)).map(m => [m.key, m.translations[code]]))]));
+    const files = [];
+    const walk = async dir => {
+      for (const entry of (await readdir(dir, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {
+        const file = path.join(dir, entry.name);
+        if (entry.name.startsWith('.') || entry.name === 'node_modules' || file === out) continue;
+        if (entry.isDirectory()) await walk(file); else if (entry.isFile()) files.push(path.relative(root, file).split(path.sep).join('/'));
+      }
+    };
+    await walk(root);
+    let pages = 0;
+    for (const file of files) {
+      if (!/\.html?$/.test(file)) { await mkdir(path.dirname(path.join(out, file)), { recursive: true }); await copyFile(path.join(root, file), path.join(out, file)); continue; }
+      const html = await readFile(path.join(root, file), 'utf8');
+      const address = code => `${base}/${code === catalog.source ? '' : code + '/'}${file.replace(/(^|\/)index\.html?$/, '$1')}`;
+      const urls = Object.fromEntries(codes.map(code => [code, address(code)]));
+      for (const code of codes) {
+        const target = code === catalog.source ? file : `${code}/${file}`;
+        const relative = code === catalog.source ? '' : path.posix.relative(path.posix.dirname(target), path.posix.dirname(file)) + '/';
+        await save(path.join(out, target), renderHtml(html, { language: code, source: catalog.source, messages: tables[code], pages: urls, languages: catalog.languages, relative, absolute: !!base }));
+        pages++;
+      }
+    }
+    for (const row of ws.rows()) if (row.missing.length || row.stale.length) console.warn(`i18nmd: ${summary(row)}; untranslated text stays in ${catalog.languages[catalog.source]}.`);
+    console.log(`Rendered ${plural(pages, 'page')} in ${plural(codes.length, 'language')} into ${path.relative(process.cwd(), out) || '.'}.`); return;
+  }
   if (command === 'join') {
     if (!options.out || !/\.md$/.test(options.out)) throw new Error('Choose the joined Markdown file with --out, e.g. --out translations.md.');
     const ws = await open(one(), options);
@@ -432,6 +478,29 @@ async function main() {
     if (!options.from) throw new Error(`Choose the format with --from ${FORMATS.join('|')}.`);
     const out = options.out || 'translations';
     const warnings = [];
+    // --merge adds source-language messages to the source file in --out: new tokens
+    // are appended, changed text replaces the old, and nothing is removed. Other
+    // languages and the lock are left alone, so status shows what to translate.
+    if (options.merge) {
+      const lock = await readLock((await findLock(out)).file);
+      const source = options.source || lock.source || 'en';
+      const file = path.join(out, `i18n-${source}.md`);
+      const catalog = await exists(file) ? parseCatalog(await readFile(file, 'utf8'), { filename: file, locale: source, formatters: lock.formatters }) : { title: 'Translations', source, syntax: 'icu', languages: { [source]: resolveLanguage(source).name }, messages: [] };
+      const index = new Map(catalog.messages.map(m => [m.key, m]));
+      let added = 0, changed = 0;
+      for (const input of positional) {
+        for (const m of importMessages(JSON.parse(await readFile(input, 'utf8')), options.from, warnings, lock.formatters)) {
+          const existing = index.get(m.key);
+          if (!existing) { const message = { key: m.key, context: m.context, optional: [], translations: { [source]: m.text } }; catalog.messages.push(message); index.set(m.key, message); added++; continue; }
+          if (existing.translations[source] !== m.text) { existing.translations[source] = m.text; changed++; }
+          if (m.context) existing.context = m.context;
+        }
+      }
+      warnings.forEach(w => console.warn(`i18nmd: ${w}`));
+      if (!added && !changed) { console.log(`${file} already has every message.`); return; }
+      await save(file, serializeCatalog(catalog, { locale: source }));
+      console.log(`Added ${plural(added, 'token')} and updated ${plural(changed, 'token')} in ${file}. Run i18nmd translate to fill the other languages.`); return;
+    }
     const byLocale = new Map();
     const files = positional.map(file => ({ file, ...localeFromJsonPath(file) }));
     const namespaces = new Set(files.map(f => f.namespace).filter(Boolean));
@@ -462,9 +531,10 @@ async function main() {
       if (await exists(file)) throw new Error(`${file} exists; import into an empty directory.`);
     }
     for (const locale of Object.keys(languages)) await save(path.join(out, `i18n-${locale}.md`), serializeCatalog(catalog, { locale, allowIncomplete: true }));
-    const lock = { source, translations: {} };
-    syncLock(catalog, lock);
-    await save(lockPathFor(out, true), serializeLock(lock));
+    // The lock that covers out, so a division shares the tree's lock and token names.
+    const ws = await open(out, { source });
+    syncLock(ws.catalog, ws.lock, { partial: ws.partial });
+    await ws.saveLock();
     [...warnings, ...(catalog.warnings || [])].forEach(w => console.warn(`i18nmd: ${w}`));
     console.log(`Imported ${plural(messages.length, 'token')} in ${plural(Object.keys(languages).length, 'language')} into ${out}.`); return;
   }
@@ -495,8 +565,8 @@ async function main() {
     const outDir = path.dirname(sourceFile);
     const namespace = namespaceFor(await divisionPrefix(outDir, (await findLock(outDir)).file));
     if (languageFromFilename(sourceFile) !== locale) throw new Error('The output filename must match --source.');
-    const files = [...new Set((await Promise.all(positional.map(filesAt))).flat())];
-    if (!files.length) throw new Error('No JavaScript or TypeScript source files found.');
+    const files = [...new Set((await Promise.all(positional.map(input => filesAt(input, { html: true })))).flat())];
+    if (!files.length) throw new Error('No HTML, JavaScript or TypeScript files found.');
     // The deepest directory holding every input; converted copies keep their paths below it.
     const dirs = await Promise.all(positional.map(async input => (await isDirectory(input)) ? path.resolve(input) : path.dirname(path.resolve(input))));
     let root = dirs[0];
@@ -513,21 +583,26 @@ async function main() {
     // Existing tokens are always kept, so extraction can be repeated as code changes.
     let catalog = await exists(sourceFile) ? parseCatalog(await readFile(sourceFile, 'utf8'), { filename: sourceFile }) : undefined;
     const outputs = [];
-    let replacements = 0;
+    let replacements = 0, updated = 0;
     for (const file of files) {
       const relative = path.relative(root, path.resolve(file));
       const output = path.join(destination, relative);
       let importPath = path.relative(path.dirname(output), runtime).split(path.sep).join('/');
       if (!importPath.startsWith('.')) importPath = './' + importPath;
-      const result = extractSource(await readFile(file, 'utf8'), { filename: path.relative(process.cwd(), file), locale, languageExpression: options['locale-expr'], existing: catalog, importPath, namespace });
-      catalog = result.catalog; replacements += result.replacements;
+      const text = await readFile(file, 'utf8'), filename = path.relative(process.cwd(), file);
+      const result = /\.html?$/.test(file)
+        ? extractHtml(text, { filename, locale, existing: catalog, namespace })
+        : extractSource(text, { filename, locale, languageExpression: options['locale-expr'], existing: catalog, importPath, namespace });
+      catalog = result.catalog; replacements += result.replacements; updated += result.updated || 0;
       result.diagnostics.forEach(message => console.warn(message));
       if (result.replacements) outputs.push([output, result.source]);
     }
     // The lock marks the top of the translations tree, so divisions keep their names.
     const lockFile = (await findLock(outDir)).file;
     if (!(await exists(lockFile))) await save(lockFile, serializeLock({ source: locale, translations: {} }));
-    if (!replacements) { console.log(`Nothing new to extract from ${plural(files.length, 'file')}. Mark other display strings with /* i18n */.`); return; }
+    if (!replacements && !updated) { console.log(`Nothing new to extract from ${plural(files.length, 'file')}. Mark other display strings with /* i18n */.`); return; }
+    // An HTML page holds its own English: an edited sentence updates the source file.
+    if (!replacements) { await save(sourceFile, serializeCatalog(catalog, { locale })); console.log(`Updated ${plural(updated, 'string')} in ${sourceFile} from the pages' English; run i18nmd status to see what to translate.`); return; }
     const markdown = serializeCatalog(catalog, { locale });
     for (const [file, text] of outputs) await save(file, text);
     await save(sourceFile, markdown);

package/lib/catalog.mjs

@@ -45,7 +45,7 @@ export function parseCatalog(markdown, options = {}) {
     if (!match || Object.hasOwn(metadata, match[1])) throw new Error(`Invalid or duplicate metadata: ${line}`);
     metadata[match[1]] = match[1] === 'languages' ? JSON.parse(match[2]) : match[2].trim();
   }
-  const catalog = { ...metadata, title: 'Translations', messages: [] };
+  const catalog = { ...metadata, title: 'Translations', messages: [], ...(options.formatters?.length && { formatters: options.formatters }) };
   let message;
   for (let i = end + 1; i < lines.length; i++) {
     const line = lines[i];
@@ -84,6 +84,9 @@ export function parseCatalog(markdown, options = {}) {
 export function validateCatalog(catalog, { allowIncomplete = false, lenient = false, single = false } = {}) {
   catalog.warnings ||= [];
   if (!['icu', 'python'].includes(catalog.syntax)) throw new Error('syntax must be icu or python.');
+  for (const name of catalog.formatters || []) {
+    if (!/^[a-zA-Z_]\w*$/.test(name) || ['number', 'date', 'time', 'plural', 'select', 'selectordinal'].includes(name)) throw new Error(`Formatter names are identifiers other than ICU's own types: ${name}`);
+  }
   if (!catalog.languages || typeof catalog.languages !== 'object' || Array.isArray(catalog.languages)) throw new Error('languages must be a JSON object mapping codes to names.');
   const locales = Object.keys(catalog.languages);
   if (!locales.length || !Object.hasOwn(catalog.languages, catalog.source)) throw new Error('Declare the source language in languages.');
@@ -107,7 +110,7 @@ export function validateCatalog(catalog, { allowIncomplete = false, lenient = fa
     for (const locale of locales) {
       if (!Object.hasOwn(table, locale) && allowIncomplete && locale !== catalog.source) continue;
       if (!Object.hasOwn(table, locale) || typeof table[locale] !== 'string' || !table[locale].trim()) throw new Error(`${message.key}: missing ${locale} translation.`);
-      try { parsed[locale] = argumentsFor(parseMessage(table[locale], catalog.syntax)); }
+      try { parsed[locale] = argumentsFor(parseMessage(table[locale], catalog.syntax, catalog.formatters)); }
       catch (error) { throw new Error(`${message.key} [${locale}]: ${error.message}`); }
     }
     const sourceArgs = parsed[catalog.source];
@@ -121,7 +124,8 @@ export function validateCatalog(catalog, { allowIncomplete = false, lenient = fa
         if (problem) break;
         const base = sourceArgs[name];
         if (!base) problem = `unknown placeholder ${type === 'tag' ? `<${name}>` : `{${name}}`}`;
-        else if (base !== type && (base === 'tag' || type === 'tag' || (base !== 'string' && type !== 'string'))) problem = `incompatible type for {${name}}`;
+        // A translation may move {len, length} but not drop or change its formatter.
+        else if (base !== type && (base === 'tag' || type === 'tag' || base.startsWith('format:') || type.startsWith('format:') || (base !== 'string' && type !== 'string'))) problem = `incompatible type for {${name}}`;
       }
       if (!problem) continue;
       if (lenient && locale !== catalog.source) {
@@ -175,7 +179,7 @@ const where = division => division ? `${division}/` : '';
 
 // Properties a division function cannot take: they exist on every function,
 // or (in) is i18nmd's own.
-const RESERVED = new Set(['in', 'name', 'length', 'call', 'apply', 'bind', 'prototype', 'caller', 'arguments', 'constructor', 'toString', 'valueOf', 'hasOwnProperty', '__proto__']);
+const RESERVED = new Set(['in', 'has', 'name', 'length', 'call', 'apply', 'bind', 'prototype', 'caller', 'arguments', 'constructor', 'toString', 'valueOf', 'hasOwnProperty', '__proto__']);
 
 /** knowledge-hub → knowledgeHub: the property for a division directory. */
 export function divisionProperty(segment) {
@@ -204,7 +208,8 @@ export function namespaceFor(division, prefix = '') {
 }
 
 /**
- * Merge one-language files into a catalog. Files may sit in subdirectories,
+ * Merge one-language files into a catalog. formatters lists the argument types
+ * the project declares in its lock ({len, length}). Files may sit in subdirectories,
  * one per division (ui/i18n-fr.md, marketing/i18n-fr.md); each division has its
  * own source file, and its tokens are namespaced by its path: ## save in
  * ui/i18n-en.md is the token ui.save. Every message records its division so it
@@ -213,7 +218,7 @@ export function namespaceFor(division, prefix = '') {
  * no longer has are set aside in catalog.orphans with a warning. strict: every
  * file must match its source exactly.
  */
-export function parseLanguageFiles(files, { source, syntax = 'icu', strict = false, prefix = '' } = {}) {
+export function parseLanguageFiles(files, { source, syntax = 'icu', strict = false, prefix = '', formatters = [] } = {}) {
   const divisions = new Map();
   for (const [filename, markdown] of Object.entries(files)) {
     const locale = languageFromFilename(filename);
@@ -221,7 +226,7 @@ export function parseLanguageFiles(files, { source, syntax = 'icu', strict = fal
     if (!divisions.has(division)) divisions.set(division, new Map());
     const byLocale = divisions.get(division);
     if (byLocale.has(locale)) throw new Error(`Duplicate language file: ${where(division)}${locale}`);
-    const parsed = parseCatalog(markdown, { locale, syntax });
+    const parsed = parseCatalog(markdown, { locale, syntax, formatters });
     const namespace = namespaceFor(division, prefix);
     for (const message of parsed.messages) message.key = namespace + message.key;
     if (parsed.source !== locale || parsed.syntax !== syntax || Object.keys(parsed.languages).length !== 1) throw new Error(`Expected a single-language ${syntax} file for ${where(division)}i18n-${locale}.md.`);
@@ -230,7 +235,7 @@ export function parseLanguageFiles(files, { source, syntax = 'icu', strict = fal
   const all = [...divisions.values()].flatMap(byLocale => [...byLocale.keys()]);
   source ||= all.includes('en') ? 'en' : all[0];
   if (!source) throw new Error('Include at least one i18n-<language>.md file.');
-  const catalog = { source, syntax, title: 'Translations', languages: Object.create(null), messages: [], orphans: Object.create(null), divisions: [...divisions.keys()], prefix };
+  const catalog = { source, syntax, title: 'Translations', languages: Object.create(null), messages: [], orphans: Object.create(null), divisions: [...divisions.keys()], prefix, ...(formatters.length && { formatters }) };
   const owner = new Map();
   for (const [division, byLocale] of divisions) {
     const base = byLocale.get(source);

package/lib/compiler.mjs

@@ -1,11 +1,15 @@
 import { parseCatalog, validateCatalog, namespaceFor, divisionProperty, accessorFor } from './catalog.mjs';
 import { argumentsFor, parseMessage } from './messages.mjs';
+import { numberOptions } from './runtime.mjs';
+import { createRequire } from 'node:module';
+
+const PLURAL_RULES = createRequire(import.meta.url)('./plural-rules.json');
 
 export function compileCatalog(input) {
   const catalog = typeof input === 'string' ? parseCatalog(input, { allowIncomplete: true }) : validateCatalog(input, { allowIncomplete: true });
   const messages = Object.create(null);
   for (const message of catalog.messages) {
-    messages[message.key] = Object.fromEntries(Object.entries(message.translations).map(([lang, text]) => [lang, parseMessage(text, catalog.syntax)]));
+    messages[message.key] = Object.fromEntries(Object.entries(message.translations).map(([lang, text]) => [lang, parseMessage(text, catalog.syntax, catalog.formatters)]));
   }
   return { source: catalog.source, languages: catalog.languages, messages };
 }
@@ -26,41 +30,116 @@ function divisionTree(catalog) {
   return tree;
 }
 
-// Plural categories for 0–199 as one letter each. CLDR rules depend on n, n % 10
-// and n % 100, so 100 + n % 100 stands in for any larger whole number.
-function pluralTable(locale, type) {
-  let rules;
-  try { rules = new Intl.PluralRules(locale, { type }); } catch { rules = new Intl.PluralRules('en', { type }); }
-  const letter = { zero: 'z', one: 'o', two: 't', few: 'f', many: 'm', other: 'x' };
-  return Array.from({ length: 200 }, (_, n) => letter[rules.select(n)]).join('');
+// The CLDR plural rules Intl.PluralRules uses for locale, as Python: a function
+// of the ICU operands n, i, v, f, t and e. Returns [function name, source].
+function pluralFunction(locale, type) {
+  let resolved;
+  try { resolved = new Intl.PluralRules(locale, { type }).resolvedOptions().locale; } catch { resolved = 'en'; }
+  const table = PLURAL_RULES[type];
+  const parts = resolved.split('-');
+  while (parts.length && !Object.hasOwn(table, parts.join('-'))) parts.pop();
+  const key = parts.join('-') || 'root';
+  const relation = text => {
+    const match = /^([nivwftce])(?:\s*%\s*(\d+))?\s*(!?=)\s*([\d.,\s]+)$/.exec(text.trim());
+    if (!match) throw new Error(`Unsupported CLDR plural rule for ${key}: ${text}`);
+    const ranges = match[4].split(',').map(r => { const [low, high = low] = r.trim().split('..'); return `(${low}, ${high})`; });
+    return `${match[3] === '=' ? '' : 'not '}_in(${match[1]}${match[2] ? ` % ${match[2]}` : ''}, (${ranges.join(', ')},))`;
+  };
+  const name = `_${type}_${key.replace(/\W/g, '_')}`;
+  let source = `def ${name}(n: Decimal, i: int, v: int, f: int, t: int, e: int) -> str:\n`;
+  for (const [category, rule] of Object.entries(table[key] || {})) {
+    source += `    if ${rule.split(/\s+or\s+/).map(and => and.split(/\s+and\s+/).map(relation).join(' and ')).join(' or ')}:\n        return "${category}"\n`;
+  }
+  return [name, source + '    return "other"\n'];
+}
+
+// How a language writes numbers, from Intl, for the Python formatter:
+// [group mark, decimal mark, primary group size, secondary group size,
+//  digits before grouping starts, { style: [positive pattern, negative pattern,
+//  most fraction digits, fewest fraction digits] }], where {n} in a pattern is
+// the grouped digits. styles lists the number styles the catalog uses.
+function numberRules(locale, styles) {
+  const make = options => { try { return new Intl.NumberFormat(locale, options); } catch { return new Intl.NumberFormat('en', options); } };
+  const format = make();
+  const parts = format.formatToParts(1234567.891);
+  const integers = parts.filter(p => p.type === 'integer').map(p => p.value);
+  const primary = integers.length > 1 ? integers.at(-1).length : 0;
+  const secondary = integers.length > 2 ? integers.at(-2).length : primary;
+  const minimum = format.formatToParts(1234).some(p => p.type === 'group') ? 1 : 2;
+  const digits = new Set(['integer', 'group', 'decimal', 'fraction']);
+  const pattern = (f, value) => f.formatToParts(value).map((p, i, all) => !digits.has(p.type) ? p.value : digits.has(all[i - 1]?.type) ? '' : '{n}').join('');
+  const patterns = Object.fromEntries(styles.map(style => {
+    const f = make(numberOptions(style));
+    const { maximumFractionDigits, minimumFractionDigits } = f.resolvedOptions();
+    return [style, [pattern(f, 1), pattern(f, -1), maximumFractionDigits, minimumFractionDigits]];
+  }));
+  return [parts.find(p => p.type === 'group')?.value ?? '', parts.find(p => p.type === 'decimal')?.value ?? '.', primary, secondary, minimum, patterns];
 }
 
 const PYTHON_RUNTIME = `
-_CATEGORY = {"z": "zero", "o": "one", "t": "two", "f": "few", "m": "many", "x": "other"}
 _log = logging.getLogger("i18nmd")
+_FORMATTERS: dict[str, Callable[[Any, str], str]] = {}
+_CONTEXT = Context(prec=1000, rounding=ROUND_HALF_UP)
 
 
-def _number(value: Any) -> int | float | None:
+def _number(value: Any) -> float | None:
+    # Numbers are doubles, as in JavaScript, so both runtimes see the same value.
     if isinstance(value, bool):
         return None
-    if isinstance(value, (int, float)):
-        return value
     try:
-        text = str(value).strip()
-        return float(text) if "." in text else int(text)
-    except (TypeError, ValueError):
+        number = float(value if isinstance(value, (int, float)) else str(value).strip())
+    except (TypeError, ValueError, OverflowError):
         return None
+    return number if math.isfinite(number) else None
+
+
+def _decimal(number: float) -> Decimal:
+    # The shortest decimal that reads back as number, which is what Intl formats.
+    return Decimal(repr(float(number)))
+
+
+def _digits(number: Decimal, most: int, fewest: int = 0) -> tuple[str, str]:
+    # Whole and fraction digits of |number|, rounded half away from zero like Intl.
+    with localcontext(_CONTEXT):
+        rounded = abs(number).quantize(Decimal(1).scaleb(-most))
+    whole, _, fraction = format(rounded, "f").partition(".")
+    return whole, fraction.rstrip("0").ljust(fewest, "0")
+
 
+def _number_text(number: float, language: str, style: str = "") -> str:
+    group, decimal, primary, secondary, minimum, styles = _NUMBERS[language]
+    positive, negative, most, fewest = styles[style]
+    value = _decimal(number)
+    if style == "percent":
+        value = value.scaleb(2)
+    whole, fraction = _digits(value, most, fewest)
+    if primary and len(whole) >= primary + minimum:
+        groups = [whole[-primary:]]
+        whole = whole[:-primary]
+        while len(whole) > secondary:
+            groups.insert(0, whole[-secondary:])
+            whole = whole[:-secondary]
+        whole = group.join([whole] + groups if whole else groups)
+    text = whole + (decimal + fraction if fraction else "")
+    return (negative if value.is_signed() else positive).replace("{n}", text)
 
-def _show(number: int | float) -> str:
-    return str(int(number)) if float(number).is_integer() else f"{number:g}"
 
+def _in(value: Any, ranges: tuple[tuple[int, int], ...]) -> bool:
+    return value == int(value) and any(low <= value <= high for low, high in ranges)
 
-def _category(table: str, number: int | float) -> str:
-    if not float(number).is_integer():
-        return "other"
-    n = abs(int(number))
-    return _CATEGORY[table[n if n < 200 else 100 + n % 100]]
+
+def _category(rules: Callable[..., str], number: float) -> str:
+    # The CLDR operands of number as Intl.PluralRules reads it: rounded to at most
+    # 3 decimals; like ICU, n is that value as a double and i keeps 18 digits.
+    whole, fraction = _digits(_decimal(number), 3)
+    n = Decimal(float(whole + "." + fraction if fraction else whole))
+    with localcontext(_CONTEXT):
+        return rules(n, int(whole[-18:]), len(fraction), int(fraction or 0), int(fraction or 0), 0)
+
+
+def _exact(options: dict[str, Any], number: float) -> Any:
+    value = _decimal(number)
+    return next((branch for key, branch in options.items() if key.startswith("=") and Decimal(key[1:]) == value), None)
 
 
 def _render(nodes: list[Any], language: str, values: dict[str, Any] | None, pound: Any = None) -> str:
@@ -71,7 +150,7 @@ def _render(nodes: list[Any], language: str, values: dict[str, Any] | None, poun
             continue
         kind = node["type"]
         if kind == "pound":
-            out.append(_show(pound))
+            out.append(_number_text(pound, language))
             continue
         if kind == "tag":
             inner = _render(node["children"], language, values, pound)
@@ -89,6 +168,20 @@ def _render(nodes: list[Any], language: str, values: dict[str, Any] | None, poun
         value = values[name]
         if kind == "argument":
             out.append("" if value is None else str(value))
+        elif kind == "format":
+            fn = _FORMATTERS.get(node["format"])
+            if value is None:
+                _log.error("Missing value for {%s}.", name)
+                out.append("{" + name + "}")
+            elif fn is None:
+                _log.error("No formatter for {%s, %s}.", name, node["format"])
+                out.append(str(value))
+            else:
+                try:
+                    out.append(str(fn(value, language)))
+                except Exception:
+                    _log.exception("Formatter %s failed for {%s}.", node["format"], name)
+                    out.append(str(value))
         elif kind == "select":
             options = node["options"]
             out.append(_render(options.get(str(value), options["other"]), language, values, pound))
@@ -99,13 +192,12 @@ def _render(nodes: list[Any], language: str, values: dict[str, Any] | None, poun
                 out.append(str(value))
                 continue
             options = node["options"]
-            exact = options.get("=" + _show(number))
-            table = (_ORDINAL if kind == "selectordinal" else _CARDINAL)[language]
-            chosen = exact or options.get(_category(table, number - node["offset"])) or options["other"]
+            rules = (_ORDINAL if kind == "selectordinal" else _CARDINAL)[language]
+            chosen = _exact(options, number) or options.get(_category(rules, number - node["offset"])) or options["other"]
             out.append(_render(chosen, language, values, number - node["offset"]))
         elif kind == "number":
             number = _number(value)
-            out.append(str(value) if number is None else f"{number * 100:g}%" if node.get("style") == "percent" else _show(number))
+            out.append(str(value) if number is None else _number_text(number, language, node.get("style", "")))
         else:
             out.append(str(value))
     return "".join(out)
@@ -132,6 +224,17 @@ def i18nmd(token: str, language: str | None = None, **values: Any) -> str:
     return _render(entry[used], used, values)
 
 
+def register_formatter(name: str, fn: Callable[[Any, str], str]) -> None:
+    # Supply the function for an argument type the project declares: {len, length}.
+    # fn(value, language) returns the text; keep it the same as the JavaScript one.
+    _FORMATTERS[name] = fn
+
+
+def has(token: str) -> bool:
+    # Whether the catalog has token, for text written after the last compile.
+    return token in _MESSAGES
+
+
 def template(token: str, language: str | None = None) -> str:
     # The message with each placeholder, plural or select shown as {name}.
     entry = _MESSAGES[token]
@@ -142,11 +245,21 @@ def template(token: str, language: str | None = None) -> str:
 // ICU messages as a dependency-free Python module: i18nmd(token, language, **values).
 function pythonModule(catalog) {
   const compiled = compileCatalog(catalog);
-  const tables = kind => Object.fromEntries(Object.keys(catalog.languages).map(l => [l, pluralTable(l, kind)]));
+  const languages = Object.keys(catalog.languages);
+  const styles = new Set(['']);
+  const visit = nodes => nodes.forEach(n => { if (n.type === 'number') styles.add(n.style); for (const branch of Object.values(n.options || {})) visit(branch); if (n.children) visit(n.children); });
+  for (const byLang of Object.values(compiled.messages)) Object.values(byLang).forEach(visit);
   const literal = value => JSON.stringify(value, null, 1).replace(/\n\s*/g, ' ');
-  let text = '# Generated from i18n.md. Edit the Markdown source.\n# ruff: noqa\nfrom __future__ import annotations\n\nimport logging\nfrom typing import Any\n\n';
+  let text = '# Generated from i18n.md. Edit the Markdown source.\n# ruff: noqa\nfrom __future__ import annotations\n\nimport logging\nimport math\nfrom decimal import ROUND_HALF_UP, Context, Decimal, localcontext\nfrom typing import Any, Callable\n\n';
   text += `SOURCE = ${JSON.stringify(catalog.source)}\nLANGS: dict[str, str] = ${literal(catalog.languages)}\n`;
-  text += `_CARDINAL: dict[str, str] = ${JSON.stringify(tables('cardinal'), null, 1)}\n_ORDINAL: dict[str, str] = ${JSON.stringify(tables('ordinal'), null, 1)}\n`;
+  // Plural rules from CLDR, the data Intl.PluralRules uses, as Python functions.
+  const functions = new Map();
+  const rules = type => Object.fromEntries(languages.map(l => { const [name, source] = pluralFunction(l, type); functions.set(name, source); return [l, name]; }));
+  const cardinal = rules('cardinal'), ordinal = rules('ordinal');
+  text += `\n\n${[...functions.values()].join('\n\n')}\n\n`;
+  const table = map => `{${Object.entries(map).map(([l, name]) => `${JSON.stringify(l)}: ${name}`).join(', ')}}`;
+  text += `_CARDINAL: dict[str, Callable[..., str]] = ${table(cardinal)}\n_ORDINAL: dict[str, Callable[..., str]] = ${table(ordinal)}\n`;
+  text += `_NUMBERS: dict[str, list[Any]] = ${JSON.stringify(Object.fromEntries(languages.map(l => [l, numberRules(l, [...styles])])), null, 1)}\n`;
   // Compiled messages are JSON of strings, objects and integers, which is also Python.
   text += `_MESSAGES: dict[str, dict[str, list[Any]]] = {\n${Object.entries(compiled.messages).map(([key, byLang]) => ` ${JSON.stringify(key)}: ${literal(byLang)},`).join('\n')}\n}\n`;
   text += 'TOKENS: frozenset[str] = frozenset(_MESSAGES)\n';
@@ -169,8 +282,8 @@ function typesFor(catalog, tree) {
     // Merge argument types across translations so a numeric source placeholder
     // can acquire plural/number formatting in a translation without weakening types.
     const params = Object.create(null);
-    for (const value of Object.values(message.translations)) argumentsFor(parseMessage(value, catalog.syntax), params);
-    types[message.key] = Object.entries(params).map(([name, type]) => `${JSON.stringify(name)}: ${type === 'number' ? 'number' : type === 'date' ? 'Date | number' : type === 'select' ? 'string' : type === 'tag' ? '(chunks: any[]) => any' : 'string | number | null | undefined'}`).join('; ');
+    for (const value of Object.values(message.translations)) argumentsFor(parseMessage(value, catalog.syntax, catalog.formatters), params);
+    types[message.key] = Object.entries(params).map(([name, type]) => `${JSON.stringify(name)}: ${type === 'number' ? 'number' : type === 'date' ? 'Date | number' : type === 'select' ? 'string' : type === 'tag' ? '(chunks: any[]) => any' : type.startsWith('format:') ? 'any' : 'string | number | null | undefined'}`).join('; ');
     if (Object.values(params).includes('tag')) rich.push(message.key);
   }
   let text = 'import type { Language } from "./language.mjs";\nexport type { Language };\n';
@@ -181,7 +294,7 @@ function typesFor(catalog, tree) {
   // A division function takes the tokens under its prefix by their short names.
   text += 'type Short<P extends string> = Token extends infer K ? (K extends `${P}${infer S}` ? S : never) : never;\n';
   text += 'type Full<P extends string, S extends string> = `${P}${S}` & Token;\n';
-  text += 'export interface Scope<P extends string> {\n  <S extends Short<P>>(token: S, ...args: Values[Full<P, S>] extends undefined ? [values?: undefined] : [values: Values[Full<P, S>]]): Full<P, S> extends RichToken ? any[] : string;\n}\n';
+  text += 'export interface Scope<P extends string> {\n  <S extends Short<P>>(token: S, ...args: Values[Full<P, S>] extends undefined ? [values?: undefined] : [values: Values[Full<P, S>]]): Full<P, S> extends RichToken ? any[] : string;\n  /** Whether the compiled catalog has this token, for text written after the last compile. */\n  has(token: string): token is Short<P>;\n}\n';
   const scopeType = (prefix, node) => `Scope<${JSON.stringify(prefix)}>${Object.keys(node).length ? ` & { ${Object.entries(node).map(([property, [segment, children]]) => `readonly ${property}: ${scopeType(`${prefix}${segment}.`, children)}`).join('; ')} }` : ''}`;
   text += `export type Translator = ${scopeType('', tree)};\n\n`;
   return text;
@@ -241,7 +354,14 @@ export function generateModules(input, { target = 'ts', eager = false } = {}) {
   if (target === 'ts') main += typesFor(catalog, tree);
   const own = eager ? catalog.messages : catalog.messages.filter(m => !m.division);
   if (own.length) main += `store.add(${table(own, catalog.source)});\n`;
-  main += `const translator = createTranslator({ store, getLanguage }, ${JSON.stringify(tree)});\n`;
+  const formatters = catalog.formatters || [];
+  if (formatters.length) {
+    main += 'const formatters = {};\n/** Supply the function for an argument type the project declares: {len, length}. */\n';
+    main += target === 'ts'
+      ? `export function registerFormatter(name: ${formatters.map(f => JSON.stringify(f)).join(' | ')}, format: (value: any, language: Language) => string): void { formatters[name] = format; }\n`
+      : 'export function registerFormatter(name, format) { formatters[name] = format; }\n';
+  }
+  main += `const translator = createTranslator({ store, getLanguage }, ${JSON.stringify(tree)}${formatters.length ? ', { formatters }' : ''});\n`;
   main += '/** Translate into the current language: i18nmd("token"), i18nmd.division("token", values), or i18nmd.in("fr")… for a given language. */\n';
   main += target === 'ts' ? 'export const i18nmd = translator as Translator & { in(language: Language | (string & {})): Translator };\n' : 'export const i18nmd = translator;\n';
   main += 'export { languages } from "./language.mjs";\nexport { ready, getLanguage, setLanguage, loadLanguage, onLanguageChange };\n';
@@ -273,10 +393,10 @@ export function generateModule(input, target = 'ts') {
 export const runtimeTypes = `/* eslint-disable */
 export interface MessageNode { type: string; name?: string; style?: string; offset?: number; options?: Record<string, (string | MessageNode)[]>; children?: (string | MessageNode)[] }
 export interface CompiledCatalog { source: string; languages: Record<string, string>; messages: Record<string, Record<string, (string | MessageNode)[]>> }
-export interface I18nOptions { onError?: (error: Error) => void }
+export interface I18nOptions { onError?: (error: Error) => void; formatters?: Record<string, (value: any, language: string) => string> }
 export interface LanguageOptions { load?: (language: string) => Record<string, (string | MessageNode)[]> | Promise<Record<string, (string | MessageNode)[]>>; divisions?: string[]; storageKey?: string | null; onError?: (error: Error) => void }
 export interface LanguageState { store: CompiledCatalog & { add(table: Record<string, (string | MessageNode)[]>, language?: string): void }; ready: Promise<string>; getLanguage(): string; setLanguage(language: string): Promise<string>; loadLanguage(language: string): Promise<void>; onLanguageChange(listener: (language: string) => void): () => void }
-export function createI18n(catalog: CompiledCatalog, options?: I18nOptions): (token: string, language?: string, values?: Record<string, unknown>) => string | any[];
+export function createI18n(catalog: CompiledCatalog, options?: I18nOptions): ((token: string, language?: string, values?: Record<string, unknown>) => string | any[]) & { has(token: string): boolean };
 export function matchLanguage(catalog: { languages: Record<string, string> }, requested: string | null | undefined): string | undefined;
 export function createLanguage(languages: Record<string, string>, source: string, options?: LanguageOptions): LanguageState;
 export function createTranslator(language: Pick<LanguageState, 'store' | 'getLanguage'>, divisions?: Record<string, unknown>, options?: I18nOptions): any;

package/lib/extractor.mjs

@@ -7,7 +7,7 @@ const sourcePlugins = ['jsx', 'typescript'];
 
 // Readable tokens from the words of the message; a numeric suffix resolves collisions.
 // Tokens never encode the text, so editing the source message keeps the token.
-function tokenFor(text, taken) {
+export function tokenFor(text, taken) {
   const words = text.toLowerCase().replace(/\{[^}]+\}|<\/?[\w-]+>/g, ' ').normalize('NFKD').replace(/[\u0300-\u036f]/g, '').split(/[^a-z0-9]+/).filter(Boolean);
   let slug = '';
   for (const word of words) { if (slug && slug.length + word.length >= 40) break; slug += (slug ? '_' : '') + word; }