@likolabs/i18nmd 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/llm.mjs ADDED
@@ -0,0 +1,138 @@
1
+ // Translate language files with an Anthropic Messages API or an
2
+ // OpenAI-compatible /chat/completions endpoint. Plain fetch keeps the tool
3
+ // dependency-free and works with any compatible base URL (proxies, gateways,
4
+ // local servers).
5
+ import { parseCatalog, serializeCatalog, validateCatalog, textOf } from './catalog.mjs';
6
+
7
+ const ANTHROPIC = 'https://api.anthropic.com';
8
+ const DEFAULT_ANTHROPIC_MODEL = 'claude-opus-5-5';
9
+
10
+ /**
11
+ * Settings come from flags, then I18NMD_* variables, then the provider's own
12
+ * variables. The key is never printed.
13
+ */
14
+ export function llmConfig(options = {}, env = process.env) {
15
+ let baseUrl = options['base-url'] || env.I18NMD_BASE_URL;
16
+ let apiKey = env.I18NMD_API_KEY;
17
+ let provider = options.provider || env.I18NMD_PROVIDER;
18
+ if (!apiKey && !baseUrl && env.ANTHROPIC_API_KEY) { apiKey = env.ANTHROPIC_API_KEY; baseUrl = env.ANTHROPIC_BASE_URL || ANTHROPIC; provider ||= 'anthropic'; }
19
+ if (!apiKey && !baseUrl && env.OPENAI_API_KEY) { apiKey = env.OPENAI_API_KEY; baseUrl = env.OPENAI_BASE_URL || 'https://api.openai.com/v1'; provider ||= 'openai'; }
20
+ apiKey ||= env.ANTHROPIC_API_KEY || env.OPENAI_API_KEY;
21
+ if (!baseUrl && !apiKey) throw new Error('Set I18NMD_API_KEY (and I18NMD_BASE_URL for a non-Anthropic endpoint), or ANTHROPIC_API_KEY / OPENAI_API_KEY.');
22
+ baseUrl = (baseUrl || ANTHROPIC).replace(/\/+$/, '');
23
+ provider ||= /anthropic\.com|\/v1\/messages$/.test(baseUrl) || apiKey?.startsWith('sk-ant-') ? 'anthropic' : 'openai';
24
+ if (!['anthropic', 'openai'].includes(provider)) throw new Error('--provider must be anthropic or openai.');
25
+ const model = options.model || env.I18NMD_MODEL || (provider === 'anthropic' ? DEFAULT_ANTHROPIC_MODEL : undefined);
26
+ if (!model) throw new Error('Choose a model with --model or I18NMD_MODEL for an OpenAI-compatible endpoint.');
27
+ return { baseUrl, apiKey, provider, model };
28
+ }
29
+
30
+ function endpoint({ baseUrl, provider }) {
31
+ if (provider === 'anthropic') return /\/messages$/.test(baseUrl) ? baseUrl : /\/v1$/.test(baseUrl) ? `${baseUrl}/messages` : `${baseUrl}/v1/messages`;
32
+ return /\/chat\/completions$/.test(baseUrl) ? baseUrl : `${baseUrl}/chat/completions`;
33
+ }
34
+
35
+ const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
36
+
37
+ /** One system + user exchange; returns the reply text. */
38
+ export async function chat(config, system, user, { fetchImpl = fetch, retries = 4 } = {}) {
39
+ const anthropic = config.provider === 'anthropic';
40
+ const official = anthropic && config.baseUrl.startsWith(ANTHROPIC);
41
+ const headers = { 'content-type': 'application/json' };
42
+ let body;
43
+ if (anthropic) {
44
+ if (config.apiKey) headers['x-api-key'] = config.apiKey;
45
+ headers['anthropic-version'] = '2023-06-01';
46
+ body = { model: config.model, max_tokens: 16000, system, messages: [{ role: 'user', content: user }] };
47
+ // On Anthropic's API, let a declined request continue on a fallback model.
48
+ if (official) { headers['anthropic-beta'] = 'server-side-fallback-2026-07-01'; body.fallbacks = 'default'; }
49
+ } else {
50
+ if (config.apiKey) headers.authorization = `Bearer ${config.apiKey}`;
51
+ body = { model: config.model, messages: [{ role: 'system', content: system }, { role: 'user', content: user }] };
52
+ }
53
+ for (let attempt = 0; ; attempt++) {
54
+ let response;
55
+ try { response = await fetchImpl(endpoint(config), { method: 'POST', headers, body: JSON.stringify(body) }); }
56
+ catch (error) { if (attempt < retries) { await sleep(1000 * 2 ** attempt); continue; } throw new Error(`Could not reach ${endpoint(config)}: ${error.message}`); }
57
+ if (response.status === 429 || response.status >= 500) {
58
+ if (attempt < retries) { await sleep(Number(response.headers.get('retry-after')) * 1000 || 1000 * 2 ** attempt); continue; }
59
+ }
60
+ const data = await response.json().catch(() => ({}));
61
+ if (!response.ok) throw new Error(`${config.provider} API ${response.status}: ${data.error?.message || response.statusText}`);
62
+ if (anthropic) {
63
+ if (data.stop_reason === 'refusal') throw new Error('The model declined to translate this batch.');
64
+ if (data.stop_reason === 'max_tokens') throw new Error('The reply was cut off; use a smaller --batch.');
65
+ return (data.content || []).filter(block => block.type === 'text').map(block => block.text).join('');
66
+ }
67
+ const choice = data.choices?.[0];
68
+ if (choice?.finish_reason === 'length') throw new Error('The reply was cut off; use a smaller --batch.');
69
+ return choice?.message?.content ?? '';
70
+ }
71
+ }
72
+
73
+ function systemPrompt(catalog, target) {
74
+ const source = catalog.languages[catalog.source];
75
+ return [
76
+ `You localize software interface strings from ${source} (${catalog.source}) into ${target.english || target.name} (${target.code}).`,
77
+ target.custom ? `"${target.english}" is not a standard language. Write in that style or variety consistently, keeping the text understandable.` : 'Write natural, idiomatic text a native speaker would expect in a polished product, using the conventional register for software in that language.',
78
+ 'Each message is ICU MessageFormat. Keep placeholder names such as {name}, tag names such as <b>…</b>, and the plural/select/selectordinal keywords and select branch names exactly as they are; translate only the human text, including the text inside branches and tags. For plurals, provide the CLDR plural categories the target language needs and always keep "other". Keep # inside plural branches. Write a literal apostrophe as \'\' only when it sits next to { or }. Preserve leading and trailing spaces.',
79
+ 'Use the Context line to choose the right meaning and length. Interface text should be concise. Keep terminology consistent with the existing translations you are given.',
80
+ 'Reply with Markdown only, in exactly the input format: for each token, a "## token" heading followed by an ```icu fenced block. Include every token you were given, in the same order, and nothing else.',
81
+ ].join('\n\n');
82
+ }
83
+
84
+ function userPrompt(catalog, target, messages, previous, glossary) {
85
+ let text = '';
86
+ if (glossary.length) text += `Existing ${target.name} translations, for consistent terminology:\n\n${glossary.map(([s, t]) => `- ${JSON.stringify(s)} → ${JSON.stringify(t)}`).join('\n')}\n\n`;
87
+ const subset = { ...catalog, messages: messages.map(m => ({ ...m, translations: { [catalog.source]: m.translations[catalog.source] }, invalid: undefined })) };
88
+ text += `Translate these ${messages.length} messages:\n\n${serializeCatalog(subset, { locale: catalog.source }).replace(/^# .*\n\n/, '')}`;
89
+ const outdated = messages.filter(m => previous[m.key]);
90
+ if (outdated.length) text += `\nThese tokens had an earlier translation for older source text; reuse its wording where it still fits:\n\n${outdated.map(m => `- ${m.key}: ${JSON.stringify(previous[m.key])}`).join('\n')}\n`;
91
+ return text;
92
+ }
93
+
94
+ /** Parse a reply and check each message against its source; returns { ok, errors }. */
95
+ export function readReply(reply, catalog, target, messages) {
96
+ const body = reply.trim().replace(/^````?(?:md|markdown)?\n([\s\S]*?)\n````?$/, '$1');
97
+ const ok = Object.create(null), errors = Object.create(null);
98
+ let parsed;
99
+ try { parsed = parseCatalog(`# ${target.name}\n\n${body}\n`, { locale: target.code, allowIncomplete: true }); }
100
+ catch (error) { for (const m of messages) errors[m.key] = `unreadable reply: ${error.message}`; return { ok, errors }; }
101
+ const replies = new Map(parsed.messages.map(m => [m.key, m.translations[target.code]]));
102
+ for (const message of messages) {
103
+ const text = replies.get(message.key);
104
+ if (text === undefined) { errors[message.key] = 'missing from the reply'; continue; }
105
+ try {
106
+ validateCatalog({ source: catalog.source, syntax: catalog.syntax, languages: { [catalog.source]: 'source', [target.code]: 'target' }, messages: [{ key: message.key, optional: message.optional, translations: { [catalog.source]: message.translations[catalog.source], [target.code]: text } }] });
107
+ ok[message.key] = text;
108
+ } catch (error) { errors[message.key] = error.message; }
109
+ }
110
+ return { ok, errors };
111
+ }
112
+
113
+ /**
114
+ * Translate the given tokens into one language, in batches, retrying invalid
115
+ * messages once with the validation error. Calls onBatch(results) after each
116
+ * batch so callers can save progress. Returns the keys that still failed.
117
+ */
118
+ export async function translateLanguage(catalog, target, keys, config, { batchSize = 40, onBatch = () => {}, chatImpl = chat } = {}) {
119
+ const messages = catalog.messages.filter(m => keys.includes(m.key));
120
+ const previous = Object.fromEntries(messages.map(m => [m.key, textOf(m, target.code)]).filter(([, t]) => t !== undefined));
121
+ const glossary = catalog.messages.filter(m => !keys.includes(m.key) && Object.hasOwn(m.translations, target.code) && m.translations[catalog.source].length < 60).slice(0, 40).map(m => [m.translations[catalog.source], m.translations[target.code]]);
122
+ const failed = Object.create(null);
123
+ for (let i = 0; i < messages.length; i += batchSize) {
124
+ const batch = messages.slice(i, i + batchSize);
125
+ const system = systemPrompt(catalog, target);
126
+ let { ok, errors } = readReply(await chatImpl(config, system, userPrompt(catalog, target, batch, previous, glossary)), catalog, target, batch);
127
+ const retry = batch.filter(m => errors[m.key]);
128
+ if (retry.length) {
129
+ const feedback = `\n\nA previous attempt had these problems; fix them:\n${retry.map(m => `- ${m.key}: ${errors[m.key]}`).join('\n')}`;
130
+ const second = readReply(await chatImpl(config, system, userPrompt(catalog, target, retry, previous, glossary) + feedback), catalog, target, retry);
131
+ Object.assign(ok, second.ok);
132
+ errors = second.errors;
133
+ }
134
+ Object.assign(failed, errors);
135
+ await onBatch(ok);
136
+ }
137
+ return failed;
138
+ }
package/lib/lock.mjs ADDED
@@ -0,0 +1,111 @@
1
+ // i18nmd.lock.json keeps bookkeeping out of the Markdown. For every translation
2
+ // it records a fingerprint of the source text it was written against and of the
3
+ // translation itself. A translation is stale when its source text changed and
4
+ // the translation has not been edited since; editing it accepts it.
5
+ import { readFile, access } from 'node:fs/promises';
6
+ import path from 'node:path';
7
+ import { hashMessage, textOf } from './catalog.mjs';
8
+
9
+ export const LOCK_FILE = 'i18nmd.lock.json';
10
+
11
+ export function lockPathFor(input, isDirectory) {
12
+ return isDirectory ? path.join(input, LOCK_FILE) : input.replace(/\.md$/, '') + '.lock.json';
13
+ }
14
+
15
+ /** The nearest enclosing directory named translations, i18n or locales, if any. */
16
+ export function namedRoot(dir) {
17
+ const parts = path.resolve(dir).split(path.sep);
18
+ const index = parts.findLastIndex(part => ['translations', 'i18n', 'locales'].includes(part));
19
+ return index < 0 ? undefined : parts.slice(0, index + 1).join(path.sep) || path.sep;
20
+ }
21
+
22
+ /**
23
+ * The lock for a directory: its own, or one in an enclosing directory when the
24
+ * directory is a division of a larger tree. The search stops at the working
25
+ * directory or a translations/, i18n/ or locales/ directory; a new lock goes in
26
+ * the latter, so every division shares one.
27
+ */
28
+ export async function findLock(dir) {
29
+ const start = path.resolve(dir), named = namedRoot(dir), cwd = path.resolve('.');
30
+ const stops = [named, start.startsWith(cwd + path.sep) ? cwd : undefined].filter(Boolean);
31
+ for (let current = start; ; current = path.dirname(current)) {
32
+ const file = path.join(current, LOCK_FILE);
33
+ try { await access(file); return { file: path.relative('.', file) || LOCK_FILE, partial: current !== start }; } catch { /* keep looking */ }
34
+ if (stops.includes(current) || current === path.dirname(current) || !stops.length) break;
35
+ }
36
+ const home = named || start;
37
+ return { file: path.relative('.', path.join(home, LOCK_FILE)), partial: home !== start };
38
+ }
39
+
40
+ export async function readLock(file) {
41
+ try {
42
+ const lock = JSON.parse(await readFile(file, 'utf8'));
43
+ return { source: lock.source, translations: lock.translations || {} };
44
+ } catch (error) {
45
+ if (error.code === 'ENOENT') return { source: undefined, translations: {} };
46
+ throw new Error(`${file}: ${error.message}`);
47
+ }
48
+ }
49
+
50
+ export function serializeLock(lock) {
51
+ const sorted = Object.fromEntries(Object.keys(lock.translations).sort().map(locale => [locale, Object.fromEntries(Object.entries(lock.translations[locale]).sort(([a], [b]) => a.localeCompare(b)))]));
52
+ return JSON.stringify({ source: lock.source, translations: sorted }, null, 2) + '\n';
53
+ }
54
+
55
+ const fingerprint = (message, catalog, locale) => `${hashMessage(message.translations[catalog.source])}:${hashMessage(textOf(message, locale))}`;
56
+
57
+ function stale(message, catalog, locale, entry) {
58
+ if (!Object.hasOwn(message.translations, locale) && message.invalid?.[locale] !== undefined) return true;
59
+ if (!entry) return false;
60
+ const [source, translation] = entry.split(':');
61
+ return source !== hashMessage(message.translations[catalog.source]) && translation === hashMessage(textOf(message, locale));
62
+ }
63
+
64
+ /** Per-language progress: tokens missing, stale, and done. */
65
+ export function report(catalog, lock) {
66
+ return Object.keys(catalog.languages).filter(l => l !== catalog.source).map(locale => {
67
+ const row = { locale, name: catalog.languages[locale], total: catalog.messages.length, missing: [], stale: [] };
68
+ for (const message of catalog.messages) {
69
+ if (textOf(message, locale) === undefined) row.missing.push(message.key);
70
+ else if (stale(message, catalog, locale, lock.translations[locale]?.[message.key])) row.stale.push(message.key);
71
+ }
72
+ row.done = row.total - row.missing.length - row.stale.length;
73
+ return row;
74
+ });
75
+ }
76
+
77
+ /**
78
+ * Bring the lock up to date with the files; returns human-readable changes.
79
+ * partial: the catalog is one division of a larger tree, so entries for other
80
+ * tokens and languages are kept.
81
+ */
82
+ export function syncLock(catalog, lock, { partial = false } = {}) {
83
+ const changes = [];
84
+ const keys = new Set(catalog.messages.map(message => message.key));
85
+ const next = partial ? { ...lock.translations } : {};
86
+ for (const locale of Object.keys(catalog.languages)) {
87
+ if (locale === catalog.source) continue;
88
+ const previous = lock.translations[locale] || {};
89
+ const entries = next[locale] = partial ? Object.fromEntries(Object.entries(previous).filter(([key]) => !keys.has(key))) : {};
90
+ for (const message of catalog.messages) {
91
+ if (textOf(message, locale) === undefined) continue;
92
+ const entry = previous[message.key];
93
+ if (stale(message, catalog, locale, entry)) {
94
+ entries[message.key] = entry || fingerprint(message, catalog, locale);
95
+ if (!message.invalid?.[locale]) changes.push(`${locale}: ${message.key} is stale; its source text changed`);
96
+ continue;
97
+ }
98
+ const current = fingerprint(message, catalog, locale);
99
+ if (entry && entry.split(':')[0] !== current.split(':')[0]) changes.push(`${locale}: ${message.key} accepted; the translation was updated after its source changed`);
100
+ entries[message.key] = current;
101
+ }
102
+ }
103
+ lock.source = catalog.source;
104
+ lock.translations = next;
105
+ return changes;
106
+ }
107
+
108
+ /** Record that a translation now matches the current source text. */
109
+ export function markCurrent(catalog, lock, locale, message) {
110
+ (lock.translations[locale] ||= {})[message.key] = fingerprint(message, catalog, locale);
111
+ }
@@ -0,0 +1,130 @@
1
+ // The compiler parses a documented ICU subset. No parsing happens in the app.
2
+ export function parseMessage(text, syntax = 'icu') {
3
+ if (typeof text !== 'string') throw new Error('Messages must be strings.');
4
+ if (syntax === 'python') {
5
+ const nodes = [];
6
+ let literal = '';
7
+ const flush = () => { if (literal) nodes.push(literal); literal = ''; };
8
+ for (let i = 0; i < text.length;) {
9
+ if (text.slice(i, i + 2) === '{{' || text.slice(i, i + 2) === '}}') {
10
+ literal += text[i]; i += 2;
11
+ } else if (text[i] === '{') {
12
+ const end = text.indexOf('}', i);
13
+ const name = text.slice(i + 1, end);
14
+ if (end < 0 || !/^[a-zA-Z_][\w]*$/.test(name)) throw new Error('Python messages support named {placeholders} only.');
15
+ flush(); nodes.push({ type: 'argument', name }); i = end + 1;
16
+ } else if (text[i] === '}') throw new Error('Unmatched closing brace.');
17
+ else literal += text[i++];
18
+ }
19
+ flush(); return nodes;
20
+ }
21
+ if (syntax !== 'icu') throw new Error(`Unknown message syntax: ${syntax}`);
22
+ let pos = 0;
23
+ const fail = message => { throw new Error(`${message} (character ${pos + 1})`); };
24
+ const space = () => { while (/\s/.test(text[pos] || '') && pos < text.length) pos++; };
25
+ const word = () => { const start = pos; while (/[\w.-]/.test(text[pos] || '') && pos < text.length) pos++; return text.slice(start, pos); };
26
+ const expect = char => { space(); if (text[pos++] !== char) fail(`Expected ${char}`); };
27
+ function sequence(nested = false, plural = false, depth = 0, closing) {
28
+ if (depth > 40) fail('Message nesting is too deep');
29
+ const nodes = [];
30
+ let literal = '';
31
+ const flush = () => { if (literal) nodes.push(literal); literal = ''; };
32
+ while (pos < text.length) {
33
+ const c = text[pos];
34
+ if (c === '}') { if (!nested) fail('Unmatched closing brace'); break; }
35
+ // Rich-text tags: <b>chunks</b>. A closing tag ends the enclosing sequence.
36
+ if (c === '<' && syntax === 'icu') {
37
+ const close = /^<\/([a-zA-Z][\w-]*)>/.exec(text.slice(pos));
38
+ if (close) { if (closing !== close[1]) fail(`Unexpected </${close[1]}>`); break; }
39
+ const open = /^<([a-zA-Z][\w-]*)>/.exec(text.slice(pos));
40
+ if (open) {
41
+ if (open[1] === '__proto__') fail('Invalid tag name');
42
+ flush(); pos += open[0].length;
43
+ const children = sequence(nested, plural, depth + 1, open[1]);
44
+ if (text.slice(pos, pos + open[1].length + 3) !== `</${open[1]}>`) fail(`Unclosed <${open[1]}>`);
45
+ pos += open[1].length + 3;
46
+ nodes.push({ type: 'tag', name: open[1], children });
47
+ continue;
48
+ }
49
+ }
50
+ if (c === "'") {
51
+ if (text[pos + 1] === "'") { literal += "'"; pos += 2; continue; }
52
+ if (['{', '}', '<', ...(plural ? ['#'] : [])].includes(text[pos + 1])) {
53
+ pos++;
54
+ while (pos < text.length) {
55
+ if (text[pos] === "'") {
56
+ if (text[pos + 1] === "'") { literal += "'"; pos += 2; }
57
+ else { pos++; break; }
58
+ } else literal += text[pos++];
59
+ }
60
+ continue;
61
+ }
62
+ }
63
+ if (c === '#' && plural) { flush(); nodes.push({ type: 'pound' }); pos++; continue; }
64
+ if (c !== '{') { literal += c; pos++; continue; }
65
+ flush(); pos++; space();
66
+ const name = word();
67
+ if (!/^[a-zA-Z_]\w*$/.test(name) || name === '__proto__') fail('Expected a named placeholder');
68
+ space();
69
+ if (text[pos] === '}') { pos++; nodes.push({ type: 'argument', name }); continue; }
70
+ expect(','); space(); const type = word(); space();
71
+ if (['plural', 'selectordinal', 'select'].includes(type)) {
72
+ expect(','); space();
73
+ let offset = 0;
74
+ if (type !== 'select' && text.slice(pos, pos + 7) === 'offset:') {
75
+ pos += 7; space(); const n = /^\d+/.exec(text.slice(pos));
76
+ if (!n) fail('Expected a nonnegative plural offset');
77
+ offset = Number(n[0]); pos += n[0].length; space();
78
+ }
79
+ const options = Object.create(null);
80
+ while (pos < text.length && text[pos] !== '}') {
81
+ let selector;
82
+ if (text[pos] === '=' && type !== 'select') {
83
+ pos++; const n = /^-?\d+(?:\.\d+)?/.exec(text.slice(pos));
84
+ if (!n) fail('Expected a numeric plural selector');
85
+ selector = '=' + Number(n[0]); pos += n[0].length;
86
+ } else selector = word();
87
+ if (!selector || selector === '__proto__') fail('Expected a branch name');
88
+ if (type !== 'select' && !selector.startsWith('=') && !['zero', 'one', 'two', 'few', 'many', 'other'].includes(selector)) fail(`Unknown plural category ${selector}`);
89
+ if (Object.hasOwn(options, selector)) fail(`Duplicate branch ${selector}`);
90
+ expect('{'); options[selector] = sequence(true, type !== 'select' || plural, depth + 1); expect('}'); space();
91
+ }
92
+ if (!Object.hasOwn(options, 'other')) fail(`${type} needs an other branch`);
93
+ expect('}'); nodes.push({ type, name, offset, options });
94
+ } else if (['number', 'date', 'time'].includes(type)) {
95
+ let style = '';
96
+ if (text[pos] === ',') {
97
+ pos++; const start = pos; while (pos < text.length && text[pos] !== '}') pos++;
98
+ style = text.slice(start, pos).trim();
99
+ }
100
+ if (type === 'number' && !['', 'integer', 'percent'].includes(style) && !/^::currency\/[A-Z]{3}$/.test(style)) fail(`Unsupported number style ${style}`);
101
+ if (type !== 'number' && !['', 'short', 'medium', 'long', 'full'].includes(style)) fail(`Unsupported ${type} style ${style}`);
102
+ expect('}'); nodes.push({ type, name, style });
103
+ } else fail(`Unsupported message type ${type}`);
104
+ }
105
+ flush(); return nodes;
106
+ }
107
+ return sequence();
108
+ }
109
+
110
+ export function argumentsFor(nodes, out = Object.create(null)) {
111
+ for (const n of nodes) {
112
+ if (typeof n === 'string' || n.type === 'pound') continue;
113
+ if (n.type === 'tag') {
114
+ if (out[n.name] && out[n.name] !== 'tag') throw new Error(`<${n.name}> is also used as a placeholder.`);
115
+ out[n.name] = 'tag'; argumentsFor(n.children, out); continue;
116
+ }
117
+ if (out[n.name] === 'tag') throw new Error(`{${n.name}} is also used as a tag.`);
118
+ const type = ['number', 'plural', 'selectordinal'].includes(n.type) ? 'number' : ['date', 'time'].includes(n.type) ? 'date' : n.type === 'select' ? 'select' : 'string';
119
+ if (out[n.name] && out[n.name] !== type && out[n.name] !== 'string' && type !== 'string') throw new Error(`Conflicting types for {${n.name}}.`);
120
+ if (!out[n.name] || type !== 'string') out[n.name] = type;
121
+ if (n.options) for (const branch of Object.values(n.options)) argumentsFor(branch, out);
122
+ }
123
+ return out;
124
+ }
125
+
126
+ // A lone apostrophe is literal (ICU 4.8+), so "isn't" stays readable; only one
127
+ // that would start a quote, before a quoted character or another apostrophe, is doubled.
128
+ export function escapeMessageLiteral(text) {
129
+ return text.replace(/'(?=['{}#<|])|[{}]|<(?=\/?[a-zA-Z])/g, match => match === "'" ? "''" : `'${match}'`);
130
+ }
@@ -0,0 +1,162 @@
1
+ /**
2
+ * Format a compiled catalog using the host's Intl implementation.
3
+ * Messages without tags return strings. Messages with rich-text tags call the
4
+ * matching function with the tag's rendered chunks and return an array, which
5
+ * frameworks such as React render directly.
6
+ * onError receives problems such as an unknown token or a missing value. By
7
+ * default they are logged and the message degrades instead of crashing the UI;
8
+ * pass `error => { throw error; }` to fail fast in tests.
9
+ */
10
+ export function createI18n(catalog, { onError = error => console.error(`i18nmd: ${error.message}`) } = {}) {
11
+ function value(values, name) {
12
+ if (!Object.hasOwn(values, name)) { onError(new Error(`Missing value for {${name}}.`)); return undefined; }
13
+ return values[name];
14
+ }
15
+ function render(nodes, locale, values, pound) {
16
+ const parts = [];
17
+ const push = part => { if (typeof part === 'string' && typeof parts.at(-1) === 'string') parts[parts.length - 1] += part; else if (part !== '') parts.push(part); };
18
+ const pushAll = rendered => Array.isArray(rendered) ? rendered.forEach(push) : push(rendered);
19
+ for (const n of nodes) {
20
+ if (typeof n === 'string') { push(n); continue; }
21
+ if (n.type === 'pound') { push(new Intl.NumberFormat(locale).format(pound)); continue; }
22
+ if (n.type === 'tag') {
23
+ const children = render(n.children, locale, values, pound);
24
+ const fn = value(values, n.name);
25
+ if (typeof fn !== 'function') { if (fn !== undefined) onError(new Error(`<${n.name}> needs a function.`)); pushAll(children); continue; }
26
+ push(fn(Array.isArray(children) ? children : [children]));
27
+ continue;
28
+ }
29
+ if (!Object.hasOwn(values, n.name)) { value(values, n.name); push(`{${n.name}}`); continue; }
30
+ const v = values[n.name];
31
+ // Like JSX, a plain placeholder renders null or undefined as nothing.
32
+ if (n.type === 'argument') { push(v == null ? '' : String(v)); continue; }
33
+ if (v == null) { onError(new Error(`Missing value for {${n.name}}.`)); push(`{${n.name}}`); continue; }
34
+ if (['number', 'plural', 'selectordinal'].includes(n.type) && (typeof v !== 'number' || !Number.isFinite(v))) { onError(new Error(`{${n.name}} must be a finite number.`)); push(String(v)); continue; }
35
+ if (n.type === 'number') {
36
+ const options = n.style === 'integer' ? { maximumFractionDigits: 0 } : n.style === 'percent' ? { style: 'percent' } : n.style.startsWith('::currency/') ? { style: 'currency', currency: n.style.slice(11) } : {};
37
+ push(new Intl.NumberFormat(locale, options).format(v)); continue;
38
+ }
39
+ if (n.type === 'date' || n.type === 'time') {
40
+ const date = v instanceof Date ? v : typeof v === 'number' ? new Date(v) : null;
41
+ if (!date || Number.isNaN(date.getTime())) { onError(new Error(`{${n.name}} must be a Date or a timestamp.`)); push(String(v)); continue; }
42
+ push(new Intl.DateTimeFormat(locale, { [n.type + 'Style']: n.style || 'medium' }).format(date)); continue;
43
+ }
44
+ if (n.type === 'select') { pushAll(render(Object.hasOwn(n.options, String(v)) ? n.options[String(v)] : n.options.other, locale, values, pound)); continue; }
45
+ const adjusted = v - n.offset;
46
+ const category = new Intl.PluralRules(locale, { type: n.type === 'selectordinal' ? 'ordinal' : 'cardinal' }).select(adjusted);
47
+ pushAll(render(n.options['=' + v] || n.options[category] || n.options.other, locale, values, adjusted));
48
+ }
49
+ return parts.length === 0 ? '' : parts.length === 1 && typeof parts[0] === 'string' ? parts[0] : parts;
50
+ }
51
+ return function i18nmd(token, language = catalog.source, values = {}) {
52
+ if (!Object.hasOwn(catalog.messages, token)) { onError(new Error(`Unknown translation token: ${token}`)); return token; }
53
+ let requested = language;
54
+ try { requested = Intl.getCanonicalLocales(language)[0]; } catch { /* Custom language names use source formatting. */ }
55
+ const entry = catalog.messages[token];
56
+ const base = requested.split('-')[0];
57
+ const languageUsed = Object.hasOwn(entry, requested) ? requested : Object.hasOwn(entry, base) ? base : catalog.source;
58
+ // Fallback content is formatted with its actual language's plural rules.
59
+ let formattingLocale = languageUsed;
60
+ try { Intl.getCanonicalLocales(formattingLocale); } catch { formattingLocale = catalog.source; }
61
+ try { Intl.getCanonicalLocales(formattingLocale); } catch { formattingLocale = 'en'; }
62
+ return render(entry[languageUsed], formattingLocale, values || {});
63
+ };
64
+ }
65
+
66
+ /** The catalog language that best matches a requested one: exact, then base language. */
67
+ export function matchLanguage(catalog, requested) {
68
+ if (!requested) return undefined;
69
+ let canonical = requested;
70
+ try { canonical = Intl.getCanonicalLocales(requested)[0]; } catch { /* custom identifiers */ }
71
+ if (Object.hasOwn(catalog.languages, canonical)) return canonical;
72
+ const base = canonical.split('-')[0];
73
+ return Object.keys(catalog.languages).find(l => l === base || l.split('-')[0] === base);
74
+ }
75
+
76
+ /**
77
+ * The current language and the messages loaded so far, shared by everything that
78
+ * translates. Source-language messages arrive with each division's module
79
+ * (store.add); another language arrives whole from load(language) before
80
+ * setLanguage switches to it, so a page never mixes languages mid-render.
81
+ * The language starts as the reader's remembered choice, then the browser's best
82
+ * match, then the source; in a browser the choice is remembered under
83
+ * options.storageKey ("i18nmd:language"; null to not remember). divisions lists
84
+ * the division names, for clearer errors.
85
+ */
86
+ export function createLanguage(languages, source, { load, divisions = [], storageKey = 'i18nmd:language', onError = e => console.error(`i18nmd: ${e.message}`) } = {}) {
87
+ const store = { source, languages, messages: Object.create(null), divisions, loaded: new Set([source]) };
88
+ store.add = (table, language = source) => {
89
+ for (const [token, nodes] of Object.entries(table)) (store.messages[token] ||= Object.create(null))[language] = nodes;
90
+ };
91
+ const fetchLanguage = language => {
92
+ if (store.loaded.has(language)) return Promise.resolve();
93
+ if (!load) return Promise.reject(new Error(`No loader for ${language}.`));
94
+ return Promise.resolve(load(language)).then(table => { store.add(table, language); store.loaded.add(language); });
95
+ };
96
+ const catalog = { languages };
97
+ const browser = typeof document !== 'undefined' && typeof navigator !== 'undefined';
98
+ const storage = (action, value) => {
99
+ if (!browser || !storageKey) return null;
100
+ try { return action === 'get' ? localStorage.getItem(storageKey) : localStorage.setItem(storageKey, value); } catch { return null; }
101
+ };
102
+ // Only a browser's languages say what the reader wants; Node also has a navigator,
103
+ // whose language is the server's.
104
+ const preferred = browser ? [storage('get'), ...(navigator.languages || [navigator.language])] : [];
105
+ const wanted = preferred.map(l => matchLanguage(catalog, l)).find(Boolean) || source;
106
+ let current = source;
107
+ const listeners = new Set();
108
+ const switchTo = next => {
109
+ if (next === current) return;
110
+ current = next;
111
+ if (browser) document.documentElement.lang = next;
112
+ for (const listener of listeners) listener(next);
113
+ };
114
+ if (browser) document.documentElement.lang = source;
115
+ // Until the reader's language has loaded, pages render in the source language.
116
+ const ready = fetchLanguage(wanted).then(() => { switchTo(wanted); return current; }, error => { onError(error); return current; });
117
+ return {
118
+ store,
119
+ ready,
120
+ getLanguage: () => current,
121
+ /** Load a language's messages, for i18nmd.in(language) on a server or in tests. */
122
+ loadLanguage: language => fetchLanguage(matchLanguage(catalog, language) || source),
123
+ /** Load a language, switch every later call to it and remember it; resolves to the language chosen. */
124
+ async setLanguage(language) {
125
+ const next = matchLanguage(catalog, language);
126
+ if (!next) { onError(new Error(`Unknown language: ${language}`)); return current; }
127
+ try { await fetchLanguage(next); } catch (error) { onError(error); return current; }
128
+ storage('set', next);
129
+ switchTo(next);
130
+ return current;
131
+ },
132
+ /** Call listener with the new language after each change; returns an unsubscribe function. */
133
+ onLanguageChange(listener) { listeners.add(listener); return () => listeners.delete(listener); },
134
+ };
135
+ }
136
+
137
+ /**
138
+ * The generated i18nmd function over a language's store. Divisions are
139
+ * properties: i18nmd.support("contact_us") is the token support.contact_us.
140
+ * i18nmd.in("fr") translates into a given (loaded) language instead, for
141
+ * servers, emails and tests. divisions maps each property to [segment, children].
142
+ */
143
+ export function createTranslator(language, divisions = {}, options = {}) {
144
+ const { store } = language;
145
+ const onError = options.onError || (error => console.error(`i18nmd: ${error.message}`));
146
+ const translate = createI18n(store, { ...options, onError: error => {
147
+ // A token from a division whose module this code never imported.
148
+ const token = /^Unknown translation token: (.+)$/.exec(error.message)?.[1];
149
+ const division = token && store.divisions.filter(d => token.startsWith(d.replace(/\//g, '.') + '.')).sort((a, b) => b.length - a.length)[0];
150
+ onError(division ? new Error(`${token} is in the ${division} division, which this page has not imported. Import i18nmd from its module (i18nmd check --in <src> --fix adds it).`) : error);
151
+ } });
152
+ const build = (prefix, tree, fixed) => {
153
+ const scoped = (token, values) => translate(prefix + token, fixed ?? language.getLanguage(), values);
154
+ for (const [property, [segment, children]] of Object.entries(tree)) {
155
+ Object.defineProperty(scoped, property, { value: build(`${prefix}${segment}.`, children, fixed), enumerable: true });
156
+ }
157
+ return scoped;
158
+ };
159
+ const i18nmd = build('', divisions);
160
+ Object.defineProperty(i18nmd, 'in', { value: fixed => build('', divisions, fixed), enumerable: false });
161
+ return i18nmd;
162
+ }
package/package.json ADDED
@@ -0,0 +1,58 @@
1
+ {
2
+ "name": "@likolabs/i18nmd",
3
+ "version": "0.1.1",
4
+ "description": "Interface strings in Markdown, one file per language. A compiler checks every language and generates typed code your app imports.",
5
+ "author": "Liko Labs (https://likolabs.com)",
6
+ "type": "module",
7
+ "publishConfig": {
8
+ "access": "public"
9
+ },
10
+ "bin": {
11
+ "i18nmd": "bin/i18nmd.mjs"
12
+ },
13
+ "exports": {
14
+ ".": "./lib/index.mjs",
15
+ "./runtime": "./lib/runtime.mjs"
16
+ },
17
+ "files": [
18
+ "bin",
19
+ "lib",
20
+ "scripts/python_catalog.py",
21
+ "README.md",
22
+ "LICENSE",
23
+ "PROMPT.md",
24
+ "CHANGELOG.md"
25
+ ],
26
+ "scripts": {
27
+ "test": "node --test tests/*.test.mjs",
28
+ "check": "npm test"
29
+ },
30
+ "engines": {
31
+ "node": ">=22"
32
+ },
33
+ "dependencies": {
34
+ "@babel/parser": "7.29.0"
35
+ },
36
+ "repository": {
37
+ "type": "git",
38
+ "url": "git+https://github.com/birep/i18n.md.git"
39
+ },
40
+ "homepage": "https://i18n.md",
41
+ "bugs": {
42
+ "url": "https://github.com/birep/i18n.md/issues"
43
+ },
44
+ "keywords": [
45
+ "i18n",
46
+ "l10n",
47
+ "internationalization",
48
+ "localization",
49
+ "translation",
50
+ "icu",
51
+ "messageformat",
52
+ "markdown",
53
+ "react",
54
+ "typescript",
55
+ "llm"
56
+ ],
57
+ "license": "MIT"
58
+ }
@@ -0,0 +1,49 @@
1
+ """Read literal translation tables with ast; never execute an input module."""
2
+ import ast
3
+ import json
4
+ import string
5
+ import sys
6
+ from pathlib import Path
7
+
8
+
9
+ def read_catalog(filename, table_name="_T", languages_name="LANGS"):
10
+ source = Path(filename).read_text()
11
+ tree = ast.parse(source)
12
+ found = {}
13
+ for node in tree.body:
14
+ if isinstance(node, (ast.Assign, ast.AnnAssign)):
15
+ targets = node.targets if isinstance(node, ast.Assign) else [node.target]
16
+ for target in targets:
17
+ if isinstance(target, ast.Name) and target.id in (table_name, languages_name):
18
+ if target.id in found:
19
+ raise ValueError(f"Duplicate assignment to {target.id}")
20
+ found[target.id] = ast.literal_eval(node.value)
21
+ tables = found[table_name]
22
+ languages = found[languages_name]
23
+ if not isinstance(tables, dict) or not isinstance(languages, dict) or set(tables) != set(languages):
24
+ raise ValueError("Translation tables must match the declared languages")
25
+ source_language = "en" if "en" in tables else next(iter(tables))
26
+ keys = set(tables[source_language])
27
+ if any(set(table) != keys for table in tables.values()):
28
+ raise ValueError("Every language must have the same tokens")
29
+ formatter = string.Formatter()
30
+ messages = []
31
+ for key, value in tables[source_language].items():
32
+ translations = {lang: table[key] for lang, table in tables.items()}
33
+ params = {name for _, name, _, _ in formatter.parse(value) if name}
34
+ optional = set()
35
+ for text in translations.values():
36
+ names = {name for _, name, _, _ in formatter.parse(text) if name}
37
+ if not names <= params:
38
+ raise ValueError(f"{key}: a translation introduces unknown placeholders")
39
+ optional |= params - names
40
+ messages.append({"key": key, "context": f"{Path(filename).name}: {key}", "optional": sorted(optional), "translations": translations})
41
+ return {"title": "Application strings", "source": source_language, "syntax": "python", "languages": languages, "messages": messages}
42
+
43
+
44
+ if __name__ == "__main__":
45
+ try:
46
+ print(json.dumps(read_catalog(*sys.argv[1:]), ensure_ascii=False))
47
+ except (ValueError, KeyError, SyntaxError, OSError) as error:
48
+ print(str(error), file=sys.stderr)
49
+ sys.exit(1)