@verbaly/compiler 0.27.0 → 0.28.0

package/README.md

@@ -26,8 +26,8 @@ npx verbaly extract --prune  # drop orphaned keys
 npx verbaly status         # translation coverage per locale, at a glance
 npx verbaly check          # exit 1 if anything is missing (CI)
 npx verbaly translate      # fill missing translations via Claude (or your provider)
-npx verbaly export         # translator files (XLIFF 2.0, CSV) or mobile resources (Android, iOS)
-npx verbaly import <files> # fill catalogs back from translated XLIFF/CSV files
+npx verbaly export         # translator files (XLIFF 2.0, CSV, gettext PO) or mobile resources (Android, iOS)
+npx verbaly import <files> # fill catalogs back from translated XLIFF/CSV/PO files
 npx verbaly pseudo         # generate a pseudo-locale catalog for i18n QA (en-XA)
 npx verbaly render         # pre-fill data-verbaly HTML per locale (SSG, kills the FOUC)
 ```
@@ -51,10 +51,11 @@ Catalogs are **flat JSON**: most TMS platforms (Crowdin, Lokalise, Phrase, …)
 ```bash
 npx verbaly export                    # verbaly-export/<locale>.xlf (XLIFF 2.0, source + target per unit)
 npx verbaly export --format csv       # spreadsheet-friendly: key,source,target,location
+npx verbaly export --format po        # gettext PO (msgctxt = key, works with any PO editor)
 npx verbaly import verbaly-export/es.xlf   # fill the catalog back
 ```
 
-`export` writes one file per target locale with the source text alongside the current translation (`--missing` exports only the untranslated entries). Every entry carries **where the text lives in your source** (XLIFF `location` notes, a `location` column in CSV), so translators and TMS tools see the context instead of guessing it. `import` reads XLIFF 2.0/1.2 or CSV back and **validates every entry like `translate` does**: a translation that drops a `{param}`, a variant block or an `<em>` tag is rejected and reported, so a translator's typo can't break your UI. Existing translations are kept unless `--overwrite`; `--dry-run` previews everything.
+`export` writes one file per target locale with the source text alongside the current translation (`--missing` exports only the untranslated entries). Every entry carries **where the text lives in your source** (XLIFF `location` notes, a `location` column in CSV, `#:` comments in PO), so translators and TMS tools see the context instead of guessing it. In XLIFF, `{params}` and rich tags travel as **protected inline codes with semantic ids** (`<ph id="name"/>`, `<pc id="em">`), so TMS editors show them as untouchable chips instead of editable raw syntax. `import` reads XLIFF 2.0/1.2, CSV or PO back (PO entries flagged `fuzzy` count as untranslated) and **validates every entry like `translate` does**: a translation that drops a `{param}`, a variant block or an `<em>` tag is rejected and reported, so a translator's typo can't break your UI. Existing translations are kept unless `--overwrite`; `--dry-run` previews everything.
 
 ## 📱 Mobile resources
 

package/dist/cli.js

@@ -331,7 +331,7 @@ function pseudoLocalize(message) {
 		}
 		const ch = message[i];
 		if (ch === "{") {
-			const end = matchBrace(message, i);
+			const end = matchBrace$1(message, i);
 			out += message.slice(i, end);
 			i = end;
 			continue;
@@ -355,7 +355,7 @@ function pseudoLocalize(message) {
 	const pad = "~".repeat(Math.ceil(letters / 3));
 	return `⟦${out}${pad ? " " + pad : ""}⟧`;
 }
-function matchBrace(message, start) {
+function matchBrace$1(message, start) {
 	let depth = 0;
 	for (let i = start; i < message.length; i++) {
 		const two = message.slice(i, i + 2);
@@ -1143,6 +1143,222 @@ function effectiveDrafts(drafts, catalogs) {
 	return out;
 }
 //#endregion
+//#region src/inline.ts
+const OPEN_TAG = /^<([a-zA-Z][a-zA-Z0-9-]*)\s*(\/?)>/;
+function messageToInline(text) {
+	return renderParts(parseParts(text, 0).parts, /* @__PURE__ */ new Map());
+}
+function inlineToMessage(xml) {
+	let out = "";
+	let text = "";
+	let i = 0;
+	const flush = () => {
+		out += unescapeXml(text);
+		text = "";
+	};
+	while (i < xml.length) {
+		if (xml[i] === "<") {
+			const slice = xml.slice(i);
+			const ph = /^<ph\b([^>]*?)\/\s*>/.exec(slice) ?? /^<ph\b([^>]*)>\s*<\/ph\s*>/.exec(slice);
+			if (ph) {
+				flush();
+				out += attr(ph[1], "disp") ?? `{${attr(ph[1], "id") ?? "ph"}}`;
+				i += ph[0].length;
+				continue;
+			}
+			const pc = /^<pc\b([^>]*)>/.exec(slice);
+			if (pc) {
+				const close = findPcClose(xml, i + pc[0].length);
+				if (close) {
+					flush();
+					const id = attr(pc[1], "id") ?? "pc";
+					out += attr(pc[1], "dispStart") ?? `<${id}>`;
+					out += inlineToMessage(xml.slice(i + pc[0].length, close.innerEnd));
+					out += attr(pc[1], "dispEnd") ?? `</${id}>`;
+					i = close.end;
+					continue;
+				}
+			}
+		}
+		text += xml[i];
+		i += 1;
+	}
+	flush();
+	return out;
+}
+function escapeXml(text) {
+	return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function unescapeXml(text) {
+	return text.replace(/&(lt|gt|amp|quot|apos|#x?[0-9a-fA-F]+);/g, (_, entity) => {
+		if (entity === "lt") return "<";
+		if (entity === "gt") return ">";
+		if (entity === "amp") return "&";
+		if (entity === "quot") return "\"";
+		if (entity === "apos") return "'";
+		const code = entity.startsWith("#x") ? parseInt(entity.slice(2), 16) : parseInt(entity.slice(1), 10);
+		return String.fromCodePoint(code);
+	});
+}
+function parseParts(text, pos, closeTag) {
+	const parts = [];
+	let plain = "";
+	const flush = () => {
+		if (plain) parts.push({
+			kind: "text",
+			text: plain
+		});
+		plain = "";
+	};
+	let i = pos;
+	while (i < text.length) {
+		const pair = text[i] + (text[i + 1] ?? "");
+		if (pair === "{{" || pair === "}}") {
+			plain += pair;
+			i += 2;
+			continue;
+		}
+		if (text[i] === "{") {
+			const end = matchBrace(text, i);
+			if (end === -1) {
+				plain += "{";
+				i += 1;
+				continue;
+			}
+			const src = text.slice(i, end + 1);
+			if (hasTopLevelPipe(src)) plain += src;
+			else {
+				flush();
+				parts.push({
+					kind: "ph",
+					name: paramName(src),
+					src
+				});
+			}
+			i = end + 1;
+			continue;
+		}
+		if (text[i] === "<") {
+			if (closeTag && text.startsWith(`</${closeTag}>`, i)) {
+				flush();
+				return {
+					parts,
+					end: i + closeTag.length + 3,
+					closed: true
+				};
+			}
+			const open = OPEN_TAG.exec(text.slice(i));
+			if (open) {
+				const [openSrc, name, selfClose] = open;
+				if (selfClose) {
+					flush();
+					parts.push({
+						kind: "ph",
+						name: idSafe(name),
+						src: openSrc
+					});
+					i += openSrc.length;
+					continue;
+				}
+				const inner = parseParts(text, i + openSrc.length, name);
+				if (inner.closed) {
+					flush();
+					parts.push({
+						kind: "pc",
+						name: idSafe(name),
+						openSrc,
+						closeSrc: `</${name}>`,
+						children: inner.parts
+					});
+					i = inner.end;
+					continue;
+				}
+			}
+			plain += "<";
+			i += 1;
+			continue;
+		}
+		plain += text[i];
+		i += 1;
+	}
+	flush();
+	return {
+		parts,
+		end: i,
+		closed: false
+	};
+}
+function matchBrace(text, start) {
+	let depth = 0;
+	for (let i = start; i < text.length; i++) {
+		const pair = text[i] + (text[i + 1] ?? "");
+		if (pair === "{{" || pair === "}}") {
+			i += 1;
+			continue;
+		}
+		if (text[i] === "{") depth += 1;
+		else if (text[i] === "}") {
+			depth -= 1;
+			if (depth === 0) return i;
+		}
+	}
+	return -1;
+}
+function hasTopLevelPipe(src) {
+	let depth = 0;
+	for (let i = 0; i < src.length; i++) {
+		const pair = src[i] + (src[i + 1] ?? "");
+		if (pair === "{{" || pair === "}}" || pair === "||") {
+			i += 1;
+			continue;
+		}
+		if (src[i] === "{") depth += 1;
+		else if (src[i] === "}") depth -= 1;
+		else if (src[i] === "|" && depth === 1) return true;
+	}
+	return false;
+}
+function paramName(src) {
+	const name = /^\{\s*([^{}|:\s]+)/.exec(src)?.[1];
+	return name ? idSafe(name) : "ph";
+}
+function idSafe(name) {
+	return name.replace(/[^a-zA-Z0-9._-]/g, "_") || "ph";
+}
+function renderParts(parts, taken) {
+	let out = "";
+	for (const part of parts) if (part.kind === "text") out += escapeXml(part.text);
+	else if (part.kind === "ph") out += `<ph id="${uniqueId(part.name, taken)}" disp="${escapeXml(part.src)}"/>`;
+	else {
+		const id = uniqueId(part.name, taken);
+		out += `<pc id="${id}" dispStart="${escapeXml(part.openSrc)}" dispEnd="${escapeXml(part.closeSrc)}">`;
+		out += renderParts(part.children, taken);
+		out += "</pc>";
+	}
+	return out;
+}
+function uniqueId(name, taken) {
+	const count = taken.get(name) ?? 0;
+	taken.set(name, count + 1);
+	return count === 0 ? name : `${name}${count + 1}`;
+}
+function findPcClose(xml, from) {
+	const tags = /<pc\b[^>]*>|<\/pc\s*>/g;
+	tags.lastIndex = from;
+	let depth = 1;
+	for (let match = tags.exec(xml); match; match = tags.exec(xml)) {
+		depth += match[0].startsWith("</") ? -1 : 1;
+		if (depth === 0) return {
+			innerEnd: match.index,
+			end: tags.lastIndex
+		};
+	}
+}
+function attr(attrs, name) {
+	const value = new RegExp(`\\b${name}\\s*=\\s*"([^"]*)"`).exec(attrs)?.[1];
+	return value === void 0 ? void 0 : unescapeXml(value);
+}
+//#endregion
 //#region src/mobile.ts
 function androidValuesDir(locale, sourceLocale) {
 	if (locale === sourceLocale) return "values";
@@ -1182,6 +1398,97 @@ function iosText(text) {
 	return text.replace(/\\/g, "\\\\").replace(/"/g, "\\\"").replace(/\n/g, "\\n").replace(/\t/g, "\\t");
 }
 //#endregion
+//#region src/po.ts
+function toPo(sourceLocale, locale, entries) {
+	return [
+		[
+			"msgid \"\"",
+			"msgstr \"\"",
+			`${poString("Project-Id-Version: verbaly\n")}`,
+			`${poString("MIME-Version: 1.0\n")}`,
+			`${poString("Content-Type: text/plain; charset=UTF-8\n")}`,
+			`${poString("Content-Transfer-Encoding: 8bit\n")}`,
+			`${poString(`Language: ${locale}\n`)}`,
+			`${poString(`X-Source-Language: ${sourceLocale}\n`)}`
+		].join("\n"),
+		...entries.map(({ key, source, target, location }) => [
+			...location?.length ? [`#: ${location.join(" ")}`] : [],
+			`msgctxt ${poString(key)}`,
+			`msgid ${poString(source)}`,
+			`msgstr ${poString(target)}`
+		].join("\n")),
+		""
+	].join("\n\n");
+}
+function parsePo(content) {
+	const entries = {};
+	let locale;
+	let msgctxt;
+	let msgid;
+	let msgstr;
+	let fuzzy = false;
+	let field;
+	const finish = () => {
+		if (msgid === "" && msgctxt === void 0) {
+			const lang = /^Language:[ \t]*(.+?)[ \t]*$/m.exec(msgstr ?? "")?.[1];
+			if (lang) locale = lang;
+		} else if (msgctxt !== void 0 && msgid !== void 0) entries[msgctxt] = fuzzy ? "" : msgstr ?? "";
+		msgctxt = msgid = msgstr = field = void 0;
+		fuzzy = false;
+	};
+	for (const raw of content.replace(/^\uFEFF/, "").split(/\r\n|[\r\n]/)) {
+		const line = raw.trim();
+		if (line === "") {
+			finish();
+			continue;
+		}
+		if (line.startsWith("#")) {
+			if (/^#,.*\bfuzzy\b/.test(line)) fuzzy = true;
+			continue;
+		}
+		const keyword = /^(msgctxt|msgid_plural|msgid|msgstr(?:\[\d+\])?)\s+(.*)$/.exec(line);
+		if (keyword) {
+			const [, name, rest] = keyword;
+			const value = poUnquote(rest);
+			if (value === void 0) continue;
+			if (name === "msgctxt") {
+				msgctxt = value;
+				field = "msgctxt";
+			} else if (name === "msgid") {
+				msgid = value;
+				field = "msgid";
+			} else if (name === "msgstr" || name === "msgstr[0]") {
+				msgstr = value;
+				field = "msgstr";
+			} else field = "other";
+			continue;
+		}
+		const continuation = poUnquote(line);
+		if (continuation === void 0 || field === void 0 || field === "other") continue;
+		if (field === "msgctxt") msgctxt = (msgctxt ?? "") + continuation;
+		else if (field === "msgid") msgid = (msgid ?? "") + continuation;
+		else msgstr = (msgstr ?? "") + continuation;
+	}
+	finish();
+	return {
+		locale,
+		entries
+	};
+}
+function poString(text) {
+	return `"${text.replace(/\\/g, "\\\\").replace(/"/g, "\\\"").replace(/\n/g, "\\n").replace(/\t/g, "\\t").replace(/\r/g, "\\r")}"`;
+}
+function poUnquote(text) {
+	const match = /^"((?:[^"\\]|\\.)*)"$/.exec(text);
+	if (!match) return void 0;
+	return match[1].replace(/\\(.)/g, (_, ch) => {
+		if (ch === "n") return "\n";
+		if (ch === "t") return "	";
+		if (ch === "r") return "\r";
+		return ch;
+	});
+}
+//#endregion
 //#region src/translate.ts
 async function translateCatalogs(cfg, catalogs, provider, options = {}) {
 	const batchSize = options.batchSize ?? 20;
@@ -1263,8 +1570,8 @@ function exportCatalogs(cfg, catalogs, options = {}) {
 		}));
 		const untranslated = all.filter((entry) => !entry.target);
 		const entries = options.missing ? untranslated : all;
-		const path = join(dir, `${locale}.${format === "csv" ? "csv" : "xlf"}`);
-		writeFileSync(path, format === "csv" ? toCsv(entries) : toXliff(cfg.sourceLocale, locale, entries));
+		const path = join(dir, `${locale}.${format === "xliff" ? "xlf" : format}`);
+		writeFileSync(path, format === "csv" ? toCsv(entries) : format === "po" ? toPo(cfg.sourceLocale, locale, entries) : toXliff(cfg.sourceLocale, locale, entries));
 		files.push({
 			locale,
 			path,
@@ -1352,7 +1659,14 @@ function parseExchangeFile(file, localeOverride) {
 		locale: localeOverride ?? basename(file).replace(/\.csv$/i, ""),
 		entries: parseCsv(content)
 	};
-	throw new Error(`[verbaly] ${file}: unsupported format, expected .xlf, .xliff or .csv.`);
+	if (/\.po$/i.test(file)) {
+		const parsed = parsePo(content);
+		return {
+			locale: localeOverride ?? parsed.locale ?? basename(file).replace(/\.po$/i, ""),
+			entries: parsed.entries
+		};
+	}
+	throw new Error(`[verbaly] ${file}: unsupported format, expected .xlf, .xliff, .csv or .po.`);
 }
 function toXliff(sourceLocale, locale, entries) {
 	const units = entries.map(({ key, source, target, location }) => [
@@ -1363,8 +1677,8 @@ function toXliff(sourceLocale, locale, entries) {
 			"      </notes>"
 		] : [],
 		`      <segment state="${target ? "translated" : "initial"}">`,
-		`        <source>${escapeXml(source)}</source>`,
-		`        <target>${escapeXml(target)}</target>`,
+		`        <source>${messageToInline(source)}</source>`,
+		`        <target>${messageToInline(target)}</target>`,
 		"      </segment>",
 		"    </unit>"
 	].join("\n")).join("\n");
@@ -1385,30 +1699,16 @@ function parseXliff(content) {
 		const id = /\bid\s*=\s*"([^"]*)"/.exec(match[1])?.[1];
 		if (!id) continue;
 		const target = /<target\b[^>]*>([\s\S]*?)<\/target>/.exec(match[2])?.[1];
-		entries[unescapeXml(id)] = target === void 0 ? "" : unescapeXml(stripCdata(target));
+		entries[unescapeXml(id)] = target === void 0 ? "" : inlineToMessage(stripCdata(target));
 	}
 	return {
 		locale,
 		entries
 	};
 }
-function escapeXml(text) {
-	return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
-}
 function stripCdata(text) {
 	return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1");
 }
-function unescapeXml(text) {
-	return text.replace(/&(lt|gt|amp|quot|apos|#x?[0-9a-fA-F]+);/g, (_, entity) => {
-		if (entity === "lt") return "<";
-		if (entity === "gt") return ">";
-		if (entity === "amp") return "&";
-		if (entity === "quot") return "\"";
-		if (entity === "apos") return "'";
-		const code = entity.startsWith("#x") ? parseInt(entity.slice(2), 16) : parseInt(entity.slice(1), 10);
-		return String.fromCodePoint(code);
-	});
-}
 function toCsv(entries) {
 	return [
 		"key,source,target,location",
@@ -1581,7 +1881,8 @@ async function renderSite(cfg, options = {}) {
 	const attribute = options.attribute ?? cfg.render.attribute;
 	const baseUrl = (options.baseUrl ?? cfg.render.baseUrl)?.replace(/\/+$/, "");
 	const wantHreflang = (options.hreflang ?? cfg.render.hreflang ?? true) && baseUrl !== void 0;
-	const wantSitemap = (options.sitemap ?? cfg.render.sitemap ?? false) && baseUrl !== void 0;
+	const sitemap = options.sitemap ?? cfg.render.sitemap ?? false;
+	const wantSitemap = sitemap !== false && baseUrl !== void 0;
 	const clean = options.clean ?? cfg.render.clean ?? false;
 	const catalogs = loadCatalogs(cfg);
 	if (clean) {
@@ -1624,7 +1925,7 @@ async function renderSite(cfg, options = {}) {
 			writeFileSync(out, result.html);
 		}
 	}
-	if (wantSitemap && urls.length) writeFileSync(join(site, typeof wantSitemap === "string" ? wantSitemap : "sitemap-i18n.xml"), buildSitemap(urls));
+	if (wantSitemap && urls.length) writeFileSync(join(site, typeof sitemap === "string" ? sitemap : "sitemap-i18n.xml"), buildSitemap(urls));
 	return {
 		files: files.length,
 		locales: [...locales],
@@ -2035,8 +2336,8 @@ Usage:
   verbaly check      verify translations are complete (CI)
   verbaly translate  fill missing translations via a provider (default: claude)
   verbaly review     list machine translations awaiting review (--approve marks them reviewed)
-  verbaly export     write translator files (XLIFF 2.0, CSV) or mobile resources (Android, iOS)
-  verbaly import <files…>  fill catalogs back from translated XLIFF/CSV files
+  verbaly export     write translator files (XLIFF 2.0, CSV, gettext PO) or mobile resources (Android, iOS)
+  verbaly import <files…>  fill catalogs back from translated XLIFF/CSV/PO files
   verbaly pseudo     generate a pseudo-locale catalog for i18n QA (default: en-XA)
   verbaly render     pre-fill data-verbaly HTML per locale (SSG, kills the FOUC)
 
@@ -2054,7 +2355,7 @@ Options:
   --reporter <name>  failure format: text (default) or github annotations (check)
   --model <id>       model override for the claude provider (translate)
   --dry-run          list what would happen, write nothing (translate, import, extract)
-  --format <f>       export format: xliff (default), csv, android-xml or ios-strings (export)
+  --format <f>       export format: xliff (default), csv, po, android-xml or ios-strings (export)
   --out <path>       export directory (export, default: verbaly-export)
   --missing          export only untranslated entries (export)
   --overwrite        replace existing translations on import (import)
@@ -2293,10 +2594,11 @@ async function runCli(args = process.argv.slice(2)) {
 		if (![
 			"xliff",
 			"csv",
+			"po",
 			"android-xml",
 			"ios-strings"
 		].includes(format)) {
-			console.error(`[verbaly] unknown format "${values.format}", use xliff, csv, android-xml or ios-strings`);
+			console.error(`[verbaly] unknown format "${values.format}", use xliff, csv, po, android-xml or ios-strings`);
 			process.exitCode = 1;
 			return;
 		}