npm - @verbaly/compiler - Versions diffs - 0.14.0 → 0.15.0 - Mend

@verbaly/compiler 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (7) hide show

package/README.md CHANGED Viewed

@@ -17,13 +17,15 @@ The compiler behind [Verbaly](https://github.com/AronSoto/verbaly): AST extracti
 ## 🧰 CLI
-```
+```bash
 npx verbaly init           # scaffold config + locale catalogs (detects your bundler)
 npx verbaly doctor         # diagnose the setup (config, catalogs, plugin, types, keys)
 npx verbaly extract        # sync catalogs + types
 npx verbaly check          # exit 1 if anything is missing (CI)
 npx verbaly extract --prune  # drop orphaned keys
 npx verbaly translate      # fill missing translations via Claude (or your provider)
+npx verbaly export         # write translator-ready XLIFF 2.0 / CSV files per locale
+npx verbaly import <files> # fill catalogs back from translated XLIFF/CSV files
 npx verbaly pseudo         # generate a pseudo-locale catalog for i18n QA (en-XA)
 npx verbaly render         # pre-fill data-verbaly HTML per locale (SSG — kills the FOUC)
 ```
@@ -34,19 +36,31 @@ Reads `verbaly.config.{js,mjs,ts,mts,json}` (TS configs need `esbuild` installed
 `verbaly translate` fills the `""` holes `check` reports. The default provider uses Claude via the official SDK — install it as a dev dependency (translation is a build-time step, not an app runtime dependency): `pnpm add -D @anthropic-ai/sdk` (or `npm i -D`), plus `ANTHROPIC_API_KEY`. Default model is `claude-sonnet-5` (balanced quality/cost); override with `translate.model` in config or `--model <id>`. Placeholders, variants and tags are validated after translation — anything not preserved verbatim stays `""` so `check` keeps failing. Plug your own provider in `verbaly.config.ts`:
-```
+```ts
 translate: {
   provider: async ({ sourceLocale, targetLocale, messages }) => ({ ...translated });
 }
 ```
+## 🌍 Human translators & TMS
+Catalogs are **flat JSON** — most TMS platforms (Crowdin, Lokalise, Phrase, …) ingest them natively; point the platform at `locales/` and you're done. For everything else there's a built-in round-trip:
+```bash
+npx verbaly export                    # verbaly-export/<locale>.xlf (XLIFF 2.0, source + target per unit)
+npx verbaly export --format csv       # spreadsheet-friendly: key,source,target
+npx verbaly import verbaly-export/es.xlf   # fill the catalog back
+```
+`export` writes one file per target locale with the source text alongside the current translation (`--missing` exports only the untranslated entries). `import` reads XLIFF 2.0/1.2 or CSV back and **validates every entry like `translate` does** — a translation that drops a `{param}`, a variant block or an `<em>` tag is rejected and reported, so a translator's typo can't break your UI. Existing translations are kept unless `--overwrite`; `--dry-run` previews everything.
 ## 📄 Static rendering (SSG)
 `verbaly render` walks your built site (`dist/` by default, `--site <path>` to change) and pre-fills every `data-verbaly` element **per locale** using the real runtime — plurals, `Intl` formatting, `data-verbaly-args`, attribute translation and `data-verbaly-rich` (same whitelist, XSS-safe). The source locale is filled in place; every other locale is mirrored to `dist/<locale>/…` with `<html lang>` set. Static HTML ships already translated — **no flash of untranslated content** — and the runtime attributes stay put, so client-side locale switching keeps working.
 Named links in rich messages render as real `<a>` elements — hrefs come from config or markup, never from messages (`javascript:` blocked):
-```
+```ts
 // verbaly.config.ts
 render: { links: { docs: { href: '/docs', target: '_blank', rel: 'noopener' } } }
 ```

package/dist/cli.js CHANGED Viewed

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { parseArgs } from "node:util";
 import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
-import { dirname, join, relative, resolve } from "node:path";
+import { basename, dirname, join, relative, resolve } from "node:path";
 import { RICH_TAGS, createVerbaly, parse, parseTags, safeHref } from "verbaly";
 import { pathToFileURL } from "node:url";
 import { glob } from "tinyglobby";
@@ -759,6 +759,247 @@ function readDeps(root) {
 	}
 }
 //#endregion
+//#region src/translate.ts
+async function translateCatalogs(cfg, catalogs, provider, options = {}) {
+	const batchSize = options.batchSize ?? 20;
+	const targets = (options.locales ?? cfg.locales).filter((locale) => locale !== cfg.sourceLocale && cfg.locales.includes(locale));
+	const source = catalogs[cfg.sourceLocale] ?? {};
+	const result = {
+		translated: {},
+		invalid: {},
+		pending: {}
+	};
+	for (const locale of targets) {
+		const catalog = catalogs[locale] ??= {};
+		const missing = Object.keys(source).filter((key) => source[key] && !catalog[key]);
+		if (missing.length === 0) continue;
+		if (options.dryRun) {
+			result.pending[locale] = missing;
+			continue;
+		}
+		for (let i = 0; i < missing.length; i += batchSize) {
+			const keys = missing.slice(i, i + batchSize);
+			const messages = Object.fromEntries(keys.map((key) => [key, source[key]]));
+			const out = await provider({
+				sourceLocale: cfg.sourceLocale,
+				targetLocale: locale,
+				messages
+			});
+			for (const key of keys) {
+				const text = out[key];
+				if (typeof text === "string" && text.trim() && structureMatches(source[key], text)) {
+					catalog[key] = text;
+					(result.translated[locale] ??= []).push(key);
+				} else (result.invalid[locale] ??= []).push(key);
+			}
+		}
+	}
+	return result;
+}
+function structureMatches(source, translated) {
+	return sameMembers(paramNames(source), paramNames(translated)) && sameMembers(tagTokens(source), tagTokens(translated));
+}
+function paramNames(message) {
+	try {
+		return [...collectParams(message).keys()].sort();
+	} catch {
+		return ["\0invalid"];
+	}
+}
+const TAG = /<(\/?)([a-zA-Z][\w-]*)(\/?)>/g;
+function tagTokens(message) {
+	const out = [];
+	for (const match of message.matchAll(TAG)) out.push(`${match[1]}${match[2]}${match[3]}`);
+	return out.sort();
+}
+function sameMembers(a, b) {
+	return a.length === b.length && a.every((value, i) => value === b[i]);
+}
+//#endregion
+//#region src/exchange.ts
+function exportCatalogs(cfg, catalogs, options = {}) {
+	const format = options.format ?? "xliff";
+	const dir = resolve(cfg.root, options.out ?? "verbaly-export");
+	const source = catalogs[cfg.sourceLocale] ?? {};
+	const targets = (options.locales ?? cfg.locales).filter((locale) => locale !== cfg.sourceLocale && cfg.locales.includes(locale));
+	const files = [];
+	mkdirSync(dir, { recursive: true });
+	for (const locale of targets) {
+		const catalog = catalogs[locale] ?? {};
+		let entries = Object.keys(source).filter((key) => source[key]).sort().map((key) => ({
+			key,
+			source: source[key],
+			target: catalog[key] ?? ""
+		}));
+		if (options.missing) entries = entries.filter((entry) => !entry.target);
+		const path = join(dir, `${locale}.${format === "csv" ? "csv" : "xlf"}`);
+		writeFileSync(path, format === "csv" ? toCsv(entries) : toXliff(cfg.sourceLocale, locale, entries));
+		files.push({
+			locale,
+			path,
+			total: entries.length,
+			untranslated: entries.filter((entry) => !entry.target).length
+		});
+	}
+	return {
+		format,
+		dir,
+		files
+	};
+}
+function importCatalogs(cfg, catalogs, files, options = {}) {
+	const source = catalogs[cfg.sourceLocale] ?? {};
+	const result = {
+		imported: {},
+		rejected: {},
+		skipped: {},
+		unknown: {}
+	};
+	for (const file of files) {
+		const parsed = parseExchangeFile(file, options.locale);
+		const locale = parsed.locale;
+		if (!/^[a-zA-Z]{2,3}([-_][a-zA-Z0-9]+)*$/.test(locale)) throw new Error(`[verbaly] ${file}: "${locale}" doesn't look like a locale — pass --locale <id>.`);
+		if (locale === cfg.sourceLocale) throw new Error(`[verbaly] ${file} targets the source locale "${locale}" — import fills translations, not the source. Pass --locale if the detection is wrong.`);
+		const catalog = catalogs[locale] ??= {};
+		for (const [key, text] of Object.entries(parsed.entries)) {
+			if (!text.trim()) continue;
+			if (!source[key]) {
+				(result.unknown[locale] ??= []).push(key);
+				continue;
+			}
+			if (catalog[key] && !options.overwrite) {
+				(result.skipped[locale] ??= []).push(key);
+				continue;
+			}
+			if (!structureMatches(source[key], text)) {
+				(result.rejected[locale] ??= []).push(key);
+				continue;
+			}
+			if (!options.dryRun) catalog[key] = text;
+			(result.imported[locale] ??= []).push(key);
+		}
+	}
+	return result;
+}
+function parseExchangeFile(file, localeOverride) {
+	const content = readFileSync(file, "utf8");
+	if (/\.(xlf|xliff)$/i.test(file)) {
+		const parsed = parseXliff(content);
+		const locale = localeOverride ?? parsed.locale;
+		if (!locale) throw new Error(`[verbaly] ${file} has no trgLang/target-language — pass --locale <id>.`);
+		return {
+			locale,
+			entries: parsed.entries
+		};
+	}
+	if (/\.csv$/i.test(file)) return {
+		locale: localeOverride ?? basename(file).replace(/\.csv$/i, ""),
+		entries: parseCsv(content)
+	};
+	throw new Error(`[verbaly] ${file}: unsupported format — expected .xlf, .xliff or .csv.`);
+}
+function toXliff(sourceLocale, locale, entries) {
+	const units = entries.map(({ key, source, target }) => [
+		`    <unit id="${escapeXml(key)}">`,
+		`      <segment state="${target ? "translated" : "initial"}">`,
+		`        <source>${escapeXml(source)}</source>`,
+		`        <target>${escapeXml(target)}</target>`,
+		"      </segment>",
+		"    </unit>"
+	].join("\n")).join("\n");
+	return [
+		"<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
+		`<xliff xmlns="urn:oasis:names:tc:xliff:document:2.0" version="2.0" srcLang="${escapeXml(sourceLocale)}" trgLang="${escapeXml(locale)}">`,
+		`  <file id="verbaly" original="${escapeXml(`${locale}.json`)}">`,
+		units,
+		"  </file>",
+		"</xliff>",
+		""
+	].join("\n");
+}
+function parseXliff(content) {
+	const locale = /\btrgLang\s*=\s*"([^"]+)"/.exec(content)?.[1] ?? /\btarget-language\s*=\s*"([^"]+)"/.exec(content)?.[1];
+	const entries = {};
+	for (const match of content.matchAll(/<(?:trans-)?unit\b([^>]*)>([\s\S]*?)<\/(?:trans-)?unit>/g)) {
+		const id = /\bid\s*=\s*"([^"]*)"/.exec(match[1])?.[1];
+		if (!id) continue;
+		const target = /<target\b[^>]*>([\s\S]*?)<\/target>/.exec(match[2])?.[1];
+		entries[unescapeXml(id)] = target === void 0 ? "" : unescapeXml(stripCdata(target));
+	}
+	return {
+		locale,
+		entries
+	};
+}
+function escapeXml(text) {
+	return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function stripCdata(text) {
+	return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1");
+}
+function unescapeXml(text) {
+	return text.replace(/&(lt|gt|amp|quot|apos|#x?[0-9a-fA-F]+);/g, (_, entity) => {
+		if (entity === "lt") return "<";
+		if (entity === "gt") return ">";
+		if (entity === "amp") return "&";
+		if (entity === "quot") return "\"";
+		if (entity === "apos") return "'";
+		const code = entity.startsWith("#x") ? parseInt(entity.slice(2), 16) : parseInt(entity.slice(1), 10);
+		return String.fromCodePoint(code);
+	});
+}
+function toCsv(entries) {
+	return [
+		"key,source,target",
+		...entries.map(({ key, source, target }) => `${csvField(key)},${csvField(source)},${csvField(target)}`),
+		""
+	].join("\r\n");
+}
+function csvField(text) {
+	return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, "\"\"")}"` : text;
+}
+function parseCsv(content) {
+	const rows = csvRows(content);
+	const header = rows[0]?.map((cell) => cell.trim().toLowerCase()) ?? [];
+	const keyCol = header.indexOf("key");
+	const targetCol = header.indexOf("target");
+	if (keyCol === -1 || targetCol === -1) throw new Error("[verbaly] CSV needs a header row with \"key\" and \"target\" columns.");
+	const entries = {};
+	for (const row of rows.slice(1)) {
+		const key = row[keyCol];
+		if (key) entries[key] = row[targetCol] ?? "";
+	}
+	return entries;
+}
+function csvRows(content) {
+	const rows = [];
+	let row = [];
+	let field = "";
+	let quoted = false;
+	for (let i = 0; i < content.length; i++) {
+		const char = content[i];
+		if (quoted) if (char === "\"") if (content[i + 1] === "\"") {
+			field += "\"";
+			i++;
+		} else quoted = false;
+		else field += char;
+		else if (char === "\"") quoted = true;
+		else if (char === ",") {
+			row.push(field);
+			field = "";
+		} else if (char === "\n" || char === "\r") {
+			if (char === "\r" && content[i + 1] === "\n") i++;
+			row.push(field);
+			field = "";
+			if (row.some((cell) => cell !== "")) rows.push(row);
+			row = [];
+		} else field += char;
+	}
+	row.push(field);
+	if (row.some((cell) => cell !== "")) rows.push(row);
+	return rows;
+}
+//#endregion
 //#region src/pseudo.ts
 const PSEUDO_LOCALE = "en-XA";
 const ACCENTS = {
@@ -908,22 +1149,19 @@ function renderHtml(html, options) {
 	const richTags = new Set(options.richTags ?? RICH_TAGS);
 	const globalLinks = options.richLinks;
 	const sourceLocale = options.sourceLocale ?? "en";
-	const messages = {};
-	for (const [locale, catalog] of Object.entries(options.catalogs)) {
-		const clean = {};
-		for (const [key, msg] of Object.entries(catalog)) if (msg) clean[key] = msg;
-		messages[locale] = clean;
-	}
 	const v = createVerbaly({
 		locale: options.locale,
 		fallback: sourceLocale,
-		messages
+		messages: options.catalogs
 	});
 	const t = v.t;
 	const ms = new MagicString(html);
 	const missing = /* @__PURE__ */ new Set();
 	const skip = protectedRanges(html);
 	const inSkip = (index) => skip.some(([from, to]) => index >= from && index < to);
+	const selfUrl = options.alternates?.find((a) => a.hreflang === options.locale)?.href;
+	const sourceUrl = options.alternates?.find((a) => a.hreflang === "x-default")?.href;
+	const rewriteUrl = selfUrl !== void 0 && selfUrl !== sourceUrl;
 	START_TAG.lastIndex = 0;
 	let m;
 	while ((m = START_TAG.exec(html)) !== null) {
@@ -937,6 +1175,8 @@ function renderHtml(html, options) {
 			continue;
 		}
 		const attrs = parseAttrs(attrChunk);
+		if (rewriteUrl && tagName === "link" && attrs.get("rel")?.toLowerCase() === "canonical" && decodeEntities(attrs.get("href") ?? "") === sourceUrl) setAttribute(ms, html, chunkStart, openEnd, attrChunk, "href", selfUrl);
+		if (rewriteUrl && tagName === "meta" && attrs.get("property") === "og:url" && decodeEntities(attrs.get("content") ?? "") === sourceUrl) setAttribute(ms, html, chunkStart, openEnd, attrChunk, "content", selfUrl);
 		const key = attrs.get(attr);
 		const attrMapRaw = attrs.get(attrsAttr);
 		if (key === void 0 && attrMapRaw === void 0) continue;
@@ -1149,63 +1389,6 @@ function decodeEntities(text) {
 	return text.replace(/&quot;/g, "\"").replace(/&#39;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&");
 }
 //#endregion
-//#region src/translate.ts
-async function translateCatalogs(cfg, catalogs, provider, options = {}) {
-	const batchSize = options.batchSize ?? 20;
-	const targets = (options.locales ?? cfg.locales).filter((locale) => locale !== cfg.sourceLocale && cfg.locales.includes(locale));
-	const source = catalogs[cfg.sourceLocale] ?? {};
-	const result = {
-		translated: {},
-		invalid: {},
-		pending: {}
-	};
-	for (const locale of targets) {
-		const catalog = catalogs[locale] ??= {};
-		const missing = Object.keys(source).filter((key) => source[key] && !catalog[key]);
-		if (missing.length === 0) continue;
-		if (options.dryRun) {
-			result.pending[locale] = missing;
-			continue;
-		}
-		for (let i = 0; i < missing.length; i += batchSize) {
-			const keys = missing.slice(i, i + batchSize);
-			const messages = Object.fromEntries(keys.map((key) => [key, source[key]]));
-			const out = await provider({
-				sourceLocale: cfg.sourceLocale,
-				targetLocale: locale,
-				messages
-			});
-			for (const key of keys) {
-				const text = out[key];
-				if (typeof text === "string" && text.trim() && structureMatches(source[key], text)) {
-					catalog[key] = text;
-					(result.translated[locale] ??= []).push(key);
-				} else (result.invalid[locale] ??= []).push(key);
-			}
-		}
-	}
-	return result;
-}
-function structureMatches(source, translated) {
-	return sameMembers(paramNames(source), paramNames(translated)) && sameMembers(tagTokens(source), tagTokens(translated));
-}
-function paramNames(message) {
-	try {
-		return [...collectParams(message).keys()].sort();
-	} catch {
-		return ["\0invalid"];
-	}
-}
-const TAG = /<(\/?)([a-zA-Z][\w-]*)(\/?)>/g;
-function tagTokens(message) {
-	const out = [];
-	for (const match of message.matchAll(TAG)) out.push(`${match[1]}${match[2]}${match[3]}`);
-	return out.sort();
-}
-function sameMembers(a, b) {
-	return a.length === b.length && a.every((value, i) => value === b[i]);
-}
-//#endregion
 //#region src/cli.ts
 const HELP = `verbaly — i18n compiler
@@ -1215,6 +1398,8 @@ Usage:
   verbaly extract    scan sources, update catalogs and types
   verbaly check      verify translations are complete (CI)
   verbaly translate  fill missing translations via a provider (default: claude)
+  verbaly export     write translator-ready files per locale (XLIFF 2.0 or CSV)
+  verbaly import <files…>  fill catalogs back from translated XLIFF/CSV files
   verbaly pseudo     generate a pseudo-locale catalog for i18n QA (default: en-XA)
   verbaly render     pre-fill data-verbaly HTML per locale (SSG, kills the FOUC)
@@ -1225,8 +1410,12 @@ Options:
   --locales <csv>    extra locales; for translate: target locales to fill
   --prune            drop keys no longer referenced (extract)
   --model <id>       model override for the claude provider (translate)
-  --dry-run          list what would be translated, write nothing (translate)
-  --locale <id>      pseudo-locale id (pseudo, default: en-XA)
+  --dry-run          list what would happen, write nothing (translate, import)
+  --format <f>       export format: xliff (default) or csv (export)
+  --out <path>       export directory (export, default: verbaly-export)
+  --missing          export only untranslated entries (export)
+  --overwrite        replace existing translations on import (import)
+  --locale <id>      pseudo-locale id (pseudo) / target-locale override (import)
   --site <path>      built site directory (render, default: dist)
   --attribute <name> base data attribute (render, default: data-verbaly)
   --base-url <url>   site origin — enables hreflang alternates (render)
@@ -1253,6 +1442,10 @@ async function main() {
 			sitemap: { type: "boolean" },
 			clean: { type: "boolean" },
 			"dry-run": { type: "boolean" },
+			format: { type: "string" },
+			out: { type: "string" },
+			missing: { type: "boolean" },
+			overwrite: { type: "boolean" },
 			help: {
 				type: "boolean",
 				short: "h"
@@ -1355,6 +1548,51 @@ async function main() {
 		if (Object.keys(result.translated).length === 0 && Object.keys(result.invalid).length === 0) console.log("[verbaly] nothing to translate ✓");
 		return;
 	}
+	if (command === "export") {
+		const format = values.format ?? "xliff";
+		if (format !== "xliff" && format !== "csv") {
+			console.error(`[verbaly] unknown format "${values.format}" — use xliff or csv`);
+			process.exitCode = 1;
+			return;
+		}
+		const result = exportCatalogs(cfg, loadCatalogs(cfg), {
+			locales: values.locales?.split(","),
+			format,
+			out: values.out,
+			missing: values.missing
+		});
+		if (result.files.length === 0) {
+			console.log("[verbaly] no target locales to export (add locales to your config)");
+			return;
+		}
+		console.log(`[verbaly] exported ${result.files.length} locales (${result.format}) → ${result.dir}`);
+		for (const file of result.files) console.log(`  ${file.locale}: ${file.total} messages (${file.untranslated} untranslated) → ${file.path}`);
+		return;
+	}
+	if (command === "import") {
+		const files = positionals.slice(1);
+		if (files.length === 0) {
+			console.error("[verbaly] import needs at least one file: verbaly import verbaly-export/es.xlf");
+			process.exitCode = 1;
+			return;
+		}
+		const catalogs = loadCatalogs(cfg);
+		const result = importCatalogs(cfg, catalogs, files, {
+			locale: values.locale,
+			overwrite: values.overwrite,
+			dryRun: values["dry-run"]
+		});
+		for (const [locale, keys] of Object.entries(result.imported)) {
+			if (!values["dry-run"]) writeCatalog(cfg, locale, catalogs[locale] ?? {});
+			const verb = values["dry-run"] ? "would import" : "imported";
+			console.log(`  ${locale}: +${keys.length} ${verb}`);
+		}
+		for (const [locale, keys] of Object.entries(result.skipped)) console.log(`  ${locale}: ${keys.length} already translated, kept (use --overwrite to replace)`);
+		for (const [locale, keys] of Object.entries(result.rejected)) console.warn(`  ${locale}: ${keys.length} rejected (params/tags not preserved): ${keys.join(", ")}`);
+		for (const [locale, keys] of Object.entries(result.unknown)) console.warn(`  ${locale}: ${keys.length} unknown keys ignored (not in the source catalog): ${keys.join(", ")}`);
+		if (Object.keys(result.imported).length === 0) console.log("[verbaly] nothing to import ✓");
+		return;
+	}
 	if (command === "render") {
 		const result = await renderSite(cfg, {
 			site: values.site,