@verbaly/compiler 0.14.0 → 0.15.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { RichLink } from "verbaly";
+import { MessageTree, RichLink } from "verbaly";
 import MagicString from "magic-string";
 //#region src/analyze.d.ts
 interface TaggedParam {
@@ -150,6 +150,44 @@ interface DoctorResult {
 }
 declare function doctor(cfg: ResolvedConfig): Promise<DoctorResult>;
 //#endregion
+//#region src/exchange.d.ts
+type ExchangeFormat = 'xliff' | 'csv';
+interface ExportOptions {
+  locales?: string[];
+  format?: ExchangeFormat;
+  out?: string;
+  missing?: boolean;
+}
+interface ExportedFile {
+  locale: string;
+  path: string;
+  total: number;
+  untranslated: number;
+}
+interface ExportResult {
+  format: ExchangeFormat;
+  dir: string;
+  files: ExportedFile[];
+}
+interface ImportOptions {
+  locale?: string;
+  overwrite?: boolean;
+  dryRun?: boolean;
+}
+interface ImportResult {
+  imported: Record<string, string[]>;
+  rejected: Record<string, string[]>;
+  skipped: Record<string, string[]>;
+  unknown: Record<string, string[]>;
+}
+declare function exportCatalogs(cfg: ResolvedConfig, catalogs: Catalogs, options?: ExportOptions): ExportResult;
+declare function importCatalogs(cfg: ResolvedConfig, catalogs: Catalogs, files: string[], options?: ImportOptions): ImportResult;
+interface ParsedFile {
+  locale: string;
+  entries: Record<string, string>;
+}
+declare function parseExchangeFile(file: string, localeOverride?: string): ParsedFile;
+//#endregion
 //#region src/extract.d.ts
 declare function extractProject(cfg: ResolvedConfig): Promise<MessageRegistry>;
 interface SyncResult {
@@ -203,7 +241,7 @@ interface Alternate {
 }
 interface RenderHtmlOptions {
   locale: string;
-  catalogs: Catalogs;
+  catalogs: Catalogs | Record<string, MessageTree>;
   sourceLocale?: string;
   attribute?: string;
   richTags?: string[];
@@ -241,5 +279,5 @@ interface TransformResult {
 }
 declare function transformCode(code: string, file: string, analysis?: Analysis): TransformResult | null;
 //#endregion
-export { type Alternate, type Analysis, type Catalog, type Catalogs, type CheckResult, type ClaudeProviderOptions, type DoctorEntry, type DoctorResult, type InitOptions, type InitResult, MessageRegistry, type MissingEntry, PSEUDO_LOCALE, type ParamType, type RenderConfig, type RenderHtmlOptions, type RenderHtmlResult, type RenderSiteOptions, type RenderSiteResult, type ResolvedConfig, type SyncResult, type TaggedMessage, type TaggedParam, type TransComponent, type TransformResult, type TranslateConfig, type TranslateOptions, type TranslateProvider, type TranslateRequest, type TranslateResult, type UnknownEntry, type UsedKey, VIRTUAL_ID, type VerbalyConfig, analyze, catalogPath, check, claudeProvider, collectParams, detectBundler, doctor, extractProject, findConfigFile, formatCheckResult, generateDts, generateLocaleModule, generateRuntimeModule, init, loadCatalogs, loadConfig, loadConfigFile, pruneCatalogs, pseudoCatalogs, pseudoLocalize, readCatalog, renderHtml, renderParamType, renderSite, resolveConfig, serializeCatalog, stableKey, structureMatches, syncCatalogs, transformCode, translateCatalogs, writeCatalog, writeDts };
+export { type Alternate, type Analysis, type Catalog, type Catalogs, type CheckResult, type ClaudeProviderOptions, type DoctorEntry, type DoctorResult, type ExchangeFormat, type ExportOptions, type ExportResult, type ExportedFile, type ImportOptions, type ImportResult, type InitOptions, type InitResult, MessageRegistry, type MissingEntry, PSEUDO_LOCALE, type ParamType, type RenderConfig, type RenderHtmlOptions, type RenderHtmlResult, type RenderSiteOptions, type RenderSiteResult, type ResolvedConfig, type SyncResult, type TaggedMessage, type TaggedParam, type TransComponent, type TransformResult, type TranslateConfig, type TranslateOptions, type TranslateProvider, type TranslateRequest, type TranslateResult, type UnknownEntry, type UsedKey, VIRTUAL_ID, type VerbalyConfig, analyze, catalogPath, check, claudeProvider, collectParams, detectBundler, doctor, exportCatalogs, extractProject, findConfigFile, formatCheckResult, generateDts, generateLocaleModule, generateRuntimeModule, importCatalogs, init, loadCatalogs, loadConfig, loadConfigFile, parseExchangeFile, pruneCatalogs, pseudoCatalogs, pseudoLocalize, readCatalog, renderHtml, renderParamType, renderSite, resolveConfig, serializeCatalog, stableKey, structureMatches, syncCatalogs, transformCode, translateCatalogs, writeCatalog, writeDts };
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,7 +1,7 @@
 import { parse } from "@babel/parser";
 import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
-import { dirname, join, relative, resolve } from "node:path";
+import { basename, dirname, join, relative, resolve } from "node:path";
 import { RICH_TAGS, createVerbaly, parse as parse$1, parseTags, safeHref } from "verbaly";
 import { pathToFileURL } from "node:url";
 import { glob } from "tinyglobby";
@@ -794,6 +794,247 @@ function readDeps(root) {
 	}
 }
 //#endregion
+//#region src/translate.ts
+async function translateCatalogs(cfg, catalogs, provider, options = {}) {
+	const batchSize = options.batchSize ?? 20;
+	const targets = (options.locales ?? cfg.locales).filter((locale) => locale !== cfg.sourceLocale && cfg.locales.includes(locale));
+	const source = catalogs[cfg.sourceLocale] ?? {};
+	const result = {
+		translated: {},
+		invalid: {},
+		pending: {}
+	};
+	for (const locale of targets) {
+		const catalog = catalogs[locale] ??= {};
+		const missing = Object.keys(source).filter((key) => source[key] && !catalog[key]);
+		if (missing.length === 0) continue;
+		if (options.dryRun) {
+			result.pending[locale] = missing;
+			continue;
+		}
+		for (let i = 0; i < missing.length; i += batchSize) {
+			const keys = missing.slice(i, i + batchSize);
+			const messages = Object.fromEntries(keys.map((key) => [key, source[key]]));
+			const out = await provider({
+				sourceLocale: cfg.sourceLocale,
+				targetLocale: locale,
+				messages
+			});
+			for (const key of keys) {
+				const text = out[key];
+				if (typeof text === "string" && text.trim() && structureMatches(source[key], text)) {
+					catalog[key] = text;
+					(result.translated[locale] ??= []).push(key);
+				} else (result.invalid[locale] ??= []).push(key);
+			}
+		}
+	}
+	return result;
+}
+function structureMatches(source, translated) {
+	return sameMembers(paramNames(source), paramNames(translated)) && sameMembers(tagTokens(source), tagTokens(translated));
+}
+function paramNames(message) {
+	try {
+		return [...collectParams(message).keys()].sort();
+	} catch {
+		return ["\0invalid"];
+	}
+}
+const TAG = /<(\/?)([a-zA-Z][\w-]*)(\/?)>/g;
+function tagTokens(message) {
+	const out = [];
+	for (const match of message.matchAll(TAG)) out.push(`${match[1]}${match[2]}${match[3]}`);
+	return out.sort();
+}
+function sameMembers(a, b) {
+	return a.length === b.length && a.every((value, i) => value === b[i]);
+}
+//#endregion
+//#region src/exchange.ts
+function exportCatalogs(cfg, catalogs, options = {}) {
+	const format = options.format ?? "xliff";
+	const dir = resolve(cfg.root, options.out ?? "verbaly-export");
+	const source = catalogs[cfg.sourceLocale] ?? {};
+	const targets = (options.locales ?? cfg.locales).filter((locale) => locale !== cfg.sourceLocale && cfg.locales.includes(locale));
+	const files = [];
+	mkdirSync(dir, { recursive: true });
+	for (const locale of targets) {
+		const catalog = catalogs[locale] ?? {};
+		let entries = Object.keys(source).filter((key) => source[key]).sort().map((key) => ({
+			key,
+			source: source[key],
+			target: catalog[key] ?? ""
+		}));
+		if (options.missing) entries = entries.filter((entry) => !entry.target);
+		const path = join(dir, `${locale}.${format === "csv" ? "csv" : "xlf"}`);
+		writeFileSync(path, format === "csv" ? toCsv(entries) : toXliff(cfg.sourceLocale, locale, entries));
+		files.push({
+			locale,
+			path,
+			total: entries.length,
+			untranslated: entries.filter((entry) => !entry.target).length
+		});
+	}
+	return {
+		format,
+		dir,
+		files
+	};
+}
+function importCatalogs(cfg, catalogs, files, options = {}) {
+	const source = catalogs[cfg.sourceLocale] ?? {};
+	const result = {
+		imported: {},
+		rejected: {},
+		skipped: {},
+		unknown: {}
+	};
+	for (const file of files) {
+		const parsed = parseExchangeFile(file, options.locale);
+		const locale = parsed.locale;
+		if (!/^[a-zA-Z]{2,3}([-_][a-zA-Z0-9]+)*$/.test(locale)) throw new Error(`[verbaly] ${file}: "${locale}" doesn't look like a locale — pass --locale <id>.`);
+		if (locale === cfg.sourceLocale) throw new Error(`[verbaly] ${file} targets the source locale "${locale}" — import fills translations, not the source. Pass --locale if the detection is wrong.`);
+		const catalog = catalogs[locale] ??= {};
+		for (const [key, text] of Object.entries(parsed.entries)) {
+			if (!text.trim()) continue;
+			if (!source[key]) {
+				(result.unknown[locale] ??= []).push(key);
+				continue;
+			}
+			if (catalog[key] && !options.overwrite) {
+				(result.skipped[locale] ??= []).push(key);
+				continue;
+			}
+			if (!structureMatches(source[key], text)) {
+				(result.rejected[locale] ??= []).push(key);
+				continue;
+			}
+			if (!options.dryRun) catalog[key] = text;
+			(result.imported[locale] ??= []).push(key);
+		}
+	}
+	return result;
+}
+function parseExchangeFile(file, localeOverride) {
+	const content = readFileSync(file, "utf8");
+	if (/\.(xlf|xliff)$/i.test(file)) {
+		const parsed = parseXliff(content);
+		const locale = localeOverride ?? parsed.locale;
+		if (!locale) throw new Error(`[verbaly] ${file} has no trgLang/target-language — pass --locale <id>.`);
+		return {
+			locale,
+			entries: parsed.entries
+		};
+	}
+	if (/\.csv$/i.test(file)) return {
+		locale: localeOverride ?? basename(file).replace(/\.csv$/i, ""),
+		entries: parseCsv(content)
+	};
+	throw new Error(`[verbaly] ${file}: unsupported format — expected .xlf, .xliff or .csv.`);
+}
+function toXliff(sourceLocale, locale, entries) {
+	const units = entries.map(({ key, source, target }) => [
+		`    <unit id="${escapeXml(key)}">`,
+		`      <segment state="${target ? "translated" : "initial"}">`,
+		`        <source>${escapeXml(source)}</source>`,
+		`        <target>${escapeXml(target)}</target>`,
+		"      </segment>",
+		"    </unit>"
+	].join("\n")).join("\n");
+	return [
+		"<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
+		`<xliff xmlns="urn:oasis:names:tc:xliff:document:2.0" version="2.0" srcLang="${escapeXml(sourceLocale)}" trgLang="${escapeXml(locale)}">`,
+		`  <file id="verbaly" original="${escapeXml(`${locale}.json`)}">`,
+		units,
+		"  </file>",
+		"</xliff>",
+		""
+	].join("\n");
+}
+function parseXliff(content) {
+	const locale = /\btrgLang\s*=\s*"([^"]+)"/.exec(content)?.[1] ?? /\btarget-language\s*=\s*"([^"]+)"/.exec(content)?.[1];
+	const entries = {};
+	for (const match of content.matchAll(/<(?:trans-)?unit\b([^>]*)>([\s\S]*?)<\/(?:trans-)?unit>/g)) {
+		const id = /\bid\s*=\s*"([^"]*)"/.exec(match[1])?.[1];
+		if (!id) continue;
+		const target = /<target\b[^>]*>([\s\S]*?)<\/target>/.exec(match[2])?.[1];
+		entries[unescapeXml(id)] = target === void 0 ? "" : unescapeXml(stripCdata(target));
+	}
+	return {
+		locale,
+		entries
+	};
+}
+function escapeXml(text) {
+	return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function stripCdata(text) {
+	return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1");
+}
+function unescapeXml(text) {
+	return text.replace(/&(lt|gt|amp|quot|apos|#x?[0-9a-fA-F]+);/g, (_, entity) => {
+		if (entity === "lt") return "<";
+		if (entity === "gt") return ">";
+		if (entity === "amp") return "&";
+		if (entity === "quot") return "\"";
+		if (entity === "apos") return "'";
+		const code = entity.startsWith("#x") ? parseInt(entity.slice(2), 16) : parseInt(entity.slice(1), 10);
+		return String.fromCodePoint(code);
+	});
+}
+function toCsv(entries) {
+	return [
+		"key,source,target",
+		...entries.map(({ key, source, target }) => `${csvField(key)},${csvField(source)},${csvField(target)}`),
+		""
+	].join("\r\n");
+}
+function csvField(text) {
+	return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, "\"\"")}"` : text;
+}
+function parseCsv(content) {
+	const rows = csvRows(content);
+	const header = rows[0]?.map((cell) => cell.trim().toLowerCase()) ?? [];
+	const keyCol = header.indexOf("key");
+	const targetCol = header.indexOf("target");
+	if (keyCol === -1 || targetCol === -1) throw new Error("[verbaly] CSV needs a header row with \"key\" and \"target\" columns.");
+	const entries = {};
+	for (const row of rows.slice(1)) {
+		const key = row[keyCol];
+		if (key) entries[key] = row[targetCol] ?? "";
+	}
+	return entries;
+}
+function csvRows(content) {
+	const rows = [];
+	let row = [];
+	let field = "";
+	let quoted = false;
+	for (let i = 0; i < content.length; i++) {
+		const char = content[i];
+		if (quoted) if (char === "\"") if (content[i + 1] === "\"") {
+			field += "\"";
+			i++;
+		} else quoted = false;
+		else field += char;
+		else if (char === "\"") quoted = true;
+		else if (char === ",") {
+			row.push(field);
+			field = "";
+		} else if (char === "\n" || char === "\r") {
+			if (char === "\r" && content[i + 1] === "\n") i++;
+			row.push(field);
+			field = "";
+			if (row.some((cell) => cell !== "")) rows.push(row);
+			row = [];
+		} else field += char;
+	}
+	row.push(field);
+	if (row.some((cell) => cell !== "")) rows.push(row);
+	return rows;
+}
+//#endregion
 //#region src/providers/claude.ts
 const DEFAULT_MODEL = "claude-sonnet-5";
 const SYSTEM = `You translate UI strings for the Verbaly i18n library.
@@ -993,22 +1234,19 @@ function renderHtml(html, options) {
 	const richTags = new Set(options.richTags ?? RICH_TAGS);
 	const globalLinks = options.richLinks;
 	const sourceLocale = options.sourceLocale ?? "en";
-	const messages = {};
-	for (const [locale, catalog] of Object.entries(options.catalogs)) {
-		const clean = {};
-		for (const [key, msg] of Object.entries(catalog)) if (msg) clean[key] = msg;
-		messages[locale] = clean;
-	}
 	const v = createVerbaly({
 		locale: options.locale,
 		fallback: sourceLocale,
-		messages
+		messages: options.catalogs
 	});
 	const t = v.t;
 	const ms = new MagicString(html);
 	const missing = /* @__PURE__ */ new Set();
 	const skip = protectedRanges(html);
 	const inSkip = (index) => skip.some(([from, to]) => index >= from && index < to);
+	const selfUrl = options.alternates?.find((a) => a.hreflang === options.locale)?.href;
+	const sourceUrl = options.alternates?.find((a) => a.hreflang === "x-default")?.href;
+	const rewriteUrl = selfUrl !== void 0 && selfUrl !== sourceUrl;
 	START_TAG.lastIndex = 0;
 	let m;
 	while ((m = START_TAG.exec(html)) !== null) {
@@ -1022,6 +1260,8 @@ function renderHtml(html, options) {
 			continue;
 		}
 		const attrs = parseAttrs(attrChunk);
+		if (rewriteUrl && tagName === "link" && attrs.get("rel")?.toLowerCase() === "canonical" && decodeEntities(attrs.get("href") ?? "") === sourceUrl) setAttribute(ms, html, chunkStart, openEnd, attrChunk, "href", selfUrl);
+		if (rewriteUrl && tagName === "meta" && attrs.get("property") === "og:url" && decodeEntities(attrs.get("content") ?? "") === sourceUrl) setAttribute(ms, html, chunkStart, openEnd, attrChunk, "content", selfUrl);
 		const key = attrs.get(attr);
 		const attrMapRaw = attrs.get(attrsAttr);
 		if (key === void 0 && attrMapRaw === void 0) continue;
@@ -1259,63 +1499,6 @@ function transformCode(code, file, analysis) {
 	};
 }
 //#endregion
-//#region src/translate.ts
-async function translateCatalogs(cfg, catalogs, provider, options = {}) {
-	const batchSize = options.batchSize ?? 20;
-	const targets = (options.locales ?? cfg.locales).filter((locale) => locale !== cfg.sourceLocale && cfg.locales.includes(locale));
-	const source = catalogs[cfg.sourceLocale] ?? {};
-	const result = {
-		translated: {},
-		invalid: {},
-		pending: {}
-	};
-	for (const locale of targets) {
-		const catalog = catalogs[locale] ??= {};
-		const missing = Object.keys(source).filter((key) => source[key] && !catalog[key]);
-		if (missing.length === 0) continue;
-		if (options.dryRun) {
-			result.pending[locale] = missing;
-			continue;
-		}
-		for (let i = 0; i < missing.length; i += batchSize) {
-			const keys = missing.slice(i, i + batchSize);
-			const messages = Object.fromEntries(keys.map((key) => [key, source[key]]));
-			const out = await provider({
-				sourceLocale: cfg.sourceLocale,
-				targetLocale: locale,
-				messages
-			});
-			for (const key of keys) {
-				const text = out[key];
-				if (typeof text === "string" && text.trim() && structureMatches(source[key], text)) {
-					catalog[key] = text;
-					(result.translated[locale] ??= []).push(key);
-				} else (result.invalid[locale] ??= []).push(key);
-			}
-		}
-	}
-	return result;
-}
-function structureMatches(source, translated) {
-	return sameMembers(paramNames(source), paramNames(translated)) && sameMembers(tagTokens(source), tagTokens(translated));
-}
-function paramNames(message) {
-	try {
-		return [...collectParams(message).keys()].sort();
-	} catch {
-		return ["\0invalid"];
-	}
-}
-const TAG = /<(\/?)([a-zA-Z][\w-]*)(\/?)>/g;
-function tagTokens(message) {
-	const out = [];
-	for (const match of message.matchAll(TAG)) out.push(`${match[1]}${match[2]}${match[3]}`);
-	return out.sort();
-}
-function sameMembers(a, b) {
-	return a.length === b.length && a.every((value, i) => value === b[i]);
-}
-//#endregion
-export { MessageRegistry, PSEUDO_LOCALE, VIRTUAL_ID, analyze, catalogPath, check, claudeProvider, collectParams, detectBundler, doctor, extractProject, findConfigFile, formatCheckResult, generateDts, generateLocaleModule, generateRuntimeModule, init, loadCatalogs, loadConfig, loadConfigFile, pruneCatalogs, pseudoCatalogs, pseudoLocalize, readCatalog, renderHtml, renderParamType, renderSite, resolveConfig, serializeCatalog, stableKey, structureMatches, syncCatalogs, transformCode, translateCatalogs, writeCatalog, writeDts };
+export { MessageRegistry, PSEUDO_LOCALE, VIRTUAL_ID, analyze, catalogPath, check, claudeProvider, collectParams, detectBundler, doctor, exportCatalogs, extractProject, findConfigFile, formatCheckResult, generateDts, generateLocaleModule, generateRuntimeModule, importCatalogs, init, loadCatalogs, loadConfig, loadConfigFile, parseExchangeFile, pruneCatalogs, pseudoCatalogs, pseudoLocalize, readCatalog, renderHtml, renderParamType, renderSite, resolveConfig, serializeCatalog, stableKey, structureMatches, syncCatalogs, transformCode, translateCatalogs, writeCatalog, writeDts };
 
 //# sourceMappingURL=index.js.map