@verbaly/studio 0.61.0 → 0.63.0

package/README.md

@@ -70,6 +70,7 @@ keep in step. Every answer is JSON, including the errors, which are `{ "error":
 |---|---|
 | `GET /api/state` | the whole project: config, catalogs, origins, status, check, drafts, triage and problems |
 | `GET /api/health` | what `verbaly doctor` reports |
+| `GET /api/commit/:key` | the commit that last changed that message, or why there is none |
 | `PUT /api/message/:locale/:key` | write a translation **and clear its draft flag**; body `{ "text": "…" }` |
 | `POST /api/approve` | approve drafts; body `{ "locale": "es", "keys": ["…"] }`, and the whole locale when `keys` is omitted |
 | `POST /api/extract` | read your code and add what your catalogs do not have; answers in the request, because it is local and free |
@@ -108,6 +109,13 @@ catalogs is a race over the same files.
 **What a machine writes stays a draft**, here as everywhere else. The panel does not get to change
 that rule, so a finished run leaves you a list to read, not a job marked done.
 
+**`GET /api/commit/:key` answers about values, not about lines.** It walks your catalogs' history
+once and remembers, per message, the newest commit where that message's text really differs from its
+parent's. `git blame` is the obvious way to ask and it is wrong here: adding a key rewrites the
+previous line's comma, so blame blames that commit, which on our own site is 26 messages in every
+100. When there is no answer the route says which of the three reasons applies, because "never
+committed", "older than the history I read" and "there is no git here" are three different facts.
+
 ## 🔎 Triage: which machine translations are worth reading
 
 Adding a language writes hundreds of drafts at once, and reading them one by one is a wall. Walls
@@ -133,7 +141,7 @@ an unfinished translation is not a second opinion.
 changed because of what the measurement said.** `digits` was reading the comma after a number as
 part of it, and it was flagging a number the translation added where the source had none, which is
 what "refresh" becoming "F5" looks like. Those were **10 of 10** of its hits, all correct text, so
-both are now excluded. The result is **23 messages** flagged in Spanish and 22 in Portuguese.
+both are now excluded. The result is **21 messages** flagged in Spanish and 20 in Portuguese.
 
 ## 🧩 Programmatic API
 

package/dist/cli.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import { parseArgs, promisify } from "node:util";
-import { DRAFTS_FILE, check, clearDrafts, collectOrigins, counted, doctor, effectiveDrafts, extractProject, formatCliError, loadCatalogs, loadConfig, loadDrafts, markDrafts, readCatalog, resolveProvider, saveDrafts, status, syncCatalogs, targetLocales, translateCatalogs, validateMessage, validatePair, writeCatalog, writeDts } from "@verbaly/compiler";
+import { DRAFTS_FILE, check, clearDrafts, collectOrigins, counted, doctor, effectiveDrafts, extractProject, formatCliError, loadCatalogs, loadConfig, loadDrafts, markDrafts, parseCatalog, readCatalog, resolveProvider, saveDrafts, status, syncCatalogs, targetLocales, translateCatalogs, validateMessage, validatePair, writeCatalog, writeDts } from "@verbaly/compiler";
 import { existsSync, readFileSync, statSync } from "node:fs";
 import { join, normalize, relative, resolve } from "node:path";
 import { createServer } from "node:http";
-import { execFile } from "node:child_process";
+import { execFile, spawn } from "node:child_process";
 import { fileURLToPath } from "node:url";
 import { randomBytes, timingSafeEqual } from "node:crypto";
 //#region src/http.ts
@@ -37,6 +37,27 @@ const bareDigits = (v) => trimPunct(v).replaceAll(/[.,]/g, ".");
 function push(out, key, reason) {
 	(out[key] ??= []).push(reason);
 }
+function clip(text, max = 42) {
+	return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+}
+const quote = (text, max) => `"${clip(text, max)}"`;
+function lost(from, to) {
+	const rest = [...to];
+	const out = [];
+	for (const item of from) {
+		const at = rest.indexOf(item);
+		if (at < 0) out.push(item);
+		else rest.splice(at, 1);
+	}
+	return out;
+}
+function movedText(kind, from, to) {
+	const gone = lost(from, to);
+	const added = lost(to, from);
+	if (gone.length && added.length) return `the source ${kind} ${quote(gone[0])}, this one ${kind} ${quote(added[0])}`;
+	if (gone.length) return `the source ${kind} ${quote(gone[0])} and this one does not`;
+	return `this one ${kind} ${quote(added[0])} and the source does not`;
+}
 function triage({ source, target, reviewed }) {
 	const out = Object.create(null);
 	const keys = Object.keys(source);
@@ -47,38 +68,61 @@ function triage({ source, target, reviewed }) {
 	}
 	for (const [text, group] of Object.entries(bySource)) {
 		if (group.length < 2) continue;
-		const divergent = reviewed.some((catalog) => new Set(group.map((k) => catalog[k]?.trim() ?? "").filter(Boolean)).size > 1);
-		if (divergent) for (const key of group) push(out, key, {
+		let split;
+		for (const { locale, catalog } of reviewed) {
+			const seen = [...new Set(group.map((k) => catalog[k]?.trim() ?? "").filter(Boolean))];
+			if (seen.length > 1) {
+				split = {
+					locale,
+					first: seen[0],
+					second: seen[1]
+				};
+				break;
+			}
+		}
+		if (split) for (const key of group) push(out, key, {
 			signal: "divergent",
-			text: `"${text}" was already translated two ways`
+			text: `${split.locale} already says this two ways: ${quote(split.first, 32)} and ${quote(split.second, 32)}`
 		});
 		const targets = new Set(group.map((key) => target[key]?.trim() ?? "").filter(Boolean));
-		if (!divergent && targets.size > 1) for (const key of group) push(out, key, {
-			signal: "collision",
-			text: `"${text}" came back as two different translations`
-		});
+		if (!split && targets.size > 1) for (const key of group) {
+			const mine = target[key]?.trim() ?? "";
+			const other = group.find((k) => k !== key && (target[k]?.trim() ?? "") !== mine && target[k]?.trim());
+			push(out, key, {
+				signal: "collision",
+				text: other ? `${other} has the same source text and says ${quote(target[other].trim())}` : `${quote(text)} came back as two different translations`
+			});
+		}
 	}
 	for (const key of keys) {
 		const src = source[key]?.trim() ?? "";
 		const trg = target[key]?.trim() ?? "";
 		if (!src || !trg) continue;
-		const opinions = reviewed.map((c) => c[key]?.trim()).filter((v) => v !== void 0 && v !== "");
-		if (trg === src && opinions.length > 0 && opinions.every((v) => v !== src)) push(out, key, {
+		const voters = reviewed.filter(({ catalog }) => {
+			const value = catalog[key]?.trim();
+			return value !== void 0 && value !== "";
+		});
+		if (trg === src && voters.length > 0 && voters.every(({ catalog }) => catalog[key]?.trim() !== src)) push(out, key, {
 			signal: "echo",
-			text: "still identical to the source"
+			text: `still the source text, and ${voters.map(({ locale }) => locale).join(" and ")} did translate it`
 		});
 		const srcDigits = bag(src, DIGITS, bareDigits);
-		if (srcDigits.length > 0 && !same(srcDigits, bag(trg, DIGITS, bareDigits))) push(out, key, {
+		const trgDigits = bag(trg, DIGITS, bareDigits);
+		if (srcDigits.length > 0 && !same(srcDigits, trgDigits)) push(out, key, {
 			signal: "digits",
-			text: "the numbers do not match the source"
+			text: movedText("says", srcDigits, trgDigits)
 		});
-		if (!same(bag(src, URLS, trimPunct), bag(trg, URLS, trimPunct))) push(out, key, {
+		const srcUrls = bag(src, URLS, trimPunct);
+		const trgUrls = bag(trg, URLS, trimPunct);
+		if (!same(srcUrls, trgUrls)) push(out, key, {
 			signal: "url",
-			text: "a URL changed"
+			text: movedText("links to", srcUrls, trgUrls)
 		});
-		if (!same(bag(src, CODE), bag(trg, CODE))) push(out, key, {
+		const srcCode = bag(src, CODE);
+		const trgCode = bag(trg, CODE);
+		if (!same(srcCode, trgCode)) push(out, key, {
 			signal: "code",
-			text: "the contents of a <code> span changed"
+			text: movedText("shows the code", srcCode, trgCode)
 		});
 	}
 	return out;
@@ -151,7 +195,10 @@ async function buildState(cfg) {
 	for (const locale of targets) perLocale[locale] = triage({
 		source,
 		target: catalogs[locale] ?? {},
-		reviewed: targets.filter((other) => other !== locale && !(live[other] ?? []).length).map((other) => catalogs[other] ?? {})
+		reviewed: targets.filter((other) => other !== locale && !(live[other] ?? []).length).map((other) => ({
+			locale: other,
+			catalog: catalogs[other] ?? {}
+		}))
 	});
 	return {
 		root: cfg.root,
@@ -313,13 +360,13 @@ async function startTranslate(cfg, locales) {
 const run = promisify(execFile);
 const FIELD = "";
 const FORMAT = `--format=%H${FIELD}%an${FIELD}%aI${FIELD}%s`;
-function parseLog(stdout) {
+function parseFullLog(stdout) {
 	const commits = [];
 	for (const line of stdout.split("\n")) {
 		const [sha, author, date, ...rest] = line.split(FIELD);
 		if (!sha || !author || !date || !rest.length) continue;
 		commits.push({
-			sha: sha.slice(0, 7),
+			sha,
 			author,
 			date,
 			subject: rest.join(FIELD)
@@ -327,6 +374,12 @@ function parseLog(stdout) {
 	}
 	return commits;
 }
+function parseLog(stdout) {
+	return parseFullLog(stdout).map((commit) => ({
+		...commit,
+		sha: commit.sha.slice(0, 7)
+	}));
+}
 async function catalogHistory(cfg, limit = 5) {
 	const dir = relative(cfg.root, cfg.dir).replaceAll("\\", "/") || ".";
 	try {
@@ -346,6 +399,182 @@ async function catalogHistory(cfg, limit = 5) {
 		return [];
 	}
 }
+function attribute(revisions, owners = /* @__PURE__ */ new Map()) {
+	for (const { commit, now, before } of revisions) {
+		if (!now || !before) continue;
+		for (const key of /* @__PURE__ */ new Set([...Object.keys(now), ...Object.keys(before)])) if (!owners.has(key) && now[key] !== before[key]) owners.set(key, commit);
+	}
+	return owners;
+}
+function mergeOwners(maps) {
+	const merged = /* @__PURE__ */ new Map();
+	for (const map of maps) for (const [key, commit] of map) {
+		const held = merged.get(key);
+		if (!held || Date.parse(commit.date) > Date.parse(held.date)) merged.set(key, commit);
+	}
+	return merged;
+}
+const BLOB = /^[0-9a-f]{40} \w+ (\d+)$/;
+function readBlobs(root, requests, take) {
+	return new Promise((resolve) => {
+		const child = spawn("git", ["cat-file", "--batch"], {
+			cwd: root,
+			windowsHide: true
+		});
+		const stop = setTimeout(() => {
+			ok = false;
+			child.kill();
+		}, 2e4);
+		let pending = Buffer.alloc(0);
+		let index = 0;
+		let ok = true;
+		const fail = () => {
+			ok = false;
+		};
+		child.on("error", fail);
+		child.stdin.on("error", fail);
+		child.stdout.on("data", (chunk) => {
+			pending = pending.length ? Buffer.concat([pending, chunk]) : chunk;
+			for (;;) {
+				const end = pending.indexOf(10);
+				if (end < 0) break;
+				const header = pending.toString("utf8", 0, end);
+				const blob = BLOB.exec(header);
+				if (!blob) {
+					take(index++, null, "");
+					pending = pending.subarray(end + 1);
+					continue;
+				}
+				const size = Number(blob[1]);
+				if (pending.length < end + size + 2) break;
+				take(index++, pending.toString("utf8", end + 1, end + 1 + size), header.slice(0, 40));
+				pending = pending.subarray(end + size + 2);
+			}
+		});
+		child.on("close", () => {
+			clearTimeout(stop);
+			resolve(ok);
+		});
+		child.stdin.end(requests.join("\n") + "\n");
+	});
+}
+const WINDOW = 200;
+async function walkFile(root, file, limit) {
+	const { stdout } = await run("git", [
+		"log",
+		`-n${limit}`,
+		FORMAT,
+		"--",
+		file
+	], {
+		cwd: root,
+		timeout: 5e3,
+		windowsHide: true,
+		maxBuffer: 8e6
+	});
+	const commits = parseFullLog(stdout);
+	if (!commits.length) return {
+		owners: /* @__PURE__ */ new Map(),
+		head: []
+	};
+	const owners = /* @__PURE__ */ new Map();
+	let head = [];
+	let waiting;
+	let last = null;
+	const parse = (body, blob) => {
+		if (body === null) return {};
+		if (last && last.blob === blob) return last.flat;
+		const flat = parseCatalog(body);
+		last = {
+			blob,
+			flat
+		};
+		return flat;
+	};
+	if (!await readBlobs(root, commits.flatMap((commit) => [`${commit.sha}:${file}`, `${commit.sha}^:${file}`]), (index, body, blob) => {
+		const flat = parse(body, blob);
+		if (index % 2 === 0) {
+			waiting = flat;
+			if (index === 0 && flat) head = Object.keys(flat);
+			return;
+		}
+		const commit = commits[(index - 1) / 2];
+		if (commit) attribute([{
+			commit,
+			now: waiting,
+			before: flat
+		}], owners);
+		waiting = void 0;
+	})) throw new Error("[verbaly] git could not read every revision of this catalog");
+	return {
+		owners,
+		head
+	};
+}
+async function catalogKeyHistory(cfg, limit = WINDOW) {
+	const maps = [];
+	const committed = /* @__PURE__ */ new Set();
+	let ok = false;
+	for (const locale of cfg.locales) {
+		const file = relative(cfg.root, join(cfg.dir, `${locale}.json`)).replaceAll("\\", "/");
+		try {
+			const walked = await walkFile(cfg.root, file, limit);
+			maps.push(walked.owners);
+			for (const key of walked.head) committed.add(key);
+			ok = true;
+		} catch {}
+	}
+	return {
+		owners: mergeOwners(maps),
+		committed,
+		ok
+	};
+}
+const walked = /* @__PURE__ */ new Map();
+async function headSha(root) {
+	try {
+		const { stdout } = await run("git", ["rev-parse", "HEAD"], {
+			cwd: root,
+			timeout: 3e3,
+			windowsHide: true
+		});
+		return stdout.trim();
+	} catch {
+		return "";
+	}
+}
+async function keyHistory(cfg) {
+	const head = await headSha(cfg.root);
+	const held = walked.get(cfg.root);
+	if (held && held.head === head) return held.history;
+	const history = await catalogKeyHistory(cfg);
+	walked.set(cfg.root, {
+		head,
+		history
+	});
+	return history;
+}
+async function keyCommit(cfg, key) {
+	const history = await keyHistory(cfg);
+	if (!history.ok) return {
+		key,
+		commit: null,
+		reason: "nogit"
+	};
+	const commit = history.owners.get(key);
+	if (commit) return {
+		key,
+		commit: {
+			...commit,
+			sha: commit.sha.slice(0, 7)
+		}
+	};
+	return {
+		key,
+		commit: null,
+		reason: history.committed.has(key) ? "older" : "uncommitted"
+	};
+}
 //#endregion
 //#region src/assets.ts
 const UI = resolve(fileURLToPath(new URL(".", import.meta.url)), "ui");
@@ -495,6 +724,10 @@ function createStudioApp(cfg, port, token) {
 				send(res, 200, { commits: await catalogHistory(cfg) });
 				return;
 			}
+			if (req.method === "GET" && url.pathname.startsWith("/api/commit/")) {
+				send(res, 200, await keyCommit(cfg, segment(url.pathname.slice(12), "key")));
+				return;
+			}
 			if (req.method === "GET" && url.pathname === "/api/health") {
 				send(res, 200, await health(cfg));
 				return;