@wrikka/create-docs 0.2.1 → 0.2.2

package/docs/translate.md

@@ -32,15 +32,40 @@ page, and a per-page Translate action on doc pages.
 
 ## How it works
 
-1. `bunx create-docs-translate --docs docs --locales th,ja` scans your docs
+1. `bunx create-docs-translate --docs <dir> --locales th,ja` scans your docs
    and writes `translation-plan.json` with one task per
    `(document, locale)` pair.
-2. A CI workflow (e.g. `.github/workflows/translate.yml`) runs the planner
-   and — once a `TRANSLATE_PROVIDER` and API key are configured — feeds the
-   plan to the provider.
-3. Translated markdown lands under `docs/<locale>/…`, gets committed, and the
-   locale switcher picks it up automatically.
+2. Pass `--apply` to run the pending tasks through an OpenAI-compatible
+   chat-completions endpoint, and `--limit N` to cap each run (cost control).
+3. When docs are generated (e.g. pulled from GitHub and gitignored), pass
+   `--i18n <dir>` so translations are written to a tracked folder
+   (`<i18n>/<locale>/<file>`) that survives re-pulls — wire that folder into
+   your `createStaticDataSource` glob next to the pulled content.
+4. A CI workflow (e.g. `.github/workflows/translate.yml`) runs the script,
+   then commits the translated markdown back to the repo.
+
+```bash
+# plan only
+bunx create-docs-translate --docs docs --locales th --out plan.json
+
+# translate up to 20 files for one collection
+TRANSLATE_PROVIDER=openai TRANSLATE_API_KEY=sk-… \
+bunx create-docs-translate \
+  --docs docs/docs/bun-packages \
+  --i18n i18n/bun-packages \
+  --locales th --apply --limit 20
+```
+
+| Env var | Default | Purpose |
+| ------- | ------- | ------- |
+| `TRANSLATE_PROVIDER` | — | Enables `--apply` (e.g. `openai`) |
+| `TRANSLATE_API_KEY` | `OPENAI_API_KEY` | Provider API key |
+| `TRANSLATE_BASE_URL` | `https://api.openai.com/v1` | OpenAI-compatible endpoint |
+| `TRANSLATE_MODEL` | `gpt-4o-mini` | Chat model |
+| `TRANSLATE_CONCURRENCY` | `3` | Parallel requests |
+| `SOURCE_LOCALE` | `en` | Source language |
 
 The `/translate` page derives per-doc status from doc ids and tags: a doc
 counts as translated for a locale when its id contains `.{locale}` or
-`--{locale}`, or when it carries a `lang:{locale}` / `locale:{locale}` tag.
+`--{locale}` (e.g. `th--index`), or when it carries a `lang:{locale}` /
+`locale:{locale}` tag.

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@wrikka/create-docs",
 	"description": "Vite plugin for documentation sites built on Functional Clean Architecture with SolidJS support",
-	"version": "0.2.1",
+	"version": "0.2.2",
 	"type": "module",
 	"bin": {
 		"create-docs": "bin/cli.js",

package/scripts/translate.ts

@@ -1,22 +1,34 @@
 #!/usr/bin/env bun
 /**
- * create-docs translate — CI-friendly AI translation planner.
+ * create-docs translate — CI-friendly AI translation pipeline.
  *
  * Scans a docs directory, computes which (document, locale) pairs are missing
- * translations, and writes a `translation-plan.json` that a CI workflow (or an
- * AI provider integration) can consume.
+ * translations, and writes a `translation-plan.json`. With `--apply` and a
+ * configured provider, it translates the pending tasks through an
+ * OpenAI-compatible chat-completions endpoint.
  *
  * Usage:
  *   bun scripts/translate.ts --docs docs --locales th,ja [--out translation-plan.json]
+ *   bun scripts/translate.ts --docs docs/docs/bun-packages --i18n i18n/bun-packages --locales th --apply [--limit 20]
  *
- * The script is intentionally a planner: it does not call an AI provider yet.
- * To wire a real provider, set TRANSLATE_PROVIDER plus the provider's API key
- * (e.g. OPENAI_API_KEY) and pipe the emitted plan into your translation step.
+ * --i18n <dir>  Write translations to <dir>/<locale>/<file> instead of
+ *               <docs>/<locale>/<file>. Use this when the docs directory is
+ *               generated (e.g. pulled from GitHub and gitignored) so the
+ *               translations live in a tracked folder that survives re-pulls.
+ *
+ * Environment:
+ *   TRANSLATE_PROVIDER   e.g. "openai" (any OpenAI-compatible endpoint works)
+ *   TRANSLATE_API_KEY    API key (falls back to OPENAI_API_KEY)
+ *   TRANSLATE_BASE_URL   default https://api.openai.com/v1
+ *   TRANSLATE_MODEL      default gpt-4o-mini
+ *   TRANSLATE_CONCURRENCY default 3
+ *   SOURCE_LOCALE        default en
  */
 
 import {
 	mkdirSync,
 	readdirSync,
+	readFileSync,
 	statSync,
 	writeFileSync,
 } from "node:fs";
@@ -24,19 +36,28 @@ import * as path from "node:path";
 
 interface CliArgs {
 	docs: string;
+	i18n: string;
 	locales: string[];
 	out: string;
+	apply: boolean;
+	limit: number;
 }
 
 function parseArgs(argv: string[]): CliArgs {
 	const args: CliArgs = {
 		docs: "docs",
+		i18n: "",
 		locales: [],
 		out: "translation-plan.json",
+		apply: false,
+		limit: Number.POSITIVE_INFINITY,
 	};
 	for (let i = 0; i < argv.length; i++) {
 		const a = argv[i];
 		if (a === "--docs") args.docs = argv[++i] ?? args.docs;
+		else if (a === "--i18n") args.i18n = argv[++i] ?? "";
+		else if (a === "--apply") args.apply = true;
+		else if (a === "--limit") args.limit = Number(argv[++i] ?? "0") || args.limit;
 		else if (a === "--locales")
 			args.locales = (argv[++i] ?? "")
 				.split(",")
@@ -63,7 +84,87 @@ function walk(dir: string, prefix = ""): string[] {
 	return out;
 }
 
-function main() {
+interface TranslateTask {
+	source: string;
+	target: string;
+	locale: string;
+	sourcePath: string;
+	targetPath: string;
+	status: "pending" | "translated" | "failed";
+	error?: string;
+}
+
+const SYSTEM_PROMPT = `You are a technical documentation translator.
+Translate the markdown document the user sends into the target language.
+
+Rules:
+- Keep YAML frontmatter keys unchanged; translate only human-readable string
+  values (title, description, section, etc.). Keep the keys themselves in English.
+- Do not translate: code blocks, inline code, URLs, import paths, identifiers,
+  component names, or frontmatter keys.
+- Preserve all markdown structure, heading levels, links, images, and
+  frontmatter delimiters exactly.
+- Output ONLY the translated document. No commentary, no wrapping fences.`;
+
+async function translateFile(
+	task: TranslateTask,
+	localeName: string,
+): Promise<void> {
+	const baseUrl = (
+		process.env.TRANSLATE_BASE_URL ?? "https://api.openai.com/v1"
+	).replace(/\/$/, "");
+	const apiKey = process.env.TRANSLATE_API_KEY ?? process.env.OPENAI_API_KEY;
+	const model = process.env.TRANSLATE_MODEL ?? "gpt-4o-mini";
+	if (!apiKey) throw new Error("TRANSLATE_API_KEY / OPENAI_API_KEY not set");
+
+	const content = readFileSync(task.sourcePath, "utf8");
+	const res = await fetch(`${baseUrl}/chat/completions`, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			Authorization: `Bearer ${apiKey}`,
+		},
+		body: JSON.stringify({
+			model,
+			temperature: 0.2,
+			messages: [
+				{ role: "system", content: SYSTEM_PROMPT },
+				{
+					role: "user",
+					content: `Target language: ${localeName}\n\n${content}`,
+				},
+			],
+		}),
+	});
+	if (!res.ok) {
+		throw new Error(`Provider ${res.status}: ${(await res.text()).slice(0, 300)}`);
+	}
+	const json = (await res.json()) as {
+		choices?: { message?: { content?: string } }[];
+	};
+	const translated = json.choices?.[0]?.message?.content;
+	if (!translated) throw new Error("Empty translation response");
+
+	mkdirSync(path.dirname(task.targetPath), { recursive: true });
+	writeFileSync(task.targetPath, translated, "utf8");
+	task.status = "translated";
+}
+
+const LOCALE_NAMES: Record<string, string> = {
+	th: "Thai",
+	ja: "Japanese",
+	zh: "Simplified Chinese",
+	ko: "Korean",
+	vi: "Vietnamese",
+	id: "Indonesian",
+	fr: "French",
+	de: "German",
+	es: "Spanish",
+	pt: "Portuguese",
+	hi: "Hindi",
+};
+
+async function main() {
 	const args = parseArgs(process.argv.slice(2));
 	const docsDir = path.resolve(args.docs);
 	if (!statSync(docsDir, { throwIfNoEntry: false })?.isDirectory()) {
@@ -71,30 +172,28 @@ function main() {
 		process.exit(1);
 	}
 
-	const files = walk(docsDir);
-	const provider = process.env.TRANSLATE_PROVIDER ?? "ai";
+	const files = walk(docsDir).filter(
+		// Never treat already-translated files (<dir>/<locale>/...) as sources.
+		(f) => !args.locales.includes(f.split("/")[0] ?? ""),
+	);
+	const provider = process.env.TRANSLATE_PROVIDER;
+	// Translations go to --i18n/<locale>/... when set, else <docs>/<locale>/...
+	const targetRoot = args.i18n ? path.resolve(args.i18n) : docsDir;
 
 	const plan = {
 		generatedAt: new Date().toISOString(),
-		provider,
+		provider: provider ?? "none",
 		sourceLocale: process.env.SOURCE_LOCALE ?? "en",
 		targetLocales: args.locales,
 		docsDir: args.docs,
-		tasks: [] as Array<{
-			source: string;
-			target: string;
-			locale: string;
-			sourcePath: string;
-			targetPath: string;
-			status: "pending" | "translated";
-		}>,
+		tasks: [] as TranslateTask[],
 	};
 
 	let pending = 0;
 	let done = 0;
 	for (const file of files) {
 		for (const locale of args.locales) {
-			const targetPath = path.join(docsDir, locale, file);
+			const targetPath = path.join(targetRoot, locale, file);
 			const exists =
 				statSync(targetPath, { throwIfNoEntry: false })?.isFile() ?? false;
 			plan.tasks.push({
@@ -110,18 +209,55 @@ function main() {
 		}
 	}
 
+	let applied = 0;
+	let failed = 0;
+	if (args.apply && provider) {
+		const queue = plan.tasks
+			.filter((t) => t.status === "pending")
+			.slice(0, args.limit);
+		const concurrency = Math.max(
+			1,
+			Number(process.env.TRANSLATE_CONCURRENCY ?? "3") || 3,
+		);
+		console.log(
+			`[create-docs translate] applying ${queue.length} task(s) via ${provider} (concurrency ${concurrency})`,
+		);
+		let cursor = 0;
+		const worker = async () => {
+			while (cursor < queue.length) {
+				const task = queue[cursor++];
+				try {
+					await translateFile(task, LOCALE_NAMES[task.locale] ?? task.locale);
+					applied++;
+					console.log(`  translated ${task.target}`);
+				} catch (err) {
+					failed++;
+					task.status = "failed";
+					task.error = err instanceof Error ? err.message : String(err);
+					console.error(`  FAILED ${task.target}: ${task.error}`);
+				}
+			}
+		};
+		await Promise.all(
+			Array.from({ length: Math.min(concurrency, queue.length) }, worker),
+		);
+		pending = plan.tasks.filter((t) => t.status === "pending").length;
+		done = plan.tasks.filter((t) => t.status === "translated").length;
+	}
+
 	mkdirSync(path.dirname(path.resolve(args.out)), { recursive: true });
 	writeFileSync(args.out, JSON.stringify(plan, null, 2), "utf8");
 
 	console.log(
-		`[create-docs translate] ${files.length} docs x ${args.locales.length} locales -> ${done} translated, ${pending} pending`,
+		`[create-docs translate] ${files.length} docs x ${args.locales.length} locales -> ${done} translated, ${pending} pending${applied ? `, ${applied} applied` : ""}${failed ? `, ${failed} failed` : ""}`,
 	);
 	console.log(`Plan written to ${path.resolve(args.out)}`);
-	if (pending > 0 && !process.env.TRANSLATE_PROVIDER) {
+	if (pending > 0 && !provider) {
 		console.log(
-			"No TRANSLATE_PROVIDER configured — plan emitted for review; set a provider + API key to run real translations.",
+			"No TRANSLATE_PROVIDER configured — plan emitted for review; set a provider + TRANSLATE_API_KEY and pass --apply to translate.",
 		);
 	}
+	if (failed > 0) process.exit(1);
 }
 
-main();
+await main();