astro-llm-translator 0.1.0

package/README.md

@@ -0,0 +1,156 @@
+# astro-llm-translator
+
+An [Astro](https://astro.build) integration that automatically translates your Markdown and MDX content using LLMs (OpenAI, Anthropic, DeepSeek, etc.) during the build process.
+
+It features incremental builds, smart caching, and automatic synchronization of deletions, ensuring you only pay for what needs to be translated.
+
+## Installation
+
+```bash
+npm install astro-llm-translator openai
+# or
+pnpm add astro-llm-translator openai
+# or
+bun add astro-llm-translator openai
+```
+
+> **Note**: You also need to install the `openai` package as a peer dependency.
+
+## Usage
+
+Add the integration to your `astro.config.mjs`:
+
+```javascript
+import { defineConfig } from 'astro/config';
+import llmTranslator from 'astro-llm-translator';
+
+export default defineConfig({
+  integrations: [
+    llmTranslator({
+      sourceLang: 'en', // or 'root'
+      targetLangs: ['es', 'fr', 'de'],
+      contentDir: 'src/content/docs',
+      openai: {
+        model: 'gpt-4o-mini', // Optional: defaults to gpt-4o-mini
+      }
+    })
+  ]
+});
+```
+
+## Configuration
+
+### Options
+
+| Option | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| `sourceLang` | `string` | **Required** | The language code of your source content (e.g., `'en'`). Set to `'root'` if your source files are directly inside `contentDir`. |
+| `targetLangs` | `string[]` | **Required** | Array of language codes to translate into (e.g., `['es', 'fr']`). |
+| `contentDir` | `string` | `'src/content/docs'` | Directory relative to project root containing your content. |
+| `customInstructions` | `string` | `undefined` | Additional instructions to guide the translator (e.g. "Use a friendly tone"). |
+| `dictionary` | `Record<string, string>` | `{}` | Key-value pairs of terms to force translation for (e.g. `{'Astro': 'AstroFramework'}`). |
+| `openai` | `object` | `{}` | Optional configuration for the LLM client. |
+
+### OpenAI Options (`openai`)
+
+| Option | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| `apiKey` | `string` | `process.env.OPENAI_API_KEY` | Your API key. |
+| `baseURL` | `string` | `process.env.OPENAI_BASE_URL` | Base URL for the API (useful for OpenRouter or local models). |
+| `model` | `string` | `'gpt-4o-mini'` | The model identifier to use. |
+
+## Environment Variables
+
+You can configure the LLM client using environment variables in your `.env` file:
+
+```bash
+# Standard OpenAI
+OPENAI_API_KEY=sk-...
+
+# Or OpenRouter / Custom Provider
+OPENROUTER_API_KEY=sk-or-...
+OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
+OPENAI_MODEL=anthropic/claude-3-haiku
+```
+
+The integration checks for keys in this order:
+1. `options.openai.apiKey`
+2. `OPENAI_API_KEY`
+3. `OPENROUTER_API_KEY`
+
+## Starlight Support
+
+If you are using [Starlight](https://starlight.astro.build/), you can use the companion plugin to automatically translate sidebar directory labels.
+
+First, the main integration collects and translates directory names during the build process. Then, the Starlight plugin uses this data to reconstruct the sidebar with translated labels.
+
+```javascript
+// astro.config.mjs
+import { defineConfig } from 'astro/config';
+import starlight from '@astrojs/starlight';
+import llmTranslator from 'astro-llm-translator';
+import starlightTranslator from 'astro-llm-translator/starlight'; // Import the plugin
+
+export default defineConfig({
+  integrations: [
+    llmTranslator({
+      sourceLang: 'en',
+      targetLangs: ['es', 'fr'],
+      // ... other options
+    }),
+    starlight({
+      title: 'My Docs',
+      defaultLocale: 'en',
+      locales: {
+        en: { label: 'English', lang: 'en' },
+        es: { label: 'Español', lang: 'es' },
+        fr: { label: 'Français', lang: 'fr' },
+      },
+      plugins: [
+        starlightTranslator(), // Add the plugin here
+      ],
+      sidebar: [
+        // Works with autogenerate!
+        {
+          label: 'Guides',
+          autogenerate: { directory: 'guides' },
+        },
+      ],
+    }),
+  ],
+});
+```
+
+> **Note**: The main `llmTranslator` integration must be present to generate the translation data (`src/.translator-segments.json`) that the Starlight plugin relies on.
+
+## Directory Structure Examples
+
+### 1. Subdirectory Mode (Default)
+
+Config: `sourceLang: 'en'`
+
+```text
+src/content/docs/
+├── en/           <-- Source
+│   └── index.md
+├── es/           <-- Auto-generated
+│   └── index.md
+└── fr/           <-- Auto-generated
+    └── index.md
+```
+
+### 2. Root Mode
+
+Config: `sourceLang: 'root'`
+
+```text
+src/content/docs/
+├── index.md      <-- Source
+├── about.md      <-- Source
+├── es/           <-- Auto-generated (ignored by scanner)
+│   ├── index.md
+│   └── about.md
+└── fr/           <-- Auto-generated (ignored by scanner)
+    ├── index.md
+    └── about.md
+```

package/dist/integration/index.d.ts

@@ -0,0 +1,50 @@
+import { AstroIntegration } from 'astro';
+import { z } from 'zod';
+
+declare const TranslatorOptionsSchema: z.ZodObject<{
+    sourceLang: z.ZodString;
+    targetLangs: z.ZodArray<z.ZodString, "many">;
+    contentDir: z.ZodOptional<z.ZodString>;
+    customInstructions: z.ZodOptional<z.ZodString>;
+    dictionary: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    openai: z.ZodOptional<z.ZodObject<{
+        baseURL: z.ZodOptional<z.ZodString>;
+        apiKey: z.ZodOptional<z.ZodString>;
+        model: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        baseURL?: string | undefined;
+        apiKey?: string | undefined;
+        model?: string | undefined;
+    }, {
+        baseURL?: string | undefined;
+        apiKey?: string | undefined;
+        model?: string | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    sourceLang: string;
+    targetLangs: string[];
+    contentDir?: string | undefined;
+    customInstructions?: string | undefined;
+    dictionary?: Record<string, string> | undefined;
+    openai?: {
+        baseURL?: string | undefined;
+        apiKey?: string | undefined;
+        model?: string | undefined;
+    } | undefined;
+}, {
+    sourceLang: string;
+    targetLangs: string[];
+    contentDir?: string | undefined;
+    customInstructions?: string | undefined;
+    dictionary?: Record<string, string> | undefined;
+    openai?: {
+        baseURL?: string | undefined;
+        apiKey?: string | undefined;
+        model?: string | undefined;
+    } | undefined;
+}>;
+type TranslatorOptions = z.infer<typeof TranslatorOptionsSchema>;
+
+declare function llmTranslator(inputOptions: TranslatorOptions): AstroIntegration;
+
+export { llmTranslator as default };

package/dist/integration/index.js

@@ -0,0 +1,488 @@
+// src/integration/index.ts
+import fs2 from "fs/promises";
+import path3 from "path";
+import { glob } from "glob";
+import "dotenv/config";
+
+// src/integration/state-manager.ts
+import fs from "fs/promises";
+import path from "path";
+var StateManager = class {
+  rootDir;
+  stateFilePath;
+  segmentStateFilePath;
+  translationState = {};
+  segmentState = {};
+  constructor(rootDir) {
+    this.rootDir = rootDir;
+    this.stateFilePath = path.join(rootDir, "src", ".translator.json");
+    this.segmentStateFilePath = path.join(
+      rootDir,
+      "src",
+      ".translator-segments.json"
+    );
+  }
+  async load() {
+    try {
+      const stateContent = await fs.readFile(this.stateFilePath, "utf-8");
+      this.translationState = JSON.parse(stateContent);
+    } catch {
+      this.translationState = {};
+    }
+    try {
+      const segContent = await fs.readFile(this.segmentStateFilePath, "utf-8");
+      this.segmentState = JSON.parse(segContent);
+    } catch {
+      this.segmentState = {};
+    }
+  }
+  async saveTranslationState() {
+    try {
+      await fs.writeFile(
+        this.stateFilePath,
+        JSON.stringify(this.translationState, null, 2)
+      );
+    } catch (e) {
+      console.error(`Failed to save translation state: ${e}`);
+    }
+  }
+  async saveSegmentState() {
+    try {
+      await fs.writeFile(
+        this.segmentStateFilePath,
+        JSON.stringify(this.segmentState, null, 2)
+      );
+    } catch (e) {
+      console.error(`Failed to save segment state: ${e}`);
+    }
+  }
+  updateTranslationEntry(key, hash, relativePath) {
+    if (!this.translationState[key]) {
+      this.translationState[key] = {
+        sourceHash: hash,
+        relativePath,
+        completedLangs: []
+      };
+    } else {
+      this.translationState[key].sourceHash = hash;
+      this.translationState[key].relativePath = relativePath;
+      this.translationState[key].completedLangs = [];
+    }
+  }
+  markLanguageCompleted(key, lang) {
+    if (this.translationState[key] && !this.translationState[key].completedLangs.includes(lang)) {
+      this.translationState[key].completedLangs.push(lang);
+    }
+  }
+  getEntry(key) {
+    return this.translationState[key];
+  }
+  removeEntry(key) {
+    delete this.translationState[key];
+  }
+  getSegmentTranslation(segment, lang) {
+    return this.segmentState[segment]?.[lang];
+  }
+  setSegmentTranslation(segment, lang, translation) {
+    if (!this.segmentState[segment]) {
+      this.segmentState[segment] = {};
+    }
+    this.segmentState[segment][lang] = translation;
+  }
+};
+
+// src/integration/translator.ts
+import matter2 from "gray-matter";
+import OpenAI from "openai";
+
+// src/integration/utils.ts
+import crypto from "crypto";
+import path2 from "path";
+import matter from "gray-matter";
+function computeHash(content) {
+  return crypto.createHash("sha256").update(content).digest("hex");
+}
+function rebaseRelativePath(sourceFilePath, targetFilePath, relativeUrl) {
+  try {
+    const sourceDir = path2.dirname(sourceFilePath);
+    const absPath = path2.resolve(sourceDir, relativeUrl);
+    const targetDir = path2.dirname(targetFilePath);
+    let newRelPath = path2.relative(targetDir, absPath);
+    if (!newRelPath.startsWith(".")) {
+      newRelPath = `./${newRelPath}`;
+    }
+    return newRelPath;
+  } catch (_e) {
+    return relativeUrl;
+  }
+}
+function getKey(content, relativePath) {
+  try {
+    const parsed = matter(content);
+    if (parsed.data.slug) {
+      return parsed.data.slug.normalize("NFC");
+    }
+  } catch (e) {
+  }
+  const match = content.match(/^---([\s\S]*?)---/);
+  if (match) {
+    const frontmatter = match[1];
+    const slugMatch = frontmatter.match(/^\s*slug:\s*(.+)$/m);
+    if (slugMatch) {
+      return slugMatch[1].trim().replace(/^['"]|['"]$/g, "").normalize("NFC");
+    }
+  }
+  return relativePath.normalize("NFC");
+}
+
+// src/integration/markdown-processor.ts
+var adjustImagePaths = (content, sourceFilePath, targetFilePath) => {
+  const regex = /!\[(.*?)\]\((.*?)\)/g;
+  return content.replace(regex, (match, alt, url) => {
+    if (url.startsWith("/") || url.startsWith("http") || url.startsWith("data:")) {
+      return match;
+    }
+    const newRelPath = rebaseRelativePath(sourceFilePath, targetFilePath, url);
+    return `![${alt}](${newRelPath})`;
+  });
+};
+var adjustImportPaths = (content, sourceFilePath, targetFilePath) => {
+  const regex = /(^import\s+(?:(?:[\w*\s{},]*)\s+from\s+)?['"])(\.[^'"]+)(['"])/gm;
+  return content.replace(regex, (match, prefix, url, suffix) => {
+    const newRelPath = rebaseRelativePath(sourceFilePath, targetFilePath, url);
+    return `${prefix}${newRelPath}${suffix}`;
+  });
+};
+var processors = [adjustImagePaths, adjustImportPaths];
+function processMarkdownContent(content, sourceFilePath, targetFilePath) {
+  return processors.reduce(
+    (currentContent, processor) => processor(currentContent, sourceFilePath, targetFilePath),
+    content
+  );
+}
+
+// src/integration/translator.ts
+var Translator = class {
+  openai;
+  model;
+  customInstructions;
+  dictionary;
+  constructor(options) {
+    const apiKey = options.openai?.apiKey || process.env.OPENAI_API_KEY || process.env.OPENROUTER_API_KEY;
+    if (!apiKey) {
+      throw new Error("No OpenAI API Key found.");
+    }
+    const baseURL = options.openai?.baseURL || process.env.OPENAI_BASE_URL || process.env.OPENROUTER_BASE_URL;
+    this.model = options.openai?.model || process.env.OPENAI_MODEL || "gpt-4o-mini";
+    this.openai = new OpenAI({ apiKey, baseURL });
+    this.customInstructions = options.customInstructions;
+    this.dictionary = options.dictionary;
+  }
+  async translateSegment(segment, targetLang) {
+    const prompt = `
+Translate the following directory name/category label into ${targetLang}.
+Return ONLY the translated string. Keep it short and title-cased if appropriate for a menu label.
+Preserve any emojis or special symbols from the original text.
+Do not include quotes or extra text.
+
+Original: "${segment}"
+    `;
+    try {
+      const completion = await this.openai.chat.completions.create({
+        messages: [{ role: "user", content: prompt }],
+        model: this.model
+      });
+      return completion.choices[0].message.content?.trim() || segment;
+    } catch (e) {
+      return segment;
+    }
+  }
+  async translateContent(content, sourceLang, targetLang, sourceFilePath, targetFilePath, logger) {
+    let parsed;
+    try {
+      parsed = matter2(content);
+    } catch (e) {
+      const msg = `Failed to parse source content: ${e}`;
+      logger ? logger.error(msg) : console.error(msg);
+      return content;
+    }
+    const { data, content: body } = parsed;
+    const payload = { body };
+    if (data.title) payload.title = data.title;
+    if (data.description) payload.description = data.description;
+    if (data.summary) payload.summary = data.summary;
+    const sourceLangName = sourceLang === "root" ? "the original language" : sourceLang;
+    let dictionaryPrompt = "";
+    if (this.dictionary && Object.keys(this.dictionary).length > 0) {
+      dictionaryPrompt = `
+USE THE FOLLOWING DICTIONARY FOR SPECIFIC TERMS:
+${Object.entries(this.dictionary).map(([key, value]) => `- "${key}": "${value}"`).join("\n")}
+`;
+    }
+    const prompt = `
+You are a professional translator. Translate the content in the following JSON object from ${sourceLangName} to ${targetLang}.
+
+${this.customInstructions || ""}
+${dictionaryPrompt}
+
+INSTRUCTIONS:
+1. Return ONLY a valid JSON object. Do not include markdown code blocks.
+2. The JSON keys must match the input exactly ("title", "description", "body", etc.).
+3. Translate the VALUES associated with the keys.
+4. For the "body" key, preserve all Markdown formatting (headers, lists, links) exactly.
+5. Do NOT translate technical terms if inappropriate.
+
+Input JSON:
+${JSON.stringify(payload, null, 2)}
+    `;
+    let translatedPayload = { ...payload };
+    try {
+      const completion = await this.openai.chat.completions.create({
+        messages: [{ role: "user", content: prompt }],
+        model: this.model,
+        response_format: { type: "json_object" }
+      });
+      const responseContent = completion.choices[0].message.content || "{}";
+      translatedPayload = JSON.parse(responseContent);
+    } catch (e) {
+      const msg = `LLM Translation failed for ${targetLang}: ${e}`;
+      logger ? logger.error(msg) : console.error(msg);
+    }
+    const translatedData = { ...data };
+    if (translatedPayload.title) translatedData.title = translatedPayload.title;
+    if (translatedPayload.description)
+      translatedData.description = translatedPayload.description;
+    if (translatedPayload.summary)
+      translatedData.summary = translatedPayload.summary;
+    if (translatedData.slug) {
+      const cleanSlug = translatedData.slug.replace(/^\/+/, "");
+      translatedData.slug = `${targetLang}/${cleanSlug}`;
+    }
+    const finalBody = processMarkdownContent(
+      translatedPayload.body || body,
+      sourceFilePath,
+      targetFilePath
+    );
+    return matter2.stringify(finalBody, translatedData);
+  }
+};
+
+// src/integration/types.ts
+import { z } from "zod";
+var TranslatorOptionsSchema = z.object({
+  sourceLang: z.string().min(1),
+  targetLangs: z.array(z.string().min(1)).min(1),
+  contentDir: z.string().optional(),
+  customInstructions: z.string().optional(),
+  dictionary: z.record(z.string()).optional(),
+  openai: z.object({
+    baseURL: z.string().url().optional(),
+    apiKey: z.string().optional(),
+    model: z.string().optional()
+  }).optional()
+});
+
+// src/integration/index.ts
+function llmTranslator(inputOptions) {
+  const options = TranslatorOptionsSchema.parse(inputOptions);
+  const { sourceLang, targetLangs, contentDir = "src/content/docs" } = options;
+  return {
+    name: "astro-llm-translator",
+    hooks: {
+      "astro:config:setup": async ({ command, logger }) => {
+        if (command !== "build" && command !== "dev") return;
+        logger.info("Starting LLM Translation...");
+        let translator;
+        try {
+          translator = new Translator(options);
+        } catch (e) {
+          logger.error(
+            `Translator init failed: ${e instanceof Error ? e.message : String(e)}`
+          );
+          return;
+        }
+        const rootDir = process.cwd();
+        const stateManager = new StateManager(rootDir);
+        await stateManager.load();
+        const isRootSource = sourceLang === "root";
+        const sourceDir = isRootSource ? path3.join(rootDir, contentDir) : path3.join(rootDir, contentDir, sourceLang);
+        const ignore = isRootSource ? targetLangs.map((lang) => `${lang}/**`) : [];
+        const sourceFiles = await glob("**/*.{md,mdx}", {
+          cwd: sourceDir,
+          ignore
+        });
+        if (sourceFiles.length === 0) {
+          logger.warn(`No source files found in ${sourceDir}`);
+        }
+        const foundKeys = /* @__PURE__ */ new Set();
+        for (const rawFile of sourceFiles) {
+          const file = rawFile.normalize("NFC");
+          const sourceFilePath = path3.join(sourceDir, file);
+          let sourceContent = "";
+          try {
+            sourceContent = await fs2.readFile(sourceFilePath, "utf-8");
+          } catch (_e) {
+            logger.error(`Failed to read source file: ${file}`);
+            continue;
+          }
+          const key = getKey(sourceContent, file);
+          foundKeys.add(key);
+          const dir = path3.dirname(file);
+          if (dir !== "." && dir !== "") {
+            const segments = dir.split(path3.sep);
+            for (const segment of segments) {
+              for (const lang of targetLangs) {
+                if (!stateManager.getSegmentTranslation(segment, lang)) {
+                  logger.info(
+                    `Translating directory segment "${segment}" to ${lang}...`
+                  );
+                  try {
+                    const translatedSeg = await translator.translateSegment(
+                      segment,
+                      lang
+                    );
+                    stateManager.setSegmentTranslation(
+                      segment,
+                      lang,
+                      translatedSeg
+                    );
+                    await stateManager.saveSegmentState();
+                  } catch (e) {
+                    logger.warn(`Failed to translate segment ${segment}: ${e}`);
+                  }
+                }
+              }
+            }
+          }
+          const currentHash = computeHash(sourceContent);
+          const stateEntry = stateManager.getEntry(key);
+          if (stateEntry && stateEntry.relativePath !== file) {
+            const oldRelativePath = stateEntry.relativePath;
+            if (stateEntry.sourceHash === currentHash) {
+              logger.info(
+                `Detected file move for slug "${key}": ${oldRelativePath} -> ${file}. Moving translations...`
+              );
+              for (const lang of stateEntry.completedLangs) {
+                const oldTargetPath = path3.join(
+                  rootDir,
+                  contentDir,
+                  lang,
+                  oldRelativePath
+                );
+                const newTargetPath = path3.join(
+                  rootDir,
+                  contentDir,
+                  lang,
+                  file
+                );
+                try {
+                  await fs2.mkdir(path3.dirname(newTargetPath), {
+                    recursive: true
+                  });
+                  await fs2.rename(oldTargetPath, newTargetPath);
+                } catch (e) {
+                  logger.warn(`Failed to move translation: ${e}`);
+                }
+              }
+              const completedLangs = [...stateEntry.completedLangs];
+              stateManager.updateTranslationEntry(key, currentHash, file);
+              completedLangs.forEach(
+                (lang) => stateManager.markLanguageCompleted(key, lang)
+              );
+              await stateManager.saveTranslationState();
+              continue;
+            } else {
+              logger.info(
+                `Detected file move with update for slug "${key}". Cleaning up old path...`
+              );
+              for (const lang of stateEntry.completedLangs) {
+                const oldTargetPath = path3.join(
+                  rootDir,
+                  contentDir,
+                  lang,
+                  oldRelativePath
+                );
+                try {
+                  await fs2.unlink(oldTargetPath);
+                } catch {
+                }
+              }
+            }
+          }
+          if (!stateEntry || currentHash !== stateEntry.sourceHash) {
+            stateManager.updateTranslationEntry(key, currentHash, file);
+            await stateManager.saveTranslationState();
+          }
+          for (const lang of targetLangs) {
+            const targetDir = path3.join(rootDir, contentDir, lang);
+            const targetFilePath = path3.join(targetDir, file);
+            let isCompleted = stateManager.getEntry(key)?.completedLangs.includes(lang);
+            if (isCompleted) {
+              try {
+                await fs2.access(targetFilePath);
+              } catch {
+                isCompleted = false;
+                logger.warn(
+                  `Missing file for ${lang}/${file}. Re-translating...`
+                );
+              }
+            }
+            if (!isCompleted) {
+              logger.info(`Translating ${file} to ${lang}...`);
+              try {
+                await fs2.mkdir(path3.dirname(targetFilePath), {
+                  recursive: true
+                });
+                const translatedContent = await translator.translateContent(
+                  sourceContent,
+                  sourceLang,
+                  lang,
+                  sourceFilePath,
+                  targetFilePath,
+                  logger
+                );
+                await fs2.writeFile(targetFilePath, translatedContent);
+                stateManager.markLanguageCompleted(key, lang);
+                await stateManager.saveTranslationState();
+                logger.info(`\u2713 Translated ${file} to ${lang}`);
+              } catch (error) {
+                logger.error(
+                  `Failed to translate ${file} to ${lang}: ${error}`
+                );
+              }
+            } else {
+              logger.info(`Skipping ${file} (${lang}) - up to date.`);
+            }
+          }
+        }
+        for (const key of Object.keys(stateManager.translationState)) {
+          if (!foundKeys.has(key)) {
+            logger.info(`Source file for key "${key}" deleted. Cleaning up...`);
+            const entry = stateManager.getEntry(key);
+            if (!entry) continue;
+            const relativePath = entry.relativePath;
+            for (const lang of targetLangs) {
+              const targetPath = path3.join(
+                rootDir,
+                contentDir,
+                lang,
+                relativePath
+              );
+              try {
+                await fs2.unlink(targetPath);
+                logger.info(`Deleted orphan translation: ${targetPath}`);
+              } catch {
+              }
+            }
+            stateManager.removeEntry(key);
+            await stateManager.saveTranslationState();
+          }
+        }
+        logger.info("LLM Translation complete.");
+      }
+    }
+  };
+}
+export {
+  llmTranslator as default
+};

package/dist/starlight/index.d.ts

@@ -0,0 +1,5 @@
+import { StarlightPlugin } from '@astrojs/starlight/types';
+
+declare function starlightTranslator(): StarlightPlugin;
+
+export { starlightTranslator as default };