@geolonia/yuuhitsu 0.1.17 → 0.2.2

package/dist/tasks/translate.js

@@ -1,21 +1,20 @@
 import { readFileSync, writeFileSync, mkdirSync } from "fs";
 import { dirname, basename, extname, join } from "path";
+import { remark } from "remark";
+import remarkGfm from "remark-gfm";
+import { visit } from "unist-util-visit";
 import { buildGlossaryPrompt } from "./glossary.js";
-export const DEFAULT_MAX_CHUNK_LINES = 300;
+export const DEFAULT_MAX_CHUNK_LINES = 150;
 const MIN_CHUNK_LINES = 50;
+export const DEFAULT_MAX_NODES_PER_BATCH = 200;
+// P-A1: minimum ratio of output characters to input characters (truncation check)
+const MIN_OUTPUT_RATIO = 0.3;
 /**
- * Separate frontmatter from Markdown content
+ * Separate frontmatter from Markdown content.
  * Handles: LF, CRLF, no trailing newline after closing ---, trailing spaces, empty frontmatter
- * @param content - Full Markdown content
- * @returns Object with separated frontmatter and body
  */
 export function separateFrontmatter(content) {
-    // Normalize CRLF to LF for regex matching
     const normalized = content.replace(/\r\n/g, "\n");
-    // Match frontmatter (two alternations to keep closing --- on its own line):
-    //   Case 1: non-empty body:  ^---\n ... \n---[ \t]*(\n|$)
-    //   Case 2: empty body:      ^---\n---[ \t]*(\n|$)
-    // Using alternation avoids the \n? ambiguity that allows --- to match mid-line.
     const frontmatterRegex = /^---\n([\s\S]*?)\n---[ \t]*(\n|$)|^---\n---[ \t]*(\n|$)/;
     const match = normalized.match(frontmatterRegex);
     if (match) {
@@ -26,359 +25,7 @@ export function separateFrontmatter(content) {
     }
     return { frontmatter: null, body: normalized };
 }
-/**
- * Replace fenced code blocks and inline code with placeholders.
- * Uses a line-by-line parser instead of regex to avoid V8 stack overflow
- * on files with many code blocks (backreference + [\s\S]*? causes recursive backtracking).
- */
-export function protectCodeBlocks(content) {
-    const map = new Map();
-    let blockIndex = 0;
-    let inlineIndex = 0;
-    // Step 1: Replace fenced code blocks using line-by-line parsing
-    const lines = content.split("\n");
-    const resultLines = [];
-    let fenceOpen = null; // the backtick sequence that opened the current block
-    let blockLines = [];
-    for (const line of lines) {
-        const fenceMatch = line.match(/^(`{3,})/);
-        if (fenceOpen === null) {
-            // Not inside a code block
-            if (fenceMatch) {
-                // Opening fence found
-                fenceOpen = fenceMatch[1];
-                blockLines = [line];
-            }
-            else {
-                resultLines.push(line);
-            }
-        }
-        else {
-            // Inside a code block — look for closing fence with same or more backticks
-            blockLines.push(line);
-            if (fenceMatch && fenceMatch[1].length >= fenceOpen.length && line.trim() === fenceMatch[1]) {
-                // Closing fence found — store as single-line placeholder (no padding).
-                // Padding was previously used to preserve line count, but it caused chunk
-                // boundaries to fall inside the placeholder's whitespace region, leading to
-                // non-deterministic LLM output when the code block exceeded --max-chunk-lines.
-                const original = blockLines.join("\n") + "\n";
-                const placeholder = `__CODE_BLOCK_${blockIndex++}__`;
-                map.set(placeholder, original);
-                resultLines.push(placeholder);
-                fenceOpen = null;
-                blockLines = [];
-            }
-        }
-    }
-    // If we ended inside an unclosed fence, emit lines as-is
-    if (fenceOpen !== null) {
-        resultLines.push(...blockLines);
-    }
-    let result = resultLines.join("\n");
-    // Step 2: Replace inline code (single backtick, not within code blocks)
-    result = result.replace(/`([^`\n]+)`/g, (match) => {
-        const placeholder = `__INLINE_CODE_${inlineIndex++}__`;
-        map.set(placeholder, match);
-        return placeholder;
-    });
-    return { text: result, map };
-}
-/**
- * Restore placeholders back to original code blocks/inline code.
- */
-export function restoreCodeBlocks(content, map) {
-    let result = content;
-    for (const [placeholder, original] of map.entries()) {
-        if (original.endsWith("\n")) {
-            // Fenced code block: original already has trailing \n — just replace, one \n total.
-            result = result.split(placeholder + "\n").join(original);
-            result = result.split(placeholder).join(original);
-        }
-        else {
-            // Inline code: original has NO trailing \n. Preserve any \n that follows the placeholder
-            // so that newlines inserted by restoreBlockBoundaries (Layer 3) are not consumed.
-            // e.g. "__INLINE_CODE_0__\n- next item" → "`code`\n- next item" (not "`code`- next item")
-            result = result.split(placeholder + "\n").join(original + "\n");
-            result = result.split(placeholder).join(original);
-        }
-    }
-    return result;
-}
-export const BLOCK_BOUNDARY_SENTINEL = "<!--BB-->";
-// Temporary escape for pre-existing <!--BB--> literals in user content.
-// Uses a character sequence unlikely to appear in markdown documents.
-const ESCAPED_SENTINEL = "\x01BB\x01";
-// Fallback regex: catches LLM-deformed variants (e.g. <!-- BB -->, <!--BB__-->, <!--BBx-->)
-const SENTINEL_FALLBACK = /<!--\s*BB[a-zA-Z0-9_-]*\s*-->/g;
-// Broad check: detects severely-deformed residuals not caught by SENTINEL_FALLBACK
-const SENTINEL_RESIDUAL_CHECK = /<!--[\s\S]*?BB[\s\S]*?-->/g;
-/**
- * Insert block boundary sentinels before structural Markdown elements
- * (list items, headings, horizontal rules, code fences, code block placeholders).
- * P-A4: prevents newline collapse around structural boundaries during LLM translation.
- * When called after protectCodeBlocks, fenced blocks appear as __CODE_BLOCK_N__ placeholders;
- * the function treats those placeholders as structural to protect fence-adjacent newlines.
- */
-export function protectBlockBoundaries(content) {
-    // Escape all sentinel-like patterns (exact + variants) to prevent control-marker confusion.
-    // SENTINEL_FALLBACK covers <!--BB-->, <!-- BB -->, <!--BBx-->, <!--BB-x-->, etc.
-    // On restore, these all round-trip back to <!--BB--> (minor cosmetic vs. content deletion).
-    const escaped = content.replace(SENTINEL_FALLBACK, ESCAPED_SENTINEL);
-    const lines = escaped.split("\n");
-    const result = [];
-    for (const line of lines) {
-        const isListItem = /^\s*[-*+]\s/.test(line) || // unordered list
-            /^\s*\d+\.\s/.test(line); // ordered list
-        const isOtherStructural = !isListItem && (/^ {0,3}#{1,6}\s/.test(line) || // heading (CommonMark: 0-3 leading spaces)
-            /^ {0,3}-{3,}\s*$/.test(line) || // hr (dash, 0-3 leading spaces)
-            /^ {0,3}\*{3,}\s*$/.test(line) || // hr (asterisk, 0-3 leading spaces)
-            /^ {0,3}_{3,}\s*$/.test(line) || // hr (underscore, 0-3 leading spaces)
-            /^\s*`{3,}/.test(line) || // fenced code (backtick)
-            /^\s*~{3,}/.test(line) || // fenced code (tilde)
-            /^__CODE_BLOCK_\d+__$/.test(line.trim()) // code block placeholder (after protectCodeBlocks)
-        );
-        if (isListItem) {
-            // P-A4 v3: double sentinel BEFORE each list item to resist LLM collapse.
-            // 0.1.16 used a single sentinel; LLM deleted it and collapsed items to one line.
-            // Two sentinels before each item mean the LLM must delete both to collapse — higher bar.
-            // After-sentinels are intentionally omitted to preserve clean round-trip (no trailing \n).
-            result.push(BLOCK_BOUNDARY_SENTINEL);
-            result.push(BLOCK_BOUNDARY_SENTINEL);
-            result.push(line);
-        }
-        else if (isOtherStructural) {
-            result.push(BLOCK_BOUNDARY_SENTINEL);
-            result.push(line);
-        }
-        else {
-            result.push(line);
-        }
-    }
-    return result.join("\n");
-}
-/**
- * Remove block boundary sentinels and restore newlines lost during LLM translation.
- * Uses a 3-pass strategy plus Layer 3 list-aware fallback (P-A4 v3):
- *   Pass 1: normalize LLM-deformed variants (e.g. <!-- BB -->) to exact sentinel form
- *   Pass 2: split + restore newlines (handles both clean and collapsed sentinel cases)
- *   Pass 3: post-restore warning for residual sentinel-like patterns (silent failure prevention)
- *   Layer 3: detect list items collapsed onto one line ("- A- B") and split them back
- *            Applied unconditionally — handles the case where LLM deleted ALL sentinels
- */
-export function restoreBlockBoundaries(content) {
-    // Pass 1: normalize variant sentinels introduced by LLM deformation (cmd_389 Root Cause A/B)
-    const normalized = content.replace(SENTINEL_FALLBACK, BLOCK_BOUNDARY_SENTINEL);
-    let restored;
-    // Pass 2: split + restore newlines
-    if (!normalized.includes(BLOCK_BOUNDARY_SENTINEL)) {
-        // No exact sentinels: fall through to Pass 3 + Layer 3 (handles fully-deleted sentinel case)
-        restored = normalized;
-    }
-    else {
-        const parts = normalized.split(BLOCK_BOUNDARY_SENTINEL);
-        restored = parts[0];
-        for (let i = 1; i < parts.length; i++) {
-            // Strip a leading newline from next part (present when LLM preserved sentinel on its own line)
-            const stripped = parts[i].replace(/^\n/, "");
-            if (restored.length === 0) {
-                // Sentinel was at the very start of content — no preceding text to separate from
-                restored = stripped;
-            }
-            else {
-                // Ensure restored ends with exactly one newline before appending next part
-                restored = restored.replace(/\n?$/, "\n") + stripped;
-            }
-        }
-    }
-    // Pass 3: post-restore warning for patterns not caught by SENTINEL_FALLBACK.
-    // Runs for BOTH the sentinel-present and no-sentinel paths so fully-deformed output
-    // is still flagged. Check before unescaping to avoid false positives from user <!--BB-->.
-    const residuals = restored.match(SENTINEL_RESIDUAL_CHECK);
-    if (residuals && residuals.length > 0) {
-        console.warn(`[yuuhitsu] restoreBlockBoundaries: ${residuals.length} residual sentinel-like pattern(s) detected after restore:`, residuals.slice(0, 5));
-    }
-    // Unescape any pre-existing <!--BB--> that were escaped before protection
-    if (restored.includes(ESCAPED_SENTINEL)) {
-        restored = restored.split(ESCAPED_SENTINEL).join(BLOCK_BOUNDARY_SENTINEL);
-    }
-    // Layer 3 (P-A4 v3): list-aware fallback — detect inline list concatenation that
-    // survived Layer 1+2 (LLM joined "- A\n- B" into "- A- B" on a single line).
-    // Applied unconditionally: handles the worst case where ALL sentinels were deleted.
-    // Uses /gm flag: ^ anchors to line start per line (multiline mode).
-    //
-    // Two sub-patterns:
-    //   (a) Spaced: "- A- B" or "- A -  B" — requires whitespace after 2nd marker (original).
-    //   (b) Placeholder-end: "- A: __INLINE_CODE_0__-B" — inline code placeholder at end of
-    //       previous item followed directly by next list marker (no space). This is the pattern
-    //       LLM produces when translating to Japanese (Japanese text has no space after marker).
-    // (?<!\s) guard: require the char immediately before the 2nd marker to be non-whitespace.
-    // This prevents false-positive splits on prose like "- Linux - macOS support" where the
-    // inline "- " is preceded by a space (valid prose) rather than collapsed item text.
-    // True collapse ("- A- B") has NO space before the 2nd marker → lookbehind passes.
-    const LIST_INLINE_MERGE_UNORDERED = /(^\s*[-*+]\s[^\n]*?)(?<!\s)([-*+]\s)/gm;
-    const LIST_INLINE_MERGE_ORDERED = /(^\s*\d+\.\s[^\n]*?)(?<!\s)(\d+\.\s)/gm;
-    // Placeholder-end pattern: matches code-placeholder end (\d+__) immediately before list marker.
-    // Inserts "\n" + space (standard list-item format: "- content") so the new line passes
-    // /^\s*[-*+]\s/ checks in integration tests.
-    // (?=[^a-z]) guard avoids false positives for "__CODE__-style" (lowercase word hyphens).
-    // e.g. "__INLINE_CODE_0__-次の項目" → "__INLINE_CODE_0__\n- 次の項目"
-    const LIST_INLINE_MERGE_PLACEHOLDER_UNORDERED = /(\d+__)([-*+])(?=[^a-z])/gm;
-    const LIST_INLINE_MERGE_PLACEHOLDER_ORDERED = /(\d+__)(\d+\.)(?=[^a-z])/gm;
-    // Apply iteratively: JavaScript replace() scans left-to-right in the original string,
-    // so "- A- B- C" needs two passes (first splits A-B, second splits B-C on the new line).
-    let layer3applied = false;
-    let prev;
-    do {
-        prev = restored;
-        restored = restored.replace(LIST_INLINE_MERGE_UNORDERED, (_match, p1, p2) => {
-            layer3applied = true;
-            return `${p1}\n${p2}`;
-        });
-        restored = restored.replace(LIST_INLINE_MERGE_ORDERED, (_match, p1, p2) => {
-            layer3applied = true;
-            return `${p1}\n${p2}`;
-        });
-        restored = restored.replace(LIST_INLINE_MERGE_PLACEHOLDER_UNORDERED, (_match, p1, p2) => {
-            layer3applied = true;
-            return `${p1}\n${p2} `; // trailing space ensures valid "- content" list-item format
-        });
-        restored = restored.replace(LIST_INLINE_MERGE_PLACEHOLDER_ORDERED, (_match, p1, p2) => {
-            layer3applied = true;
-            return `${p1}\n${p2} `;
-        });
-    } while (restored !== prev);
-    if (layer3applied) {
-        console.warn("[yuuhitsu] restoreBlockBoundaries: Layer 3 list-aware fallback applied — " +
-            "LLM concatenated list items inline. Layer 1+2 sentinels were insufficient.");
-    }
-    return restored;
-}
-const DEFAULT_TEMPLATE = `You are a professional translator. Translate the following Markdown document to {{targetLanguage}}.
-
-Rules:
-- Preserve all Markdown formatting (headings, links, code blocks, tables, lists)
-- Do not translate code blocks, URLs, or file paths
-- Do not translate frontmatter keys (only translate values where appropriate)
-- Maintain the same document structure
-- Produce natural, fluent text in the target language
-- Every opening \`\`\` you write MUST be followed by a language identifier on the same line (e.g., \`\`\`json, \`\`\`bash, \`\`\`typescript)
-- If the language is unknown, use \`\`\`text — never emit a bare opening \`\`\`
-
-CRITICAL - Link and URL preservation:
-- NEVER modify any URLs or link paths. Keep all href/src values exactly as-is.
-- NEVER change internal link paths (e.g., /ja/..., /en/..., ./relative-path). Preserve them verbatim.
-- NEVER convert external URLs to different language versions.
-- If the source has [text](/ja/changelog), the output must keep the same path, only translate the link text if needed.
-- Example: [紹介](/ja/intro) → translate "紹介" but keep "/ja/intro" unchanged
-- Example: [MDN](https://developer.mozilla.org/ja/) → keep the /ja/ in URL, translate "MDN" if needed
-
-Additional rules for Japanese translation:
-- Use full-width punctuation: 。、？！ (not .,?!)
-- Add half-width spaces around English words and numbers (e.g., "Vela とは", "NGSIv2 は", "3 つの")
-- Use natural Japanese terms for technical words where appropriate (e.g., "registration" → "登録", "subscription" → "サブスクリプション")
-- Keep product names, proper nouns, and abbreviations unchanged (e.g., Vela, FIWARE, NGSIv2, NGSI-LD, MCP)
-
-Example — code fence with language identifier:
-  Bad:  \`\`\` echo hello \`\`\`
-  Good: \`\`\`bash echo hello \`\`\``;
-function buildPrompt(content, targetLang, hasPlaceholders, hasSentinels, templateContent, glossaryConfig) {
-    const template = templateContent || DEFAULT_TEMPLATE;
-    let systemPrompt = template
-        .replace(/\{\{targetLanguage\}\}/g, targetLang)
-        .replace(/\{\{content\}\}/g, "");
-    if (glossaryConfig) {
-        const glossarySection = buildGlossaryPrompt(glossaryConfig, targetLang);
-        if (glossarySection) {
-            systemPrompt += glossarySection;
-        }
-    }
-    if (hasSentinels) {
-        systemPrompt +=
-            "\n\n## Block boundary markers (P-A4 v2)\n\n" +
-                "Lines containing the marker `<!--BB-->` are **block boundary markers** inserted\n" +
-                "by the translation pipeline to preserve newlines around structural elements.\n\n" +
-                "Rules for `<!--BB-->` markers (HTML comment form):\n" +
-                "- Output every `<!--BB-->` marker **verbatim and unchanged** in your translation.\n" +
-                "- Each marker must remain on its own line, in the same position relative to\n" +
-                "  surrounding content.\n" +
-                "- Do not translate, remove, paraphrase, modify, normalize whitespace inside, or\n" +
-                "  rename these markers.\n" +
-                "- Do not add new `<!--BB-->` markers; only preserve existing ones.\n\n" +
-                "Good example (correct preservation):\n" +
-                "  Input:\n" +
-                "    <!--BB-->\n" +
-                "    - List item one\n" +
-                "    <!--BB-->\n" +
-                "    - List item two\n" +
-                "    <!--BB-->\n" +
-                "    ## Section heading\n" +
-                "  Output:\n" +
-                "    <!--BB-->\n" +
-                "    - リスト項目その一\n" +
-                "    <!--BB-->\n" +
-                "    - リスト項目その二\n" +
-                "    <!--BB-->\n" +
-                "    ## セクション見出し\n\n" +
-                "Bad examples (DO NOT do these):\n" +
-                "  - <!--BB-->  ❌ → <!--BB__-->     (added suffix — FORBIDDEN)\n" +
-                "  - <!--BB-->  ❌ → <!-- BB -->     (added internal whitespace — FORBIDDEN)\n" +
-                "  - <!--BB-->  ❌ → <!--bb-->       (case change — FORBIDDEN)\n" +
-                "  - <!--BB-->  ❌ → (omitted)       (deleted — FORBIDDEN)\n" +
-                "  - <!--BB-->  ❌ → <!--BB-x-->     (added suffix — FORBIDDEN)\n\n" +
-                "Preserve the marker exactly: 9 characters, opening `<!--`, content `BB`,\n" +
-                "closing `-->`, no whitespace, no case changes, no suffixes.\n\n" +
-                "## List boundary protection (P-A4 v3 list addendum)\n\n" +
-                "For list items (lines starting with `-`, `*`, `+`, or `1.`),\n" +
-                "`<!--BB-->` markers appear **multiple times in a row** (e.g., two consecutive\n" +
-                "`<!--BB-->` lines). This is intentional — preserve ALL of them.\n\n" +
-                "Bad example (DO NOT do this):\n" +
-                "  <!--BB-->\n" +
-                "  <!--BB-->\n" +
-                "  - Item A\n" +
-                "  <!--BB-->\n" +
-                "  <!--BB-->\n" +
-                "  - Item B\n\n" +
-                "  ❌ becomes: - Item A- Item B   (list items on one line — FORBIDDEN)\n\n" +
-                "Good example (keep each item on its own line):\n" +
-                "  <!--BB-->\n" +
-                "  <!--BB-->\n" +
-                "  - アイテムA\n" +
-                "  <!--BB-->\n" +
-                "  <!--BB-->\n" +
-                "  - アイテムB\n\n" +
-                "Key rules for lists:\n" +
-                "- Each list item MUST remain on its own line.\n" +
-                "- NEVER join two list items into one line (e.g., `- A- B` is FORBIDDEN).\n" +
-                "- Preserve ALL `<!--BB-->` markers, even when they appear consecutively.\n\n" +
-                "Bad example (DO NOT do this) — inline code list collapse:\n" +
-                "  - Item A: `value 1`\n" +
-                "  - Item B: `value 2`\n\n" +
-                "  ❌ becomes: - Item A: `value 1`- Item B: `value 2`    (FORBIDDEN, space before marker)\n" +
-                "  ❌ becomes: - Item A: `value 1`-Item B: `value 2`     (FORBIDDEN, no space)\n\n" +
-                "Good example: each list item must remain on its own line, even when items contain inline code:\n" +
-                "  - アイテムA: `value 1`\n" +
-                "  - アイテムB: `value 2`";
-    }
-    if (hasPlaceholders) {
-        systemPrompt +=
-            "\n\nIMPORTANT - Placeholder preservation:\n" +
-                "- Tokens matching __CODE_BLOCK_N__ or __INLINE_CODE_N__ are placeholders for code blocks/inline code.\n" +
-                "- Output them VERBATIM and UNCHANGED. Do NOT translate, modify, or remove them.\n" +
-                "- Example: if input has __CODE_BLOCK_0__, output must contain __CODE_BLOCK_0__ exactly.";
-    }
-    return [
-        { role: "system", content: systemPrompt },
-        { role: "user", content },
-    ];
-}
-function resolveOutputPath(inputPath, targetLang, outputPath) {
-    if (outputPath)
-        return outputPath;
-    const dir = dirname(inputPath);
-    const ext = extname(inputPath);
-    const base = basename(inputPath, ext);
-    return join(dir, `${base}.${targetLang}${ext}`);
-}
+// ─── Chunking utilities (kept for splitIntoChunks export) ───────────────────
 /**
  * Find positions (line indices) of Markdown headings at the given level,
  * excluding lines inside fenced code blocks or table rows.
@@ -389,7 +36,7 @@ export function findHeadingPositions(lines, level) {
     const prefix = "#".repeat(level) + " ";
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i];
-        if (/^`{3,}/.test(line))
+        if (/^(`{3,}|~{3,})/.test(line))
             inCodeBlock = !inCodeBlock;
         if (inCodeBlock)
             continue;
@@ -401,8 +48,8 @@ export function findHeadingPositions(lines, level) {
     return positions;
 }
 /**
- * Split lines at the given positions into chunks (each position starts a new chunk).
- * Segments exceeding maxChunkLines are further split using ### headings or safeSplitLines.
+ * Split lines at the given positions into chunks. Segments exceeding maxChunkLines
+ * are further split using ### headings or safeSplitLines.
  */
 export function splitAtPositions(lines, positions, maxChunkLines) {
     const result = [];
@@ -417,8 +64,6 @@ export function splitAtPositions(lines, positions, maxChunkLines) {
             continue;
         if (segmentLines.length > maxChunkLines) {
             const subPositions = findHeadingPositions(segmentLines, 3);
-            // Filter out position 0: splitting at the start doesn't reduce segment size
-            // and causes infinite recursion when the only heading is at position 0.
             const effectivePositions = subPositions.filter((p) => p > 0);
             if (effectivePositions.length > 0) {
                 result.push(...splitAtPositions(segmentLines, effectivePositions, maxChunkLines));
@@ -443,12 +88,8 @@ export function safeSplitLines(lines, maxChunkLines) {
     let inCodeBlock = false;
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i];
-        const isFence = /^`{3,}/.test(line);
+        const isFence = /^(`{3,}|~{3,})/.test(line);
         const isTableLine = line.startsWith("|");
-        // Check split eligibility BEFORE toggling fence state:
-        // - never split inside a code block
-        // - never split ON a fence line (would separate opening/closing ``` from their block)
-        // - never split ON a table row
         const canSplitHere = !inCodeBlock && !isFence && !isTableLine;
         if (isFence)
             inCodeBlock = !inCodeBlock;
@@ -463,8 +104,7 @@ export function safeSplitLines(lines, maxChunkLines) {
     return chunks.filter((c) => c.trim().length > 0);
 }
 /**
- * Merge chunks smaller than MIN_CHUNK_LINES into the previous chunk,
- * as long as the merged result does not exceed maxLines.
+ * Merge chunks smaller than MIN_CHUNK_LINES into the previous chunk.
  */
 export function mergeSmallChunks(chunks, maxLines) {
     if (chunks.length <= 1)
@@ -500,9 +140,294 @@ export function splitIntoChunks(content, maxChunkLines = DEFAULT_MAX_CHUNK_LINES
     }
     return safeSplitLines(lines, maxChunkLines);
 }
+/**
+ * Replace fenced code blocks and inline code with placeholders.
+ * Used by glossary-fix.ts to protect code blocks before text replacement.
+ */
+export function protectCodeBlocks(content) {
+    const map = new Map();
+    let blockIndex = 0;
+    let inlineIndex = 0;
+    const lines = content.split("\n");
+    const resultLines = [];
+    let fenceOpen = null;
+    let blockLines = [];
+    for (const line of lines) {
+        const fenceMatch = line.match(/^(`{3,})/);
+        if (fenceOpen === null) {
+            if (fenceMatch) {
+                fenceOpen = fenceMatch[1];
+                blockLines = [line];
+            }
+            else {
+                resultLines.push(line);
+            }
+        }
+        else {
+            blockLines.push(line);
+            if (fenceMatch && fenceMatch[1].length >= fenceOpen.length && line.trim() === fenceMatch[1]) {
+                const original = blockLines.join("\n") + "\n";
+                const placeholder = `__CODE_BLOCK_${blockIndex++}__`;
+                map.set(placeholder, original);
+                resultLines.push(placeholder);
+                fenceOpen = null;
+                blockLines = [];
+            }
+        }
+    }
+    if (fenceOpen !== null) {
+        resultLines.push(...blockLines);
+    }
+    let result = resultLines.join("\n");
+    result = result.replace(/`([^`\n]+)`/g, (match) => {
+        const placeholder = `__INLINE_CODE_${inlineIndex++}__`;
+        map.set(placeholder, match);
+        return placeholder;
+    });
+    return { text: result, map };
+}
+/**
+ * Restore placeholders back to original code blocks/inline code.
+ */
+export function restoreCodeBlocks(content, map) {
+    let result = content;
+    for (const [placeholder, original] of map.entries()) {
+        if (original.endsWith("\n")) {
+            result = result.split(placeholder + "\n").join(original);
+            result = result.split(placeholder).join(original);
+        }
+        else {
+            result = result.split(placeholder + "\n").join(original + "\n");
+            result = result.split(placeholder).join(original);
+        }
+    }
+    return result;
+}
+/**
+ * Extract all translatable text nodes from an mdast AST.
+ * remark AST guarantees text nodes cannot be inside code/inlineCode nodes,
+ * so we only skip empty/whitespace-only nodes.
+ */
+function extractTextNodes(ast) {
+    const nodes = [];
+    visit(ast, "text", (node) => {
+        if (!node.value.trim())
+            return;
+        nodes.push({ node, id: nodes.length });
+    });
+    return nodes;
+}
+/**
+ * Build system prompt for text-mode batch translation (Gemini / Ollama fallback).
+ * Includes JSON format instructions since we're relying on the LLM to output JSON.
+ */
+function buildBatchSystemPrompt(targetLang, templateContent, glossaryConfig) {
+    const basePrompt = templateContent
+        ? templateContent.replace(/\{\{targetLanguage\}\}/g, targetLang)
+        : `You are a professional translator. Translate text segments to ${targetLang}.
+
+Rules:
+- Translate only the text content; do not add or remove punctuation structure
+- Preserve proper nouns, code identifiers, URLs, and file paths unchanged
+- Produce natural, fluent text in the target language
+- For Japanese: use full-width punctuation (。、？！), add half-width spaces around English words/numbers
+- Keep product names, abbreviations, and technical terms unchanged (e.g., NGSI-LD, MCP, GeoJSON)`;
+    let prompt = basePrompt;
+    if (glossaryConfig) {
+        const glossarySection = buildGlossaryPrompt(glossaryConfig, targetLang);
+        if (glossarySection)
+            prompt += glossarySection;
+    }
+    prompt += `
+
+## Translation format
+
+You will receive a JSON object with a "segments" array.
+Each segment has an "id" (integer) and "text" (string to translate).
+
+Return ONLY a valid JSON object with a "translations" array.
+Each translation must have the same "id" and the translated "text".
+Do not include any explanation, markdown, or text outside the JSON object.
+
+Example input:
+{"segments": [{"id": 0, "text": "Hello world"}, {"id": 1, "text": "This is a test."}]}
+
+Example output:
+{"translations": [{"id": 0, "text": "こんにちは世界"}, {"id": 1, "text": "これはテストです。"}]}`;
+    return prompt;
+}
+/**
+ * Build system prompt for structured output translation (Claude tool_use path).
+ * No JSON format instructions needed — the tool schema enforces the response shape.
+ */
+function buildStructuredSystemPrompt(targetLang, templateContent, glossaryConfig) {
+    const basePrompt = templateContent
+        ? templateContent.replace(/\{\{targetLanguage\}\}/g, targetLang)
+        : `You are a professional translator. Translate each text segment to ${targetLang}.
+
+Rules:
+- Translate only the text content; do not alter structure or punctuation outside the text
+- Preserve proper nouns, code identifiers, URLs, and file paths unchanged
+- Produce natural, fluent text in the target language
+- For Japanese: use full-width punctuation (。、？！), add half-width spaces around English words/numbers
+- Keep product names, abbreviations, and technical terms unchanged (e.g., NGSI-LD, MCP, GeoJSON)
+- Each segment is independent; translate it on its own`;
+    let prompt = basePrompt;
+    if (glossaryConfig) {
+        const glossarySection = buildGlossaryPrompt(glossaryConfig, targetLang);
+        if (glossarySection)
+            prompt += glossarySection;
+    }
+    return prompt;
+}
+/**
+ * Parse translation JSON response from LLM (text mode fallback).
+ * Handles both clean JSON and JSON embedded in prose (extracts first {...} block).
+ */
+function parseTranslationResponse(raw) {
+    const trimmed = raw.trim();
+    try {
+        return JSON.parse(trimmed);
+    }
+    catch {
+        // Extract first JSON object if LLM added prose around it
+        const jsonMatch = trimmed.match(/\{[\s\S]*\}/);
+        if (jsonMatch) {
+            return JSON.parse(jsonMatch[0]);
+        }
+        throw new Error(`Failed to parse translation response: ${trimmed.slice(0, 200)}`);
+    }
+}
+function assertValidTranslations(value) {
+    if (!Array.isArray(value) ||
+        value.some((item) => {
+            if (typeof item !== "object" || item === null)
+                return true;
+            const candidate = item;
+            return typeof candidate.id !== "number" || typeof candidate.text !== "string";
+        })) {
+        throw new Error("[yuuhitsu] translateBatch: invalid translation payload — expected Array<{id: number, text: string}>");
+    }
+}
+/**
+ * Translate a batch of text segments using the provider.
+ *
+ * - If the provider implements `translateStructured` (Claude): uses tool_use to
+ *   enforce the JSON schema at the API level, guaranteeing 1:1 ID mapping.
+ * - Otherwise: falls back to text-mode JSON prompt (Gemini / Ollama).
+ *
+ * In both paths the 1:1 ID mapping is validated and an error is thrown on mismatch.
+ */
+async function translateBatch(provider, nodes, targetLang, templateContent, glossaryConfig) {
+    if (nodes.length === 0) {
+        return { usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 } };
+    }
+    const segments = nodes.map(({ node, id }) => ({ id, text: node.value }));
+    // P-A1: track total input character count for truncation detection
+    const totalInputChars = segments.reduce((sum, s) => sum + s.text.length, 0);
+    let translations;
+    let usage;
+    if (provider.translateStructured) {
+        // Structured output path: provider (Claude) enforces JSON schema via tool_use
+        const systemPrompt = buildStructuredSystemPrompt(targetLang, templateContent, glossaryConfig);
+        const result = await provider.translateStructured({ segments, systemPrompt });
+        assertValidTranslations(result.translations);
+        translations = result.translations;
+        usage = result.usage;
+        // P-A1: warn on truncation (compare translated chars to input chars)
+        const totalOutputChars = translations.reduce((sum, t) => sum + t.text.length, 0);
+        if (totalInputChars > 0 && totalOutputChars < totalInputChars * MIN_OUTPUT_RATIO) {
+            console.warn(`[yuuhitsu] translateBatch: output may be truncated ` +
+                `(input chars: ${totalInputChars}, output chars: ${totalOutputChars})`);
+        }
+    }
+    else {
+        // Text mode fallback: Gemini / Ollama
+        const systemPrompt = buildBatchSystemPrompt(targetLang, templateContent, glossaryConfig);
+        const messages = [
+            { role: "system", content: systemPrompt },
+            { role: "user", content: JSON.stringify({ segments }) },
+        ];
+        const response = await provider.chat({ model: "", messages });
+        const parsed = parseTranslationResponse(response.content);
+        assertValidTranslations(parsed.translations);
+        // P-A1: warn on truncation (compare translated text chars, not raw JSON string length)
+        const totalOutputChars = parsed.translations.reduce((sum, t) => sum + t.text.length, 0);
+        if (totalInputChars > 0 && totalOutputChars < totalInputChars * MIN_OUTPUT_RATIO) {
+            console.warn(`[yuuhitsu] translateBatch: output may be truncated ` +
+                `(input chars: ${totalInputChars}, output chars: ${totalOutputChars})`);
+        }
+        translations = parsed.translations;
+        usage = response.usage;
+    }
+    // Validate strict 1:1 ID mapping (both paths):
+    // - no duplicate output IDs (Map silently overwrites; we must detect before)
+    // - no unexpected IDs (IDs not present in the input set)
+    // - no missing IDs (input ID absent from output)
+    const inputIds = new Set(nodes.map(({ id }) => id));
+    const seenOutputIds = new Set();
+    const duplicateIds = [];
+    const unexpectedIds = [];
+    for (const t of translations) {
+        if (seenOutputIds.has(t.id)) {
+            duplicateIds.push(t.id);
+        }
+        else {
+            seenOutputIds.add(t.id);
+        }
+        if (!inputIds.has(t.id)) {
+            unexpectedIds.push(t.id);
+        }
+    }
+    if (duplicateIds.length > 0) {
+        throw new Error(`[yuuhitsu] translateBatch: duplicate IDs in response (IDs: ${duplicateIds.join(", ")})`);
+    }
+    if (unexpectedIds.length > 0) {
+        throw new Error(`[yuuhitsu] translateBatch: unexpected IDs in response (IDs: ${unexpectedIds.join(", ")})`);
+    }
+    const translationMap = new Map(translations.map((t) => [t.id, t.text]));
+    const missingIds = nodes.filter(({ id }) => !translationMap.has(id)).map(({ id }) => id);
+    if (missingIds.length > 0) {
+        throw new Error(`[yuuhitsu] translateBatch: partial translation — ${missingIds.length} node(s) missing` +
+            ` from response (IDs: ${missingIds.join(", ")})`);
+    }
+    // Apply translations back to AST nodes
+    for (const { node, id } of nodes) {
+        const translated = translationMap.get(id);
+        if (translated !== undefined && translated.trim()) {
+            node.value = translated;
+        }
+    }
+    return { usage };
+}
+function resolveOutputPath(inputPath, targetLang, outputPath) {
+    if (outputPath)
+        return outputPath;
+    const dir = dirname(inputPath);
+    const ext = extname(inputPath);
+    const base = basename(inputPath, ext);
+    return join(dir, `${base}.${targetLang}${ext}`);
+}
+/**
+ * Translate a Markdown file using AST-based approach.
+ *
+ * Architecture (0.2.0):
+ *   1. Separate frontmatter (preserved verbatim)
+ *   2. Split body into chunks by heading boundaries
+ *   3. For each chunk: parse to AST → extract text nodes → translate via LLM → write back
+ *   4. Serialize AST → markdown → concatenate → write output
+ *
+ * Key properties:
+ *   - Code blocks (fenced and inline) are never sent to LLM (AST handles them deterministically)
+ *   - Markdown structure (headings, lists, tables, HR) is preserved by AST round-trip
+ *   - No sentinel injection or removal needed
+ */
 export async function translateFile(options) {
-    const { provider, inputPath, targetLang, templateContent, glossaryConfig, maxChunkLines } = options;
-    // Read input file
+    const { provider, inputPath, targetLang, templateContent, glossaryConfig, maxChunkLines, maxNodesPerBatch, } = options;
+    if (maxNodesPerBatch !== undefined && (!Number.isInteger(maxNodesPerBatch) || maxNodesPerBatch < 1)) {
+        throw new Error(`maxNodesPerBatch must be a positive integer, got: ${maxNodesPerBatch}`);
+    }
+    const resolvedMaxNodes = maxNodesPerBatch ?? DEFAULT_MAX_NODES_PER_BATCH;
     let content;
     try {
         content = readFileSync(inputPath, "utf-8");
@@ -513,46 +438,46 @@ export async function translateFile(options) {
         }
         throw err;
     }
-    // Check for empty file
     if (content.trim().length === 0) {
         throw new Error(`Input file is empty: ${inputPath}`);
     }
     const resolvedOutput = resolveOutputPath(inputPath, targetLang, options.outputPath);
-    // Ensure output directory exists
     mkdirSync(dirname(resolvedOutput), { recursive: true });
-    // Separate frontmatter from body
     const { frontmatter, body } = separateFrontmatter(content);
-    // Protect code blocks: replace with placeholders before sending to LLM
-    const { text: protectedBody, map: codeMap } = protectCodeBlocks(body);
-    const hasPlaceholders = codeMap.size > 0;
-    // Protect block boundaries: insert %%BB%% sentinels before structural elements (P-A4)
-    const bodyWithSentinels = protectBlockBoundaries(protectedBody);
-    const hasSentinels = bodyWithSentinels.includes(BLOCK_BOUNDARY_SENTINEL);
-    // Split body into chunks if needed (frontmatter is never sent to LLM)
-    const chunks = splitIntoChunks(bodyWithSentinels, maxChunkLines);
-    const translatedParts = [];
+    // Split body into text chunks (heading-based, same as before)
+    const chunks = splitIntoChunks(body, maxChunkLines ?? DEFAULT_MAX_CHUNK_LINES);
+    const translatedChunks = [];
     let totalUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
     for (const chunk of chunks) {
-        const messages = buildPrompt(chunk, targetLang, hasPlaceholders, hasSentinels, templateContent, glossaryConfig);
-        const response = await provider.chat({
-            model: "",
-            messages,
-        });
-        translatedParts.push(response.content);
-        totalUsage.promptTokens += response.usage.promptTokens;
-        totalUsage.completionTokens += response.usage.completionTokens;
-        totalUsage.totalTokens += response.usage.totalTokens;
+        // Parse chunk to AST (code blocks, inline code become typed nodes → never reach LLM)
+        const processor = remark().use(remarkGfm);
+        const ast = processor.parse(chunk);
+        // Extract translatable text nodes
+        const textNodes = extractTextNodes(ast);
+        if (textNodes.length > 0) {
+            // Split into sub-batches when node count exceeds maxNodesPerBatch (BUG-421-dense-chunk fix).
+            // Dense files (e.g. changelog) may have 300+ nodes/chunk, causing Claude ID hallucination.
+            for (let batchStart = 0; batchStart < textNodes.length; batchStart += resolvedMaxNodes) {
+                const batch = textNodes.slice(batchStart, batchStart + resolvedMaxNodes);
+                const { usage } = await translateBatch(provider, batch, targetLang, templateContent, glossaryConfig);
+                totalUsage.promptTokens += usage.promptTokens;
+                totalUsage.completionTokens += usage.completionTokens;
+                totalUsage.totalTokens += usage.totalTokens;
+            }
+        }
+        // Serialize AST back to markdown
+        const translatedChunk = processor.stringify(ast);
+        translatedChunks.push(translatedChunk);
     }
-    // Restore block boundaries (sentinels → newlines) then restore code block placeholders.
-    // join("\n") ensures chunk boundaries always have a separator even if LLM drops trailing newlines.
-    const translatedBodyWithSentinels = translatedParts.join("\n");
-    const translatedBodyWithPlaceholders = restoreBlockBoundaries(translatedBodyWithSentinels);
-    const translatedBody = restoreCodeBlocks(translatedBodyWithPlaceholders, codeMap);
-    // Recombine frontmatter with translated body
+    // Join chunks: normalize each chunk to exactly one trailing newline
+    // (preserves trailing spaces for Markdown hard line breaks), then join
+    // with a single "\n" so chunk boundaries produce exactly one blank line.
+    const translatedBody = translatedChunks
+        .map((c) => c.replace(/\n+$/, "\n"))
+        .join("\n");
     const translatedContent = frontmatter
         ? frontmatter + translatedBody
         : translatedBody;
-    // Write output
     writeFileSync(resolvedOutput, translatedContent, "utf-8");
     return {
         outputPath: resolvedOutput,