@aigne/doc-smith 0.9.7-beta.3 → 0.9.8-alpha.0

package/utils/d2-utils.mjs

@@ -1,189 +1,26 @@
 import path from "node:path";
 
-import { D2 } from "@terrastruct/d2";
 import fs from "fs-extra";
-import { glob } from "glob";
-import pMap from "p-map";
 
-import {
-  D2_CONCURRENCY,
-  D2_CONFIG,
-  DOC_SMITH_DIR,
-  FILE_CONCURRENCY,
-  TMP_ASSETS_DIR,
-  TMP_DIR,
-} from "./constants/index.mjs";
-import { debug } from "./debug.mjs";
-import { iconMap } from "./icon-map.mjs";
-import { getContentHash } from "./utils.mjs";
+import { DOC_SMITH_DIR, TMP_DIR } from "./constants/index.mjs";
 
-const codeBlockRegex = /```d2.*\n([\s\S]*?)```/g;
+// Note: .* matches title or other text after ```d2 (e.g., ```d2 Vault 驗證流程)
+// Export regex for reuse across the codebase to avoid duplication
+export const d2CodeBlockRegex = /```d2.*\n([\s\S]*?)```/g;
 
 export const DIAGRAM_PLACEHOLDER = "DIAGRAM_PLACEHOLDER";
 
-export async function getChart({ content, strict }) {
-  const d2 = new D2();
-  const iconUrlList = Object.keys(iconMap);
-  const escapedUrls = iconUrlList.map((url) => url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
-  const regexPattern = escapedUrls.join("|");
-  const regex = new RegExp(regexPattern, "g");
+// Diagram image regex patterns for reuse across the codebase
+// Pattern 1: Match only the start marker (for checking existence)
+export const diagramImageStartRegex = /<!--\s*DIAGRAM_IMAGE_START:[^>]+-->/g;
 
-  const contentWithBase64Img = content.replace(regex, (match) => {
-    return iconMap[match];
-  });
-  try {
-    const { diagram, renderOptions, graph } = await d2.compile(contentWithBase64Img);
+// Pattern 2: Match full diagram image block without capturing image path (for finding/replacing)
+export const diagramImageBlockRegex =
+  /<!--\s*DIAGRAM_IMAGE_START:[^>]+-->\s*[\s\S]*?<!--\s*DIAGRAM_IMAGE_END\s*-->/g;
 
-    // Do not apply a stroke-dash to sequence diagrams.
-    if (
-      graph?.root?.attributes?.shape &&
-      graph.root.attributes.shape.value !== "sequence_diagram"
-    ) {
-      // Save the first-level container.
-      const firstLevelContainer = new Set();
-      diagram.shapes.forEach((x) => {
-        const idList = x.id.split(".");
-        if (idList.length > 1) {
-          const targetShape = diagram.shapes.find((x) => x.id === idList[0]);
-          if (targetShape && !["c4-person", "cylinder", "queue"].includes(targetShape.type)) {
-            firstLevelContainer.add(targetShape);
-          }
-        }
-      });
-      firstLevelContainer.forEach((shape) => {
-        if (!shape.strokeDash) {
-          // Note: The data structure here is different from the d2 source code.
-          shape.strokeDash = 3;
-        }
-      });
-    }
-
-    const svg = await d2.render(diagram, renderOptions);
-
-    return svg;
-  } catch (err) {
-    if (strict) throw err;
-
-    console.error("Failed to generate D2 diagram. Content:", content, "Error:", err);
-    return null;
-  } finally {
-    d2.worker.terminate();
-  }
-}
-
-export async function saveAssets({ markdown, docsDir }) {
-  if (!markdown) {
-    return markdown;
-  }
-
-  const { replaced } = await runIterator({
-    input: markdown,
-    regexp: codeBlockRegex,
-    replace: true,
-    fn: async ([_match, _code]) => {
-      const assetDir = path.join(docsDir, "../", TMP_ASSETS_DIR, "d2");
-      await fs.ensureDir(assetDir);
-      const d2Content = [D2_CONFIG, _code].join("\n");
-      const fileName = `${getContentHash(d2Content)}.svg`;
-      const svgPath = path.join(assetDir, fileName);
-
-      if (await fs.pathExists(svgPath)) {
-        debug("Asset cache found, skipping generation", svgPath);
-      } else {
-        try {
-          debug("Generating d2 diagram", svgPath);
-          if (debug.enabled) {
-            const d2FileName = `${getContentHash(d2Content)}.d2`;
-            const d2Path = path.join(assetDir, d2FileName);
-            await fs.writeFile(d2Path, d2Content, { encoding: "utf8" });
-          }
-
-          const svg = await getChart({ content: d2Content });
-          if (svg) {
-            await fs.writeFile(svgPath, svg, { encoding: "utf8" });
-          }
-        } catch (error) {
-          debug("Failed to generate D2 diagram. Content:", d2Content, "Error:", error);
-          return _code;
-        }
-      }
-      return `![](${path.posix.join("..", TMP_ASSETS_DIR, "d2", fileName)})`;
-    },
-    options: { concurrency: D2_CONCURRENCY },
-  });
-
-  return replaced;
-}
-
-export async function beforePublishHook({ docsDir }) {
-  // Process each markdown file to save d2 svg assets.
-  const mdFilePaths = await glob("**/*.md", { cwd: docsDir });
-  await pMap(
-    mdFilePaths,
-    async (filePath) => {
-      let finalContent = await fs.readFile(path.join(docsDir, filePath), { encoding: "utf8" });
-      finalContent = await saveAssets({ markdown: finalContent, docsDir });
-
-      await fs.writeFile(path.join(docsDir, filePath), finalContent, { encoding: "utf8" });
-    },
-    { concurrency: FILE_CONCURRENCY },
-  );
-}
-
-async function runIterator({ input, regexp, fn = () => {}, options, replace = false }) {
-  if (!input) return input;
-  const matches = [...input.matchAll(regexp)];
-  const results = [];
-  await pMap(
-    matches,
-    async (...args) => {
-      const resultItem = await fn(...args);
-      results.push(resultItem);
-    },
-    options,
-  );
-
-  let replaced = input;
-  if (replace) {
-    let index = 0;
-    replaced = replaced.replace(regexp, () => {
-      return results[index++];
-    });
-  }
-
-  return {
-    results,
-    replaced,
-  };
-}
-
-export async function checkContent({ content: _content }) {
-  const matches = Array.from(_content.matchAll(codeBlockRegex));
-  let content = _content;
-  if (matches.length > 0) {
-    content = matches[0][1];
-  }
-  await ensureTmpDir();
-  const assetDir = path.join(DOC_SMITH_DIR, TMP_DIR, TMP_ASSETS_DIR, "d2");
-  await fs.ensureDir(assetDir);
-  const d2Content = [D2_CONFIG, content].join("\n");
-  const fileName = `${getContentHash(d2Content)}.svg`;
-  const svgPath = path.join(assetDir, fileName);
-
-  if (debug.enabled) {
-    const d2FileName = `${getContentHash(d2Content)}.d2`;
-    const d2Path = path.join(assetDir, d2FileName);
-    await fs.writeFile(d2Path, d2Content, { encoding: "utf8" });
-  }
-
-  if (await fs.pathExists(svgPath)) {
-    debug("Asset cache found, skipping generation", svgPath);
-    return;
-  }
-
-  const svg = await getChart({ content: d2Content, strict: true });
-  await fs.writeFile(svgPath, svg, { encoding: "utf8" });
-}
+// Pattern 3: Match full diagram image block with image path capture (for extracting paths)
+export const diagramImageWithPathRegex =
+  /<!--\s*DIAGRAM_IMAGE_START:[^>]+-->\s*!\[[^\]]*\]\(([^)]+)\)\s*<!--\s*DIAGRAM_IMAGE_END\s*-->/g;
 
 export async function ensureTmpDir() {
   const tmpDir = path.join(DOC_SMITH_DIR, TMP_DIR);
@@ -198,7 +35,7 @@ export function isValidCode(lang) {
 }
 
 export function wrapCode({ content }) {
-  const matches = Array.from(content.matchAll(codeBlockRegex));
+  const matches = Array.from(content.matchAll(d2CodeBlockRegex));
   if (matches.length > 0) {
     return content;
   }
@@ -212,7 +49,7 @@ export function wrapCode({ content }) {
  * @returns {Array} - [contentWithPlaceholder, originalCodeBlock]
  */
 export function replaceD2WithPlaceholder({ content }) {
-  const [firstMatch] = Array.from(content.matchAll(codeBlockRegex));
+  const [firstMatch] = Array.from(content.matchAll(d2CodeBlockRegex));
   if (firstMatch) {
     const matchContent = firstMatch[0];
     const cleanContent = content.replace(matchContent, DIAGRAM_PLACEHOLDER);
@@ -255,3 +92,107 @@ export function replacePlaceholderWithD2({ content, diagramSourceCode }) {
 
   return content.replace(DIAGRAM_PLACEHOLDER, replacement);
 }
+
+/**
+ * Replace all diagrams (D2 code blocks and generated images) with DIAGRAM_PLACEHOLDER
+ * Used for deletion operations to normalize all diagram types to a single placeholder
+ * @param {string} content - Document content containing diagrams
+ * @param {number} [diagramIndex] - Optional index of diagram to replace (0-based). If not provided, replaces all diagrams.
+ * @returns {string} - Content with diagrams replaced by DIAGRAM_PLACEHOLDER
+ */
+export function replaceDiagramsWithPlaceholder({ content, diagramIndex }) {
+  if (!content) {
+    return content;
+  }
+
+  // Use exported regex to find all diagram locations
+  // We'll use a similar approach to findAllDiagramLocations
+  const mermaidCodeBlockRegex = /```mermaid.*\n([\s\S]*?)```/g;
+
+  // Find all diagram locations
+  const locations = [];
+
+  // 1. Find DIAGRAM_PLACEHOLDER (already a placeholder, keep as is)
+  let placeholderIndex = content.indexOf(DIAGRAM_PLACEHOLDER);
+  while (placeholderIndex !== -1) {
+    locations.push({
+      type: "placeholder",
+      start: placeholderIndex,
+      end: placeholderIndex + DIAGRAM_PLACEHOLDER.length,
+    });
+    placeholderIndex = content.indexOf(DIAGRAM_PLACEHOLDER, placeholderIndex + 1);
+  }
+
+  // 2. Find DIAGRAM_IMAGE_START markers (generated images)
+  const imageMatches = Array.from(content.matchAll(diagramImageBlockRegex));
+  for (const match of imageMatches) {
+    locations.push({
+      type: "image",
+      start: match.index,
+      end: match.index + match[0].length,
+    });
+  }
+
+  // 3. Find D2 code blocks
+  const d2Matches = Array.from(content.matchAll(d2CodeBlockRegex));
+  for (const match of d2Matches) {
+    locations.push({
+      type: "d2",
+      start: match.index,
+      end: match.index + match[0].length,
+    });
+  }
+
+  // 4. Find Mermaid code blocks
+  const mermaidMatches = Array.from(content.matchAll(mermaidCodeBlockRegex));
+  for (const match of mermaidMatches) {
+    locations.push({
+      type: "mermaid",
+      start: match.index,
+      end: match.index + match[0].length,
+    });
+  }
+
+  // Sort by position (top to bottom)
+  locations.sort((a, b) => a.start - b.start);
+
+  if (locations.length === 0) {
+    return content;
+  }
+
+  // If diagramIndex is provided, only replace that specific diagram
+  if (diagramIndex !== undefined && diagramIndex >= 0 && diagramIndex < locations.length) {
+    const targetLocation = locations[diagramIndex];
+    const before = content.substring(0, targetLocation.start);
+    const after = content.substring(targetLocation.end);
+    // Add newlines if needed
+    let replacement = DIAGRAM_PLACEHOLDER;
+    if (before && !before.endsWith("\n")) {
+      replacement = `\n${replacement}`;
+    }
+    if (after && !after.startsWith("\n")) {
+      replacement = `${replacement}\n`;
+    }
+    return before + replacement + after;
+  }
+
+  // Replace all diagrams with placeholder (for deletion)
+  // Process from end to start to preserve indices
+  let result = content;
+  for (let i = locations.length - 1; i >= 0; i--) {
+    const location = locations[i];
+    const before = result.substring(0, location.start);
+    const after = result.substring(location.end);
+    // Add newlines if needed
+    let replacement = DIAGRAM_PLACEHOLDER;
+    if (before && !before.endsWith("\n")) {
+      replacement = `\n${replacement}`;
+    }
+    if (after && !after.startsWith("\n")) {
+      replacement = `${replacement}\n`;
+    }
+    result = before + replacement + after;
+  }
+
+  return result;
+}

package/utils/delete-diagram-images.mjs

@@ -0,0 +1,99 @@
+import { unlink } from "node:fs/promises";
+import { join, dirname, normalize } from "node:path";
+import fs from "fs-extra";
+import { debug } from "./debug.mjs";
+
+/**
+ * Extract image file paths from markdown content
+ * Finds all diagram image references and extracts their file paths
+ * @param {string} content - Markdown content
+ * @param {string} path - Document path (e.g., "guides/getting-started.md")
+ * @param {string} docsDir - Documentation directory
+ * @returns {Promise<Array<string>>} Array of absolute paths to image files
+ */
+export async function extractDiagramImagePaths(content, path, docsDir) {
+  if (!content || !path || !docsDir) {
+    return [];
+  }
+
+  const imagePaths = [];
+
+  // Pattern to match: <!-- DIAGRAM_IMAGE_START:... -->![alt](path)<!-- DIAGRAM_IMAGE_END -->
+  const { diagramImageWithPathRegex } = await import("./d2-utils.mjs");
+  const matches = Array.from(content.matchAll(diagramImageWithPathRegex));
+
+  for (const match of matches) {
+    const imagePath = match[1];
+
+    // Resolve absolute path
+    // If imagePath is relative, resolve from document location
+    // If imagePath is absolute or starts with http, skip
+    if (imagePath.startsWith("http://") || imagePath.startsWith("https://")) {
+      continue; // Skip remote URLs
+    }
+
+    // Calculate relative path from document to image
+    const docDir = dirname(path);
+    const imageRelativePath = imagePath.startsWith("../")
+      ? imagePath
+      : join(docDir, imagePath).replace(/\\/g, "/");
+
+    // Resolve absolute path
+    const absolutePath = join(process.cwd(), docsDir, imageRelativePath);
+
+    // Normalize path (remove .. and .)
+    const normalizedPath = normalize(absolutePath);
+
+    if (await fs.pathExists(normalizedPath)) {
+      imagePaths.push(normalizedPath);
+    }
+  }
+
+  return imagePaths;
+}
+
+/**
+ * Delete diagram image files associated with a document
+ * @param {string} content - Markdown content (before deletion)
+ * @param {string} path - Document path
+ * @param {string} docsDir - Documentation directory
+ * @returns {Promise<{deleted: number, failed: number}>}
+ */
+export async function deleteDiagramImages(content, path, docsDir) {
+  if (!content || !path || !docsDir) {
+    return { deleted: 0, failed: 0 };
+  }
+
+  try {
+    const imagePaths = await extractDiagramImagePaths(content, path, docsDir);
+
+    if (imagePaths.length === 0) {
+      return { deleted: 0, failed: 0 };
+    }
+
+    let deleted = 0;
+    let failed = 0;
+
+    for (const imagePath of imagePaths) {
+      try {
+        await unlink(imagePath);
+        debug(`Deleted diagram image: ${imagePath}`);
+        deleted++;
+      } catch (error) {
+        if (error.code !== "ENOENT") {
+          // File not found is ok, other errors should be logged
+          console.warn(`Failed to delete diagram image ${imagePath}: ${error.message}`);
+          failed++;
+        } else {
+          // File already doesn't exist, count as deleted
+          deleted++;
+        }
+      }
+    }
+
+    return { deleted, failed };
+  } catch (error) {
+    console.warn(`Error deleting diagram images: ${error.message}`);
+    return { deleted: 0, failed: 0 };
+  }
+}

package/utils/docs-finder-utils.mjs

@@ -2,6 +2,7 @@ import { access, readdir, readFile } from "node:fs/promises";
 import { join } from "node:path";
 import chalk from "chalk";
 import pLimit from "p-limit";
+import yaml from "yaml";
 import { pathExists } from "./file-utils.mjs";
 
 /**
@@ -107,10 +108,43 @@ export async function findItemByPath(documentStructure, docPath, boardId, docsDi
  * @param {string} fileName - File name to read
  * @returns {Promise<string|null>} File content or null if failed
  */
+/**
+ * Remove base64 encoded images from markdown content
+ * This prevents large binary data from being included in document content
+ * Base64 images are completely removed (not replaced with placeholders) because:
+ * 1. They significantly increase token usage without providing useful information to LLM
+ * 2. Normal image references (file paths) are preserved and should be used instead
+ * 3. Base64 images are typically temporary or erroneous entries
+ * @param {string} content - Markdown content that may contain base64 images
+ * @returns {string} - Content with base64 images completely removed
+ */
+function removeBase64Images(content) {
+  if (!content || typeof content !== "string") {
+    return content;
+  }
+
+  // Match markdown image syntax with data URLs: ![alt](data:image/...;base64,...)
+  // This regex matches:
+  // - ![alt text](data:image/type;base64,base64data...)
+  // - ![alt](data:image/type;base64,base64data...)
+  // - [![alt](data:image/type;base64,base64data...)](link)
+  const base64ImageRegex = /!\[([^\]]*)\]\(data:image\/[^)]+\)/g;
+
+  // Completely remove base64 images (including the entire markdown image syntax)
+  // This maximizes token reduction while preserving normal image references
+  const cleanedContent = content.replace(base64ImageRegex, "");
+
+  return cleanedContent;
+}
+
 export async function readFileContent(docsDir, fileName) {
   try {
     const filePath = join(docsDir, fileName);
-    return await readFile(filePath, "utf-8");
+    const content = await readFile(filePath, "utf-8");
+
+    // Remove base64 encoded images to reduce token usage
+    // Base64 image data is not useful for LLM processing and significantly increases token count
+    return removeBase64Images(content);
   } catch (readError) {
     console.warn(`⚠️  Could not read content from ${fileName}:`, readError.message);
     return null;
@@ -146,7 +180,8 @@ export async function getMainLanguageFiles(docsDir, locale, documentStructure =
     }
 
     // If main language is English, return files without language suffix
-    if (locale === "en") {
+    // FIXME: 临时修改为 zh，后续需要优化
+    if (locale === "zh") {
       // Return files that don't have language suffixes (e.g., overview.md, not overview.zh.md)
       return !file.match(/\.\w+(-\w+)?\.md$/);
     } else {
@@ -274,8 +309,58 @@ export function addFeedbackToItems(items, feedback) {
 }
 
 /**
- * Load document execution structure from structure-plan.json
- * @param {string} outputDir - Output directory containing structure-plan.json
+ * Convert YAML document structure to flat array format
+ * @param {Object} yamlData - Parsed YAML data with documents array
+ * @returns {Array} Flat array of document structure items
+ */
+function convertYamlToStructure(yamlData) {
+  const result = [];
+
+  function flattenDocuments(documents, parentId = null) {
+    if (!Array.isArray(documents)) {
+      return;
+    }
+
+    for (const doc of documents) {
+      if (!doc.path || !doc.title) {
+        continue;
+      }
+
+      // Create structure item
+      const item = {
+        title: doc.title,
+        description: doc.description || "",
+        path: doc.path,
+      };
+
+      if (parentId) {
+        item.parentId = parentId;
+      }
+
+      if (doc.icon) {
+        item.icon = doc.icon;
+      }
+
+      if (doc.sourcePaths) {
+        item.sourcePaths = doc.sourcePaths;
+      }
+
+      result.push(item);
+
+      // Recursively process children
+      if (doc.children && Array.isArray(doc.children)) {
+        flattenDocuments(doc.children, doc.path);
+      }
+    }
+  }
+
+  flattenDocuments(yamlData.documents);
+  return result;
+}
+
+/**
+ * Load document execution structure from structure-plan.json or document_structure.yaml
+ * @param {string} outputDir - Output directory containing structure files
  * @returns {Promise<Array|null>} Document execution structure array or null if not found/failed
  */
 export async function loadDocumentStructure(outputDir) {
@@ -283,41 +368,64 @@ export async function loadDocumentStructure(outputDir) {
     return null;
   }
 
+  // Try loading structure-plan.json first
   try {
     const structurePlanPath = join(outputDir, "structure-plan.json");
     const structureExists = await pathExists(structurePlanPath);
 
-    if (!structureExists) {
-      return null;
+    if (structureExists) {
+      const structureContent = await readFile(structurePlanPath, "utf8");
+      if (structureContent?.trim()) {
+        try {
+          // Validate that the content looks like JSON before parsing
+          const trimmedContent = structureContent.trim();
+          if (trimmedContent.startsWith("[") || trimmedContent.startsWith("{")) {
+            const parsed = JSON.parse(structureContent);
+            // Return array if it's an array, otherwise return null
+            if (Array.isArray(parsed)) {
+              return parsed;
+            }
+          } else {
+            console.warn("structure-plan.json contains non-JSON content, skipping parse");
+          }
+        } catch (parseError) {
+          console.error(`Failed to parse structure-plan.json: ${parseError.message}`);
+        }
+      }
     }
-
-    const structureContent = await readFile(structurePlanPath, "utf8");
-    if (!structureContent?.trim()) {
-      return null;
+  } catch (readError) {
+    // Only warn if it's not a "file not found" error
+    if (readError.code !== "ENOENT") {
+      console.warn(`Error reading structure-plan.json: ${readError.message}`);
     }
+  }
 
-    try {
-      // Validate that the content looks like JSON before parsing
-      const trimmedContent = structureContent.trim();
-      if (!trimmedContent.startsWith("[") && !trimmedContent.startsWith("{")) {
-        console.warn("structure-plan.json contains non-JSON content, skipping parse");
-        return null;
+  // Try loading document_structure.yaml as fallback
+  try {
+    const yamlPath = join(outputDir, "document_structure.yaml");
+    const yamlExists = await pathExists(yamlPath);
+
+    if (yamlExists) {
+      const yamlContent = await readFile(yamlPath, "utf8");
+      if (yamlContent?.trim()) {
+        try {
+          const parsed = yaml.parse(yamlContent);
+          if (parsed && parsed.documents) {
+            return convertYamlToStructure(parsed);
+          }
+        } catch (parseError) {
+          console.error(`Failed to parse document_structure.yaml: ${parseError.message}`);
+        }
       }
-
-      const parsed = JSON.parse(structureContent);
-      // Return array if it's an array, otherwise return null
-      return Array.isArray(parsed) ? parsed : null;
-    } catch (parseError) {
-      console.error(`Failed to parse structure-plan.json: ${parseError.message}`);
-      return null;
     }
   } catch (readError) {
     // Only warn if it's not a "file not found" error
     if (readError.code !== "ENOENT") {
-      console.warn(`Error reading structure-plan.json: ${readError.message}`);
+      console.warn(`Error reading document_structure.yaml: ${readError.message}`);
     }
-    return null;
   }
+
+  return null;
 }
 
 /**