@aigne/doc-smith 0.9.7 → 0.9.8-beta.1

package/utils/d2-utils.mjs

@@ -1,189 +1,26 @@
 import path from "node:path";
 
-import { D2 } from "@terrastruct/d2";
 import fs from "fs-extra";
-import { glob } from "glob";
-import pMap from "p-map";
 
-import {
-  D2_CONCURRENCY,
-  D2_CONFIG,
-  DOC_SMITH_DIR,
-  FILE_CONCURRENCY,
-  TMP_ASSETS_DIR,
-  TMP_DIR,
-} from "./constants/index.mjs";
-import { debug } from "./debug.mjs";
-import { iconMap } from "./icon-map.mjs";
-import { getContentHash } from "./utils.mjs";
+import { DOC_SMITH_DIR, TMP_DIR } from "./constants/index.mjs";
 
-const codeBlockRegex = /```d2.*\n([\s\S]*?)```/g;
+// Note: .* matches title or other text after ```d2 (e.g., ```d2 Vault 驗證流程)
+// Export regex for reuse across the codebase to avoid duplication
+export const d2CodeBlockRegex = /```d2.*\n([\s\S]*?)```/g;
 
 export const DIAGRAM_PLACEHOLDER = "DIAGRAM_PLACEHOLDER";
 
-export async function getChart({ content, strict }) {
-  const d2 = new D2();
-  const iconUrlList = Object.keys(iconMap);
-  const escapedUrls = iconUrlList.map((url) => url.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
-  const regexPattern = escapedUrls.join("|");
-  const regex = new RegExp(regexPattern, "g");
+// Diagram image regex patterns for reuse across the codebase
+// Pattern 1: Match only the start marker (for checking existence)
+export const diagramImageStartRegex = /<!--\s*DIAGRAM_IMAGE_START:[^>]+-->/g;
 
-  const contentWithBase64Img = content.replace(regex, (match) => {
-    return iconMap[match];
-  });
-  try {
-    const { diagram, renderOptions, graph } = await d2.compile(contentWithBase64Img);
+// Pattern 2: Match full diagram image block without capturing image path (for finding/replacing)
+export const diagramImageBlockRegex =
+  /<!--\s*DIAGRAM_IMAGE_START:[^>]+-->\s*[\s\S]*?<!--\s*DIAGRAM_IMAGE_END\s*-->/g;
 
-    // Do not apply a stroke-dash to sequence diagrams.
-    if (
-      graph?.root?.attributes?.shape &&
-      graph.root.attributes.shape.value !== "sequence_diagram"
-    ) {
-      // Save the first-level container.
-      const firstLevelContainer = new Set();
-      diagram.shapes.forEach((x) => {
-        const idList = x.id.split(".");
-        if (idList.length > 1) {
-          const targetShape = diagram.shapes.find((x) => x.id === idList[0]);
-          if (targetShape && !["c4-person", "cylinder", "queue"].includes(targetShape.type)) {
-            firstLevelContainer.add(targetShape);
-          }
-        }
-      });
-      firstLevelContainer.forEach((shape) => {
-        if (!shape.strokeDash) {
-          // Note: The data structure here is different from the d2 source code.
-          shape.strokeDash = 3;
-        }
-      });
-    }
-
-    const svg = await d2.render(diagram, renderOptions);
-
-    return svg;
-  } catch (err) {
-    if (strict) throw err;
-
-    console.error("Failed to generate D2 diagram. Content:", content, "Error:", err);
-    return null;
-  } finally {
-    d2.worker.terminate();
-  }
-}
-
-export async function saveAssets({ markdown, docsDir }) {
-  if (!markdown) {
-    return markdown;
-  }
-
-  const { replaced } = await runIterator({
-    input: markdown,
-    regexp: codeBlockRegex,
-    replace: true,
-    fn: async ([_match, _code]) => {
-      const assetDir = path.join(docsDir, "../", TMP_ASSETS_DIR, "d2");
-      await fs.ensureDir(assetDir);
-      const d2Content = [D2_CONFIG, _code].join("\n");
-      const fileName = `${getContentHash(d2Content)}.svg`;
-      const svgPath = path.join(assetDir, fileName);
-
-      if (await fs.pathExists(svgPath)) {
-        debug("Asset cache found, skipping generation", svgPath);
-      } else {
-        try {
-          debug("Generating d2 diagram", svgPath);
-          if (debug.enabled) {
-            const d2FileName = `${getContentHash(d2Content)}.d2`;
-            const d2Path = path.join(assetDir, d2FileName);
-            await fs.writeFile(d2Path, d2Content, { encoding: "utf8" });
-          }
-
-          const svg = await getChart({ content: d2Content });
-          if (svg) {
-            await fs.writeFile(svgPath, svg, { encoding: "utf8" });
-          }
-        } catch (error) {
-          debug("Failed to generate D2 diagram. Content:", d2Content, "Error:", error);
-          return _code;
-        }
-      }
-      return `![](${path.posix.join("..", TMP_ASSETS_DIR, "d2", fileName)})`;
-    },
-    options: { concurrency: D2_CONCURRENCY },
-  });
-
-  return replaced;
-}
-
-export async function beforePublishHook({ docsDir }) {
-  // Process each markdown file to save d2 svg assets.
-  const mdFilePaths = await glob("**/*.md", { cwd: docsDir });
-  await pMap(
-    mdFilePaths,
-    async (filePath) => {
-      let finalContent = await fs.readFile(path.join(docsDir, filePath), { encoding: "utf8" });
-      finalContent = await saveAssets({ markdown: finalContent, docsDir });
-
-      await fs.writeFile(path.join(docsDir, filePath), finalContent, { encoding: "utf8" });
-    },
-    { concurrency: FILE_CONCURRENCY },
-  );
-}
-
-async function runIterator({ input, regexp, fn = () => {}, options, replace = false }) {
-  if (!input) return input;
-  const matches = [...input.matchAll(regexp)];
-  const results = [];
-  await pMap(
-    matches,
-    async (...args) => {
-      const resultItem = await fn(...args);
-      results.push(resultItem);
-    },
-    options,
-  );
-
-  let replaced = input;
-  if (replace) {
-    let index = 0;
-    replaced = replaced.replace(regexp, () => {
-      return results[index++];
-    });
-  }
-
-  return {
-    results,
-    replaced,
-  };
-}
-
-export async function checkContent({ content: _content }) {
-  const matches = Array.from(_content.matchAll(codeBlockRegex));
-  let content = _content;
-  if (matches.length > 0) {
-    content = matches[0][1];
-  }
-  await ensureTmpDir();
-  const assetDir = path.join(DOC_SMITH_DIR, TMP_DIR, TMP_ASSETS_DIR, "d2");
-  await fs.ensureDir(assetDir);
-  const d2Content = [D2_CONFIG, content].join("\n");
-  const fileName = `${getContentHash(d2Content)}.svg`;
-  const svgPath = path.join(assetDir, fileName);
-
-  if (debug.enabled) {
-    const d2FileName = `${getContentHash(d2Content)}.d2`;
-    const d2Path = path.join(assetDir, d2FileName);
-    await fs.writeFile(d2Path, d2Content, { encoding: "utf8" });
-  }
-
-  if (await fs.pathExists(svgPath)) {
-    debug("Asset cache found, skipping generation", svgPath);
-    return;
-  }
-
-  const svg = await getChart({ content: d2Content, strict: true });
-  await fs.writeFile(svgPath, svg, { encoding: "utf8" });
-}
+// Pattern 3: Match full diagram image block with image path capture (for extracting paths)
+export const diagramImageWithPathRegex =
+  /<!--\s*DIAGRAM_IMAGE_START:[^>]+-->\s*!\[[^\]]*\]\(([^)]+)\)\s*<!--\s*DIAGRAM_IMAGE_END\s*-->/g;
 
 export async function ensureTmpDir() {
   const tmpDir = path.join(DOC_SMITH_DIR, TMP_DIR);
@@ -198,7 +35,7 @@ export function isValidCode(lang) {
 }
 
 export function wrapCode({ content }) {
-  const matches = Array.from(content.matchAll(codeBlockRegex));
+  const matches = Array.from(content.matchAll(d2CodeBlockRegex));
   if (matches.length > 0) {
     return content;
   }
@@ -212,7 +49,7 @@ export function wrapCode({ content }) {
  * @returns {Array} - [contentWithPlaceholder, originalCodeBlock]
  */
 export function replaceD2WithPlaceholder({ content }) {
-  const [firstMatch] = Array.from(content.matchAll(codeBlockRegex));
+  const [firstMatch] = Array.from(content.matchAll(d2CodeBlockRegex));
   if (firstMatch) {
     const matchContent = firstMatch[0];
     const cleanContent = content.replace(matchContent, DIAGRAM_PLACEHOLDER);
@@ -255,3 +92,107 @@ export function replacePlaceholderWithD2({ content, diagramSourceCode }) {
 
   return content.replace(DIAGRAM_PLACEHOLDER, replacement);
 }
+
+/**
+ * Replace all diagrams (D2 code blocks and generated images) with DIAGRAM_PLACEHOLDER
+ * Used for deletion operations to normalize all diagram types to a single placeholder
+ * @param {string} content - Document content containing diagrams
+ * @param {number} [diagramIndex] - Optional index of diagram to replace (0-based). If not provided, replaces all diagrams.
+ * @returns {string} - Content with diagrams replaced by DIAGRAM_PLACEHOLDER
+ */
+export function replaceDiagramsWithPlaceholder({ content, diagramIndex }) {
+  if (!content) {
+    return content;
+  }
+
+  // Use exported regex to find all diagram locations
+  // We'll use a similar approach to findAllDiagramLocations
+  const mermaidCodeBlockRegex = /```mermaid.*\n([\s\S]*?)```/g;
+
+  // Find all diagram locations
+  const locations = [];
+
+  // 1. Find DIAGRAM_PLACEHOLDER (already a placeholder, keep as is)
+  let placeholderIndex = content.indexOf(DIAGRAM_PLACEHOLDER);
+  while (placeholderIndex !== -1) {
+    locations.push({
+      type: "placeholder",
+      start: placeholderIndex,
+      end: placeholderIndex + DIAGRAM_PLACEHOLDER.length,
+    });
+    placeholderIndex = content.indexOf(DIAGRAM_PLACEHOLDER, placeholderIndex + 1);
+  }
+
+  // 2. Find DIAGRAM_IMAGE_START markers (generated images)
+  const imageMatches = Array.from(content.matchAll(diagramImageBlockRegex));
+  for (const match of imageMatches) {
+    locations.push({
+      type: "image",
+      start: match.index,
+      end: match.index + match[0].length,
+    });
+  }
+
+  // 3. Find D2 code blocks
+  const d2Matches = Array.from(content.matchAll(d2CodeBlockRegex));
+  for (const match of d2Matches) {
+    locations.push({
+      type: "d2",
+      start: match.index,
+      end: match.index + match[0].length,
+    });
+  }
+
+  // 4. Find Mermaid code blocks
+  const mermaidMatches = Array.from(content.matchAll(mermaidCodeBlockRegex));
+  for (const match of mermaidMatches) {
+    locations.push({
+      type: "mermaid",
+      start: match.index,
+      end: match.index + match[0].length,
+    });
+  }
+
+  // Sort by position (top to bottom)
+  locations.sort((a, b) => a.start - b.start);
+
+  if (locations.length === 0) {
+    return content;
+  }
+
+  // If diagramIndex is provided, only replace that specific diagram
+  if (diagramIndex !== undefined && diagramIndex >= 0 && diagramIndex < locations.length) {
+    const targetLocation = locations[diagramIndex];
+    const before = content.substring(0, targetLocation.start);
+    const after = content.substring(targetLocation.end);
+    // Add newlines if needed
+    let replacement = DIAGRAM_PLACEHOLDER;
+    if (before && !before.endsWith("\n")) {
+      replacement = `\n${replacement}`;
+    }
+    if (after && !after.startsWith("\n")) {
+      replacement = `${replacement}\n`;
+    }
+    return before + replacement + after;
+  }
+
+  // Replace all diagrams with placeholder (for deletion)
+  // Process from end to start to preserve indices
+  let result = content;
+  for (let i = locations.length - 1; i >= 0; i--) {
+    const location = locations[i];
+    const before = result.substring(0, location.start);
+    const after = result.substring(location.end);
+    // Add newlines if needed
+    let replacement = DIAGRAM_PLACEHOLDER;
+    if (before && !before.endsWith("\n")) {
+      replacement = `\n${replacement}`;
+    }
+    if (after && !after.startsWith("\n")) {
+      replacement = `${replacement}\n`;
+    }
+    result = before + replacement + after;
+  }
+
+  return result;
+}

package/utils/delete-diagram-images.mjs

@@ -0,0 +1,99 @@
+import { unlink } from "node:fs/promises";
+import { join, dirname, normalize } from "node:path";
+import fs from "fs-extra";
+import { debug } from "./debug.mjs";
+
+/**
+ * Extract image file paths from markdown content
+ * Finds all diagram image references and extracts their file paths
+ * @param {string} content - Markdown content
+ * @param {string} path - Document path (e.g., "guides/getting-started.md")
+ * @param {string} docsDir - Documentation directory
+ * @returns {Promise<Array<string>>} Array of absolute paths to image files
+ */
+export async function extractDiagramImagePaths(content, path, docsDir) {
+  if (!content || !path || !docsDir) {
+    return [];
+  }
+
+  const imagePaths = [];
+
+  // Pattern to match: <!-- DIAGRAM_IMAGE_START:... -->![alt](path)<!-- DIAGRAM_IMAGE_END -->
+  const { diagramImageWithPathRegex } = await import("./d2-utils.mjs");
+  const matches = Array.from(content.matchAll(diagramImageWithPathRegex));
+
+  for (const match of matches) {
+    const imagePath = match[1];
+
+    // Resolve absolute path
+    // If imagePath is relative, resolve from document location
+    // If imagePath is absolute or starts with http, skip
+    if (imagePath.startsWith("http://") || imagePath.startsWith("https://")) {
+      continue; // Skip remote URLs
+    }
+
+    // Calculate relative path from document to image
+    const docDir = dirname(path);
+    const imageRelativePath = imagePath.startsWith("../")
+      ? imagePath
+      : join(docDir, imagePath).replace(/\\/g, "/");
+
+    // Resolve absolute path
+    const absolutePath = join(process.cwd(), docsDir, imageRelativePath);
+
+    // Normalize path (remove .. and .)
+    const normalizedPath = normalize(absolutePath);
+
+    if (await fs.pathExists(normalizedPath)) {
+      imagePaths.push(normalizedPath);
+    }
+  }
+
+  return imagePaths;
+}
+
+/**
+ * Delete diagram image files associated with a document
+ * @param {string} content - Markdown content (before deletion)
+ * @param {string} path - Document path
+ * @param {string} docsDir - Documentation directory
+ * @returns {Promise<{deleted: number, failed: number}>}
+ */
+export async function deleteDiagramImages(content, path, docsDir) {
+  if (!content || !path || !docsDir) {
+    return { deleted: 0, failed: 0 };
+  }
+
+  try {
+    const imagePaths = await extractDiagramImagePaths(content, path, docsDir);
+
+    if (imagePaths.length === 0) {
+      return { deleted: 0, failed: 0 };
+    }
+
+    let deleted = 0;
+    let failed = 0;
+
+    for (const imagePath of imagePaths) {
+      try {
+        await unlink(imagePath);
+        debug(`Deleted diagram image: ${imagePath}`);
+        deleted++;
+      } catch (error) {
+        if (error.code !== "ENOENT") {
+          // File not found is ok, other errors should be logged
+          console.warn(`Failed to delete diagram image ${imagePath}: ${error.message}`);
+          failed++;
+        } else {
+          // File already doesn't exist, count as deleted
+          deleted++;
+        }
+      }
+    }
+
+    return { deleted, failed };
+  } catch (error) {
+    console.warn(`Error deleting diagram images: ${error.message}`);
+    return { deleted: 0, failed: 0 };
+  }
+}

package/utils/docs-finder-utils.mjs

@@ -107,10 +107,43 @@ export async function findItemByPath(documentStructure, docPath, boardId, docsDi
  * @param {string} fileName - File name to read
  * @returns {Promise<string|null>} File content or null if failed
  */
+/**
+ * Remove base64 encoded images from markdown content
+ * This prevents large binary data from being included in document content
+ * Base64 images are completely removed (not replaced with placeholders) because:
+ * 1. They significantly increase token usage without providing useful information to LLM
+ * 2. Normal image references (file paths) are preserved and should be used instead
+ * 3. Base64 images are typically temporary or erroneous entries
+ * @param {string} content - Markdown content that may contain base64 images
+ * @returns {string} - Content with base64 images completely removed
+ */
+function removeBase64Images(content) {
+  if (!content || typeof content !== "string") {
+    return content;
+  }
+
+  // Match markdown image syntax with data URLs: ![alt](data:image/...;base64,...)
+  // This regex matches:
+  // - ![alt text](data:image/type;base64,base64data...)
+  // - ![alt](data:image/type;base64,base64data...)
+  // - [![alt](data:image/type;base64,base64data...)](link)
+  const base64ImageRegex = /!\[([^\]]*)\]\(data:image\/[^)]+\)/g;
+
+  // Completely remove base64 images (including the entire markdown image syntax)
+  // This maximizes token reduction while preserving normal image references
+  const cleanedContent = content.replace(base64ImageRegex, "");
+
+  return cleanedContent;
+}
+
 export async function readFileContent(docsDir, fileName) {
   try {
     const filePath = join(docsDir, fileName);
-    return await readFile(filePath, "utf-8");
+    const content = await readFile(filePath, "utf-8");
+
+    // Remove base64 encoded images to reduce token usage
+    // Base64 image data is not useful for LLM processing and significantly increases token count
+    return removeBase64Images(content);
   } catch (readError) {
     console.warn(`⚠️  Could not read content from ${fileName}:`, readError.message);
     return null;

package/utils/image-compress.mjs

@@ -0,0 +1,75 @@
+import sharp from "sharp";
+import path from "node:path";
+import { debug } from "./debug.mjs";
+
+/**
+ * Compress an image using sharp
+ * Supports JPEG, PNG, and WebP formats
+ * @param {string} inputPath - Path to the input image file
+ * @param {object} options - Compression options
+ * @param {number} options.quality - Compression quality (0-100, default: 80)
+ * @param {string} options.outputFormat - Output format: 'jpeg', 'png', 'webp' (default: auto-detect from input)
+ * @param {string} options.outputPath - Output path for compressed image (if not provided, creates temp file)
+ * @returns {Promise<string>} - Path to the compressed image (outputPath if provided, or temp path, or inputPath if compression fails)
+ */
+export async function compressImage(inputPath, options = {}) {
+  const { quality = 80, outputFormat, outputPath } = options;
+
+  try {
+    const inputExt = path.extname(inputPath).toLowerCase();
+
+    // Determine output format
+    let format = outputFormat;
+    if (!format) {
+      // Auto-detect from input extension
+      if (inputExt === ".jpg" || inputExt === ".jpeg") {
+        format = "jpeg";
+      } else if (inputExt === ".png") {
+        format = "png";
+      } else if (inputExt === ".webp") {
+        format = "webp";
+      } else {
+        // Default to JPEG for unknown formats
+        format = "jpeg";
+        debug(`Unknown image format ${inputExt}, defaulting to JPEG`);
+      }
+    }
+
+    // Determine output path
+    let finalOutputPath = outputPath;
+    if (!finalOutputPath) {
+      // If no output path provided, create temp file in same directory as input
+      const outputExt = format === "jpeg" ? ".jpg" : format === "png" ? ".png" : ".webp";
+      const inputDir = path.dirname(inputPath);
+      const inputBase = path.basename(inputPath, path.extname(inputPath));
+      finalOutputPath = path.join(inputDir, `${inputBase}.compressed${outputExt}`);
+    }
+
+    // Create sharp instance and compress
+    let sharpInstance = sharp(inputPath);
+
+    // Apply format-specific compression options
+    if (format === "jpeg") {
+      // mozjpeg is a valid sharp option for better JPEG compression
+      const jpegOptions = { quality, mozjpeg: true };
+      sharpInstance = sharpInstance.jpeg(jpegOptions);
+    } else if (format === "png") {
+      sharpInstance = sharpInstance.png({ quality, compressionLevel: 9 });
+    } else if (format === "webp") {
+      sharpInstance = sharpInstance.webp({ quality });
+    }
+
+    // Write compressed image directly to output path
+    await sharpInstance.toFile(finalOutputPath);
+
+    debug(
+      `✅ Image compressed: ${inputPath} -> ${finalOutputPath} (format: ${format}, quality: ${quality})`,
+    );
+
+    return finalOutputPath;
+  } catch (error) {
+    debug(`⚠️  Failed to compress image ${inputPath}: ${error.message}`);
+    // Return original path if compression fails
+    return inputPath;
+  }
+}

package/utils/kroki-utils.mjs

@@ -15,6 +15,7 @@ import {
   TMP_DIR,
 } from "./constants/index.mjs";
 import { getContentHash } from "./utils.mjs";
+import { d2CodeBlockRegex } from "./d2-utils.mjs";
 
 const debug = Debug("doc-smith");
 
@@ -59,11 +60,9 @@ export async function saveD2Assets({ markdown, docsDir }) {
     return markdown;
   }
 
-  const codeBlockRegex = /```d2.*\n([\s\S]*?)```/g;
-
   const { replaced } = await runIterator({
     input: markdown,
-    regexp: codeBlockRegex,
+    regexp: d2CodeBlockRegex,
     replace: true,
     fn: async ([_match, _code]) => {
       const assetDir = path.join(docsDir, "../", TMP_ASSETS_DIR, "d2");