@aigne/doc-smith 0.5.1 → 0.7.0

package/utils/markdown-checker.mjs

@@ -1,9 +1,14 @@
+import fs from "node:fs";
+import path from "node:path";
+import pMap from "p-map";
 import remarkGfm from "remark-gfm";
 import remarkLint from "remark-lint";
 import remarkParse from "remark-parse";
 import { unified } from "unified";
 import { visit } from "unist-util-visit";
 import { VFile } from "vfile";
+import { KROKI_CONCURRENCY } from "./constants.mjs";
+import { checkD2Content } from "./kroki-utils.mjs";
 import { validateMermaidSyntax } from "./mermaid-validator.mjs";
 
 /**
@@ -65,7 +70,10 @@ function checkDeadLinks(markdown, source, allowedLinks, errorMessages) {
   const linkRegex = /(?<!!)\[([^\]]+)\]\(([^)]+)\)/g;
   let match;
 
-  while ((match = linkRegex.exec(markdown)) !== null) {
+  while (true) {
+    match = linkRegex.exec(markdown);
+    if (match === null) break;
+
     const link = match[2];
     const trimLink = link.trim();
 
@@ -159,6 +167,74 @@ function checkCodeBlockIndentation(codeBlockContent, codeBlockIndent, source, er
   }
 }
 
+/**
+ * Check for local images and verify their existence
+ * @param {string} markdown - The markdown content
+ * @param {string} source - Source description for error reporting
+ * @param {Array} errorMessages - Array to push error messages to
+ * @param {string} [markdownFilePath] - Path to the markdown file for resolving relative paths
+ * @param {string} [baseDir] - Base directory for resolving relative paths (alternative to markdownFilePath)
+ */
+function checkLocalImages(markdown, source, errorMessages, markdownFilePath, baseDir) {
+  const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
+  let match;
+
+  while (true) {
+    match = imageRegex.exec(markdown);
+    if (match === null) break;
+    const imagePath = match[2].trim();
+    const altText = match[1];
+
+    // Skip external URLs (http/https)
+    if (/^https?:\/\//.test(imagePath)) continue;
+
+    // Skip data URLs
+    if (/^data:/.test(imagePath)) continue;
+
+    // Check if it's a local path
+    if (!imagePath.startsWith("/") && !imagePath.includes("://")) {
+      // It's a relative local path, check if file exists
+      try {
+        let resolvedPath;
+        if (markdownFilePath) {
+          // Resolve relative to the markdown file's directory
+          const markdownDir = path.dirname(markdownFilePath);
+          resolvedPath = path.resolve(markdownDir, imagePath);
+        } else if (baseDir) {
+          // Resolve relative to the provided base directory
+          resolvedPath = path.resolve(baseDir, imagePath);
+        } else {
+          // Fallback to current working directory
+          resolvedPath = path.resolve(imagePath);
+        }
+
+        if (!fs.existsSync(resolvedPath)) {
+          errorMessages.push(
+            `Found invalid local image in ${source}: ![${altText}](${imagePath}) - only valid media resources can be used`,
+          );
+        }
+      } catch {
+        errorMessages.push(
+          `Found invalid local image in ${source}: ![${altText}](${imagePath}) - only valid media resources can be used`,
+        );
+      }
+    } else if (imagePath.startsWith("/")) {
+      // Absolute local path
+      try {
+        if (!fs.existsSync(imagePath)) {
+          errorMessages.push(
+            `Found invalid local image in ${source}: ![${altText}](${imagePath}) - only valid media resources can be used`,
+          );
+        }
+      } catch {
+        errorMessages.push(
+          `Found invalid local image in ${source}: ![${altText}](${imagePath}) - only valid media resources can be used`,
+        );
+      }
+    }
+  }
+}
+
 /**
  * Check content structure and formatting issues
  * @param {string} markdown - The markdown content
@@ -224,7 +300,7 @@ function checkContentStructure(markdown, source, errorMessages) {
   }
 
   // Check if content ends with proper punctuation (indicating completeness)
-  const validEndingPunctuation = [".", "。", ")", "|"];
+  const validEndingPunctuation = [".", "。", ")", "|", "*"];
   const trimmedText = markdown.trim();
   const hasValidEnding = validEndingPunctuation.some((punct) => trimmedText.endsWith(punct));
 
@@ -241,6 +317,8 @@ function checkContentStructure(markdown, source, errorMessages) {
  * @param {string} [source] - Source description for error reporting (e.g., "result")
  * @param {Object} [options] - Additional options for validation
  * @param {Array} [options.allowedLinks] - Set of allowed links for link validation
+ * @param {string} [options.filePath] - Path to the markdown file for resolving relative image paths
+ * @param {string} [options.baseDir] - Base directory for resolving relative image paths (alternative to filePath)
  * @returns {Promise<Array<string>>} - Array of error messages in check-detail-result format
  */
 export async function checkMarkdown(markdown, source = "content", options = {}) {
@@ -248,8 +326,8 @@ export async function checkMarkdown(markdown, source = "content", options = {})
   const errorMessages = [];
 
   try {
-    // Extract allowed links from options
-    const { allowedLinks } = options;
+    // Extract allowed links, file path, and base directory from options
+    const { allowedLinks, filePath, baseDir } = options;
 
     // Create unified processor with markdown parsing and linting
     // Use individual rules instead of presets to have better control
@@ -292,88 +370,110 @@ export async function checkMarkdown(markdown, source = "content", options = {})
       checkDeadLinks(markdown, source, allowedLinks, errorMessages);
     }
 
-    // 2. Check content structure and formatting issues
+    // 2. Check local images existence
+    checkLocalImages(markdown, source, errorMessages, filePath, baseDir);
+
+    // 3. Check content structure and formatting issues
     checkContentStructure(markdown, source, errorMessages);
 
     // Check mermaid code blocks and other custom validations
     const mermaidChecks = [];
+    const d2ChecksList = [];
     visit(ast, "code", (node) => {
-      if (node.lang && node.lang.toLowerCase() === "mermaid") {
-        // Check for mermaid syntax errors
-        mermaidChecks.push(
-          validateMermaidSyntax(node.value).catch((error) => {
-            const errorMessage = error?.message || String(error) || "Unknown mermaid syntax error";
-
-            // Format mermaid error in check-detail-result style
-            const line = node.position?.start?.line || "unknown";
-            errorMessages.push(
-              `Found Mermaid syntax error in ${source} at line ${line}: ${errorMessage}`,
-            );
-          }),
-        );
-
-        // Check for specific mermaid rendering issues
-        const mermaidContent = node.value;
+      if (node.lang) {
         const line = node.position?.start?.line || "unknown";
 
-        // Check for backticks in node labels
-        const nodeLabelRegex = /[A-Za-z0-9_]+\["([^"]*`[^"]*)"\]|[A-Za-z0-9_]+{"([^}]*`[^}]*)"}/g;
-        let match;
-        while ((match = nodeLabelRegex.exec(mermaidContent)) !== null) {
-          const label = match[1] || match[2];
-          errorMessages.push(
-            `Found backticks in Mermaid node label in ${source} at line ${line}: "${label}" - backticks in node labels cause rendering issues in Mermaid diagrams`,
+        if (node.lang.toLowerCase() === "mermaid") {
+          // Check for mermaid syntax errors
+          mermaidChecks.push(
+            validateMermaidSyntax(node.value).catch((error) => {
+              const errorMessage =
+                error?.message || String(error) || "Unknown mermaid syntax error";
+
+              // Format mermaid error in check-detail-result style
+              errorMessages.push(
+                `Found Mermaid syntax error in ${source} at line ${line}: ${errorMessage}`,
+              );
+            }),
           );
-        }
 
-        // Check for numbered list format in edge descriptions
-        const edgeDescriptionRegex = /--\s*"([^"]*)"\s*-->/g;
-        let edgeMatch;
-        while ((edgeMatch = edgeDescriptionRegex.exec(mermaidContent)) !== null) {
-          const description = edgeMatch[1];
-          if (/^\d+\.\s/.test(description)) {
-            errorMessages.push(
-              `Unsupported markdown: list - Found numbered list format in Mermaid edge description in ${source} at line ${line}: "${description}" - numbered lists in edge descriptions are not supported`,
-            );
-          }
-        }
+          // Check for specific mermaid rendering issues
+          const mermaidContent = node.value;
 
-        // Check for numbered list format in node labels (for both [] and {} syntax)
-        const nodeLabelWithNumberRegex =
-          /[A-Za-z0-9_]+\["([^"]*\d+\.\s[^"]*)"\]|[A-Za-z0-9_]+{"([^}]*\d+\.\s[^}]*)"}/g;
-        let numberMatch;
-        while ((numberMatch = nodeLabelWithNumberRegex.exec(mermaidContent)) !== null) {
-          const label = numberMatch[1] || numberMatch[2];
-          // Check if the label contains numbered list format
-          if (/\d+\.\s/.test(label)) {
+          // Check for backticks in node labels
+          const nodeLabelRegex = /[A-Za-z0-9_]+\["([^"]*`[^"]*)"\]|[A-Za-z0-9_]+{"([^}]*`[^}]*)"}/g;
+          let match;
+          match = nodeLabelRegex.exec(mermaidContent);
+          while (match !== null) {
+            const label = match[1] || match[2];
             errorMessages.push(
-              `Unsupported markdown: list - Found numbered list format in Mermaid node label in ${source} at line ${line}: "${label}" - numbered lists in node labels cause rendering issues`,
+              `Found backticks in Mermaid node label in ${source} at line ${line}: "${label}" - backticks in node labels cause rendering issues in Mermaid diagrams`,
             );
+            match = nodeLabelRegex.exec(mermaidContent);
           }
-        }
 
-        // Check for special characters in node labels that should be quoted
-        const nodeWithSpecialCharsRegex = /([A-Za-z0-9_]+)\[([^\]]*[(){}:;,\-\s.][^\]]*)\]/g;
-        let specialCharMatch;
-        while ((specialCharMatch = nodeWithSpecialCharsRegex.exec(mermaidContent)) !== null) {
-          const nodeId = specialCharMatch[1];
-          const label = specialCharMatch[2];
-
-          // Check if label contains special characters but is not quoted
-          if (!/^".*"$/.test(label)) {
-            // List of characters that typically need quoting
-            const specialChars = ["(", ")", "{", "}", ":", ";", ",", "-", "."];
-            const foundSpecialChars = specialChars.filter((char) => label.includes(char));
+          // Check for numbered list format in edge descriptions
+          const edgeDescriptionRegex = /--\s*"([^"]*)"\s*-->/g;
+          let edgeMatch;
+          edgeMatch = edgeDescriptionRegex.exec(mermaidContent);
+          while (edgeMatch !== null) {
+            const description = edgeMatch[1];
+            if (/^\d+\.\s/.test(description)) {
+              errorMessages.push(
+                `Unsupported markdown: list - Found numbered list format in Mermaid edge description in ${source} at line ${line}: "${description}" - numbered lists in edge descriptions are not supported`,
+              );
+            }
+            edgeMatch = edgeDescriptionRegex.exec(mermaidContent);
+          }
 
-            if (foundSpecialChars.length > 0) {
+          // Check for numbered list format in node labels (for both [] and {} syntax)
+          const nodeLabelWithNumberRegex =
+            /[A-Za-z0-9_]+\["([^"]*\d+\.\s[^"]*)"\]|[A-Za-z0-9_]+{"([^}]*\d+\.\s[^}]*)"}/g;
+          let numberMatch;
+          numberMatch = nodeLabelWithNumberRegex.exec(mermaidContent);
+          while (numberMatch !== null) {
+            const label = numberMatch[1] || numberMatch[2];
+            // Check if the label contains numbered list format
+            if (/\d+\.\s/.test(label)) {
               errorMessages.push(
-                `Found unquoted special characters in Mermaid node label in ${source} at line ${line}: "${label}" contains ${foundSpecialChars.join(
-                  ", ",
-                )} - node labels with special characters should be quoted like ${nodeId}["${label}"]`,
+                `Unsupported markdown: list - Found numbered list format in Mermaid node label in ${source} at line ${line}: "${label}" - numbered lists in node labels cause rendering issues`,
               );
             }
+            numberMatch = nodeLabelWithNumberRegex.exec(mermaidContent);
           }
+
+          // Check for special characters in node labels that should be quoted
+          const nodeWithSpecialCharsRegex = /([A-Za-z0-9_]+)\[([^\]]*[(){}:;,\-\s.][^\]]*)\]/g;
+          let specialCharMatch;
+          specialCharMatch = nodeWithSpecialCharsRegex.exec(mermaidContent);
+          while (specialCharMatch !== null) {
+            const nodeId = specialCharMatch[1];
+            const label = specialCharMatch[2];
+
+            // Check if label contains special characters but is not quoted
+            if (!/^".*"$/.test(label)) {
+              // List of characters that typically need quoting
+              const specialChars = ["(", ")", "{", "}", ":", ";", ",", "-", "."];
+              const foundSpecialChars = specialChars.filter((char) => label.includes(char));
+
+              if (foundSpecialChars.length > 0) {
+                errorMessages.push(
+                  `Found unquoted special characters in Mermaid node label in ${source} at line ${line}: "${label}" contains ${foundSpecialChars.join(
+                    ", ",
+                  )} - node labels with special characters should be quoted like ${nodeId}["${label}"]`,
+                );
+              }
+            }
+            specialCharMatch = nodeWithSpecialCharsRegex.exec(mermaidContent);
+          }
+        }
+        if (node.lang.toLowerCase() === "d2") {
+          d2ChecksList.push({
+            content: node.value,
+            line,
+          });
         }
+        // TODO: @zhanghan need to check correctness of every code language
       }
     });
 
@@ -424,6 +524,15 @@ export async function checkMarkdown(markdown, source = "content", options = {})
 
     // Wait for all mermaid checks to complete
     await Promise.all(mermaidChecks);
+    await pMap(
+      d2ChecksList,
+      async ({ content, line }) =>
+        checkD2Content({ content }).catch((err) => {
+          const errorMessage = err?.message || String(err) || "Unknown d2 syntax error";
+          errorMessages.push(`Found D2 syntax error in ${source} at line ${line}: ${errorMessage}`);
+        }),
+      { concurrency: KROKI_CONCURRENCY },
+    );
 
     // Run markdown linting rules
     await processor.run(ast, file);
@@ -431,7 +540,6 @@ export async function checkMarkdown(markdown, source = "content", options = {})
     // Format messages in check-detail-result style
     file.messages.forEach((message) => {
       const line = message.line || "unknown";
-      const _column = message.column || "unknown";
       const reason = message.reason || "Unknown markdown issue";
       const ruleId = message.ruleId || message.source || "markdown";
 

package/utils/utils.mjs

@@ -1,8 +1,13 @@
 import { execSync } from "node:child_process";
+import crypto from "node:crypto";
 import { accessSync, constants, existsSync, mkdirSync, readdirSync, statSync } from "node:fs";
 import fs from "node:fs/promises";
 import path from "node:path";
-import { parse } from "yaml";
+import { parse, stringify as yamlStringify } from "yaml";
+import {
+  detectResolvableConflicts,
+  generateConflictResolutionRules,
+} from "./conflict-detector.mjs";
 import {
   DEFAULT_EXCLUDE_PATTERNS,
   DEFAULT_INCLUDE_PATTERNS,
@@ -32,6 +37,16 @@ export function toRelativePath(filePath) {
   return path.isAbsolute(filePath) ? path.relative(process.cwd(), filePath) : filePath;
 }
 
+/**
+ * Check if a string looks like a glob pattern
+ * @param {string} pattern - The string to check
+ * @returns {boolean} - True if the string contains glob pattern characters
+ */
+export function isGlobPattern(pattern) {
+  if (pattern == null) return false;
+  return /[*?[\]]|(\*\*)/.test(pattern);
+}
+
 export function processContent({ content }) {
   // Match markdown regular links [text](link), exclude images ![text](link)
   return content.replace(/(?<!!)\[([^\]]+)\]\(([^)]+)\)/g, (match, text, link) => {
@@ -95,6 +110,7 @@ export async function saveDocWithTranslations({
 
       // Add labels front matter if labels are provided
       let finalContent = processContent({ content });
+
       if (labels && labels.length > 0) {
         const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
         finalContent = frontMatter + finalContent;
@@ -113,6 +129,7 @@ export async function saveDocWithTranslations({
       let finalTranslationContent = processContent({
         content: translate.translation,
       });
+
       if (labels && labels.length > 0) {
         const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
         finalTranslationContent = frontMatter + finalTranslationContent;
@@ -149,8 +166,8 @@ export function getCurrentGitHead() {
  * @param {string} gitHead - The current git HEAD commit hash
  */
 export async function saveGitHeadToConfig(gitHead) {
-  if (!gitHead) {
-    return; // Skip if no git HEAD available
+  if (!gitHead || process.env.NODE_ENV === "test" || process.env.BUN_TEST) {
+    return; // Skip if no git HEAD available or in test environment
   }
 
   try {
@@ -169,7 +186,9 @@ export async function saveGitHeadToConfig(gitHead) {
 
     // Check if lastGitHead already exists in the file
     const lastGitHeadRegex = /^lastGitHead:\s*.*$/m;
-    const newLastGitHeadLine = `lastGitHead: ${gitHead}`;
+    // Use yaml library to safely serialize the git head value
+    const yamlContent = yamlStringify({ lastGitHead: gitHead }).trim();
+    const newLastGitHeadLine = yamlContent;
 
     if (lastGitHeadRegex.test(fileContent)) {
       // Replace existing lastGitHead line
@@ -287,8 +306,10 @@ export function hasFileChangesBetweenCommits(
     return addedOrDeletedFiles.some((filePath) => {
       // Check if file matches any include pattern
       const matchesInclude = includePatterns.some((pattern) => {
-        // Convert glob pattern to regex for matching
-        const regexPattern = pattern.replace(/\./g, "\\.").replace(/\*/g, ".*").replace(/\?/g, ".");
+        // First escape all regex special characters except * and ?
+        const escapedPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&");
+        // Then convert glob wildcards to regex
+        const regexPattern = escapedPattern.replace(/\*/g, ".*").replace(/\?/g, ".");
         const regex = new RegExp(regexPattern);
         return regex.test(filePath);
       });
@@ -299,8 +320,10 @@ export function hasFileChangesBetweenCommits(
 
       // Check if file matches any exclude pattern
       const matchesExclude = excludePatterns.some((pattern) => {
-        // Convert glob pattern to regex for matching
-        const regexPattern = pattern.replace(/\./g, "\\.").replace(/\*/g, ".*").replace(/\?/g, ".");
+        // First escape all regex special characters except * and ?
+        const escapedPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&");
+        // Then convert glob wildcards to regex
+        const regexPattern = escapedPattern.replace(/\*/g, ".*").replace(/\?/g, ".");
         const regex = new RegExp(regexPattern);
         return regex.test(filePath);
       });
@@ -351,9 +374,10 @@ export async function loadConfigFromFile() {
  * @returns {string} Updated file content
  */
 function handleArrayValueUpdate(key, value, comment, fileContent) {
-  // Format array value
-  const formattedValue =
-    value.length === 0 ? `${key}: []` : `${key}:\n${value.map((item) => `  - ${item}`).join("\n")}`;
+  // Use yaml library to safely serialize the key-value pair
+  const yamlObject = { [key]: value };
+  const yamlContent = yamlStringify(yamlObject).trim();
+  const formattedValue = yamlContent;
 
   const lines = fileContent.split("\n");
 
@@ -435,7 +459,10 @@ function handleArrayValueUpdate(key, value, comment, fileContent) {
  * @returns {string} Updated file content
  */
 function handleStringValueUpdate(key, value, comment, fileContent) {
-  const formattedValue = `${key}: "${value}"`;
+  // Use yaml library to safely serialize the key-value pair
+  const yamlObject = { [key]: value };
+  const yamlContent = yamlStringify(yamlObject).trim();
+  const formattedValue = yamlContent;
   const lines = fileContent.split("\n");
 
   // Handle string values (original logic)
@@ -849,6 +876,29 @@ export function processConfigFields(config) {
   const processed = {};
   const allRulesContent = [];
 
+  // Set default values for missing or empty fields
+  const defaults = {
+    nodeName: "Section",
+    locale: "en",
+    sourcesPath: ["./"],
+    docsDir: "./.aigne/doc-smith/docs",
+    outputDir: "./.aigne/doc-smith/output",
+    translateLanguages: [],
+    rules: "",
+    targetAudience: "",
+  };
+
+  // Apply defaults for missing or empty fields
+  for (const [key, defaultValue] of Object.entries(defaults)) {
+    if (
+      !config[key] ||
+      (Array.isArray(defaultValue) && (!config[key] || config[key].length === 0)) ||
+      (typeof defaultValue === "string" && (!config[key] || config[key].trim() === ""))
+    ) {
+      processed[key] = defaultValue;
+    }
+  }
+
   // Check if original rules field has content
   if (config.rules) {
     if (typeof config.rules === "string") {
@@ -868,27 +918,35 @@ export function processConfigFields(config) {
   }
 
   // Process document purpose (array)
-  let purposeContents = "";
   if (config.documentPurpose && Array.isArray(config.documentPurpose)) {
-    purposeContents = config.documentPurpose
-      .map((key) => DOCUMENT_STYLES[key]?.content)
-      .filter(Boolean)
-      .join("\n\n");
+    const purposeRules = config.documentPurpose
+      .map((key) => {
+        const style = DOCUMENT_STYLES[key];
+        if (!style) return null;
+        return `Document Purpose - ${style.name}:\n${style.description}\n${style.content}`;
+      })
+      .filter(Boolean);
 
-    if (purposeContents) {
-      allRulesContent.push(purposeContents);
+    if (purposeRules.length > 0) {
+      allRulesContent.push(purposeRules.join("\n\n"));
     }
   }
 
   // Process target audience types (array)
-  let audienceContents = "";
   let audienceNames = "";
   if (config.targetAudienceTypes && Array.isArray(config.targetAudienceTypes)) {
-    // Get content for rules
-    audienceContents = config.targetAudienceTypes
-      .map((key) => TARGET_AUDIENCES[key]?.content)
-      .filter(Boolean)
-      .join("\n\n");
+    // Get structured content for rules
+    const audienceRules = config.targetAudienceTypes
+      .map((key) => {
+        const audience = TARGET_AUDIENCES[key];
+        if (!audience) return null;
+        return `Target Audience - ${audience.name}:\n${audience.description}\n${audience.content}`;
+      })
+      .filter(Boolean);
+
+    if (audienceRules.length > 0) {
+      allRulesContent.push(audienceRules.join("\n\n"));
+    }
 
     // Get names for targetAudience field
     audienceNames = config.targetAudienceTypes
@@ -896,10 +954,6 @@ export function processConfigFields(config) {
       .filter(Boolean)
       .join(", ");
 
-    if (audienceContents) {
-      allRulesContent.push(audienceContents);
-    }
-
     if (audienceNames) {
       // Check if original targetAudience field has content
       const existingTargetAudience = config.targetAudience?.trim();
@@ -933,6 +987,16 @@ export function processConfigFields(config) {
     }
   }
 
+  // Detect and handle conflicts in user selections
+  const conflicts = detectResolvableConflicts(config);
+  if (conflicts.length > 0) {
+    const conflictResolutionRules = generateConflictResolutionRules(conflicts);
+    allRulesContent.push(conflictResolutionRules);
+
+    // Store conflict information for debugging/logging
+    processed.detectedConflicts = conflicts;
+  }
+
   // Combine all content into rules field
   if (allRulesContent.length > 0) {
     processed.rules = allRulesContent.join("\n\n");
@@ -1087,3 +1151,7 @@ export function detectSystemLanguage() {
     return "en";
   }
 }
+
+export function getContentHash(str) {
+  return crypto.createHash("sha256").update(str).digest("hex");
+}