@aigne/doc-smith 0.8.15-beta → 0.8.15-beta.10

package/utils/file-utils.mjs

@@ -8,9 +8,11 @@ import { isBinaryFile } from "isbinaryfile";
 import { encode } from "gpt-tokenizer";
 import { fileTypeFromBuffer } from "file-type";
 import { gunzipSync } from "node:zlib";
+
+import { debug } from "./debug.mjs";
 import { isGlobPattern } from "./utils.mjs";
-import { INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD } from "./constants/index.mjs";
 import { uploadFiles } from "./upload-files.mjs";
+import { extractApi } from "./extract-api.mjs";
 
 /**
  * Check if a directory is inside a git repository using git command
@@ -284,6 +286,11 @@ export async function loadFilesFromPaths(sourcesPath, options = {}) {
         continue;
       }
 
+      if (isRemoteFile(dir)) {
+        allFiles.push(dir);
+        continue;
+      }
+
       // First try to access as a file or directory
       const stats = await stat(dir);
 
@@ -313,7 +320,13 @@ export async function loadFilesFromPaths(sourcesPath, options = {}) {
             : [];
 
           finalIncludePatterns = [...defaultIncludePatterns, ...userInclude];
-          finalExcludePatterns = [...defaultExcludePatterns, ...userExclude];
+          finalExcludePatterns = [
+            ...defaultExcludePatterns,
+            ...userExclude.map((x) => {
+              const prefix = `${dir}/`;
+              return x.startsWith(prefix) ? x.slice(prefix.length) : x;
+            }),
+          ];
         } else {
           // Use only user patterns
           if (includePatterns) {
@@ -374,6 +387,10 @@ export async function loadFilesFromPaths(sourcesPath, options = {}) {
  * @returns {Promise<boolean>} True if file appears to be a text file
  */
 async function isTextFile(filePath) {
+  if (isRemoteFile(filePath)) {
+    return isRemoteTextFile(filePath);
+  }
+
   try {
     const isBinary = await isBinaryFile(filePath);
     return !isBinary;
@@ -383,6 +400,81 @@ async function isTextFile(filePath) {
   }
 }
 
+/**
+ * Check if a string is an HTTP/HTTPS URL
+ * @param {string} fileUrl - The string to check
+ * @returns {boolean} - True if the string starts with http:// or https://
+ */
+export function isRemoteFile(fileUrl) {
+  if (typeof fileUrl !== "string") return false;
+
+  try {
+    const url = new URL(fileUrl);
+    // Only accept http and https url
+    if (["http:", "https:"].includes(url.protocol)) {
+      return true;
+    }
+    // other protocol will be treated as bad url
+    return false;
+  } catch {
+    return false;
+  }
+}
+
+export async function isRemoteFileAvailable(fileUrl) {
+  if (!isRemoteFile(fileUrl)) return false;
+
+  try {
+    const res = await fetch(fileUrl, {
+      method: "HEAD",
+    });
+    return res.ok;
+  } catch (error) {
+    debug(`Failed to check HTTP file availability: ${fileUrl} - ${error.message}`);
+    return false;
+  }
+}
+
+export async function isRemoteTextFile(fileUrl) {
+  try {
+    const res = await fetch(fileUrl, {
+      method: "HEAD",
+    });
+    const contentType = res.headers.get("content-type") || "";
+    const textMimeTypes = [
+      "application/json",
+      "application/ld+json",
+      "application/graphql+json",
+      "application/xml",
+      "application/xhtml+xml",
+      "application/javascript",
+      "application/ecmascript",
+      "application/x-www-form-urlencoded",
+      "application/rss+xml",
+      "application/atom+xml",
+    ];
+    if (contentType.startsWith("text/") || textMimeTypes.includes(contentType)) {
+      return true;
+    }
+    return false;
+  } catch (error) {
+    debug(`Failed to check HTTP file content type: ${fileUrl} - ${error.message}`);
+    return null;
+  }
+}
+
+export async function getRemoteFileContent(fileUrl) {
+  if (!fileUrl) return null;
+  try {
+    const res = await fetch(fileUrl);
+    const text = await res.text();
+    return text;
+  } catch (error) {
+    debug(`Failed to fetch HTTP file content: ${fileUrl} - ${error.message}`);
+    return null;
+  }
+}
+
 /**
  * Read and parse file contents from an array of file paths
  * @param {string[]} files - Array of file paths to read
@@ -405,12 +497,26 @@ export async function readFileContents(files, baseDir = process.cwd(), options =
       }
 
       try {
-        const content = await readFile(file, "utf8");
-        const relativePath = path.relative(baseDir, file);
-        return {
-          sourceId: relativePath,
-          content,
-        };
+        if (isRemoteFile(file)) {
+          const content = await getRemoteFileContent(file);
+          if (content) {
+            return {
+              sourceId: file,
+              content,
+            };
+          }
+
+          return null;
+        } else {
+          const content = await extractApi(file);
+          if (!content) return null;
+
+          const relativePath = path.relative(baseDir, file);
+          return {
+            sourceId: relativePath,
+            content,
+          };
+        }
       } catch (error) {
         // If reading as text fails (e.g., binary file), skip it
         console.warn(`Failed to read file as text: ${file} - ${error.message}`);
@@ -423,6 +529,11 @@ export async function readFileContents(files, baseDir = process.cwd(), options =
   return results.filter((result) => result !== null);
 }
 
+export function calculateTokens(text) {
+  const tokens = encode(text);
+  return tokens.length;
+}
+
 /**
  * Calculate total lines and tokens from file contents
  * @param {Array<{content: string}>} sourceFiles - Array of objects containing content property
@@ -448,97 +559,17 @@ export function calculateFileStats(sourceFiles) {
 }
 
 /**
- * Build sources content string based on context size
- * For large contexts, only include core project files to avoid token limit issues
+ * Build sources content string
  * @param {Array<{sourceId: string, content: string}>} sourceFiles - Array of source file objects
- * @param {boolean} isLargeContext - Whether the context is large
  * @returns {string} Concatenated sources content with sourceId comments
  */
-export function buildSourcesContent(sourceFiles, isLargeContext = false) {
-  // Define core file patterns that represent project structure and key information
-  const coreFilePatterns = [
-    // Configuration files
-    /package\.json$/,
-    /tsconfig\.json$/,
-    /jsconfig\.json$/,
-    /\.env\.example$/,
-    /Cargo\.toml$/,
-    /go\.mod$/,
-    /pom\.xml$/,
-    /build\.gradle$/,
-    /Gemfile$/,
-    /requirements\.txt$/,
-    /Pipfile$/,
-    /composer\.json$/,
-    /pyproject\.toml$/,
-
-    // Documentation
-    /README\.md$/i,
-    /CHANGELOG\.md$/i,
-    /CONTRIBUTING\.md$/i,
-    /\.github\/.*\.md$/i,
-
-    // Entry points and main files
-    /index\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /main\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /app\.(js|ts|jsx|tsx|py)$/,
-    /server\.(js|ts|jsx|tsx|py)$/,
-
-    // API definitions
-    /api\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /routes\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /controllers\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-
-    // Type definitions and schemas
-    /types\.(ts|d\.ts)$/,
-    /schema\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /.*\.d\.ts$/,
-
-    // Core utilities
-    /utils\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /lib\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /helpers\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-  ];
-
-  // Function to check if a file is a core file
-  const isCoreFile = (filePath) => {
-    return coreFilePatterns.some((pattern) => pattern.test(filePath));
-  };
-
+export function buildSourcesContent(sourceFiles) {
   // Build sources string
   let allSources = "";
 
-  if (isLargeContext) {
-    // Only include core files for large contexts
-    const coreFiles = sourceFiles.filter((source) => isCoreFile(source.sourceId));
-
-    // Determine which files to use and set appropriate message
-    const filesToInclude = coreFiles.length > 0 ? coreFiles : sourceFiles;
-    const noteMessage =
-      coreFiles.length > 0
-        ? "// Note: Context is large, showing only core project files.\n"
-        : "// Note: Context is large, showing a sample of files.\n";
-
-    allSources += noteMessage;
-    let accumulatedTokens = 0;
-
-    for (const source of filesToInclude) {
-      const fileContent = `// sourceId: ${source.sourceId}\n${source.content}\n`;
-      const fileTokens = encode(fileContent);
-
-      // Check if adding this file would exceed the token limit
-      if (accumulatedTokens + fileTokens.length > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD) {
-        break;
-      }
-
-      allSources += fileContent;
-      accumulatedTokens += fileTokens.length;
-    }
-  } else {
-    // Include all files for normal contexts
-    for (const source of sourceFiles) {
-      allSources += `// sourceId: ${source.sourceId}\n${source.content}\n`;
-    }
+  // Include all files for normal contexts
+  for (const source of sourceFiles) {
+    allSources += `\n// sourceId: ${source.sourceId}\n${source.content}\n`;
   }
 
   return allSources;

package/utils/history-utils.mjs

@@ -99,7 +99,7 @@ function recordUpdateGit({ feedback }) {
 /**
  * Records an update in the YAML file.
  */
-function recordUpdateYaml({ operation, feedback, documentPath = null }) {
+function recordUpdateYaml({ operation, feedback, docPaths = null }) {
   try {
     const docSmithDir = join(process.cwd(), DOC_SMITH_DIR);
     if (!existsSync(docSmithDir)) {
@@ -125,9 +125,9 @@ function recordUpdateYaml({ operation, feedback, documentPath = null }) {
       feedback,
     };
 
-    // Add document path if provided
-    if (documentPath) {
-      entry.documentPath = documentPath;
+    // Add document paths if provided
+    if (Array.isArray(docPaths) && docPaths.length > 0) {
+      entry.docPaths = docPaths;
     }
 
     // Add to beginning (newest first)
@@ -153,14 +153,14 @@ function recordUpdateYaml({ operation, feedback, documentPath = null }) {
  * @param {Object} params
  * @param {string} params.operation - The type of operation (e.g., 'document_update', 'structure_update', 'translation_update').
  * @param {string} params.feedback - The user's feedback text.
- * @param {string} params.documentPath - The document path.
+ * @param {string[]} [params.docPaths] - Document path list for updates.
  */
-export function recordUpdate({ operation, feedback, documentPath = null }) {
+export function recordUpdate({ operation, feedback, docPaths = null }) {
   // Skip if no feedback
   if (!feedback?.trim()) return;
 
   // Always record in YAML
-  recordUpdateYaml({ operation, feedback, documentPath });
+  recordUpdateYaml({ operation, feedback, docPaths });
 
   // Also record in git if git is available and not in a git repository
   if (isGitAvailable() && !isInGitRepository(process.cwd())) {
@@ -183,7 +183,19 @@ export function getHistory() {
 
   try {
     const content = readFileSync(historyPath, "utf8");
-    return parse(content) || { entries: [] };
+    const parsed = parse(content) || { entries: [] };
+    const entries = Array.isArray(parsed.entries) ? parsed.entries : [];
+
+    const normalized = entries.map((entry) => {
+      if (!entry) return entry;
+      // Normalize legacy entries: documentPath -> docPaths: [documentPath]
+      if (!entry.docPaths && entry.documentPath) {
+        return { ...entry, docPaths: [entry.documentPath] };
+      }
+      return entry;
+    });
+
+    return { ...parsed, entries: normalized };
   } catch (error) {
     console.warn("Could not read the history:", error.message);
     return { entries: [] };

package/utils/load-config.mjs

@@ -28,7 +28,7 @@ export default async function loadConfig({ config, appUrl }) {
     }
 
     // Parse new configuration fields and convert keys to actual content
-    const processedConfig = processConfigFields(parsedConfig);
+    const processedConfig = await processConfigFields(parsedConfig);
 
     return {
       lastGitHead: parsedConfig.lastGitHead || "",

package/utils/markdown-checker.mjs

@@ -3,9 +3,12 @@ import path from "node:path";
 import remarkGfm from "remark-gfm";
 import remarkLint from "remark-lint";
 import remarkParse from "remark-parse";
+import { isRelative } from "ufo";
 import { unified } from "unified";
 import { visit } from "unist-util-visit";
 import { VFile } from "vfile";
+
+import { isRemoteFile, isRemoteFileAvailable } from "./file-utils.mjs";
 import { validateMermaidSyntax } from "./mermaid-validator.mjs";
 
 /**
@@ -232,6 +235,34 @@ function checkLocalImages(markdown, source, errorMessages, markdownFilePath, bas
   }
 }
 
+async function checkRemoteImages(markdown, source, errorMessages) {
+  const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
+  let match;
+
+  while (true) {
+    match = imageRegex.exec(markdown);
+    if (match === null) break;
+    const imagePath = match[2].trim();
+    const altText = match[1];
+
+    if (isRelative(imagePath)) continue;
+    if (imagePath.startsWith("/")) continue;
+
+    // Skip data URLs
+    if (/^data:/.test(imagePath)) continue;
+
+    if (isRemoteFile(imagePath)) {
+      const isAvailable = await isRemoteFileAvailable(imagePath);
+      if (isAvailable) continue;
+      else {
+        errorMessages.push(
+          `Found invalid remote image in ${source}: ![${altText}](${imagePath}) - only valid media resources can be used`,
+        );
+      }
+    }
+  }
+}
+
 /**
  * Check content structure and formatting issues
  * @param {string} markdown - The markdown content
@@ -370,7 +401,10 @@ export async function checkMarkdown(markdown, source = "content", options = {})
     // 2. Check local images existence
     checkLocalImages(markdown, source, errorMessages, filePath, baseDir);
 
-    // 3. Check content structure and formatting issues
+    // 3. Check remote images existence
+    await checkRemoteImages(markdown, source, errorMessages);
+
+    // 4. Check content structure and formatting issues
     checkContentStructure(markdown, source, errorMessages);
 
     // Check mermaid code blocks and other custom validations

package/utils/openapi/index.mjs

@@ -0,0 +1,24 @@
+import { parse } from "yaml";
+
+export function isOpenAPISpecFile(content) {
+  const trimmedContent = content.trim();
+  try {
+    const parsed = parse(trimmedContent, {
+      logLevel: "silent",
+    });
+    if (parsed.openapi || parsed.swagger) {
+      return true;
+    }
+  } catch {
+    //
+  }
+  try {
+    const parsed = JSON.parse(trimmedContent);
+    if (parsed.openapi || parsed.swagger) {
+      return true;
+    }
+  } catch {
+    //
+  }
+  return false;
+}

package/utils/utils.mjs

@@ -18,6 +18,7 @@ import {
   SUPPORTED_LANGUAGES,
   TARGET_AUDIENCES,
 } from "./constants/index.mjs";
+import { isRemoteFile, getRemoteFileContent } from "./file-utils.mjs";
 
 /**
  * Normalize path to absolute path for consistent comparison
@@ -47,16 +48,6 @@ export function isGlobPattern(pattern) {
   return /[*?[\]]|(\*\*)/.test(pattern);
 }
 
-/**
- * Check if a string is an HTTP/HTTPS URL
- * @param {string} url - The string to check
- * @returns {boolean} - True if the string starts with http:// or https://
- */
-export function isHttp(url) {
-  if (typeof url !== "string") return false;
-  return url.startsWith("http://") || url.startsWith("https://");
-}
-
 export function processContent({ content }) {
   // Match markdown regular links [text](link), exclude images ![text](link)
   return content.replace(/(?<!!)\[([^\]]+)\]\(([^)]+)\)/g, (match, text, link) => {
@@ -81,77 +72,72 @@ export function processContent({ content }) {
   });
 }
 
+// Helper function to generate filename based on language
+export function getFileName(docPath, language) {
+  // Flatten path: remove leading /, replace all / with -
+  const flatName = docPath.replace(/^\//, "").replace(/\//g, "-");
+  const isEnglish = language === "en";
+  return isEnglish ? `${flatName}.md` : `${flatName}.${language}.md`;
+}
+
 /**
- * Save a single document and its translations to files
+ * Save a single document to files
  * @param {Object} params
  * @param {string} params.path - Relative path (without extension)
  * @param {string} params.content - Main document content
  * @param {string} params.docsDir - Root directory
  * @param {string} params.locale - Main content language (e.g., 'en', 'zh', 'fr')
- * @param {Array<{language: string, translation: string}>} [params.translates] - Translation content
  * @param {Array<string>} [params.labels] - Document labels for front matter
- * @returns {Promise<Array<{ path: string, success: boolean, error?: string }>>}
+ * @returns {Promise<{ path: string, success: boolean, error?: string }>}
  */
-export async function saveDocWithTranslations({
-  path: docPath,
-  content,
-  docsDir,
-  locale,
-  translates = [],
-  labels,
-  isTranslate = false,
-}) {
-  const results = [];
+export async function saveDoc({ path: docPath, content, docsDir, locale, labels }) {
   try {
-    // Flatten path: remove leading /, replace all / with -
-    const flatName = docPath.replace(/^\//, "").replace(/\//g, "-");
     await fs.mkdir(docsDir, { recursive: true });
+    const mainFileName = getFileName(docPath, locale);
+    const mainFilePath = path.join(docsDir, mainFileName);
 
-    // Helper function to generate filename based on language
-    const getFileName = (language) => {
-      const isEnglish = language === "en";
-      return isEnglish ? `${flatName}.md` : `${flatName}.${language}.md`;
-    };
-
-    // Save main content with appropriate filename based on locale (skip if isTranslate is true)
-    if (!isTranslate) {
-      const mainFileName = getFileName(locale);
-      const mainFilePath = path.join(docsDir, mainFileName);
-
-      // Add labels front matter if labels are provided
-      let finalContent = processContent({ content });
+    // Add labels front matter if labels are provided
+    let finalContent = processContent({ content });
 
-      if (labels && labels.length > 0) {
-        const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
-        finalContent = frontMatter + finalContent;
-      }
-
-      await fs.writeFile(mainFilePath, finalContent, "utf8");
-      results.push({ path: mainFilePath, success: true });
+    if (labels && labels.length > 0) {
+      const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
+      finalContent = frontMatter + finalContent;
     }
 
-    // Process all translations
-    for (const translate of translates) {
-      const translateFileName = getFileName(translate.language);
-      const translatePath = path.join(docsDir, translateFileName);
+    await fs.writeFile(mainFilePath, finalContent, "utf8");
+    return { path: mainFilePath, success: true };
+  } catch (err) {
+    return { path: docPath, success: false, error: err.message };
+  }
+}
 
-      // Add labels front matter to translation content if labels are provided
-      let finalTranslationContent = processContent({
-        content: translate.translation,
-      });
+export async function saveDocTranslation({
+  path: docPath,
+  docsDir,
+  translation,
+  language,
+  labels,
+}) {
+  try {
+    await fs.mkdir(docsDir, { recursive: true });
+    const translateFileName = getFileName(docPath, language);
+    const translatePath = path.join(docsDir, translateFileName);
 
-      if (labels && labels.length > 0) {
-        const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
-        finalTranslationContent = frontMatter + finalTranslationContent;
-      }
+    // Add labels front matter to translation content if labels are provided
+    let finalTranslationContent = processContent({
+      content: translation,
+    });
 
-      await fs.writeFile(translatePath, finalTranslationContent, "utf8");
-      results.push({ path: translatePath, success: true });
+    if (labels && labels.length > 0) {
+      const frontMatter = `---\nlabels: ${JSON.stringify(labels)}\n---\n\n`;
+      finalTranslationContent = frontMatter + finalTranslationContent;
     }
+
+    await fs.writeFile(translatePath, finalTranslationContent, "utf8");
+    return { path: translatePath, success: true };
   } catch (err) {
-    results.push({ path: docPath, success: false, error: err.message });
+    return { path: docPath, success: false, error: err.message };
   }
-  return results;
 }
 
 /**
@@ -963,7 +949,7 @@ export function processTargetAudience(targetAudienceTypes, existingTargetAudienc
  * @param {Object} config - Parsed configuration
  * @returns {Object} Processed configuration with content fields
  */
-export function processConfigFields(config) {
+export async function processConfigFields(config) {
   const processed = {};
   const allRulesContent = [];
 
@@ -995,7 +981,15 @@ export function processConfigFields(config) {
     if (typeof config.rules === "string") {
       const existingRules = config.rules.trim();
       if (existingRules) {
-        allRulesContent.push(existingRules);
+        // load rules from remote url
+        if (isRemoteFile(existingRules)) {
+          const remoteFileContent = await getRemoteFileContent(existingRules);
+          if (remoteFileContent) {
+            allRulesContent.push(remoteFileContent);
+          }
+        } else {
+          allRulesContent.push(existingRules);
+        }
       }
     } else if (Array.isArray(config.rules)) {
       // Handle array of rules - join them with newlines
@@ -1054,6 +1048,12 @@ export function processConfigFields(config) {
     }
   }
 
+  if (config.glossary) {
+    if (isRemoteFile(config.glossary)) {
+      processed.glossary = await getRemoteFileContent(config.glossary);
+    }
+  }
+
   // Detect and handle conflicts in user selections
   const conflicts = detectResolvableConflicts(config);
   if (conflicts.length > 0) {
@@ -1097,8 +1097,10 @@ export function processConfigFields(config) {
  * @returns {Promise<any>} - The processed configuration with file content loaded in place of references.
  */
 export async function resolveFileReferences(obj, basePath = process.cwd()) {
-  if (typeof obj === "string" && obj.startsWith("@")) {
-    return await loadFileContent(obj.slice(1), basePath);
+  if (typeof obj === "string") {
+    if (obj.startsWith("@")) {
+      return await loadFileContent(obj.slice(1), basePath);
+    }
   }
 
   if (Array.isArray(obj)) {