@aigne/doc-smith 0.9.10 → 0.9.11-beta

package/utils/document-paths.mjs

@@ -0,0 +1,172 @@
+import { readFile, access } from "node:fs/promises";
+import { constants } from "node:fs";
+import { parse as yamlParse } from "yaml";
+import { PATHS, ERROR_CODES } from "./agent-constants.mjs";
+
+/**
+ * Normalize path
+ * @param {string} rawPath - Raw path (may or may not have leading slash)
+ * @returns {Object} - { filePath: path without slash, displayPath: path with slash }
+ */
+export function normalizePath(rawPath) {
+  if (!rawPath || typeof rawPath !== "string") {
+    throw new Error("Path must be a non-empty string");
+  }
+
+  let normalized = rawPath.trim();
+
+  // Remove leading slash
+  if (normalized.startsWith("/")) {
+    normalized = normalized.slice(1);
+  }
+
+  // Remove trailing slash
+  if (normalized.endsWith("/")) {
+    normalized = normalized.slice(0, -1);
+  }
+
+  return {
+    filePath: normalized, // "overview" or "api/authentication"
+    displayPath: `/${normalized}`, // "/overview" or "/api/authentication"
+  };
+}
+
+/**
+ * Recursively collect document paths
+ * @param {Array} docs - Document array
+ * @param {Object} options - Collection options
+ * @param {boolean} options.includeBothFormats - Whether to include both formats with and without leading slash
+ * @param {boolean} options.collectMetadata - Whether to collect additional metadata
+ * @returns {Set|Array} - Path set or path object array
+ */
+export function collectDocumentPaths(docs, options = {}) {
+  const { includeBothFormats = false, collectMetadata = false } = options;
+
+  const paths = collectMetadata ? [] : new Set();
+
+  function collect(documents) {
+    for (const doc of documents) {
+      if (doc.path) {
+        // Normalize path
+        const normalized = doc.path.startsWith("/") ? doc.path.slice(1) : doc.path;
+
+        if (collectMetadata) {
+          // Collect path and metadata
+          paths.push({
+            path: normalized,
+            displayPath: `/${normalized}`,
+            title: doc.title || "",
+            description: doc.description || "",
+          });
+        } else {
+          // Only collect paths
+          paths.add(normalized);
+          if (includeBothFormats) {
+            paths.add(`/${normalized}`);
+          }
+        }
+      }
+
+      // Recursively process child documents
+      if (doc.children && Array.isArray(doc.children)) {
+        collect(doc.children);
+      }
+    }
+  }
+
+  collect(docs);
+  return paths;
+}
+
+/**
+ * Load all paths from document structure
+ * @param {Object} options - Loading options
+ * @param {string} options.yamlPath - YAML file path
+ * @param {boolean} options.includeBothFormats - Whether to include both formats with and without leading slash
+ * @param {boolean} options.collectMetadata - Whether to collect additional metadata
+ * @param {boolean} options.throwOnInvalid - Whether to throw error when document format is invalid
+ * @returns {Promise<Set|Array>} - Set of all valid paths or path object array
+ */
+export async function loadDocumentPaths(options = {}) {
+  const {
+    yamlPath = PATHS.DOCUMENT_STRUCTURE,
+    includeBothFormats = false,
+    collectMetadata = false,
+    throwOnInvalid = true,
+  } = options;
+
+  // Check if file exists
+  try {
+    await access(yamlPath, constants.F_OK | constants.R_OK);
+  } catch (_error) {
+    throw new Error(ERROR_CODES.MISSING_STRUCTURE_FILE);
+  }
+
+  // Read and parse YAML
+  const content = await readFile(yamlPath, "utf8");
+  const data = yamlParse(content);
+
+  if (!data.documents || !Array.isArray(data.documents)) {
+    if (throwOnInvalid) {
+      throw new Error(ERROR_CODES.INVALID_STRUCTURE_FILE);
+    } else {
+      throw new Error(ERROR_CODES.MISSING_STRUCTURE_FILE);
+    }
+  }
+
+  // Recursively collect all paths
+  return collectDocumentPaths(data.documents, {
+    includeBothFormats,
+    collectMetadata,
+  });
+}
+
+/**
+ * Validate whether a path exists in the document structure
+ * @param {string} path - The path to validate
+ * @param {Set|Array} validPaths - Set of valid paths
+ * @returns {boolean} - Whether the path is valid
+ */
+export function isValidDocumentPath(path, validPaths) {
+  if (!path) return false;
+
+  const { filePath, displayPath } = normalizePath(path);
+
+  if (validPaths instanceof Set) {
+    return validPaths.has(filePath) || validPaths.has(displayPath);
+  }
+
+  if (Array.isArray(validPaths)) {
+    return validPaths.some((p) => {
+      const pathValue = typeof p === "string" ? p : p.path;
+      return pathValue === filePath || pathValue === displayPath;
+    });
+  }
+
+  return false;
+}
+
+/**
+ * Filter valid paths from a path array
+ * @param {string[]} paths - Array of paths to validate
+ * @param {Set|Array} validPaths - Set of valid paths
+ * @returns {Object} - { validPaths: [], invalidPaths: [] }
+ */
+export function filterValidPaths(paths, validPaths) {
+  const result = {
+    validPaths: [],
+    invalidPaths: [],
+  };
+
+  for (const path of paths) {
+    const { filePath } = normalizePath(path);
+
+    if (isValidDocumentPath(path, validPaths)) {
+      result.validPaths.push(filePath);
+    } else {
+      result.invalidPaths.push(path);
+    }
+  }
+
+  return result;
+}

package/utils/files.mjs

@@ -0,0 +1,74 @@
+import { existsSync, mkdirSync } from "node:fs";
+import path from "node:path";
+import { PATHS } from "./agent-constants.mjs";
+
+// Shared extension → MIME type mapping table
+const EXT_TO_MIME = {
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".png": "image/png",
+  ".gif": "image/gif",
+  ".bmp": "image/bmp",
+  ".webp": "image/webp",
+  ".svg": "image/svg+xml",
+  ".heic": "image/heic",
+  ".heif": "image/heif",
+  ".mp4": "video/mp4",
+  ".mpeg": "video/mpeg",
+  ".mpg": "video/mpg",
+  ".mov": "video/mov",
+  ".avi": "video/avi",
+  ".flv": "video/x-flv",
+  ".mkv": "video/x-matroska",
+  ".webm": "video/webm",
+  ".wmv": "video/wmv",
+  ".m4v": "video/x-m4v",
+  ".3gpp": "video/3gpp",
+  ".mp3": "audio/mpeg",
+  ".wav": "audio/wav",
+  ".pdf": "application/pdf",
+  ".doc": "application/msword",
+  ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  ".xls": "application/vnd.ms-excel",
+  ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ".ppt": "application/vnd.ms-powerpoint",
+  ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  ".txt": "text/plain",
+  ".json": "application/json",
+  ".xml": "application/xml",
+  ".html": "text/html",
+  ".css": "text/css",
+  ".js": "application/javascript",
+  ".zip": "application/zip",
+  ".rar": "application/x-rar-compressed",
+  ".7z": "application/x-7z-compressed",
+};
+
+/**
+ * Get MIME type from file path based on extension
+ * @param {string} filePath - File path
+ * @returns {string} MIME type
+ */
+export function getMimeType(filePath) {
+  const ext = path.extname(filePath || "").toLowerCase();
+  return EXT_TO_MIME[ext] || "application/octet-stream";
+}
+
+/**
+ * Ensure temporary directory exists
+ * @returns {Promise<void>}
+ */
+export async function ensureTmpDir() {
+  if (!existsSync(PATHS.TMP_DIR)) {
+    mkdirSync(PATHS.TMP_DIR, { recursive: true });
+  }
+}
+
+/**
+ * Check if a file is a remote URL
+ * @param {string} file - File path or URL
+ * @returns {boolean}
+ */
+export function isRemoteFile(file) {
+  return file && (file.startsWith("http://") || file.startsWith("https://"));
+}

package/utils/git.mjs

@@ -0,0 +1,65 @@
+import { execSync } from "node:child_process";
+
+/**
+ * Validate if a URL is a valid GitHub repository URL
+ * @param {string} url - The URL to validate
+ * @returns {boolean} - True if valid GitHub URL
+ */
+export function isValidGithubUrl(url) {
+  if (!url || typeof url !== "string") return false;
+  // Match both HTTPS and SSH GitHub URLs
+  return /^(https:\/\/github\.com\/|git@github\.com:)[^/]+\/[^/]+/.test(url);
+}
+
+/**
+ * Get GitHub repository URL
+ * @returns {string} - GitHub repository URL or empty string
+ */
+export function getGithubRepoUrl() {
+  try {
+    const gitRemote = execSync("git remote get-url origin", {
+      encoding: "utf8",
+      stdio: ["pipe", "pipe", "ignore"],
+    }).trim();
+
+    if (isValidGithubUrl(gitRemote)) {
+      return gitRemote;
+    }
+
+    return "";
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * Get GitHub repository information
+ * @param {string} repoUrl - The repository URL
+ * @returns {Promise<Object>} - Repository information
+ */
+export async function getGitHubRepoInfo(repoUrl) {
+  try {
+    const match = repoUrl.match(/github\.com[/:]([^/]+)\/([^/]+?)(?:\.git)?$/);
+    if (!match) return null;
+
+    const [, owner, repo] = match;
+    const apiUrl = `https://api.github.com/repos/${owner}/${repo}`;
+
+    const response = await fetch(apiUrl);
+
+    if (!response.ok) {
+      console.warn("Failed to fetch GitHub repository info:", repoUrl, response.statusText);
+      return null;
+    }
+
+    const data = await response.json();
+    return {
+      name: data.name,
+      description: data.description || "",
+      icon: data.owner?.avatar_url || "",
+    };
+  } catch (error) {
+    console.warn("Failed to fetch GitHub repository info:", error.message);
+    return null;
+  }
+}

package/utils/{blocklet.mjs → http.mjs}

@@ -102,3 +102,21 @@ export async function getComponentInfoWithMountPoint(appUrl, did) {
     mountPoint: component.mountPoint,
   };
 }
+
+/**
+ * Perform HTTP request with authentication token
+ * @param {string} url - Request URL
+ * @param {Object} options - Fetch options
+ * @param {string} authToken - Authentication token
+ * @returns {Promise<Object>} Response JSON
+ */
+export async function requestWithAuthToken(url, options, authToken) {
+  if (!authToken) {
+    console.error("No authentication token provided");
+  }
+  const response = await fetch(url, {
+    ...options,
+    headers: { ...options.headers, Authorization: `Bearer ${authToken}` },
+  });
+  return response.json();
+}

package/utils/image-slots.mjs

@@ -0,0 +1,57 @@
+/**
+ * AFS Image Slot utility functions
+ * For parsing and processing image slots in documents
+ */
+
+/**
+ * Slot regex (for replacement operations)
+ * Supports the following formats:
+ * - <!-- afs:image id="..." key="..." desc="..." -->
+ * - <!-- afs:image id="..." desc="..." -->
+ * - <!-- afs:image id=\"...\" key=\"...\" desc=\"...\" -->
+ * - <!-- afs:image id=\"...\" desc=\"...\" -->
+ */
+export const SLOT_REGEX =
+  /<!--\s*afs:image\s+id=\\?"([^\\"]+)\\?"(?:\s+key=\\?"([^\\"]+)\\?")?\s+desc=\\?"([^\\"]+)\\?"\s*-->/g;
+
+/**
+ * Generate key (if slot doesn't provide one)
+ * @param {string} docPath - Document path (e.g., "/overview")
+ * @param {string} id - slot id
+ * @returns {string} - Generated key
+ */
+export function generateKey(docPath, id) {
+  // Remove leading /
+  const normalizedPath = docPath.startsWith("/") ? docPath.slice(1) : docPath;
+  // Replace / with -
+  const pathPart = normalizedPath.replace(/\//g, "-");
+  return `${pathPart}-${id}`;
+}
+
+/**
+ * Parse AFS image slots from document
+ * @param {string} content - Document content
+ * @param {string} docPath - Document path (used for generating key)
+ * @returns {Array<{id: string, key: string, desc: string, raw: string}>} - Slot array
+ */
+export function parseSlots(content, docPath) {
+  // Slot format: <!-- afs:image id="..." key="..." desc="..." -->
+  // key is optional
+  const slotRegex = SLOT_REGEX;
+
+  const slots = [];
+
+  for (const match of content.matchAll(slotRegex)) {
+    const id = match[1];
+    const userKey = match[2]; // May be undefined
+    const desc = match[3];
+    const raw = match[0]; // Complete slot string
+
+    // Auto-generate key if user didn't provide one
+    const key = userKey || generateKey(docPath, id);
+
+    slots.push({ id, key, desc, raw });
+  }
+
+  return slots;
+}

package/utils/image-utils.mjs

@@ -0,0 +1,114 @@
+/**
+ * Common utility functions for image processing
+ */
+
+import { readFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import fs from "fs-extra";
+
+/**
+ * Calculate SHA256 hash of a file
+ * @param {string} filePath - File path
+ * @returns {Promise<string>} - SHA256 hash (hex)
+ */
+export async function calculateFileHash(filePath) {
+  const content = await readFile(filePath);
+  return createHash("sha256").update(content).digest("hex");
+}
+
+/**
+ * Calculate SHA256 hash of string content
+ * @param {string} content - String content
+ * @returns {string} - SHA256 hash (hex)
+ */
+export function calculateContentHash(content) {
+  return createHash("sha256").update(content, "utf8").digest("hex");
+}
+
+/**
+ * Supported image extensions
+ */
+export const IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".gif", ".webp"];
+
+/**
+ * Find image file (supports multiple extensions)
+ * @param {string} imagesDir - Image directory path
+ * @param {string} locale - Language code
+ * @param {string[]} extensions - Supported extension list (optional, defaults to IMAGE_EXTENSIONS)
+ * @returns {Promise<string|null>} - Image file path, returns null if not found
+ */
+export async function findImageFile(imagesDir, locale, extensions = IMAGE_EXTENSIONS) {
+  for (const ext of extensions) {
+    const imagePath = join(imagesDir, `${locale}${ext}`);
+    if (await fs.pathExists(imagePath)) {
+      return imagePath;
+    }
+  }
+  return null;
+}
+
+/**
+ * Find image file (with language fallback)
+ * @param {string} key - Image key
+ * @param {string} locale - Current language code
+ * @param {string} mainLocale - Primary language code (for fallback)
+ * @param {string} assetsDir - Assets directory path
+ * @returns {Promise<string|null>} - Image relative path (relative to assets), returns null if not found
+ */
+export async function findImageWithFallback(key, locale, mainLocale, assetsDir = "./assets") {
+  const keyDir = join(assetsDir, key, "images");
+
+  // 1. Try to find image for current language
+  const currentLocaleImage = await findImageFile(keyDir, locale);
+  if (currentLocaleImage) {
+    // Return path relative to assets
+    const filename = currentLocaleImage.split("/").pop();
+    return join(key, "images", filename);
+  }
+
+  // 2. If current language doesn't exist, fall back to primary language
+  if (mainLocale && locale !== mainLocale) {
+    const mainLocaleImage = await findImageFile(keyDir, mainLocale);
+    if (mainLocaleImage) {
+      const filename = mainLocaleImage.split("/").pop();
+      return join(key, "images", filename);
+    }
+  }
+
+  // 3. Image doesn't exist
+  return null;
+}
+
+/**
+ * Get image MIME type
+ * @param {string} filePath - Image file path
+ * @returns {string} - MIME type
+ */
+export function getImageMimeType(filePath) {
+  const ext = filePath.toLowerCase().split(".").pop();
+  const mimeTypes = {
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    png: "image/png",
+    gif: "image/gif",
+    webp: "image/webp",
+  };
+  return mimeTypes[ext] || "image/jpeg";
+}
+
+/**
+ * Get file extension from MIME type
+ * @param {string} mimeType - MIME type
+ * @returns {string} - File extension
+ */
+export function getExtensionFromMimeType(mimeType) {
+  const mimeToExt = {
+    "image/jpeg": "jpg",
+    "image/jpg": "jpg",
+    "image/png": "png",
+    "image/gif": "gif",
+    "image/webp": "webp",
+  };
+  return mimeToExt[mimeType] || "png";
+}

package/utils/project.mjs

@@ -0,0 +1,95 @@
+import { execSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { SUPPORTED_LANGUAGES } from "./constants.mjs";
+import { getGitHubRepoInfo, isValidGithubUrl } from "./git.mjs";
+
+/**
+ * Get project information from Git or directory
+ * @returns {Promise<Object>} - Project information
+ */
+export async function getProjectInfo() {
+  let repoInfo = null;
+  let defaultName = path.basename(process.cwd());
+  let defaultDescription = "";
+  let defaultIcon = "";
+  let fromGitHub = false;
+  let sourceDir = null;
+
+  try {
+    // Get the first folder in sources directory
+    const sourcesPath = path.join(process.cwd(), "sources");
+
+    if (fs.existsSync(sourcesPath)) {
+      const entries = fs.readdirSync(sourcesPath, { withFileTypes: true });
+      const firstFolder = entries.find((entry) => entry.isDirectory());
+
+      if (firstFolder) {
+        sourceDir = path.join(sourcesPath, firstFolder.name);
+        defaultName = firstFolder.name;
+      }
+    }
+
+    // If no source directory found, fall back to current directory
+    const targetDir = sourceDir || process.cwd();
+
+    const gitRemote = execSync("git remote get-url origin", {
+      encoding: "utf8",
+      cwd: targetDir,
+      stdio: ["pipe", "pipe", "ignore"],
+    }).trim();
+
+    const repoName = gitRemote.split("/").pop().replace(".git", "");
+    defaultName = repoName;
+
+    if (isValidGithubUrl(gitRemote)) {
+      repoInfo = await getGitHubRepoInfo(gitRemote);
+      if (repoInfo) {
+        defaultDescription = repoInfo.description;
+        defaultIcon = repoInfo.icon;
+        fromGitHub = true;
+      }
+    }
+  } catch (_error) {
+    console.warn("No git repository found, using current directory name");
+  }
+
+  return {
+    name: defaultName,
+    description: defaultDescription,
+    icon: defaultIcon,
+    fromGitHub,
+  };
+}
+
+/**
+ * Detect system language
+ * @returns {string} - Language code (e.g., 'en', 'zh')
+ */
+export function detectSystemLanguage() {
+  try {
+    let systemLocale = null;
+
+    systemLocale = process.env.LANG || process.env.LANGUAGE || process.env.LC_ALL;
+
+    if (!systemLocale) {
+      try {
+        systemLocale = Intl.DateTimeFormat().resolvedOptions().locale;
+      } catch (_error) {
+        // Intl API failed
+      }
+    }
+
+    if (!systemLocale) {
+      return "en";
+    }
+
+    const languageCode = systemLocale.split(/[_.-]/)[0].toLowerCase();
+    const supportedCode = SUPPORTED_LANGUAGES.find((lang) => lang.code === languageCode);
+
+    return supportedCode ? supportedCode.code : "en";
+  } catch (error) {
+    console.warn("Failed to detect system language:", error.message);
+    return "en";
+  }
+}

package/utils/sources-path-resolver.mjs

@@ -0,0 +1,76 @@
+import { resolve } from "node:path";
+import fs from "fs-extra";
+
+/**
+ * Check if path is a /sources/... absolute path
+ * @param {string} imagePath - Image path
+ * @returns {boolean}
+ */
+export function isSourcesAbsolutePath(imagePath) {
+  return imagePath.startsWith("/sources/");
+}
+
+/**
+ * Parse /sources/... absolute path and extract relative path portion
+ * @param {string} absolutePath - Absolute path, format: /sources/<relativePath>
+ * @returns {string | null} - Relative path, returns null if parsing fails or path is unsafe
+ */
+export function parseSourcesPath(absolutePath) {
+  // /sources/assets/screenshot.png → assets/screenshot.png
+  const match = absolutePath.match(/^\/sources\/(.+)$/);
+  if (!match) return null;
+
+  const relativePath = match[1];
+
+  // Security check: reject paths containing path traversal sequences
+  // Prevent accessing files outside sources directory via paths like /sources/../../../etc/passwd
+  if (relativePath.includes("..")) {
+    return null;
+  }
+
+  return relativePath;
+}
+
+/**
+ * Resolve virtual absolute path to physical path based on config.yaml sources configuration
+ * Searches each source in order and returns the first existing path
+ *
+ * Execution layer perspective (after AFS mount):
+ *   modules/workspace/ and modules/sources/ are at same level
+ *   Documents reference using /sources/<path> format
+ *
+ * Physical disk perspective:
+ *   - local-path: path relative to workspace
+ *   - git-clone: workspace/sources/<name>/ directory
+ *
+ * @param {string} absolutePath - Virtual absolute path, format: /sources/<relativePath>
+ * @param {Array} sourcesConfig - sources configuration array from config.yaml
+ * @param {string} workspaceBase - Workspace physical root directory
+ * @returns {Promise<{physicalPath: string, sourceName: string} | null>} - Physical path and source name, returns null if resolution fails
+ */
+export async function resolveSourcesPath(absolutePath, sourcesConfig, workspaceBase) {
+  const relativePath = parseSourcesPath(absolutePath);
+  if (!relativePath) return null;
+
+  // Search in each source in order
+  for (const source of sourcesConfig) {
+    let physicalPath;
+
+    if (source.type === "local-path") {
+      // local-path: path relative to workspace
+      physicalPath = resolve(workspaceBase, source.path, relativePath);
+    } else if (source.type === "git-clone") {
+      // git-clone: cloned to workspace/sources/<name>/ directory
+      physicalPath = resolve(workspaceBase, "sources", source.name, relativePath);
+    } else {
+      continue;
+    }
+
+    // Check if file exists
+    if (await fs.pathExists(physicalPath)) {
+      return { physicalPath, sourceName: source.name };
+    }
+  }
+
+  return null;
+}

package/utils/{upload-files.mjs → upload.mjs}

@@ -5,9 +5,9 @@ import path from "node:path";
 import pLimit from "p-limit";
 import pRetry from "p-retry";
 
-import { getComponentMountPoint } from "./blocklet.mjs";
-import { DISCUSS_KIT_DID, MEDIA_KIT_DID } from "./constants/index.mjs";
-import { getMimeType } from "./file-utils.mjs";
+import { getComponentMountPoint } from "./http.mjs";
+import { DISCUSS_KIT_DID, MEDIA_KIT_DID } from "./constants.mjs";
+import { getMimeType } from "./files.mjs";
 
 /**
  * Perform single file upload