@aigne/doc-smith 0.8.15-beta.6 → 0.8.15-beta.8

package/prompts/structure/review/structure-review-system.md

@@ -0,0 +1,73 @@
+<role_and_goal>
+You are an AI document strategist with the personality of an **INTJ (The Architect)**. Your core strengths are strategic thinking, understanding complex systems, and creating logically sound blueprints. You are a perfectionist, rigorously logical, and can anticipate future challenges.
+
+</role_and_goal>
+
+<document_structure>
+projectName: |
+  {{projectName}}
+projectDesc: |
+  {{projectDesc}}
+
+documentStructure:
+{{ documentStructure | yaml.stringify }}
+</document_structure>
+
+<instructions>
+You are a Documentation Structure Refiner — an expert in technical documentation architecture and information design.
+
+Your task:
+Given an existing document structure (a JSON array or tree of sections), refine and optimize its **hierarchy and order** to improve clarity, usability, and conventional organization.
+️ You must not add, delete, rename, or rewrite any nodes. Only adjust the **order** and **nesting levels** of existing nodes.
+
+---
+
+## Optimization Goals
+
+1. **Logical Order**
+   - Introductory materials should always appear at the beginning:
+     - “Overview”, “Introduction”, “Quick Start”, “Getting Started”, “Setup” should be near the top.
+   - Meta and community-related sections (e.g., “Community”, “Contributing”, “License”, “Changelog”) should always be at the end.
+   - Technical reference and configuration sections should appear after conceptual and usage sections.
+
+2. **Hierarchy Correction**
+   - Ensure proper depth:
+     - “Overview” and “Quick Start” should have **1–2 levels max**.
+     - Remove deeply nested technical details from “Overview” or “Quick Start”.
+     - Relocate such details under “Architecture”, “API Reference”, or “Modules”.
+   - Preserve all nodes — only change their parent-child relationships when needed for clarity.
+
+3. **Grouping and Alignment**
+   - Align similar nodes logically (e.g., group “Usage”, “Examples”, “Tutorials” together).
+   - Avoid duplication or overlap by reordering, not by deletion.
+
+4. **Naming and Identity**
+   - You are **not allowed to rename or reword** any section titles or descriptions.
+   - Keep all existing keys, identifiers, and text intact.
+
+5. **Balance**
+   - Maintain a clean, well-organized hierarchy.
+   - Keep top-level nodes concise (≤ 8 preferred).
+   - Avoid over-nesting (≤ 4 levels deep).
+
+---
+
+## Behavior Rules
+
+- Do **not** add new nodes.
+- Do **not** delete existing nodes.
+- Do **not** rename or rewrite content.
+- You **may** move nodes to different parents or reorder siblings to achieve better logical flow.
+- You **must** maintain all data and structural integrity.
+- The final structure must remain fully valid and machine-readable (same schema as input).
+
+---
+
+## Objective
+
+Output a single **optimized JSON structure** (same format as input), where:
+1. The hierarchy and order are improved.
+2. All nodes are preserved exactly as given.
+3. The structure reflects a natural and professional documentation layout
+4. Only return the nodes need to be changed to achieve the above goals.
+</instructions>

package/prompts/translate/code-block.md

@@ -2,13 +2,23 @@
 The following formats are considered Code Blocks:
 
 - Wrapped with ```
-- Supports configurations: language, title, icon, where title and icon are optional
+- Supports configurations: language, optional title, optional icon (icon uses key=value)
+- title is free text placed after the language (not as title=xxx), may contain spaces, and **must NEVER be wrapped in quotes**
 - content can be code, command line examples, text or any other content
 
 <code_block_sample>
 
-```{language} [{title}] [icon={icon}]
-{content}
+- `language`: javascript
+- `title`: Modern: Using createRoot()
+- `icon`: logos:javascript
+
+```javascript Modern: Using createRoot() icon=logos:javascript
+import { createRoot } from 'react-dom/client'
+
+const container = document.getElementById('root')
+const root = createRoot(container)
+
+root.unmount()
 ```
 
 </code_block_sample>

package/types/document-structure-schema.mjs

@@ -6,7 +6,7 @@ export const documentItemSchema = z.object({
   title: z.string().min(1, "Title is required"),
   description: z.string().min(1, "Description is required"),
   path: z.string().startsWith("/", 'Path must start with "/"'),
-  parentId: z.string().nullable(),
+  parentId: z.string().nullish(),
   sourceIds: z.array(z.string()).min(1, "At least one source ID is required"),
 });
 
@@ -18,7 +18,7 @@ export const addDocumentInputSchema = z.object({
   title: z.string().min(1, "Title is required"),
   description: z.string().min(1, "Description is required"),
   path: z.string().startsWith("/", 'Path must start with "/"'),
-  parentId: z.string().nullable().optional(),
+  parentId: z.string().nullish(),
   sourceIds: z.array(z.string()).min(1, "At least one source ID is required"),
 });
 
@@ -44,7 +44,7 @@ export const deleteDocumentOutputSchema = z.object({
 // Move document schemas
 export const moveDocumentInputSchema = z.object({
   path: z.string().min(1, "Path is required"),
-  newParentId: z.string().nullable().optional(),
+  newParentId: z.string().nullish(),
 });
 
 export const moveDocumentOutputSchema = z.object({

package/utils/docs-finder-utils.mjs

@@ -1,5 +1,6 @@
 import { access, readdir, readFile } from "node:fs/promises";
 import { join } from "node:path";
+import { pathExists } from "./file-utils.mjs";
 
 /**
  * Get action-specific text based on isTranslate flag
@@ -276,3 +277,50 @@ export function addFeedbackToItems(items, feedback) {
     feedback: feedback.trim(),
   }));
 }
+
+/**
+ * Load document execution structure from structure-plan.json
+ * @param {string} outputDir - Output directory containing structure-plan.json
+ * @returns {Promise<Array|null>} Document execution structure array or null if not found/failed
+ */
+export async function loadDocumentStructure(outputDir) {
+  if (!outputDir) {
+    return null;
+  }
+
+  try {
+    const structurePlanPath = join(outputDir, "structure-plan.json");
+    const structureExists = await pathExists(structurePlanPath);
+
+    if (!structureExists) {
+      return null;
+    }
+
+    const structureContent = await readFile(structurePlanPath, "utf8");
+    if (!structureContent?.trim()) {
+      return null;
+    }
+
+    try {
+      // Validate that the content looks like JSON before parsing
+      const trimmedContent = structureContent.trim();
+      if (!trimmedContent.startsWith("[") && !trimmedContent.startsWith("{")) {
+        console.warn("structure-plan.json contains non-JSON content, skipping parse");
+        return null;
+      }
+
+      const parsed = JSON.parse(structureContent);
+      // Return array if it's an array, otherwise return null
+      return Array.isArray(parsed) ? parsed : null;
+    } catch (parseError) {
+      console.error(`Failed to parse structure-plan.json: ${parseError.message}`);
+      return null;
+    }
+  } catch (readError) {
+    // Only warn if it's not a "file not found" error
+    if (readError.code !== "ENOENT") {
+      console.warn(`Error reading structure-plan.json: ${readError.message}`);
+    }
+    return null;
+  }
+}

package/utils/extract-api.mjs

@@ -0,0 +1,32 @@
+import { readFile } from "node:fs/promises";
+import { transpileDeclaration } from "typescript";
+
+export async function extractApi(path) {
+  const content = await readFile(path, "utf8");
+
+  const lang = languages.find((lang) => lang.match(path, content));
+  if (lang) {
+    return lang.extract(path, content);
+  }
+
+  return content;
+}
+
+const languages = [
+  {
+    match: (path) => /\.m?(js|ts)x?$/.test(path),
+    extract: extractJsApi,
+  },
+];
+
+async function extractJsApi(_path, content) {
+  const res = transpileDeclaration(content, {
+    compilerOptions: {
+      declaration: true,
+      emitDeclarationOnly: true,
+      allowJs: true,
+    },
+  });
+
+  return res.outputText.trim();
+}

package/utils/file-utils.mjs

@@ -11,8 +11,8 @@ import { gunzipSync } from "node:zlib";
 
 import { debug } from "./debug.mjs";
 import { isGlobPattern } from "./utils.mjs";
-import { INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD } from "./constants/index.mjs";
 import { uploadFiles } from "./upload-files.mjs";
+import { extractApi } from "./extract-api.mjs";
 
 /**
  * Check if a directory is inside a git repository using git command
@@ -508,7 +508,9 @@ export async function readFileContents(files, baseDir = process.cwd(), options =
 
           return null;
         } else {
-          const content = await readFile(file, "utf8");
+          const content = await extractApi(file);
+          if (!content) return null;
+
           const relativePath = path.relative(baseDir, file);
           return {
             sourceId: relativePath,
@@ -527,6 +529,11 @@ export async function readFileContents(files, baseDir = process.cwd(), options =
   return results.filter((result) => result !== null);
 }
 
+export function calculateTokens(text) {
+  const tokens = encode(text);
+  return tokens.length;
+}
+
 /**
  * Calculate total lines and tokens from file contents
  * @param {Array<{content: string}>} sourceFiles - Array of objects containing content property
@@ -552,97 +559,17 @@ export function calculateFileStats(sourceFiles) {
 }
 
 /**
- * Build sources content string based on context size
- * For large contexts, only include core project files to avoid token limit issues
+ * Build sources content string
  * @param {Array<{sourceId: string, content: string}>} sourceFiles - Array of source file objects
- * @param {boolean} isLargeContext - Whether the context is large
  * @returns {string} Concatenated sources content with sourceId comments
  */
-export function buildSourcesContent(sourceFiles, isLargeContext = false) {
-  // Define core file patterns that represent project structure and key information
-  const coreFilePatterns = [
-    // Configuration files
-    /package\.json$/,
-    /tsconfig\.json$/,
-    /jsconfig\.json$/,
-    /\.env\.example$/,
-    /Cargo\.toml$/,
-    /go\.mod$/,
-    /pom\.xml$/,
-    /build\.gradle$/,
-    /Gemfile$/,
-    /requirements\.txt$/,
-    /Pipfile$/,
-    /composer\.json$/,
-    /pyproject\.toml$/,
-
-    // Documentation
-    /README\.md$/i,
-    /CHANGELOG\.md$/i,
-    /CONTRIBUTING\.md$/i,
-    /\.github\/.*\.md$/i,
-
-    // Entry points and main files
-    /index\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /main\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /app\.(js|ts|jsx|tsx|py)$/,
-    /server\.(js|ts|jsx|tsx|py)$/,
-
-    // API definitions
-    /api\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /routes\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /controllers\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-
-    // Type definitions and schemas
-    /types\.(ts|d\.ts)$/,
-    /schema\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /.*\.d\.ts$/,
-
-    // Core utilities
-    /utils\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /lib\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-    /helpers\/.*\.(js|ts|jsx|tsx|py|go|rs|java|rb|php)$/,
-  ];
-
-  // Function to check if a file is a core file
-  const isCoreFile = (filePath) => {
-    return coreFilePatterns.some((pattern) => pattern.test(filePath));
-  };
-
+export function buildSourcesContent(sourceFiles) {
   // Build sources string
   let allSources = "";
 
-  if (isLargeContext) {
-    // Only include core files for large contexts
-    const coreFiles = sourceFiles.filter((source) => isCoreFile(source.sourceId));
-
-    // Determine which files to use and set appropriate message
-    const filesToInclude = coreFiles.length > 0 ? coreFiles : sourceFiles;
-    const noteMessage =
-      coreFiles.length > 0
-        ? "// Note: Context is large, showing only core project files.\n"
-        : "// Note: Context is large, showing a sample of files.\n";
-
-    allSources += noteMessage;
-    let accumulatedTokens = 0;
-
-    for (const source of filesToInclude) {
-      const fileContent = `// sourceId: ${source.sourceId}\n${source.content}\n`;
-      const fileTokens = encode(fileContent);
-
-      // Check if adding this file would exceed the token limit
-      if (accumulatedTokens + fileTokens.length > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD) {
-        break;
-      }
-
-      allSources += fileContent;
-      accumulatedTokens += fileTokens.length;
-    }
-  } else {
-    // Include all files for normal contexts
-    for (const source of sourceFiles) {
-      allSources += `// sourceId: ${source.sourceId}\n${source.content}\n`;
-    }
+  // Include all files for normal contexts
+  for (const source of sourceFiles) {
+    allSources += `\n// sourceId: ${source.sourceId}\n${source.content}\n`;
   }
 
   return allSources;

package/utils/history-utils.mjs

@@ -99,7 +99,7 @@ function recordUpdateGit({ feedback }) {
 /**
  * Records an update in the YAML file.
  */
-function recordUpdateYaml({ operation, feedback, documentPath = null }) {
+function recordUpdateYaml({ operation, feedback, docPaths = null }) {
   try {
     const docSmithDir = join(process.cwd(), DOC_SMITH_DIR);
     if (!existsSync(docSmithDir)) {
@@ -125,9 +125,9 @@ function recordUpdateYaml({ operation, feedback, documentPath = null }) {
       feedback,
     };
 
-    // Add document path if provided
-    if (documentPath) {
-      entry.documentPath = documentPath;
+    // Add document paths if provided
+    if (Array.isArray(docPaths) && docPaths.length > 0) {
+      entry.docPaths = docPaths;
     }
 
     // Add to beginning (newest first)
@@ -153,14 +153,14 @@ function recordUpdateYaml({ operation, feedback, documentPath = null }) {
  * @param {Object} params
  * @param {string} params.operation - The type of operation (e.g., 'document_update', 'structure_update', 'translation_update').
  * @param {string} params.feedback - The user's feedback text.
- * @param {string} params.documentPath - The document path.
+ * @param {string[]} [params.docPaths] - Document path list for updates.
  */
-export function recordUpdate({ operation, feedback, documentPath = null }) {
+export function recordUpdate({ operation, feedback, docPaths = null }) {
   // Skip if no feedback
   if (!feedback?.trim()) return;
 
   // Always record in YAML
-  recordUpdateYaml({ operation, feedback, documentPath });
+  recordUpdateYaml({ operation, feedback, docPaths });
 
   // Also record in git if git is available and not in a git repository
   if (isGitAvailable() && !isInGitRepository(process.cwd())) {
@@ -183,7 +183,19 @@ export function getHistory() {
 
   try {
     const content = readFileSync(historyPath, "utf8");
-    return parse(content) || { entries: [] };
+    const parsed = parse(content) || { entries: [] };
+    const entries = Array.isArray(parsed.entries) ? parsed.entries : [];
+
+    const normalized = entries.map((entry) => {
+      if (!entry) return entry;
+      // Normalize legacy entries: documentPath -> docPaths: [documentPath]
+      if (!entry.docPaths && entry.documentPath) {
+        return { ...entry, docPaths: [entry.documentPath] };
+      }
+      return entry;
+    });
+
+    return { ...parsed, entries: normalized };
   } catch (error) {
     console.warn("Could not read the history:", error.message);
     return { entries: [] };

package/agents/generate/document-structure-tools/generate-sub-structure.mjs

@@ -1,131 +0,0 @@
-import {
-  buildSourcesContent,
-  calculateFileStats,
-  loadFilesFromPaths,
-  readFileContents,
-} from "../../../utils/file-utils.mjs";
-import {
-  INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD,
-  DEFAULT_EXCLUDE_PATTERNS,
-  DEFAULT_INCLUDE_PATTERNS,
-} from "../../../utils/constants/index.mjs";
-import { toRelativePath } from "../../../utils/utils.mjs";
-
-export default async function generateSubStructure(
-  {
-    parentDocument,
-    subSourcePaths,
-    includePatterns,
-    excludePatterns,
-    useDefaultPatterns = true,
-    ...rest
-  },
-  options,
-) {
-  const sourcePaths = subSourcePaths?.map((item) => item.path);
-  if (!sourcePaths || sourcePaths.length === 0) {
-    return {
-      subStructure: [],
-    };
-  }
-
-  let files = await loadFilesFromPaths(sourcePaths, {
-    includePatterns,
-    excludePatterns,
-    useDefaultPatterns,
-    defaultIncludePatterns: DEFAULT_INCLUDE_PATTERNS,
-    defaultExcludePatterns: DEFAULT_EXCLUDE_PATTERNS,
-  });
-  files = [...new Set(files)];
-
-  // all files path
-  const allFilesPaths = files.map((file) => `- ${toRelativePath(file)}`).join("\n");
-
-  // Read all source files using the utility function
-  const sourceFiles = await readFileContents(files, process.cwd());
-
-  // Count tokens and lines using utility function
-  const { totalTokens } = calculateFileStats(sourceFiles);
-
-  // check if totalTokens is too large
-  let isLargeContext = false;
-  if (totalTokens > INTELLIGENT_SUGGESTION_TOKEN_THRESHOLD) {
-    isLargeContext = true;
-  }
-
-  // Build allSources string using utility function
-  const allSources = buildSourcesContent(sourceFiles, isLargeContext);
-
-  // Performance optimization:
-  // Using both structured output and Tool with Gemini model causes redundant calls
-  // Only use Tool when context is very large
-  const generateStructureAgent = isLargeContext
-    ? options.context.agents["generateStructure"]
-    : options.context.agents["generateStructureWithoutTools"];
-  const result = await options.context.invoke(generateStructureAgent, {
-    ...rest,
-    isSubStructure: true,
-    parentDocument,
-    datasources: allSources,
-    allFilesPaths,
-    isLargeContext,
-    files,
-    totalTokens,
-  });
-
-  return {
-    subStructure: result.documentStructure || [],
-    message: `Generated a sub structure for '${parentDocument.path}' successfully. Please merge all sub-structures to output the complete document structure.`,
-  };
-}
-
-generateSubStructure.description = `
-Generates a sub-structure. 
-Handles large file sets by splitting them into smaller sub-document structures when the context size exceeds limits. This approach ensures more focused and complete documentation generation.
-`;
-
-generateSubStructure.inputSchema = {
-  type: "object",
-  properties: {
-    parentDocument: {
-      type: "object",
-      description: "The parent node to generate a sub structure for",
-      properties: {
-        title: { type: "string", description: "The title of the parent node" },
-        description: { type: "string", description: "The description of the parent node" },
-        path: {
-          type: "string",
-          description:
-            "The path of the parent node, Path in URL format, cannot be empty, cannot contain spaces or special characters, must start with /, no need to include language level, e.g., /zh/about should return /about ",
-        },
-        parentId: { type: "string", description: "The parent ID of the parent node" },
-        sourceIds: { type: "array", description: "The source IDs of the parent node" },
-      },
-    },
-    subSourcePaths: {
-      type: "array",
-      description: "The source paths of the sub structure",
-      items: {
-        type: "object",
-        properties: {
-          path: { type: "string", description: "The source path of the sub structure" },
-          reason: { type: "string", description: "The reason for selecting the source path" },
-        },
-        required: ["path", "reason"],
-      },
-    },
-  },
-};
-
-generateSubStructure.outputSchema = {
-  type: "object",
-  properties: {
-    subStructure: {
-      type: "array",
-      description:
-        "The sub structure of the parent node, need merge all sub-structures and output the complete document structure.",
-    },
-    message: { type: "string", description: "The message of the sub structure" },
-  },
-  required: ["subStructure"],
-};

package/agents/generate/generate-structure-without-tools.yaml

@@ -1,65 +0,0 @@
-name: generateStructureWithoutTools
-description: Generate the structure and organization of your documentation
-instructions:
-  - role: system
-    url: ../../prompts/structure/generate/system-prompt.md
-  - role: user
-    url: ../../prompts/structure/generate/user-prompt.md
-task_render_mode: collapse
-task_title: Generate the structure of the documentation
-input_schema:
-  type: object
-  properties:
-    rules:
-      type: string
-      description: Your specific requirements for documentation structure
-    locale:
-      type: string
-      description: Primary language for documentation (e.g., zh, en, ja)
-    datasources:
-      type: string
-      description: Project content and context to help generate documentation structure
-    targetAudience:
-      type: string
-      description: Target audience for the documentation
-    nodeName:
-      type: string
-      description: Specific section or page name to focus on
-    glossary:
-      type: string
-      description: Glossary for consistent terminology
-    feedback:
-      type: string
-      description: Tell us how to improve the documentation structure
-    userPreferences:
-      type: string
-      description: Your saved preferences for structure and documentation style
-    docsType:
-      type: string
-      description: "Documentation type (options: general, getting-started, reference, faq)"
-      default: general
-  required:
-    - rules
-    - datasources
-output_schema:
-  type: object
-  properties:
-    projectName:
-      type: string
-      description: Project name identified from your content sources
-    projectDesc:
-      type: string
-      description: Brief project description generated from content analysis (under 50 words)
-    documentStructure: ../schema/document-structure.yaml
-    documentStructureTree:
-      type: string
-      description: |
-        Visual tree structure showing documentation hierarchy with indented levels for easy review:
-        ```
-        - Home
-          - Getting Started
-            - Installation
-              - Requirements
-        ```
-  required:
-    - documentStructure