@geminilight/mindos 0.5.69 → 0.6.0

package/app/lib/agent/tools.ts

@@ -1,3 +1,4 @@
+import path from 'path';
 import { Type, type Static } from '@sinclair/typebox';
 import type { AgentTool, AgentToolResult } from '@mariozechner/pi-agent-core';
 import {
@@ -6,6 +7,8 @@ import {
   deleteFile, renameFile, moveFile, findBacklinks, gitLog, gitShowFile, appendCsvRow,
   getMindRoot,
 } from '@/lib/fs';
+import { readSkillContentByName, scanSkillDirs } from '@/lib/pi-integration/skills';
+import { callMcporterTool, createMcporterAgentTools, listMcporterServers, listMcporterTools } from '@/lib/pi-integration/mcporter';
 
 // Max chars per file to avoid token overflow (~100k chars ≈ ~25k tokens)
 const MAX_FILE_CHARS = 20_000;
@@ -94,6 +97,14 @@ const AppendParams = Type.Object({
   content: Type.String({ description: 'Content to append' }),
 });
 
+const FetchUrlParams = Type.Object({
+  url: Type.String({ description: 'The HTTP/HTTPS URL to fetch' }),
+});
+
+const WebSearchParams = Type.Object({
+  query: Type.String({ description: 'The search query or keywords to look up on the internet' }),
+});
+
 const InsertHeadingParams = Type.Object({
   path: Type.String({ description: 'Relative file path' }),
   heading: Type.String({ description: 'Heading text to find (e.g. "## Tasks" or just "Tasks")' }),
@@ -138,6 +149,22 @@ const CsvAppendParams = Type.Object({
   row: Type.Array(Type.String(), { description: 'Array of cell values for the new row' }),
 });
 
+const ListSkillsParams = Type.Object({});
+
+const LoadSkillParams = Type.Object({
+  name: Type.String({ description: 'Skill name, e.g. "mindos" or "context7"' }),
+});
+
+const ListMcpToolsParams = Type.Object({
+  server: Type.Optional(Type.String({ description: 'Optional MCP server name. Omit to list discovered servers only.' })),
+});
+
+const CallMcpToolParams = Type.Object({
+  server: Type.String({ description: 'MCP server name discovered via list_mcp_tools' }),
+  tool: Type.String({ description: 'Exact MCP tool name to invoke' }),
+  arguments_json: Type.Optional(Type.String({ description: 'Optional JSON object string of tool arguments. Example: {"query":"react hooks"}' })),
+});
+
 // ─── Tool Definitions (AgentTool interface) ─────────────────────────────────
 
 // Write-operation tool names — used by beforeToolCall for write-protection
@@ -146,6 +173,28 @@ export const WRITE_TOOLS = new Set([
   'update_section', 'edit_lines', 'delete_file', 'rename_file', 'move_file', 'append_csv',
 ]);
 
+export async function getRequestScopedTools(): Promise<AgentTool<any>[]> {
+  try {
+    const result = await listMcporterServers();
+    const okServers = (result.servers ?? []).filter((server) => server.status === 'ok');
+    if (okServers.length === 0) return knowledgeBaseTools;
+
+    const detailedServers = await Promise.all(okServers.map(async (server) => {
+      try {
+        return await listMcporterTools(server.name);
+      } catch {
+        return server;
+      }
+    }));
+
+    const dynamicMcpTools = createMcporterAgentTools(detailedServers);
+    if (dynamicMcpTools.length === 0) return knowledgeBaseTools;
+    return [...knowledgeBaseTools, ...dynamicMcpTools];
+  } catch {
+    return knowledgeBaseTools;
+  }
+}
+
 export const knowledgeBaseTools: AgentTool<any>[] = [
   {
     name: 'list_files',
@@ -243,6 +292,209 @@ export const knowledgeBaseTools: AgentTool<any>[] = [
     }),
   },
 
+  {
+    name: 'list_skills',
+    label: 'List Skills',
+    description: 'List available MindOS skills discovered from app/data/skills, skills, {mindRoot}/.skills, and ~/.mindos/skills. Use this before load_skill when you need a skill by name.',
+    parameters: ListSkillsParams,
+    execute: safeExecute(async () => {
+      const projectRoot = process.env.MINDOS_PROJECT_ROOT || path.resolve(process.cwd(), '..');
+      const skills = scanSkillDirs({ projectRoot, mindRoot: getMindRoot() });
+      if (skills.length === 0) return textResult('No skills found.');
+      return textResult(skills.map((skill) => `- **${skill.name}** [${skill.origin}]${skill.enabled ? '' : ' (disabled)'} — ${skill.description || 'No description'}\n  Path: ${skill.path}`).join('\n'));
+    }),
+  },
+
+  {
+    name: 'load_skill',
+    label: 'Load Skill',
+    description: 'Load the full content of a specific skill by name. Use list_skills first if you do not know the exact skill name.',
+    parameters: LoadSkillParams,
+    execute: safeExecute(async (_id, params: Static<typeof LoadSkillParams>) => {
+      const projectRoot = process.env.MINDOS_PROJECT_ROOT || path.resolve(process.cwd(), '..');
+      const content = readSkillContentByName(params.name, { projectRoot, mindRoot: getMindRoot() });
+      if (!content) return textResult(`Skill not found: ${params.name}`);
+      return textResult(truncate(content));
+    }),
+  },
+
+  {
+    name: 'list_mcp_tools',
+    label: 'List MCP Tools',
+    description: 'List MCP servers configured in ~/.mindos/mcp.json. Without `server`, lists discovered servers and their health. With `server`, lists that server\'s tools and JSON schemas.',
+    parameters: ListMcpToolsParams,
+    execute: safeExecute(async (_id, params: Static<typeof ListMcpToolsParams>) => {
+      if (!params.server) {
+        const result = await listMcporterServers();
+        if (!result.servers || result.servers.length === 0) return textResult('No external MCP servers configured. The MindOS built-in MCP server is always available.');
+        return textResult(result.servers.map((server) => `- **${server.name}** — status: ${server.status}${server.transport ? ` | transport: ${server.transport}` : ''}${server.error ? ` | error: ${server.error}` : ''}`).join('\n'));
+      }
+
+      const server = await listMcporterTools(params.server);
+      if (!server.tools || server.tools.length === 0) {
+        return textResult(`No tools found for MCP server: ${params.server}`);
+      }
+      return textResult(server.tools.map((tool) => `## ${tool.name}\n${tool.description || 'No description'}\n\nSchema:\n${JSON.stringify(tool.inputSchema ?? {}, null, 2)}`).join('\n\n'));
+    }),
+  },
+
+  {
+    name: 'call_mcp_tool',
+    label: 'Call MCP Tool',
+    description: 'Call a specific MCP tool by server and tool name. Pass `arguments_json` as a JSON object string. Use list_mcp_tools first to discover names and schemas.',
+    parameters: CallMcpToolParams,
+    execute: safeExecute(async (_id, params: Static<typeof CallMcpToolParams>) => {
+      let parsedArgs: Record<string, unknown> = {};
+      if (params.arguments_json?.trim()) {
+        try {
+          parsedArgs = JSON.parse(params.arguments_json) as Record<string, unknown>;
+        } catch (error) {
+          return textResult(`Invalid arguments_json. Expected a JSON object string. Error: ${formatToolError(error)}`);
+        }
+      }
+      const output = await callMcporterTool(params.server, params.tool, parsedArgs);
+      return textResult(output || '(empty MCP response)');
+    }),
+  },
+
+  {
+    name: 'web_search',
+    label: 'Web Search',
+    description: 'Search the internet for up-to-date information. Uses DuckDuckGo HTML search. Returns top search results with titles, snippets, and URLs.',
+    parameters: WebSearchParams,
+    execute: safeExecute(async (_id, params: Static<typeof WebSearchParams>) => {
+      try {
+        const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(params.query)}`;
+        const res = await fetch(url, {
+          headers: {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept-Language': 'en-US,en;q=0.9',
+          },
+          signal: AbortSignal.timeout(10000)
+        });
+        
+        if (!res.ok) {
+          return textResult(`Failed to search: HTTP ${res.status}`);
+        }
+        
+        const html = await res.text();
+        const results: string[] = [];
+        
+        // Simple regex parsing for DuckDuckGo HTML results
+        const resultBlocks = html.split('class="result__body"').slice(1);
+        
+        for (let i = 0; i < Math.min(resultBlocks.length, 5); i++) {
+          const block = resultBlocks[i];
+          
+          const titleMatch = block.match(/class="result__title"[^>]*>[\s\S]*?<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
+          const snippetMatch = block.match(/class="result__snippet[^>]*>([\s\S]*?)(?:<\/a>|<\/div>)/i);
+          
+          if (titleMatch) {
+            let link = titleMatch[1];
+            // Decode DuckDuckGo redirect URL if necessary
+            if (link.startsWith('//duckduckgo.com/l/?uddg=')) {
+              const urlParam = new URL('https:' + link).searchParams.get('uddg');
+              if (urlParam) link = decodeURIComponent(urlParam);
+            }
+            
+            // Clean up tags
+            const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
+            const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
+            
+            results.push(`### ${i+1}. ${title}\n**URL:** ${link}\n**Snippet:** ${snippet}\n`);
+          }
+        }
+        
+        if (results.length === 0) {
+          return textResult(`No web search results found for: ${params.query}`);
+        }
+        
+        return textResult(`## Web Search Results for: "${params.query}"\n\n${results.join('\n')}\n\n*Note: Use web_fetch tool with any of the URLs above to read the full page content.*`);
+      } catch (err) {
+        return textResult(`Web search failed: ${formatToolError(err)}`);
+      }
+    }),
+  },
+
+  {
+    name: 'web_fetch',
+    label: 'Web Fetch',
+    description: 'Fetch the text content of any public URL. Extracts main text from HTML and converts it to Markdown. Use this to read external docs, repos, or articles.',
+    parameters: FetchUrlParams,
+    execute: safeExecute(async (_id, params: Static<typeof FetchUrlParams>) => {
+      let url = params.url;
+      if (!url.startsWith('http://') && !url.startsWith('https://')) {
+        url = 'https://' + url;
+      }
+      
+      try {
+        const res = await fetch(url, {
+          headers: {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+          },
+          // Don't wait forever
+          signal: AbortSignal.timeout(10000)
+        });
+        
+        if (!res.ok) {
+          return textResult(`Failed to fetch URL: HTTP ${res.status} ${res.statusText}`);
+        }
+        
+        const contentType = res.headers.get('content-type') || '';
+        
+        // If it's a raw file (like raw.githubusercontent.com or a raw text file)
+        if (contentType.includes('text/plain') || contentType.includes('application/json') || url.includes('raw.githubusercontent.com')) {
+          const text = await res.text();
+          return textResult(truncate(text));
+        }
+        
+        // For HTML, we do a basic extraction (in a real app you might use JSDOM/Readability, but we'll do a robust regex cleanup here to avoid new dependencies)
+        let html = await res.text();
+        
+        // Extract title if possible
+        const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+        const title = titleMatch ? titleMatch[1].trim() : url;
+        
+        // Strip out scripts, styles, svg, and headers/footers roughly
+        html = html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, ' ')
+                   .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, ' ')
+                   .replace(/<svg\b[^<]*(?:(?!<\/svg>)<[^<]*)*<\/svg>/gi, ' ')
+                   .replace(/<header\b[^<]*(?:(?!<\/header>)<[^<]*)*<\/header>/gi, ' ')
+                   .replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, ' ')
+                   .replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, ' ');
+
+        // Convert some basic tags to markdown equivalents roughly before stripping all HTML
+        html = html.replace(/<h[1-2][^>]*>(.*?)<\/h[1-2]>/gi, '\n\n# $1\n\n')
+                   .replace(/<h[3-6][^>]*>(.*?)<\/h[3-6]>/gi, '\n\n## $1\n\n')
+                   .replace(/<p[^>]*>(.*?)<\/p>/gi, '\n\n$1\n\n')
+                   .replace(/<li[^>]*>(.*?)<\/li>/gi, '\n- $1')
+                   .replace(/<br\s*\/?>/gi, '\n');
+                   
+        // Strip remaining HTML tags
+        let text = html.replace(/<[^>]+>/g, ' ');
+        
+        // Decode common HTML entities
+        text = text.replace(/&nbsp;/g, ' ')
+                   .replace(/&amp;/g, ' ')
+                   .replace(/&lt;/g, '<')
+                   .replace(/&gt;/g, '>')
+                   .replace(/&quot;/g, '"')
+                   .replace(/&#39;/g, "'");
+
+        // Clean up whitespace: remove empty lines and extra spaces
+        text = text.replace(/[ \t]+/g, ' ')
+                   .replace(/\n\s*\n\s*\n/g, '\n\n')
+                   .trim();
+                   
+        const result = `# ${title}\nSource: ${url}\n\n${text}`;
+        return textResult(truncate(result));
+      } catch (err) {
+        return textResult(`Failed to fetch URL: ${formatToolError(err)}`);
+      }
+    }),
+  },
+
   {
     name: 'get_recent',
     label: 'Recent Files',

package/app/lib/core/create-space.ts

@@ -1,6 +1,16 @@
 import { MindOSError, ErrorCodes } from '@/lib/errors';
 import { createFile } from './fs-ops';
 
+/**
+ * Generate the template README.md content for a new space.
+ * Extracted so both createSpaceFilesystem and revert can produce identical content.
+ */
+export function generateReadmeTemplate(fullPath: string, name: string, description: string): string {
+  const cleanName = name.replace(/^[\p{Emoji_Presentation}\p{Extended_Pictographic}\s]+/u, '') || name;
+  const desc = description.trim() || '(Describe the purpose and usage of this space.)';
+  return `# ${cleanName}\n\n${desc}\n\n## 📁 Structure\n\n\`\`\`bash\n${fullPath}/\n├── INSTRUCTION.md\n├── README.md\n└── (your files here)\n\`\`\`\n\n## 💡 Usage\n\n(Add usage guidelines for this space.)\n`;
+}
+
 /**
  * Create a Mind Space on disk: `{fullPath}/README.md` plus scaffold from {@link createFile} / scaffoldIfNewSpace.
  * Caller must invalidate app file-tree cache (e.g. `invalidateCache()` in `lib/fs.ts`).
@@ -26,10 +36,7 @@ export function createSpaceFilesystem(
 
   const prefix = cleanParent ? `${cleanParent}/` : '';
   const fullPath = `${prefix}${trimmed}`;
-
-  const cleanName = trimmed.replace(/^[\p{Emoji_Presentation}\p{Extended_Pictographic}\s]+/u, '') || trimmed;
-  const desc = description.trim() || '(Describe the purpose and usage of this space.)';
-  const readmeContent = `# ${cleanName}\n\n${desc}\n\n## 📁 Structure\n\n\`\`\`bash\n${fullPath}/\n├── INSTRUCTION.md\n├── README.md\n└── (your files here)\n\`\`\`\n\n## 💡 Usage\n\n(Add usage guidelines for this space.)\n`;
+  const readmeContent = generateReadmeTemplate(fullPath, trimmed, description);
 
   createFile(mindRoot, `${fullPath}/README.md`, readmeContent);
   return { path: fullPath };

package/app/lib/core/file-convert.ts

@@ -0,0 +1,97 @@
+import path from 'path';
+
+export const ALLOWED_IMPORT_EXTENSIONS = new Set([
+  '.txt', '.md', '.markdown', '.csv', '.json', '.yaml', '.yml', '.xml', '.html', '.htm', '.pdf',
+]);
+
+export interface ConvertResult {
+  content: string;
+  originalName: string;
+  targetName: string;
+  metadata?: Record<string, string>;
+}
+
+export function sanitizeFileName(name: string): string {
+  let base = name.replace(/\\/g, '/').split('/').pop() ?? '';
+  base = base.replace(/\.\./g, '').replace(/^\/+/, '');
+  base = base.replace(/[\\:*?"<>|]/g, '-');
+  base = base.replace(/-{2,}/g, '-');
+  base = base.replace(/^[-\s]+|[-\s]+$/g, '');
+  return base || 'imported-file';
+}
+
+export function titleFromFileName(name: string): string {
+  const ext = path.extname(name);
+  const stem = (ext ? name.slice(0, -ext.length) : name).replace(/^\.+/, '');
+  const words = stem.replace(/[-_]+/g, ' ').trim().split(/\s+/);
+  if (words.length === 0 || (words.length === 1 && !words[0])) return 'Untitled';
+  return words.map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
+}
+
+function stripHtmlTags(html: string): string {
+  return html
+    .replace(/<script[\s\S]*?<\/script>/gi, '')
+    .replace(/<style[\s\S]*?<\/style>/gi, '')
+    .replace(/<[^>]+>/g, '')
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+export function convertToMarkdown(fileName: string, rawContent: string): ConvertResult {
+  const originalName = fileName;
+  const ext = path.extname(fileName).toLowerCase();
+  const stem = path.basename(fileName, ext) || 'note';
+  const title = titleFromFileName(fileName);
+
+  if (ext === '.md' || ext === '.markdown') {
+    return { content: rawContent, originalName, targetName: sanitizeFileName(fileName) };
+  }
+
+  if (ext === '.csv' || ext === '.json') {
+    return { content: rawContent, originalName, targetName: sanitizeFileName(fileName) };
+  }
+
+  if (ext === '.txt') {
+    return {
+      content: `# ${title}\n\n${rawContent}`,
+      originalName,
+      targetName: sanitizeFileName(`${stem}.md`),
+    };
+  }
+
+  if (ext === '.yaml' || ext === '.yml') {
+    return {
+      content: `# ${title}\n\n\`\`\`yaml\n${rawContent}\n\`\`\`\n`,
+      originalName,
+      targetName: sanitizeFileName(`${stem}.md`),
+    };
+  }
+
+  if (ext === '.html' || ext === '.htm') {
+    const text = stripHtmlTags(rawContent);
+    return {
+      content: `# ${title}\n\n${text}\n`,
+      originalName,
+      targetName: sanitizeFileName(`${stem}.md`),
+    };
+  }
+
+  if (ext === '.xml') {
+    return {
+      content: `# ${title}\n\n\`\`\`xml\n${rawContent}\n\`\`\`\n`,
+      originalName,
+      targetName: sanitizeFileName(`${stem}.md`),
+    };
+  }
+
+  return {
+    content: `# ${title}\n\n${rawContent}`,
+    originalName,
+    targetName: sanitizeFileName(`${stem}.md`),
+  };
+}

package/app/lib/core/index.ts

@@ -61,7 +61,7 @@ export { findBacklinks } from './backlinks';
 export { isGitRepo, gitLog, gitShowFile } from './git';
 
 // Mind Space
-export { createSpaceFilesystem } from './create-space';
+export { createSpaceFilesystem, generateReadmeTemplate } from './create-space';
 export { summarizeTopLevelSpaces } from './list-spaces';
 export type { MindSpaceSummary } from './list-spaces';
 

package/app/lib/core/organize.ts

@@ -0,0 +1,105 @@
+import fs from 'fs';
+import path from 'path';
+import { resolveSafe } from './security';
+import { appendContentChange } from './content-changes';
+
+export interface OrganizeResult {
+  readmeUpdated: boolean;
+  relatedFiles: Array<{ path: string; matchType: 'backlink' | 'keyword' }>;
+}
+
+const STOP_WORDS = new Set([
+  'the', 'a', 'an', 'is', 'of', 'and', 'for', 'to', 'in', 'on', 'at', 'by', 'or',
+  'not', 'but', 'with', 'from', 'this', 'that', 'was', 'are', 'be', 'has', 'had',
+  'readme', 'instruction', 'md',
+]);
+
+const SKIP_DIRS = new Set(['.git', 'node_modules', '.next', '.DS_Store']);
+
+function extractKeywords(filePath: string): string[] {
+  const stem = path.basename(filePath, path.extname(filePath));
+  return stem
+    .split(/[-_\s]+/)
+    .map(w => w.toLowerCase())
+    .filter(w => w.length >= 3 && !STOP_WORDS.has(w));
+}
+
+function collectMdFiles(dir: string, limit: number): string[] {
+  const results: string[] = [];
+  function walk(d: string) {
+    if (results.length >= limit) return;
+    let entries: fs.Dirent[];
+    try { entries = fs.readdirSync(d, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries) {
+      if (results.length >= limit) return;
+      if (entry.isDirectory()) {
+        if (!SKIP_DIRS.has(entry.name)) walk(path.join(d, entry.name));
+      } else if (entry.isFile() && entry.name.endsWith('.md')) {
+        results.push(path.relative(dir, path.join(d, entry.name)).replace(/\\/g, '/'));
+      }
+    }
+  }
+  walk(dir);
+  return results;
+}
+
+export function organizeAfterImport(
+  mindRoot: string,
+  createdFiles: string[],
+  targetSpace: string,
+): OrganizeResult {
+  for (const fp of createdFiles) {
+    try {
+      appendContentChange(mindRoot, {
+        op: 'import_file',
+        path: fp,
+        source: 'user',
+        summary: 'Imported file into knowledge base',
+      });
+    } catch { /* ignore */ }
+  }
+
+  let readmeUpdated = false;
+  const space = targetSpace.replace(/\\/g, '/').replace(/^\/+|\/+$/g, '').trim();
+
+  if (space && createdFiles.length > 0) {
+    const readmePath = path.posix.join(space, 'README.md');
+    try {
+      const resolved = resolveSafe(mindRoot, readmePath);
+      if (fs.existsSync(resolved)) {
+        const existing = fs.readFileSync(resolved, 'utf-8');
+        const bullets = createdFiles.map(f => {
+          const base = path.posix.basename(f);
+          return `- [${base}](./${base})`;
+        }).join('\n');
+        fs.writeFileSync(resolved, `${existing.trimEnd()}\n\n${bullets}\n`, 'utf-8');
+        readmeUpdated = true;
+      }
+    } catch { /* README missing or write failed */ }
+  }
+
+  const createdSet = new Set(createdFiles);
+  const allKeywords = new Set<string>();
+  for (const f of createdFiles) {
+    for (const kw of extractKeywords(f)) allKeywords.add(kw);
+  }
+
+  const relatedFiles: OrganizeResult['relatedFiles'] = [];
+  if (allKeywords.size > 0) {
+    const candidates = collectMdFiles(mindRoot, 50);
+    const kwArray = [...allKeywords];
+    for (const candidate of candidates) {
+      if (createdSet.has(candidate)) continue;
+      if (relatedFiles.length >= 10) break;
+      try {
+        const resolved = resolveSafe(mindRoot, candidate);
+        const content = fs.readFileSync(resolved, 'utf-8').toLowerCase();
+        if (kwArray.some(kw => content.includes(kw))) {
+          relatedFiles.push({ path: candidate, matchType: 'keyword' });
+        }
+      } catch { /* ignore */ }
+    }
+  }
+
+  return { readmeUpdated, relatedFiles };
+}