@nahisaho/shikigami 1.1.0

package/mcp-server/src/tools/file-parser.ts

@@ -0,0 +1,332 @@
+/**
+ * File Parser Tool
+ *
+ * Implements REQ-DR-003: ファイル入力
+ * Implements REQ-CS-003: コード/ドキュメント解析
+ *
+ * Supports:
+ * - PDF (via pdf-parse or external API)
+ * - Text files (txt, md, json, yaml, csv)
+ * - Code files (ts, js, py, etc.)
+ *
+ * Note: For advanced parsing (complex PDF, Excel), use docling or external API
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+export interface ParsedFile {
+  /** ファイルパス */
+  filePath: string;
+  /** ファイル名 */
+  fileName: string;
+  /** 拡張子 */
+  extension: string;
+  /** MIMEタイプ */
+  mimeType: string;
+  /** 抽出されたテキスト */
+  content: string;
+  /** メタデータ */
+  metadata: FileMetadata;
+  /** エラー（あれば） */
+  error?: string;
+}
+
+export interface FileMetadata {
+  /** ファイルサイズ（bytes） */
+  size: number;
+  /** 最終更新日時 */
+  modifiedAt: string;
+  /** 行数（テキストファイルの場合） */
+  lineCount?: number;
+  /** 文字数 */
+  charCount?: number;
+  /** 言語（コードファイルの場合） */
+  language?: string;
+}
+
+/** 対応拡張子とMIMEタイプのマッピング */
+const EXTENSION_MIME_MAP: Record<string, string> = {
+  // Text
+  '.txt': 'text/plain',
+  '.md': 'text/markdown',
+  '.markdown': 'text/markdown',
+  // Data
+  '.json': 'application/json',
+  '.yaml': 'application/yaml',
+  '.yml': 'application/yaml',
+  '.csv': 'text/csv',
+  '.tsv': 'text/tab-separated-values',
+  '.xml': 'application/xml',
+  // Code
+  '.ts': 'text/typescript',
+  '.tsx': 'text/typescript',
+  '.js': 'text/javascript',
+  '.jsx': 'text/javascript',
+  '.py': 'text/x-python',
+  '.rb': 'text/x-ruby',
+  '.go': 'text/x-go',
+  '.rs': 'text/x-rust',
+  '.java': 'text/x-java',
+  '.c': 'text/x-c',
+  '.cpp': 'text/x-c++',
+  '.h': 'text/x-c',
+  '.hpp': 'text/x-c++',
+  '.cs': 'text/x-csharp',
+  '.swift': 'text/x-swift',
+  '.kt': 'text/x-kotlin',
+  '.scala': 'text/x-scala',
+  '.php': 'text/x-php',
+  '.sh': 'text/x-shellscript',
+  '.bash': 'text/x-shellscript',
+  '.zsh': 'text/x-shellscript',
+  '.sql': 'text/x-sql',
+  // Config
+  '.ini': 'text/plain',
+  '.conf': 'text/plain',
+  '.cfg': 'text/plain',
+  '.toml': 'application/toml',
+  '.env': 'text/plain',
+  // Document
+  '.html': 'text/html',
+  '.htm': 'text/html',
+  '.css': 'text/css',
+  '.scss': 'text/x-scss',
+  '.less': 'text/x-less',
+  // Binary (will fail gracefully)
+  '.pdf': 'application/pdf',
+  '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+};
+
+/** 言語判定 */
+const EXTENSION_LANGUAGE_MAP: Record<string, string> = {
+  '.ts': 'typescript',
+  '.tsx': 'typescript',
+  '.js': 'javascript',
+  '.jsx': 'javascript',
+  '.py': 'python',
+  '.rb': 'ruby',
+  '.go': 'go',
+  '.rs': 'rust',
+  '.java': 'java',
+  '.c': 'c',
+  '.cpp': 'c++',
+  '.h': 'c',
+  '.hpp': 'c++',
+  '.cs': 'csharp',
+  '.swift': 'swift',
+  '.kt': 'kotlin',
+  '.scala': 'scala',
+  '.php': 'php',
+  '.sh': 'shell',
+  '.bash': 'bash',
+  '.zsh': 'zsh',
+  '.sql': 'sql',
+  '.md': 'markdown',
+  '.json': 'json',
+  '.yaml': 'yaml',
+  '.yml': 'yaml',
+  '.html': 'html',
+  '.css': 'css',
+};
+
+/**
+ * テキストファイルかどうか判定
+ */
+function isTextFile(extension: string): boolean {
+  const textExtensions = new Set([
+    '.txt', '.md', '.markdown',
+    '.json', '.yaml', '.yml', '.csv', '.tsv', '.xml', '.toml',
+    '.ts', '.tsx', '.js', '.jsx',
+    '.py', '.rb', '.go', '.rs', '.java',
+    '.c', '.cpp', '.h', '.hpp', '.cs',
+    '.swift', '.kt', '.scala', '.php',
+    '.sh', '.bash', '.zsh', '.sql',
+    '.ini', '.conf', '.cfg', '.env',
+    '.html', '.htm', '.css', '.scss', '.less',
+  ]);
+  return textExtensions.has(extension.toLowerCase());
+}
+
+/**
+ * ファイルを解析してテキストを抽出
+ */
+export async function parseFile(filePath: string): Promise<ParsedFile> {
+  const absolutePath = path.resolve(filePath);
+  const fileName = path.basename(absolutePath);
+  const extension = path.extname(absolutePath).toLowerCase();
+  const mimeType = EXTENSION_MIME_MAP[extension] ?? 'application/octet-stream';
+
+  // ファイル存在確認
+  if (!fs.existsSync(absolutePath)) {
+    return {
+      filePath: absolutePath,
+      fileName,
+      extension,
+      mimeType,
+      content: '',
+      metadata: { size: 0, modifiedAt: '' },
+      error: `File not found: ${absolutePath}`,
+    };
+  }
+
+  // ファイル情報取得
+  const stat = fs.statSync(absolutePath);
+  const metadata: FileMetadata = {
+    size: stat.size,
+    modifiedAt: stat.mtime.toISOString(),
+  };
+
+  // テキストファイルの場合
+  if (isTextFile(extension)) {
+    try {
+      const content = fs.readFileSync(absolutePath, 'utf-8');
+      const lines = content.split('\n');
+
+      metadata.lineCount = lines.length;
+      metadata.charCount = content.length;
+      metadata.language = EXTENSION_LANGUAGE_MAP[extension];
+
+      // 大きすぎる場合は切り詰め
+      const maxLength = 100000; // 100KB
+      const truncatedContent = content.length > maxLength
+        ? content.slice(0, maxLength) + '\n\n[Content truncated...]'
+        : content;
+
+      return {
+        filePath: absolutePath,
+        fileName,
+        extension,
+        mimeType,
+        content: truncatedContent,
+        metadata,
+      };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      return {
+        filePath: absolutePath,
+        fileName,
+        extension,
+        mimeType,
+        content: '',
+        metadata,
+        error: `Failed to read file: ${message}`,
+      };
+    }
+  }
+
+  // バイナリファイル（PDF, Office等）の場合
+  // 注: 本格的な解析には docling や専用ライブラリが必要
+  return {
+    filePath: absolutePath,
+    fileName,
+    extension,
+    mimeType,
+    content: '',
+    metadata,
+    error: `Binary file parsing not supported for ${extension}. Use docling or external API for PDF/Office documents.`,
+  };
+}
+
+/**
+ * 複数ファイルを解析
+ */
+export async function parseFiles(filePaths: string[]): Promise<ParsedFile[]> {
+  const results: ParsedFile[] = [];
+
+  for (const filePath of filePaths) {
+    const result = await parseFile(filePath);
+    results.push(result);
+  }
+
+  return results;
+}
+
+/**
+ * ディレクトリ内のファイルを再帰的に解析
+ */
+export async function parseDirectory(
+  dirPath: string,
+  options?: {
+    /** 解析する拡張子（指定しない場合は全て） */
+    extensions?: string[];
+    /** 除外パターン（glob形式ではなく、パスに含まれる文字列） */
+    exclude?: string[];
+    /** 最大ファイル数 */
+    maxFiles?: number;
+  }
+): Promise<ParsedFile[]> {
+  const absolutePath = path.resolve(dirPath);
+  const results: ParsedFile[] = [];
+
+  const extensions = options?.extensions?.map(e => e.startsWith('.') ? e : `.${e}`);
+  const exclude = options?.exclude ?? ['node_modules', '.git', 'dist', 'build'];
+  const maxFiles = options?.maxFiles ?? 100;
+
+  function walk(dir: string): void {
+    if (results.length >= maxFiles) return;
+
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+    for (const entry of entries) {
+      if (results.length >= maxFiles) break;
+
+      const fullPath = path.join(dir, entry.name);
+
+      // 除外パターンチェック
+      if (exclude.some(pattern => fullPath.includes(pattern))) {
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        walk(fullPath);
+      } else if (entry.isFile()) {
+        const ext = path.extname(entry.name).toLowerCase();
+
+        // 拡張子フィルタ
+        if (extensions && !extensions.includes(ext)) {
+          continue;
+        }
+
+        // テキストファイルのみ
+        if (!isTextFile(ext)) {
+          continue;
+        }
+
+        // 同期的に解析（簡易実装）
+        const stat = fs.statSync(fullPath);
+        try {
+          const content = fs.readFileSync(fullPath, 'utf-8');
+          const lines = content.split('\n');
+
+          const maxLength = 100000;
+          const truncatedContent = content.length > maxLength
+            ? content.slice(0, maxLength) + '\n\n[Content truncated...]'
+            : content;
+
+          results.push({
+            filePath: fullPath,
+            fileName: entry.name,
+            extension: ext,
+            mimeType: EXTENSION_MIME_MAP[ext] ?? 'text/plain',
+            content: truncatedContent,
+            metadata: {
+              size: stat.size,
+              modifiedAt: stat.mtime.toISOString(),
+              lineCount: lines.length,
+              charCount: content.length,
+              language: EXTENSION_LANGUAGE_MAP[ext],
+            },
+          });
+        } catch {
+          // エラー時はスキップ
+        }
+      }
+    }
+  }
+
+  walk(absolutePath);
+  return results;
+}

package/mcp-server/src/tools/search.ts

@@ -0,0 +1,181 @@
+/**
+ * DuckDuckGo Search Tool
+ * 
+ * Implements REQ-DR-002: Web検索
+ * Implements REQ-ERR-001: Web検索失敗時のリトライ（最大3回、指数バックオフ）
+ * Implements REQ-NF-007: プロバイダー設定ファイル対応
+ * Uses DuckDuckGo HTML search (no API key required)
+ */
+
+import * as cheerio from 'cheerio';
+import { getConfig } from '../config/loader.js';
+
+export interface SearchResult {
+  title: string;
+  url: string;
+  snippet: string;
+}
+
+export interface SearchError {
+  query: string;
+  error: string;
+  retries: number;
+  lastStatusCode?: number;
+}
+
+const USER_AGENT =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+// Rate limiting (will be initialized from config)
+let lastRequestTime = 0;
+
+/**
+ * Get rate limit interval from config
+ */
+function getMinRequestInterval(): number {
+  const config = getConfig();
+  return config.search?.options?.rateLimitMs ?? 1500;
+}
+
+/**
+ * Get max retries from config
+ */
+function getMaxRetries(): number {
+  const config = getConfig();
+  return config.search?.options?.maxRetries ?? 3;
+}
+
+/**
+ * Get base backoff from config (default 2000ms)
+ */
+function getBaseBackoff(): number {
+  return 2000;
+}
+
+async function rateLimit(): Promise<void> {
+  const now = Date.now();
+  const minInterval = getMinRequestInterval();
+  const timeSinceLastRequest = now - lastRequestTime;
+  
+  if (timeSinceLastRequest < minInterval) {
+    await new Promise((resolve) =>
+      setTimeout(resolve, minInterval - timeSinceLastRequest)
+    );
+  }
+  
+  lastRequestTime = Date.now();
+}
+
+/**
+ * Sleep for exponential backoff
+ */
+async function exponentialBackoff(attempt: number): Promise<void> {
+  const baseBackoff = getBaseBackoff();
+  const maxRetries = getMaxRetries();
+  const delay = baseBackoff * Math.pow(2, attempt);
+  console.error(`[SHIKIGAMI] Retry ${attempt + 1}/${maxRetries} - waiting ${delay}ms`);
+  await new Promise((resolve) => setTimeout(resolve, delay));
+}
+
+/**
+ * Search DuckDuckGo with retry logic (REQ-ERR-001)
+ */
+export async function searchDuckDuckGo(
+  query: string,
+  maxResults: number = 10
+): Promise<SearchResult[]> {
+  const config = getConfig();
+  const maxRetries = getMaxRetries();
+  const locale = config.search?.options?.locale ?? 'ja,en-US;q=0.9,en;q=0.8';
+
+  let lastError: Error | null = null;
+  let lastStatusCode: number | undefined;
+
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      await rateLimit();
+
+      const encodedQuery = encodeURIComponent(query);
+      const url = `https://html.duckduckgo.com/html/?q=${encodedQuery}`;
+
+      const response = await fetch(url, {
+        headers: {
+          'User-Agent': USER_AGENT,
+          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+          'Accept-Language': locale,
+        },
+      });
+
+      lastStatusCode = response.status;
+
+      // Handle rate limiting (429) with exponential backoff
+      if (response.status === 429) {
+        console.error(`[SHIKIGAMI] Rate limited (429) for query: "${query}"`);
+        if (attempt < maxRetries - 1) {
+          await exponentialBackoff(attempt);
+          continue;
+        }
+        throw new Error(`Rate limited after ${maxRetries} retries`);
+      }
+
+      if (!response.ok) {
+        throw new Error(`DuckDuckGo search failed: ${response.status}`);
+      }
+
+      const html = await response.text();
+      const $ = cheerio.load(html);
+      const results: SearchResult[] = [];
+
+      // Parse search results
+      $('.result').each((index, element) => {
+        if (index >= maxResults) return false;
+
+        const $el = $(element);
+        const titleEl = $el.find('.result__title a');
+        const snippetEl = $el.find('.result__snippet');
+
+        const title = titleEl.text().trim();
+        let resultUrl = titleEl.attr('href') || '';
+        const snippet = snippetEl.text().trim();
+
+        // DuckDuckGo uses redirect URLs, extract the actual URL
+        if (resultUrl.includes('uddg=')) {
+          const match = resultUrl.match(/uddg=([^&]+)/);
+          if (match) {
+            resultUrl = decodeURIComponent(match[1]);
+          }
+        }
+
+        if (title && resultUrl) {
+          results.push({
+            title,
+            url: resultUrl,
+            snippet,
+          });
+        }
+      });
+
+      // Success - log and return
+      if (attempt > 0) {
+        console.error(`[SHIKIGAMI] Search succeeded after ${attempt + 1} attempts`);
+      }
+      return results;
+
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      console.error(`[SHIKIGAMI] Search attempt ${attempt + 1} failed: ${lastError.message}`);
+
+      // Retry on network errors
+      if (attempt < maxRetries - 1) {
+        await exponentialBackoff(attempt);
+      }
+    }
+  }
+
+  // All retries exhausted - log and throw
+  console.error(`[SHIKIGAMI] Search failed after ${maxRetries} retries for query: "${query}"`);
+  throw new Error(
+    `Search failed after ${maxRetries} retries: ${lastError?.message || 'Unknown error'}` +
+    (lastStatusCode ? ` (last status: ${lastStatusCode})` : '')
+  );
+}

package/mcp-server/src/tools/visit.ts

@@ -0,0 +1,168 @@
+/**
+ * Page Visit Tool
+ * 
+ * Implements REQ-DR-002: Web検索 (page fetching)
+ * Implements REQ-NF-007: プロバイダー設定ファイル対応
+ * Uses Jina AI Reader for LLM-optimized text extraction
+ */
+
+import { getConfig } from '../config/loader.js';
+
+export interface PageContent {
+  url: string;
+  title: string;
+  content: string;
+  fetchedAt: string;
+  error?: string;
+}
+
+const USER_AGENT =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+// Rate limiting (will be initialized from config)
+let lastRequestTime = 0;
+
+/**
+ * Get rate limit interval from config
+ */
+function getMinRequestInterval(): number {
+  const config = getConfig();
+  return config.pageFetcher?.options?.rateLimitMs ?? 1000;
+}
+
+/**
+ * Get Jina API key from config
+ */
+function getJinaApiKey(): string | undefined {
+  const config = getConfig();
+  return config.pageFetcher?.options?.apiKey;
+}
+
+async function rateLimit(): Promise<void> {
+  const now = Date.now();
+  const minInterval = getMinRequestInterval();
+  const timeSinceLastRequest = now - lastRequestTime;
+  
+  if (timeSinceLastRequest < minInterval) {
+    await new Promise((resolve) =>
+      setTimeout(resolve, minInterval - timeSinceLastRequest)
+    );
+  }
+  
+  lastRequestTime = Date.now();
+}
+
+/**
+ * Visit a page and extract content using Jina AI Reader
+ */
+export async function visitPage(url: string, goal?: string): Promise<PageContent> {
+  await rateLimit();
+
+  const fetchedAt = new Date().toISOString();
+  const apiKey = getJinaApiKey();
+  
+  try {
+    // Use Jina AI Reader for clean text extraction
+    const jinaUrl = `https://r.jina.ai/${url}`;
+    
+    const headers: Record<string, string> = {
+      'User-Agent': USER_AGENT,
+      'Accept': 'text/plain',
+    };
+
+    // Add API key if configured (for higher rate limits)
+    if (apiKey) {
+      headers['Authorization'] = `Bearer ${apiKey}`;
+    }
+    
+    const response = await fetch(jinaUrl, { headers });
+
+    if (!response.ok) {
+      // Fallback to direct fetch if Jina fails
+      return await directFetch(url, fetchedAt);
+    }
+
+    const content = await response.text();
+    
+    // Extract title from the first line (Jina format)
+    const lines = content.split('\n');
+    let title = '';
+    let bodyContent = content;
+    
+    if (lines[0]?.startsWith('Title:')) {
+      title = lines[0].replace('Title:', '').trim();
+      bodyContent = lines.slice(1).join('\n').trim();
+    }
+
+    // Truncate if too long (for context management)
+    const maxLength = 50000;
+    if (bodyContent.length > maxLength) {
+      bodyContent = bodyContent.slice(0, maxLength) + '\n\n[Content truncated...]';
+    }
+
+    return {
+      url,
+      title,
+      content: bodyContent,
+      fetchedAt,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      url,
+      title: '',
+      content: '',
+      fetchedAt,
+      error: `Failed to fetch page: ${message}`,
+    };
+  }
+}
+
+/**
+ * Direct fetch fallback (basic HTML text extraction)
+ */
+async function directFetch(url: string, fetchedAt: string): Promise<PageContent> {
+  try {
+    const response = await fetch(url, {
+      headers: {
+        'User-Agent': USER_AGENT,
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
+
+    const html = await response.text();
+    
+    // Basic HTML to text conversion
+    const { load } = await import('cheerio');
+    const $ = load(html);
+    
+    // Remove scripts and styles
+    $('script, style, nav, header, footer, aside').remove();
+    
+    const title = $('title').text().trim() || $('h1').first().text().trim();
+    const content = $('body').text()
+      .replace(/\s+/g, ' ')
+      .trim()
+      .slice(0, 50000);
+
+    return {
+      url,
+      title,
+      content,
+      fetchedAt,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      url,
+      title: '',
+      content: '',
+      fetchedAt,
+      error: `Direct fetch failed: ${message}`,
+    };
+  }
+}

package/mcp-server/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}