gencode-ai 0.1.0 → 0.1.1

package/src/tools/builtin/webfetch.ts

@@ -0,0 +1,264 @@
+/**
+ * WebFetch Tool - Fetch and convert web content
+ */
+
+import TurndownService from 'turndown';
+import { z } from 'zod';
+import type { Tool, ToolContext, ToolResult } from '../types.js';
+import { getErrorMessage } from '../types.js';
+import { validateUrl } from '../utils/ssrf.js';
+
+// Constants
+const MAX_RESPONSE_SIZE = 5 * 1024 * 1024; // 5MB
+const DEFAULT_TIMEOUT = 30 * 1000; // 30 seconds
+const MAX_TIMEOUT = 120 * 1000; // 2 minutes
+const MAX_LINE_LENGTH = 2000;
+const MAX_OUTPUT_LENGTH = 50000;
+
+// Input schema
+export const WebFetchInputSchema = z.object({
+  url: z.string().describe('The URL to fetch content from (http:// or https://)'),
+  format: z
+    .enum(['text', 'markdown', 'html'])
+    .optional()
+    .describe('Output format: markdown (default), text, or html'),
+  timeout: z.number().optional().describe('Timeout in seconds (default: 30, max: 120)'),
+});
+export type WebFetchInput = z.infer<typeof WebFetchInputSchema>;
+
+/**
+ * Get Accept header based on requested format
+ */
+function getAcceptHeader(format: string): string {
+  switch (format) {
+    case 'markdown':
+      return 'text/markdown, text/plain, text/html;q=0.9, */*;q=0.1';
+    case 'text':
+      return 'text/plain, text/html;q=0.8, */*;q=0.1';
+    case 'html':
+      return 'text/html, application/xhtml+xml, */*;q=0.1';
+    default:
+      return 'text/html, */*;q=0.1';
+  }
+}
+
+/**
+ * Convert HTML to Markdown using Turndown
+ */
+function convertHtmlToMarkdown(html: string): string {
+  const turndown = new TurndownService({
+    headingStyle: 'atx',
+    hr: '---',
+    bulletListMarker: '-',
+    codeBlockStyle: 'fenced',
+    emDelimiter: '*',
+  });
+
+  // Remove script, style, meta, link, noscript tags
+  turndown.remove(['script', 'style', 'meta', 'link', 'noscript']);
+
+  return turndown.turndown(html);
+}
+
+/**
+ * Extract plain text from HTML
+ */
+function extractTextFromHtml(html: string): string {
+  return (
+    html
+      // Remove script and style content
+      .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+      .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+      .replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, '')
+      // Remove all tags
+      .replace(/<[^>]+>/g, ' ')
+      // Decode common HTML entities
+      .replace(/&nbsp;/g, ' ')
+      .replace(/&lt;/g, '<')
+      .replace(/&gt;/g, '>')
+      .replace(/&amp;/g, '&')
+      .replace(/&quot;/g, '"')
+      .replace(/&#(\d+);/g, (_, num) => String.fromCharCode(parseInt(num)))
+      // Normalize whitespace
+      .replace(/\s+/g, ' ')
+      .trim()
+  );
+}
+
+/**
+ * Process content based on content type and requested format
+ */
+function processContent(content: string, contentType: string, format: string): string {
+  const isHtml = contentType.includes('text/html') || contentType.includes('application/xhtml');
+
+  switch (format) {
+    case 'markdown':
+      if (isHtml) {
+        return convertHtmlToMarkdown(content);
+      }
+      return content;
+
+    case 'text':
+      if (isHtml) {
+        return extractTextFromHtml(content);
+      }
+      return content;
+
+    case 'html':
+      return content;
+
+    default:
+      return content;
+  }
+}
+
+/**
+ * Format bytes to human-readable size
+ */
+function formatSize(bytes: number): string {
+  if (bytes < 1024) return `${bytes}B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+}
+
+/**
+ * Truncate output to prevent excessive content
+ */
+function truncateOutput(output: string): string {
+  // Truncate long lines
+  const lines = output.split('\n').map((line) => {
+    if (line.length > MAX_LINE_LENGTH) {
+      return line.slice(0, MAX_LINE_LENGTH) + '... (truncated)';
+    }
+    return line;
+  });
+
+  let result = lines.join('\n');
+
+  // Truncate overall output
+  if (result.length > MAX_OUTPUT_LENGTH) {
+    result = result.slice(0, MAX_OUTPUT_LENGTH) + '\n\n... (output truncated)';
+  }
+
+  return result;
+}
+
+/**
+ * WebFetch Tool
+ */
+export const webfetchTool: Tool<WebFetchInput> = {
+  name: 'WebFetch',
+  description: `Fetch content from a URL and return it in the specified format.
+- Converts HTML to Markdown by default for easier reading
+- Supports text, markdown, and html output formats
+- Maximum response size: 5MB
+- Timeout: 30 seconds (configurable up to 120 seconds)`,
+  parameters: WebFetchInputSchema,
+
+  async execute(input: WebFetchInput, context: ToolContext): Promise<ToolResult> {
+    const startTime = Date.now();
+
+    try {
+      // Validate URL (SSRF protection)
+      validateUrl(input.url);
+
+      // Calculate timeout
+      const timeoutMs = input.timeout
+        ? Math.min(input.timeout * 1000, MAX_TIMEOUT)
+        : DEFAULT_TIMEOUT;
+
+      // Create abort controller for timeout
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+      // Combine with context abort signal if present
+      const signal = context.abortSignal
+        ? AbortSignal.any([controller.signal, context.abortSignal])
+        : controller.signal;
+
+      try {
+        // Fetch with appropriate headers
+        const response = await fetch(input.url, {
+          signal,
+          headers: {
+            'User-Agent': 'GenCode/1.0 (+https://github.com/gencode)',
+            Accept: getAcceptHeader(input.format ?? 'markdown'),
+            'Accept-Language': 'en-US,en;q=0.9',
+          },
+          redirect: 'follow',
+        });
+
+        clearTimeout(timeoutId);
+
+        if (!response.ok) {
+          return {
+            success: false,
+            output: '',
+            error: `HTTP ${response.status}: ${response.statusText}`,
+          };
+        }
+
+        // Check content length header
+        const contentLength = response.headers.get('content-length');
+        if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
+          return {
+            success: false,
+            output: '',
+            error: `Response too large: ${contentLength} bytes (max: ${MAX_RESPONSE_SIZE})`,
+          };
+        }
+
+        // Read response body with size limit
+        const arrayBuffer = await response.arrayBuffer();
+        if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
+          return {
+            success: false,
+            output: '',
+            error: `Response too large: ${arrayBuffer.byteLength} bytes (max: ${MAX_RESPONSE_SIZE})`,
+          };
+        }
+
+        const content = new TextDecoder().decode(arrayBuffer);
+        const contentType = response.headers.get('content-type') || '';
+
+        // Process content based on format
+        let output = processContent(content, contentType, input.format ?? 'markdown');
+
+        // Truncate long lines and overall output
+        output = truncateOutput(output);
+
+        // Build result with metadata for improved display
+        const size = arrayBuffer.byteLength;
+        const duration = Date.now() - startTime;
+
+        return {
+          success: true,
+          output: output,
+          metadata: {
+            title: `Fetch(${input.url})`,
+            subtitle: `Received ${formatSize(size)} (${response.status} ${response.statusText})`,
+            size,
+            statusCode: response.status,
+            contentType: contentType,
+            duration,
+          },
+        };
+      } finally {
+        clearTimeout(timeoutId);
+      }
+    } catch (error) {
+      if (error instanceof Error && error.name === 'AbortError') {
+        return {
+          success: false,
+          output: '',
+          error: 'Request timed out',
+        };
+      }
+      return {
+        success: false,
+        output: '',
+        error: `Fetch failed: ${getErrorMessage(error)}`,
+      };
+    }
+  },
+};

package/src/tools/builtin/websearch.ts

@@ -0,0 +1,117 @@
+/**
+ * WebSearch Tool - Search the web for current information
+ */
+
+import { z } from 'zod';
+import type { Tool, ToolContext, ToolResult } from '../types.js';
+import { getErrorMessage } from '../types.js';
+import {
+  createSearchProvider,
+  getCurrentSearchProviderName,
+  type SearchResult,
+} from '../../providers/search/index.js';
+
+// Constants
+const DEFAULT_NUM_RESULTS = 10;
+
+// Input schema
+export const WebSearchInputSchema = z.object({
+  query: z
+    .string()
+    .min(2)
+    .describe('The search query (minimum 2 characters)'),
+  allowed_domains: z
+    .array(z.string())
+    .optional()
+    .describe('Only include results from these domains'),
+  blocked_domains: z
+    .array(z.string())
+    .optional()
+    .describe('Exclude results from these domains'),
+  num_results: z
+    .number()
+    .optional()
+    .describe(`Number of results to return (default: ${DEFAULT_NUM_RESULTS})`),
+});
+export type WebSearchInput = z.infer<typeof WebSearchInputSchema>;
+
+/**
+ * Format search results as markdown
+ */
+function formatResults(results: SearchResult[], query: string): string {
+  if (results.length === 0) {
+    return `No results found for "${query}".`;
+  }
+
+  const lines: string[] = [`Found ${results.length} results for "${query}":\n`];
+
+  results.forEach((result, index) => {
+    lines.push(`${index + 1}. [${result.title}](${result.url})`);
+    if (result.snippet) {
+      lines.push(`   ${result.snippet}\n`);
+    } else {
+      lines.push('');
+    }
+  });
+
+  return lines.join('\n');
+}
+
+/**
+ * WebSearch Tool
+ */
+export const websearchTool: Tool<WebSearchInput> = {
+  name: 'WebSearch',
+  description: `Search the web for current information.
+
+Use this tool when you need:
+- Up-to-date information beyond your knowledge cutoff
+- Current documentation or release notes
+- Recent solutions to technical problems
+- Current best practices
+
+IMPORTANT: After answering, include a "Sources:" section with all relevant URLs as markdown hyperlinks.
+
+Example:
+  [Your answer]
+
+  Sources:
+  - [Title 1](https://url1)
+  - [Title 2](https://url2)`,
+  parameters: WebSearchInputSchema,
+
+  async execute(input: WebSearchInput, context: ToolContext): Promise<ToolResult> {
+    const startTime = Date.now();
+
+    try {
+      const provider = createSearchProvider();
+
+      const results = await provider.search(input.query, {
+        numResults: input.num_results ?? DEFAULT_NUM_RESULTS,
+        allowedDomains: input.allowed_domains,
+        blockedDomains: input.blocked_domains,
+        abortSignal: context.abortSignal,
+      });
+
+      const output = formatResults(results, input.query);
+      const duration = Date.now() - startTime;
+      const providerName = getCurrentSearchProviderName();
+
+      return {
+        success: true,
+        output,
+        metadata: {
+          title: `Search("${input.query}")`,
+          subtitle: `Found ${results.length} results via ${providerName}`,
+          duration,
+        },
+      };
+    } catch (error) {
+      return {
+        success: false,
+        output: '',
+        error: `Search failed: ${getErrorMessage(error)}`,
+      };
+    }
+  },
+};

package/src/tools/index.ts

@@ -12,6 +12,8 @@ export { editTool } from './builtin/edit.js';
 export { bashTool } from './builtin/bash.js';
 export { globTool } from './builtin/glob.js';
 export { grepTool } from './builtin/grep.js';
+export { webfetchTool } from './builtin/webfetch.js';
+export { websearchTool } from './builtin/websearch.js';
 
 import { ToolRegistry } from './registry.js';
 import { readTool } from './builtin/read.js';
@@ -20,17 +22,37 @@ import { editTool } from './builtin/edit.js';
 import { bashTool } from './builtin/bash.js';
 import { globTool } from './builtin/glob.js';
 import { grepTool } from './builtin/grep.js';
+import { webfetchTool } from './builtin/webfetch.js';
+import { websearchTool } from './builtin/websearch.js';
 
 /**
  * Create a registry with all built-in tools
  */
 export function createDefaultRegistry(): ToolRegistry {
   const registry = new ToolRegistry();
-  registry.registerAll([readTool, writeTool, editTool, bashTool, globTool, grepTool]);
+  registry.registerAll([
+    readTool,
+    writeTool,
+    editTool,
+    bashTool,
+    globTool,
+    grepTool,
+    webfetchTool,
+    websearchTool,
+  ]);
   return registry;
 }
 
 /**
  * All built-in tools
  */
-export const builtinTools = [readTool, writeTool, editTool, bashTool, globTool, grepTool];
+export const builtinTools = [
+  readTool,
+  writeTool,
+  editTool,
+  bashTool,
+  globTool,
+  grepTool,
+  webfetchTool,
+  websearchTool,
+];

package/src/tools/types.ts

@@ -14,10 +14,20 @@ export interface ToolContext {
   abortSignal?: AbortSignal;
 }
 
+export interface ToolResultMetadata {
+  title?: string; // Short title, e.g., "Fetch(url)"
+  subtitle?: string; // Subtitle, e.g., "Received 540.3KB (200 OK)"
+  size?: number; // Response size in bytes
+  statusCode?: number; // HTTP status code
+  contentType?: string; // Content-Type header
+  duration?: number; // Duration in milliseconds
+}
+
 export interface ToolResult {
   success: boolean;
   output: string;
   error?: string;
+  metadata?: ToolResultMetadata;
 }
 
 export interface Tool<TInput = unknown> {
@@ -88,6 +98,16 @@ export const GrepInputSchema = z.object({
 });
 export type GrepInput = z.infer<typeof GrepInputSchema>;
 
+export const WebFetchInputSchema = z.object({
+  url: z.string().describe('The URL to fetch content from (http:// or https://)'),
+  format: z
+    .enum(['text', 'markdown', 'html'])
+    .optional()
+    .describe('Output format: markdown (default), text, or html'),
+  timeout: z.number().optional().describe('Timeout in seconds (default: 30, max: 120)'),
+});
+export type WebFetchInput = z.infer<typeof WebFetchInputSchema>;
+
 // ============================================================================
 // JSON Schema Conversion
 // ============================================================================

package/src/tools/utils/ssrf.ts

@@ -0,0 +1,79 @@
+/**
+ * SSRF Protection Utilities
+ * Prevents Server-Side Request Forgery by blocking internal/private addresses
+ */
+
+// Private IP ranges (RFC 1918 + loopback + link-local + cloud metadata)
+const PRIVATE_IP_PATTERNS = [
+  /^127\./,                            // Loopback (127.0.0.0/8)
+  /^10\./,                             // Class A private (10.0.0.0/8)
+  /^172\.(1[6-9]|2[0-9]|3[01])\./,    // Class B private (172.16.0.0/12)
+  /^192\.168\./,                       // Class C private (192.168.0.0/16)
+  /^169\.254\./,                       // Link-local (169.254.0.0/16)
+  /^0\./,                              // "This" network
+  /^::1$/,                             // IPv6 loopback
+  /^fe80:/i,                           // IPv6 link-local
+  /^fc00:/i,                           // IPv6 unique local
+  /^fd[0-9a-f]{2}:/i,                  // IPv6 unique local
+];
+
+const BLOCKED_HOSTNAMES = [
+  'localhost',
+  'localhost.localdomain',
+  'metadata.google.internal',   // GCP metadata
+  '169.254.169.254',            // AWS/GCP/Azure metadata
+];
+
+/**
+ * Check if an IP address is in a private range
+ */
+export function isPrivateIP(ip: string): boolean {
+  return PRIVATE_IP_PATTERNS.some((pattern) => pattern.test(ip));
+}
+
+/**
+ * Check if a hostname is blocked
+ */
+export function isBlockedHostname(hostname: string): boolean {
+  const lower = hostname.toLowerCase();
+
+  // Direct match
+  if (BLOCKED_HOSTNAMES.includes(lower)) {
+    return true;
+  }
+
+  // Check .local suffix
+  if (lower.endsWith('.local')) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Validate a URL for SSRF protection
+ * Throws an error if the URL is not allowed
+ */
+export function validateUrl(urlString: string): void {
+  let parsed: URL;
+  try {
+    parsed = new URL(urlString);
+  } catch {
+    throw new Error('Invalid URL format');
+  }
+
+  // Only allow http/https protocols
+  if (!['http:', 'https:'].includes(parsed.protocol)) {
+    throw new Error('Only http:// and https:// URLs are supported');
+  }
+
+  // Check hostname blocklist
+  if (isBlockedHostname(parsed.hostname)) {
+    throw new Error('Access to internal/local addresses is not allowed');
+  }
+
+  // Check if hostname is a private IP
+  if (isPrivateIP(parsed.hostname)) {
+    throw new Error('Access to private IP addresses is not allowed');
+  }
+}

package/CLAUDE.md

@@ -1,70 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## Project Overview
-
-**GenCode** (npm: `gencode`) is an open-source, provider-agnostic AI coding assistant. It brings Claude Code's excellent interactive CLI experience while allowing flexible switching between different LLM providers (OpenAI, Anthropic, Google Gemini).
-
-## Build & Run Commands
-
-```bash
-npm install          # Install dependencies
-npm run build        # Compile TypeScript to dist/
-npm run dev          # Watch mode compilation
-npm start            # Run CLI directly via tsx
-npm run example      # Run examples/basic.ts
-```
-
-## Architecture
-
-### Provider Abstraction Layer (`src/providers/`)
-
-Unified `LLMProvider` interface abstracts API differences:
-- `complete()` - Non-streaming completion
-- `stream()` - Streaming completion (AsyncGenerator)
-
-Each provider (OpenAI, Anthropic, Gemini) translates the unified message format to its native API format and back. The `createProvider()` factory instantiates providers by name.
-
-### Tool System (`src/tools/`)
-
-Tools are defined with Zod schemas for input validation:
-```typescript
-interface Tool<TInput> {
-  name: string;
-  description: string;
-  parameters: z.ZodSchema<TInput>;
-  execute(input: TInput, context: ToolContext): Promise<ToolResult>;
-}
-```
-
-`ToolRegistry` manages tools and converts Zod schemas to JSON Schema for LLM consumption via `zodToJsonSchema()`.
-
-### Agent Loop (`src/agent/agent.ts`)
-
-The `Agent` class implements the core conversation loop:
-1. User message → LLM with tools
-2. If `stopReason === 'tool_use'`: execute tools, append results, loop back
-3. If `stopReason !== 'tool_use'`: done
-
-Events are yielded as `AgentEvent` (text, tool_start, tool_result, done, error).
-
-### Session Management (`src/session/`)
-
-Sessions persist conversation history to `~/.gencode/sessions/` as JSON files. Supports resume, fork, list, and delete operations.
-
-## Configuration
-
-Provider/model selection priority:
-1. `GENCODE_PROVIDER` / `GENCODE_MODEL` env vars
-2. Auto-detect from available API keys (ANTHROPIC_API_KEY → OPENAI_API_KEY → GOOGLE_API_KEY)
-3. Default: Gemini
-
-Proxy: Set `HTTP_PROXY` or `HTTPS_PROXY` for network proxy support.
-
-## Key Patterns
-
-- All file paths in tools should be resolved relative to `ToolContext.cwd`
-- Tool input validation uses Zod; errors returned as `ToolResult.error`
-- Provider implementations handle message format conversion internally
-- CLI commands start with `/` (e.g., `/sessions`, `/resume`, `/help`)