@clinebot/core 0.0.37 → 0.0.38

package/src/extensions/tools/executors/search.ts

@@ -1,470 +0,0 @@
-/**
- * Search Executor
- *
- * Built-in implementation for searching the codebase using ripgrep (if available) or regex.
- */
-
-import { spawn } from "node:child_process";
-import * as fs from "node:fs/promises";
-import * as path from "node:path";
-import type { ToolContext } from "@clinebot/shared";
-import { getFileIndex } from "../../../services/workspace";
-import type { SearchExecutor } from "../types";
-
-/**
- * Options for the search executor
- */
-export interface SearchExecutorOptions {
-	/**
-	 * File extensions to include in search (without dot)
-	 * @default common code extensions
-	 */
-	includeExtensions?: string[];
-
-	/**
-	 * Directories to exclude from search
-	 * @default ["node_modules", ".git", "dist", "build", ".next", "coverage"]
-	 */
-	excludeDirs?: string[];
-
-	/**
-	 * Maximum number of results to return
-	 * @default 100
-	 */
-	maxResults?: number;
-
-	/**
-	 * Number of context lines before and after match
-	 * @default 2
-	 */
-	contextLines?: number;
-
-	/**
-	 * Maximum depth to traverse
-	 * @default 20
-	 */
-	maxDepth?: number;
-}
-
-const DEFAULT_INCLUDE_EXTENSIONS = [
-	"ts",
-	"tsx",
-	"js",
-	"jsx",
-	"mjs",
-	"cjs",
-	"json",
-	"md",
-	"mdx",
-	"txt",
-	"yaml",
-	"yml",
-	"toml",
-	"py",
-	"rb",
-	"go",
-	"rs",
-	"java",
-	"kt",
-	"swift",
-	"c",
-	"cpp",
-	"h",
-	"hpp",
-	"css",
-	"scss",
-	"less",
-	"html",
-	"vue",
-	"svelte",
-	"sql",
-	"sh",
-	"bash",
-	"zsh",
-	"fish",
-	"ps1",
-	"env",
-	"gitignore",
-	"dockerignore",
-	"editorconfig",
-];
-
-const DEFAULT_EXCLUDE_DIRS = [
-	"node_modules",
-	".git",
-	"dist",
-	"build",
-	".next",
-	"coverage",
-	"__pycache__",
-	".venv",
-	"venv",
-	".cache",
-	".turbo",
-	".output",
-	"out",
-	"target",
-	"bin",
-	"obj",
-];
-
-/**
- * Search result for a single file match
- */
-interface SearchMatch {
-	file: string;
-	line: number;
-	column: number;
-	match: string;
-	context: string[];
-}
-
-let rgAvailable: boolean | null = null;
-
-function checkRipgrepAvailable(): Promise<boolean> {
-	if (rgAvailable !== null) {
-		return Promise.resolve(rgAvailable);
-	}
-
-	return new Promise((resolve) => {
-		const child = spawn("rg", ["--version"], {
-			stdio: ["ignore", "pipe", "pipe"],
-		});
-
-		child.on("close", (code) => {
-			rgAvailable = code === 0;
-			resolve(rgAvailable);
-		});
-
-		child.on("error", () => {
-			rgAvailable = false;
-			resolve(false);
-		});
-
-		setTimeout(() => {
-			if (!child.killed) {
-				child.kill("SIGTERM");
-			}
-			if (rgAvailable === null) {
-				rgAvailable = false;
-				resolve(false);
-			}
-		}, 1000);
-	});
-}
-
-function searchWithRipgrep(
-	query: string,
-	cwd: string,
-	maxResults: number,
-	contextLines: number,
-	timeoutMs: number = 5000,
-	abortSignal?: AbortSignal,
-): Promise<SearchMatch[] | null> {
-	return new Promise((resolve) => {
-		const child = spawn(
-			"rg",
-			["--json", `--context=${contextLines}`, "--max-count=1", "-i", query],
-			{
-				cwd,
-				stdio: ["ignore", "pipe", "pipe"],
-			},
-		);
-
-		let stdout = "";
-		let resolved = false;
-
-		const cleanup = () => {
-			if (!child.killed) {
-				child.kill("SIGTERM");
-			}
-		};
-
-		const timeout = setTimeout(() => {
-			if (!resolved) {
-				resolved = true;
-				cleanup();
-				resolve(null);
-			}
-		}, timeoutMs);
-
-		const finalize = (result: SearchMatch[] | null) => {
-			if (!resolved) {
-				resolved = true;
-				clearTimeout(timeout);
-				cleanup();
-				resolve(result);
-			}
-		};
-
-		if (abortSignal?.aborted) {
-			cleanup();
-			resolve(null);
-			return;
-		}
-
-		abortSignal?.addEventListener("abort", () => {
-			finalize(null);
-		});
-
-		child.stdout.on("data", (chunk: Buffer | string) => {
-			stdout += chunk.toString();
-		});
-
-		child.stderr.on("data", () => {
-			// Ignore stderr
-		});
-
-		child.on("close", (code: number | null) => {
-			if (code === 0 || code === 1) {
-				try {
-					const matches: SearchMatch[] = [];
-					const lines = stdout.split("\n").filter((line) => line.trim());
-
-					for (const line of lines) {
-						if (matches.length >= maxResults) break;
-
-						const json = JSON.parse(line);
-						if (json.type === "match") {
-							const matchData = json.data;
-							const contextLines: string[] = [];
-
-							if (json.data.submatches && json.data.submatches.length > 0) {
-								const submatch = json.data.submatches[0];
-								matches.push({
-									file: matchData.path.text,
-									line: matchData.line_number,
-									column: (submatch?.start ?? 0) + 1,
-									match: submatch?.match?.text ?? "",
-									context: contextLines,
-								});
-							}
-						} else if (json.type === "context" && matches.length > 0) {
-							const lastMatch = matches[matches.length - 1];
-							const prefix =
-								json.data.line_number === lastMatch.line ? ">" : " ";
-							lastMatch.context.push(
-								`${prefix} ${json.data.line_number}: ${json.data.lines?.text ?? json.data.line?.text ?? ""}`,
-							);
-						}
-					}
-
-					finalize(matches.length > 0 ? matches : null);
-				} catch {
-					finalize(null);
-				}
-				return;
-			}
-
-			finalize(null);
-		});
-
-		child.on("error", () => {
-			finalize(null);
-		});
-	});
-}
-
-function shouldIncludeFile(
-	relativePath: string,
-	excludeDirs: Set<string>,
-	includeExtensions: Set<string>,
-	maxDepth: number,
-): boolean {
-	const segments = relativePath.split("/");
-	const fileName = segments[segments.length - 1] ?? "";
-	const directoryDepth = segments.length - 1;
-
-	if (directoryDepth > maxDepth) {
-		return false;
-	}
-
-	for (let i = 0; i < segments.length - 1; i++) {
-		if (excludeDirs.has(segments[i] ?? "")) {
-			return false;
-		}
-	}
-
-	const ext = path.posix.extname(fileName).slice(1).toLowerCase();
-	return includeExtensions.has(ext) || (!ext && !fileName.startsWith("."));
-}
-
-/**
- * Create a search executor using regex pattern matching
- *
- * @example
- * ```typescript
- * const search = createSearchExecutor({
- *   maxResults: 50,
- *   contextLines: 3,
- * })
- *
- * const results = await search("function\\s+handleClick", "/path/to/project", context)
- * ```
- */
-export function createSearchExecutor(
-	options: SearchExecutorOptions = {},
-): SearchExecutor {
-	const {
-		includeExtensions = DEFAULT_INCLUDE_EXTENSIONS,
-		excludeDirs = DEFAULT_EXCLUDE_DIRS,
-		maxResults = 100,
-		contextLines = 2,
-		maxDepth = 20,
-	} = options;
-	const excludeDirsSet = new Set(excludeDirs);
-	const includeExtensionsSet = new Set(
-		includeExtensions.map((extension) => extension.toLowerCase()),
-	);
-
-	return async (
-		query: string,
-		cwd: string,
-		context: ToolContext,
-	): Promise<string> => {
-		// Check for abort before starting
-		if (context.abortSignal?.aborted) {
-			throw new Error("Search operation aborted");
-		}
-
-		// Try ripgrep first if available
-		const isRgAvailable = await checkRipgrepAvailable();
-		let rgMatches: SearchMatch[] | null = null;
-		if (isRgAvailable) {
-			rgMatches = await searchWithRipgrep(
-				query,
-				cwd,
-				maxResults,
-				contextLines,
-				5000,
-				context.abortSignal,
-			);
-		}
-
-		if (rgMatches) {
-			const resultLines: string[] = [
-				`Found ${rgMatches.length} result${rgMatches.length === 1 ? "" : "s"} for pattern: ${query}`,
-				"",
-			];
-
-			for (const match of rgMatches) {
-				resultLines.push(`${match.file}:${match.line}:${match.column}`);
-				resultLines.push(...match.context);
-				resultLines.push("");
-			}
-
-			if (rgMatches.length >= maxResults) {
-				resultLines.push(
-					`(Showing first ${maxResults} results. Refine your search for more specific results.)`,
-				);
-			}
-
-			return resultLines.join("\n");
-		}
-
-		// Fallback to manual regex search
-		let regex: RegExp;
-		try {
-			regex = new RegExp(query, "gim");
-		} catch (error) {
-			throw new Error(
-				`Invalid regex pattern: ${query}. ${error instanceof Error ? error.message : ""}`,
-			);
-		}
-
-		const matches: SearchMatch[] = [];
-		let totalFilesSearched = 0;
-
-		const fileList = await getFileIndex(cwd);
-
-		// Search files from the fast index.
-		for (const relativePath of fileList) {
-			// Check for abort signal
-			if (context.abortSignal?.aborted) {
-				throw new Error("Search operation aborted");
-			}
-
-			if (
-				!shouldIncludeFile(
-					relativePath,
-					excludeDirsSet,
-					includeExtensionsSet,
-					maxDepth,
-				)
-			) {
-				continue;
-			}
-
-			if (matches.length >= maxResults) break;
-
-			totalFilesSearched++;
-			const filePath = path.join(cwd, relativePath);
-
-			try {
-				const content = await fs.readFile(filePath, "utf-8");
-				const lines = content.split("\n");
-
-				for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
-					const line = lines[lineIdx];
-					regex.lastIndex = 0; // Reset regex state
-
-					let match: RegExpExecArray | null;
-					while ((match = regex.exec(line)) !== null) {
-						if (matches.length >= maxResults) break;
-
-						// Get context lines
-						const contextStart = Math.max(0, lineIdx - contextLines);
-						const contextEnd = Math.min(
-							lines.length - 1,
-							lineIdx + contextLines,
-						);
-						const contextLinesArr: string[] = [];
-
-						for (let i = contextStart; i <= contextEnd; i++) {
-							const prefix = i === lineIdx ? ">" : " ";
-							contextLinesArr.push(`${prefix} ${i + 1}: ${lines[i]}`);
-						}
-
-						matches.push({
-							file: relativePath,
-							line: lineIdx + 1,
-							column: match.index + 1,
-							match: match[0],
-							context: contextLinesArr,
-						});
-
-						// Prevent infinite loop on zero-length matches
-						if (match.index === regex.lastIndex) {
-							regex.lastIndex++;
-						}
-					}
-				}
-			} catch {}
-		}
-
-		// Format results
-		if (matches.length === 0) {
-			return `No results found for pattern: ${query}\nSearched ${totalFilesSearched} files.`;
-		}
-
-		const resultLines: string[] = [
-			`Found ${matches.length} result${matches.length === 1 ? "" : "s"} for pattern: ${query}`,
-			`Searched ${totalFilesSearched} files.`,
-			"",
-		];
-
-		for (const match of matches) {
-			resultLines.push(`${match.file}:${match.line}:${match.column}`);
-			resultLines.push(...match.context);
-			resultLines.push("");
-		}
-
-		if (matches.length >= maxResults) {
-			resultLines.push(
-				`(Showing first ${maxResults} results. Refine your search for more specific results.)`,
-			);
-		}
-
-		return resultLines.join("\n");
-	};
-}

package/src/extensions/tools/executors/web-fetch.ts

@@ -1,259 +0,0 @@
-/**
- * Web Fetch Executor
- *
- * Built-in implementation for fetching web content using native fetch.
- */
-
-import type { ToolContext } from "@clinebot/shared";
-import type { WebFetchExecutor } from "../types";
-
-/**
- * Options for the web fetch executor
- */
-export interface WebFetchExecutorOptions {
-	/**
-	 * Timeout for fetch requests in milliseconds
-	 * @default 30000 (30 seconds)
-	 */
-	timeoutMs?: number;
-
-	/**
-	 * Maximum response size in bytes
-	 * @default 5_000_000 (5MB)
-	 */
-	maxResponseBytes?: number;
-
-	/**
-	 * User agent string
-	 * @default "Mozilla/5.0 (compatible; AgentBot/1.0)"
-	 */
-	userAgent?: string;
-
-	/**
-	 * Additional headers
-	 */
-	headers?: Record<string, string>;
-
-	/**
-	 * Whether to follow redirects
-	 * @default true
-	 */
-	followRedirects?: boolean;
-
-	/**
-	 * Maximum number of redirects to follow
-	 * @default 5
-	 */
-	maxRedirects?: number;
-}
-
-/**
- * Extract text content from HTML
- * Simple implementation - strips tags and normalizes whitespace
- */
-function htmlToText(html: string): string {
-	return (
-		html
-			// Remove script and style elements
-			.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
-			.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
-			// Remove HTML comments
-			.replace(/<!--[\s\S]*?-->/g, "")
-			// Replace block elements with newlines
-			.replace(/<(p|div|br|hr|h[1-6]|li|tr)[^>]*>/gi, "\n")
-			// Remove all remaining tags
-			.replace(/<[^>]+>/g, " ")
-			// Decode HTML entities
-			.replace(/&nbsp;/g, " ")
-			.replace(/&amp;/g, "&")
-			.replace(/&lt;/g, "<")
-			.replace(/&gt;/g, ">")
-			.replace(/&quot;/g, '"')
-			.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)))
-			// Normalize whitespace
-			.replace(/\s+/g, " ")
-			.replace(/\n\s+/g, "\n")
-			.replace(/\n{3,}/g, "\n\n")
-			.trim()
-	);
-}
-
-/**
- * Create a web fetch executor using native fetch
- *
- * @example
- * ```typescript
- * const webFetch = createWebFetchExecutor({
- *   timeoutMs: 15000,
- *   maxResponseBytes: 2_000_000,
- * })
- *
- * const content = await webFetch(
- *   "https://docs.example.com/api",
- *   "Extract the authentication section",
- *   context
- * )
- * ```
- */
-export function createWebFetchExecutor(
-	options: WebFetchExecutorOptions = {},
-): WebFetchExecutor {
-	const {
-		timeoutMs = 30000,
-		maxResponseBytes = 5_000_000,
-		userAgent = "Mozilla/5.0 (compatible; AgentBot/1.0)",
-		headers = {},
-		followRedirects = true,
-		// maxRedirects is available in options but native fetch handles it automatically
-	} = options;
-
-	return async (
-		url: string,
-		prompt: string,
-		context: ToolContext,
-	): Promise<string> => {
-		// Validate URL
-		let parsedUrl: URL;
-		try {
-			parsedUrl = new URL(url);
-		} catch {
-			throw new Error(`Invalid URL: ${url}`);
-		}
-
-		// Only allow http and https
-		if (!["http:", "https:"].includes(parsedUrl.protocol)) {
-			throw new Error(
-				`Invalid protocol: ${parsedUrl.protocol}. Only http and https are supported.`,
-			);
-		}
-
-		// Create abort controller for timeout
-		const controller = new AbortController();
-		const timeout = setTimeout(() => controller.abort(), timeoutMs);
-		let contextAbortHandler: (() => void) | undefined;
-
-		// Combine with context abort signal
-		if (context.abortSignal) {
-			contextAbortHandler = () => controller.abort();
-			context.abortSignal.addEventListener("abort", contextAbortHandler);
-		}
-
-		try {
-			const response = await fetch(url, {
-				method: "GET",
-				headers: {
-					"User-Agent": userAgent,
-					Accept:
-						"text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.7",
-					"Accept-Language": "en-US,en;q=0.9",
-					...headers,
-				},
-				redirect: followRedirects ? "follow" : "manual",
-				signal: controller.signal,
-			});
-
-			clearTimeout(timeout);
-
-			// Check for redirect limit (if we're checking manually)
-			if (!followRedirects && response.status >= 300 && response.status < 400) {
-				const location = response.headers.get("location");
-				return `Redirect to: ${location}`;
-			}
-
-			// Check response status
-			if (!response.ok) {
-				throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-			}
-
-			// Get content type
-			const contentType = response.headers.get("content-type") || "";
-
-			// Read response body with size limit
-			const reader = response.body?.getReader();
-			if (!reader) {
-				throw new Error("Failed to read response body");
-			}
-
-			const chunks: Uint8Array[] = [];
-			let totalSize = 0;
-
-			while (true) {
-				const { done, value } = await reader.read();
-				if (done) break;
-
-				totalSize += value.length;
-				if (totalSize > maxResponseBytes) {
-					reader.cancel();
-					throw new Error(
-						`Response too large: exceeded ${maxResponseBytes} bytes`,
-					);
-				}
-
-				chunks.push(value);
-			}
-
-			// Combine chunks
-			const buffer = new Uint8Array(totalSize);
-			let offset = 0;
-			for (const chunk of chunks) {
-				buffer.set(chunk, offset);
-				offset += chunk.length;
-			}
-
-			// Decode as text
-			const text = new TextDecoder("utf-8").decode(buffer);
-
-			// Process content based on type
-			let content: string;
-			if (
-				contentType.includes("text/html") ||
-				contentType.includes("application/xhtml")
-			) {
-				content = htmlToText(text);
-			} else if (contentType.includes("application/json")) {
-				try {
-					const json = JSON.parse(text);
-					content = JSON.stringify(json, null, 2);
-				} catch {
-					content = text;
-				}
-			} else {
-				content = text;
-			}
-
-			// Format output with metadata
-			const outputLines = [
-				`URL: ${url}`,
-				`Content-Type: ${contentType}`,
-				`Size: ${totalSize} bytes`,
-				``,
-				`--- Content ---`,
-				content.slice(0, 50000), // Limit content size for output
-			];
-
-			if (content.length > 50000) {
-				outputLines.push(
-					`\n[Content truncated: showing first 50000 of ${content.length} characters]`,
-				);
-			}
-
-			outputLines.push(``, `--- Analysis Request ---`, `Prompt: ${prompt}`);
-
-			return outputLines.join("\n");
-		} catch (error) {
-			clearTimeout(timeout);
-
-			if (error instanceof Error) {
-				if (error.name === "AbortError") {
-					throw new Error(`Request timed out after ${timeoutMs}ms`);
-				}
-				throw error;
-			}
-			throw new Error(`Fetch failed: ${String(error)}`);
-		} finally {
-			if (context.abortSignal && contextAbortHandler) {
-				context.abortSignal.removeEventListener("abort", contextAbortHandler);
-			}
-		}
-	};
-}