@clinebot/core 0.0.22 → 0.0.24

package/src/tools/executors/search.ts

@@ -0,0 +1,278 @@
+/**
+ * Search Executor
+ *
+ * Built-in implementation for searching the codebase using regex.
+ */
+
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import type { ToolContext } from "@clinebot/agents";
+import { getFileIndex } from "../../input";
+import type { SearchExecutor } from "../types";
+
+/**
+ * Options for the search executor
+ */
+export interface SearchExecutorOptions {
+	/**
+	 * File extensions to include in search (without dot)
+	 * @default common code extensions
+	 */
+	includeExtensions?: string[];
+
+	/**
+	 * Directories to exclude from search
+	 * @default ["node_modules", ".git", "dist", "build", ".next", "coverage"]
+	 */
+	excludeDirs?: string[];
+
+	/**
+	 * Maximum number of results to return
+	 * @default 100
+	 */
+	maxResults?: number;
+
+	/**
+	 * Number of context lines before and after match
+	 * @default 2
+	 */
+	contextLines?: number;
+
+	/**
+	 * Maximum depth to traverse
+	 * @default 20
+	 */
+	maxDepth?: number;
+}
+
+const DEFAULT_INCLUDE_EXTENSIONS = [
+	"ts",
+	"tsx",
+	"js",
+	"jsx",
+	"mjs",
+	"cjs",
+	"json",
+	"md",
+	"mdx",
+	"txt",
+	"yaml",
+	"yml",
+	"toml",
+	"py",
+	"rb",
+	"go",
+	"rs",
+	"java",
+	"kt",
+	"swift",
+	"c",
+	"cpp",
+	"h",
+	"hpp",
+	"css",
+	"scss",
+	"less",
+	"html",
+	"vue",
+	"svelte",
+	"sql",
+	"sh",
+	"bash",
+	"zsh",
+	"fish",
+	"ps1",
+	"env",
+	"gitignore",
+	"dockerignore",
+	"editorconfig",
+];
+
+const DEFAULT_EXCLUDE_DIRS = [
+	"node_modules",
+	".git",
+	"dist",
+	"build",
+	".next",
+	"coverage",
+	"__pycache__",
+	".venv",
+	"venv",
+	".cache",
+	".turbo",
+	".output",
+	"out",
+	"target",
+	"bin",
+	"obj",
+];
+
+/**
+ * Search result for a single file match
+ */
+interface SearchMatch {
+	file: string;
+	line: number;
+	column: number;
+	match: string;
+	context: string[];
+}
+
+function shouldIncludeFile(
+	relativePath: string,
+	excludeDirs: Set<string>,
+	includeExtensions: Set<string>,
+	maxDepth: number,
+): boolean {
+	const segments = relativePath.split("/");
+	const fileName = segments[segments.length - 1] ?? "";
+	const directoryDepth = segments.length - 1;
+
+	if (directoryDepth > maxDepth) {
+		return false;
+	}
+
+	for (let i = 0; i < segments.length - 1; i++) {
+		if (excludeDirs.has(segments[i] ?? "")) {
+			return false;
+		}
+	}
+
+	const ext = path.posix.extname(fileName).slice(1).toLowerCase();
+	return includeExtensions.has(ext) || (!ext && !fileName.startsWith("."));
+}
+
+/**
+ * Create a search executor using regex pattern matching
+ *
+ * @example
+ * ```typescript
+ * const search = createSearchExecutor({
+ *   maxResults: 50,
+ *   contextLines: 3,
+ * })
+ *
+ * const results = await search("function\\s+handleClick", "/path/to/project", context)
+ * ```
+ */
+export function createSearchExecutor(
+	options: SearchExecutorOptions = {},
+): SearchExecutor {
+	const {
+		includeExtensions = DEFAULT_INCLUDE_EXTENSIONS,
+		excludeDirs = DEFAULT_EXCLUDE_DIRS,
+		maxResults = 100,
+		contextLines = 2,
+		maxDepth = 20,
+	} = options;
+	const excludeDirsSet = new Set(excludeDirs);
+	const includeExtensionsSet = new Set(
+		includeExtensions.map((extension) => extension.toLowerCase()),
+	);
+
+	return async (
+		query: string,
+		cwd: string,
+		_context: ToolContext,
+	): Promise<string> => {
+		// Compile regex
+		let regex: RegExp;
+		try {
+			regex = new RegExp(query, "gim");
+		} catch (error) {
+			throw new Error(
+				`Invalid regex pattern: ${query}. ${error instanceof Error ? error.message : ""}`,
+			);
+		}
+
+		const matches: SearchMatch[] = [];
+		let totalFilesSearched = 0;
+
+		const fileList = await getFileIndex(cwd);
+
+		// Search files from the fast index.
+		for (const relativePath of fileList) {
+			if (
+				!shouldIncludeFile(
+					relativePath,
+					excludeDirsSet,
+					includeExtensionsSet,
+					maxDepth,
+				)
+			) {
+				continue;
+			}
+
+			if (matches.length >= maxResults) break;
+
+			totalFilesSearched++;
+			const filePath = path.join(cwd, relativePath);
+
+			try {
+				const content = await fs.readFile(filePath, "utf-8");
+				const lines = content.split("\n");
+
+				for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
+					const line = lines[lineIdx];
+					regex.lastIndex = 0; // Reset regex state
+
+					let match: RegExpExecArray | null;
+					while ((match = regex.exec(line)) !== null) {
+						if (matches.length >= maxResults) break;
+
+						// Get context lines
+						const contextStart = Math.max(0, lineIdx - contextLines);
+						const contextEnd = Math.min(
+							lines.length - 1,
+							lineIdx + contextLines,
+						);
+						const contextLinesArr: string[] = [];
+
+						for (let i = contextStart; i <= contextEnd; i++) {
+							const prefix = i === lineIdx ? ">" : " ";
+							contextLinesArr.push(`${prefix} ${i + 1}: ${lines[i]}`);
+						}
+
+						matches.push({
+							file: relativePath,
+							line: lineIdx + 1,
+							column: match.index + 1,
+							match: match[0],
+							context: contextLinesArr,
+						});
+
+						// Prevent infinite loop on zero-length matches
+						if (match.index === regex.lastIndex) {
+							regex.lastIndex++;
+						}
+					}
+				}
+			} catch {}
+		}
+
+		// Format results
+		if (matches.length === 0) {
+			return `No results found for pattern: ${query}\nSearched ${totalFilesSearched} files.`;
+		}
+
+		const resultLines: string[] = [
+			`Found ${matches.length} result${matches.length === 1 ? "" : "s"} for pattern: ${query}`,
+			`Searched ${totalFilesSearched} files.`,
+			"",
+		];
+
+		for (const match of matches) {
+			resultLines.push(`${match.file}:${match.line}:${match.column}`);
+			resultLines.push(...match.context);
+			resultLines.push("");
+		}
+
+		if (matches.length >= maxResults) {
+			resultLines.push(
+				`(Showing first ${maxResults} results. Refine your search for more specific results.)`,
+			);
+		}
+
+		return resultLines.join("\n");
+	};
+}

package/src/tools/executors/web-fetch.ts

@@ -0,0 +1,259 @@
+/**
+ * Web Fetch Executor
+ *
+ * Built-in implementation for fetching web content using native fetch.
+ */
+
+import type { ToolContext } from "@clinebot/agents";
+import type { WebFetchExecutor } from "../types";
+
+/**
+ * Options for the web fetch executor
+ */
+export interface WebFetchExecutorOptions {
+	/**
+	 * Timeout for fetch requests in milliseconds
+	 * @default 30000 (30 seconds)
+	 */
+	timeoutMs?: number;
+
+	/**
+	 * Maximum response size in bytes
+	 * @default 5_000_000 (5MB)
+	 */
+	maxResponseBytes?: number;
+
+	/**
+	 * User agent string
+	 * @default "Mozilla/5.0 (compatible; AgentBot/1.0)"
+	 */
+	userAgent?: string;
+
+	/**
+	 * Additional headers
+	 */
+	headers?: Record<string, string>;
+
+	/**
+	 * Whether to follow redirects
+	 * @default true
+	 */
+	followRedirects?: boolean;
+
+	/**
+	 * Maximum number of redirects to follow
+	 * @default 5
+	 */
+	maxRedirects?: number;
+}
+
+/**
+ * Extract text content from HTML
+ * Simple implementation - strips tags and normalizes whitespace
+ */
+function htmlToText(html: string): string {
+	return (
+		html
+			// Remove script and style elements
+			.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
+			.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
+			// Remove HTML comments
+			.replace(/<!--[\s\S]*?-->/g, "")
+			// Replace block elements with newlines
+			.replace(/<(p|div|br|hr|h[1-6]|li|tr)[^>]*>/gi, "\n")
+			// Remove all remaining tags
+			.replace(/<[^>]+>/g, " ")
+			// Decode HTML entities
+			.replace(/&nbsp;/g, " ")
+			.replace(/&amp;/g, "&")
+			.replace(/&lt;/g, "<")
+			.replace(/&gt;/g, ">")
+			.replace(/&quot;/g, '"')
+			.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)))
+			// Normalize whitespace
+			.replace(/\s+/g, " ")
+			.replace(/\n\s+/g, "\n")
+			.replace(/\n{3,}/g, "\n\n")
+			.trim()
+	);
+}
+
+/**
+ * Create a web fetch executor using native fetch
+ *
+ * @example
+ * ```typescript
+ * const webFetch = createWebFetchExecutor({
+ *   timeoutMs: 15000,
+ *   maxResponseBytes: 2_000_000,
+ * })
+ *
+ * const content = await webFetch(
+ *   "https://docs.example.com/api",
+ *   "Extract the authentication section",
+ *   context
+ * )
+ * ```
+ */
+export function createWebFetchExecutor(
+	options: WebFetchExecutorOptions = {},
+): WebFetchExecutor {
+	const {
+		timeoutMs = 30000,
+		maxResponseBytes = 5_000_000,
+		userAgent = "Mozilla/5.0 (compatible; AgentBot/1.0)",
+		headers = {},
+		followRedirects = true,
+		// maxRedirects is available in options but native fetch handles it automatically
+	} = options;
+
+	return async (
+		url: string,
+		prompt: string,
+		context: ToolContext,
+	): Promise<string> => {
+		// Validate URL
+		let parsedUrl: URL;
+		try {
+			parsedUrl = new URL(url);
+		} catch {
+			throw new Error(`Invalid URL: ${url}`);
+		}
+
+		// Only allow http and https
+		if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+			throw new Error(
+				`Invalid protocol: ${parsedUrl.protocol}. Only http and https are supported.`,
+			);
+		}
+
+		// Create abort controller for timeout
+		const controller = new AbortController();
+		const timeout = setTimeout(() => controller.abort(), timeoutMs);
+		let contextAbortHandler: (() => void) | undefined;
+
+		// Combine with context abort signal
+		if (context.abortSignal) {
+			contextAbortHandler = () => controller.abort();
+			context.abortSignal.addEventListener("abort", contextAbortHandler);
+		}
+
+		try {
+			const response = await fetch(url, {
+				method: "GET",
+				headers: {
+					"User-Agent": userAgent,
+					Accept:
+						"text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.7",
+					"Accept-Language": "en-US,en;q=0.9",
+					...headers,
+				},
+				redirect: followRedirects ? "follow" : "manual",
+				signal: controller.signal,
+			});
+
+			clearTimeout(timeout);
+
+			// Check for redirect limit (if we're checking manually)
+			if (!followRedirects && response.status >= 300 && response.status < 400) {
+				const location = response.headers.get("location");
+				return `Redirect to: ${location}`;
+			}
+
+			// Check response status
+			if (!response.ok) {
+				throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+			}
+
+			// Get content type
+			const contentType = response.headers.get("content-type") || "";
+
+			// Read response body with size limit
+			const reader = response.body?.getReader();
+			if (!reader) {
+				throw new Error("Failed to read response body");
+			}
+
+			const chunks: Uint8Array[] = [];
+			let totalSize = 0;
+
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+
+				totalSize += value.length;
+				if (totalSize > maxResponseBytes) {
+					reader.cancel();
+					throw new Error(
+						`Response too large: exceeded ${maxResponseBytes} bytes`,
+					);
+				}
+
+				chunks.push(value);
+			}
+
+			// Combine chunks
+			const buffer = new Uint8Array(totalSize);
+			let offset = 0;
+			for (const chunk of chunks) {
+				buffer.set(chunk, offset);
+				offset += chunk.length;
+			}
+
+			// Decode as text
+			const text = new TextDecoder("utf-8").decode(buffer);
+
+			// Process content based on type
+			let content: string;
+			if (
+				contentType.includes("text/html") ||
+				contentType.includes("application/xhtml")
+			) {
+				content = htmlToText(text);
+			} else if (contentType.includes("application/json")) {
+				try {
+					const json = JSON.parse(text);
+					content = JSON.stringify(json, null, 2);
+				} catch {
+					content = text;
+				}
+			} else {
+				content = text;
+			}
+
+			// Format output with metadata
+			const outputLines = [
+				`URL: ${url}`,
+				`Content-Type: ${contentType}`,
+				`Size: ${totalSize} bytes`,
+				``,
+				`--- Content ---`,
+				content.slice(0, 50000), // Limit content size for output
+			];
+
+			if (content.length > 50000) {
+				outputLines.push(
+					`\n[Content truncated: showing first 50000 of ${content.length} characters]`,
+				);
+			}
+
+			outputLines.push(``, `--- Analysis Request ---`, `Prompt: ${prompt}`);
+
+			return outputLines.join("\n");
+		} catch (error) {
+			clearTimeout(timeout);
+
+			if (error instanceof Error) {
+				if (error.name === "AbortError") {
+					throw new Error(`Request timed out after ${timeoutMs}ms`);
+				}
+				throw error;
+			}
+			throw new Error(`Fetch failed: ${String(error)}`);
+		} finally {
+			if (context.abortSignal && contextAbortHandler) {
+				context.abortSignal.removeEventListener("abort", contextAbortHandler);
+			}
+		}
+	};
+}

package/src/tools/helpers.ts

@@ -0,0 +1,130 @@
+import { validateWithZod } from "@clinebot/shared";
+import {
+	type EditFileInput,
+	INPUT_ARG_CHAR_LIMIT,
+	type ReadFileRequest,
+	ReadFilesInputUnionSchema,
+	type StructuredCommandInput,
+	StructuredCommandsInputUnionSchema,
+} from "./schemas";
+
+/**
+ * Format an error into a string message
+ */
+export function formatError(error: unknown): string {
+	if (error instanceof Error) {
+		return error.message;
+	}
+	return String(error);
+}
+
+export function getEditorSizeError(input: EditFileInput): string | null {
+	if (
+		typeof input.old_text === "string" &&
+		input.old_text.length > INPUT_ARG_CHAR_LIMIT
+	) {
+		return `Editor input too large: old_text was ${input.old_text.length} characters, exceeding the recommended limit of ${INPUT_ARG_CHAR_LIMIT}. Split the edit into smaller tool calls so later tool calls are less likely to be truncated or time out.`;
+	}
+
+	if (input.new_text.length > INPUT_ARG_CHAR_LIMIT) {
+		return `Editor input too large: new_text was ${input.new_text.length} characters, exceeding the recommended limit of ${INPUT_ARG_CHAR_LIMIT}. Split the edit into smaller tool calls so later tool calls are less likely to be truncated or time out.`;
+	}
+
+	return null;
+}
+
+/**
+ * Create a timeout-wrapped promise
+ */
+export function withTimeout<T>(
+	promise: Promise<T>,
+	ms: number,
+	message: string,
+): Promise<T> {
+	return Promise.race([
+		promise,
+		new Promise<never>((_, reject) => {
+			setTimeout(() => reject(new Error(message)), ms);
+		}),
+	]);
+}
+
+export function normalizeReadFileRequests(input: unknown): ReadFileRequest[] {
+	const validate = validateWithZod(ReadFilesInputUnionSchema, input);
+
+	if (typeof validate === "string") {
+		return [{ path: validate }];
+	}
+
+	if (Array.isArray(validate)) {
+		return validate.map((value) =>
+			typeof value === "string" ? { path: value } : value,
+		);
+	}
+
+	if ("files" in validate) {
+		const files = Array.isArray(validate.files)
+			? validate.files
+			: [validate.files];
+		return files;
+	}
+
+	if ("file_paths" in validate) {
+		const filePaths = Array.isArray(validate.file_paths)
+			? validate.file_paths
+			: [validate.file_paths];
+		return filePaths.map((filePath) => ({ path: filePath }));
+	}
+
+	return [validate];
+}
+
+export function formatReadFileQuery(request: ReadFileRequest): string {
+	const { path, start_line, end_line } = request;
+	if (start_line == null && end_line == null) {
+		return path;
+	}
+	const start = start_line ?? 1;
+	const end = end_line ?? "EOF";
+	return `${path}:${start}-${end}`;
+}
+
+export function normalizeRunCommandsInput(
+	input: unknown,
+): Array<string | StructuredCommandInput> {
+	const validate = validateWithZod(StructuredCommandsInputUnionSchema, input);
+
+	if (typeof validate === "string") {
+		return [validate];
+	}
+
+	if (Array.isArray(validate)) {
+		return validate;
+	}
+
+	if ("commands" in validate) {
+		return Array.isArray(validate.commands)
+			? validate.commands
+			: [validate.commands];
+	}
+
+	return [validate];
+}
+
+export function formatRunCommandQuery(
+	command: string | StructuredCommandInput,
+): string {
+	if (typeof command === "string") {
+		return command;
+	}
+
+	const args = command.args ?? [];
+	if (args.length === 0) {
+		return command.command;
+	}
+
+	const renderedArgs = args.map((arg) =>
+		/[\s"]/u.test(arg) ? JSON.stringify(arg) : arg,
+	);
+	return `${command.command} ${renderedArgs.join(" ")}`;
+}