@skroyc/librarian 0.1.0 → 0.2.1

package/src/tools/grep-content.tool.ts

@@ -1,325 +1,405 @@
+import { stat } from "node:fs/promises";
+import path from "node:path";
+import { Glob } from "bun";
 import { tool } from "langchain";
 import { z } from "zod";
-import { readdir, stat } from "node:fs/promises";
-import path from "node:path";
-import { logger } from "../utils/logger.js";
+import { formatToolError, getToolSuggestion } from "../utils/error-utils.js";
 import { isTextFile } from "../utils/file-utils.js";
+import {
+  formatSearchResults,
+  type SearchMatch,
+} from "../utils/format-utils.js";
+import { GitIgnoreService } from "../utils/gitignore-service.js";
+import { logger } from "../utils/logger.js";
 
-// Safe file read with encoding detection
-async function readFileContent(filePath: string): Promise<string> {
-	try {
-		return await Bun.file(filePath).text();
-	} catch (error: unknown) {
-		const errorMessage =
-			error instanceof Error ? error.message : "Unknown error";
-		throw new Error(`Failed to read file: ${errorMessage}`);
-	}
-}
-
-// Search for content in a single file
-function searchFileContent(content: string, regex: RegExp) {
-	const lines = content.split("\n");
-	const matches: Array<{ line: number; text: string; column: number; match: RegExpExecArray | null }> = [];
-
-	for (let i = 0; i < lines.length; i++) {
-		const line = lines[i];
-		if (line === undefined) {
-			continue;
-		}
-
-		const match = regex.exec(line);
-
-		if (match) {
-			matches.push({
-				line: i + 1,
-				column: match.index + 1,
-				text: line,
-				match,
-			});
-			regex.lastIndex = 0;
-		}
-	}
-
-	return matches;
-}
-
-// Find files matching a pattern in a directory
-async function findFiles(
-	dirPath: string,
-	pattern: string,
-	recursive = true,
-): Promise<string[]> {
-	const foundFiles: string[] = [];
-
-	const entries = await readdir(dirPath, { withFileTypes: true });
-
-	for (const entry of entries) {
-		const fullPath = path.join(dirPath, entry.name);
-
-		if (entry.isDirectory()) {
-			if (recursive) {
-				const subDirFiles = await findFiles(fullPath, pattern, recursive);
-				foundFiles.push(...subDirFiles);
-			}
-		} else if (entry.isFile() && (pattern === "*" || entry.name.includes(pattern.replace(/\*/g, "")))) {
-			foundFiles.push(fullPath);
-		}
-	}
-
-	return foundFiles;
-}
-
-async function validateAndResolvePath(workingDir: string, searchPath: string): Promise<string> {
-	const resolvedPath = path.resolve(workingDir, searchPath);
-	const resolvedWorkingDir = path.resolve(workingDir);
-	const relativePath = path.relative(resolvedWorkingDir, resolvedPath);
-
-	logger.debug("TOOL", "Path validation", {
-		resolvedPath: resolvedPath.replace(Bun.env.HOME || "", "~"),
-		resolvedWorkingDir: resolvedWorkingDir.replace(Bun.env.HOME || "", "~"),
-		relativePath,
-		validated: !relativePath.startsWith(".."),
-	});
-
-	if (relativePath.startsWith("..")) {
-		logger.error(
-			"PATH",
-			"Search path escapes working directory sandbox",
-			undefined,
-			{ searchPath, relativePath },
-		);
-		throw new Error(
-			`Search path "${searchPath}" attempts to escape the working directory sandbox`,
-		);
-	}
-
-	const stats = await stat(resolvedPath);
-	if (!stats.isDirectory()) {
-		logger.error("TOOL", "Search path is not a directory", undefined, {
-			searchPath,
-		});
-		throw new Error(`Search path "${searchPath}" is not a directory`);
-	}
-
-	return resolvedPath;
+/**
+ * Robust search for context-aware grep.
+ * While streaming is good for huge files, contextAfter requires looking ahead or buffering.
+ * Since Librarian typically works on source code files (KB to low MBs), loading the file into memory
+ * but processing it efficiently is a balanced trade-off for accurate context.
+ */
+async function searchFileWithContext(
+  filePath: string,
+  regex: RegExp,
+  contextBefore = 0,
+  contextAfter = 0
+): Promise<SearchMatch[]> {
+  const file = Bun.file(filePath);
+  const stream = file.stream();
+  const reader = stream.getReader();
+  const decoder = new TextDecoder();
+
+  const matches: SearchMatch[] = [];
+  const beforeBuffer: string[] = [];
+  let currentLineNum = 1;
+  let partialLine = "";
+
+  // Track matches that are still waiting for their contextAfter lines
+  const pendingMatches: Array<{
+    match: SearchMatch;
+    linesRemaining: number;
+  }> = [];
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        if (partialLine) {
+          processLine(partialLine);
+        }
+        break;
+      }
+
+      const chunk = decoder.decode(value, { stream: true });
+      const lines = (partialLine + chunk).split("\n");
+      partialLine = lines.pop() || "";
+
+      for (const line of lines) {
+        processLine(line);
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  function processLine(line: string) {
+    // Update pending matches
+    for (const pending of pendingMatches) {
+      if (pending.linesRemaining > 0) {
+        pending.match.context?.after.push(line);
+        pending.linesRemaining--;
+      }
+    }
+
+    // Remove completed matches from pendingMatches to prevent memory leak
+    // This keeps the array small for files with many matches
+    for (let i = pendingMatches.length - 1; i >= 0; i--) {
+      const pending = pendingMatches[i];
+      if (pending && pending.linesRemaining === 0) {
+        pendingMatches.splice(i, 1);
+      }
+    }
+
+    // Check for new match
+    regex.lastIndex = 0;
+    const matchExec = regex.exec(line);
+
+    // We only take the first match on a line for context-aware grep to avoid duplication
+    // or complex context merging if there are multiple matches on the same line.
+    if (matchExec !== null) {
+      const searchMatch: SearchMatch = {
+        line: currentLineNum,
+        column: matchExec.index + 1,
+        text: line,
+      };
+
+      if (contextBefore > 0 || contextAfter > 0) {
+        const context: { before: string[]; after: string[] } = {
+          before: contextBefore > 0 ? [...beforeBuffer] : [],
+          after: [],
+        };
+        searchMatch.context = context;
+
+        if (contextAfter > 0) {
+          pendingMatches.push({
+            match: searchMatch,
+            linesRemaining: contextAfter,
+          });
+        }
+      }
+
+      matches.push(searchMatch);
+    }
+
+    // Update before buffer
+    if (contextBefore > 0) {
+      beforeBuffer.push(line);
+      if (beforeBuffer.length > contextBefore) {
+        beforeBuffer.shift();
+      }
+    }
+
+    currentLineNum++;
+  }
+
+  return matches;
 }
 
+// Find files matching a pattern in a directory using Bun.Glob
 async function findFilesToSearch(
-	resolvedPath: string,
-	patterns: string[],
-	recursive: boolean,
+  dirPath: string,
+  patterns: string[],
+  recursive: boolean,
+  includeHidden: boolean
 ): Promise<string[]> {
-	let filesToSearch: string[] = [];
-	for (const pattern of patterns) {
-		const foundFiles = await findFiles(resolvedPath, pattern, recursive);
-		filesToSearch = [...filesToSearch, ...foundFiles];
-	}
-
-	logger.debug("TOOL", "Files to search", {
-		count: filesToSearch.length,
-		patterns,
-	});
-
-	return filesToSearch;
-}
-
-function compileSearchRegex(query: string, regex: boolean, caseSensitive: boolean): RegExp {
-	const flags = caseSensitive ? "gm" : "gim";
-
-	if (regex) {
-		try {
-			const searchRegex = new RegExp(query, flags);
-			logger.debug("TOOL", "Regex pattern compiled", { query, flags });
-			return searchRegex;
-		} catch (e) {
-			logger.error(
-				"TOOL",
-				"Invalid regex pattern",
-				e instanceof Error ? e : new Error(String(e)),
-				{ query },
-			);
-			throw new Error(`Invalid regex pattern: ${(e as Error).message}`);
-		}
-	}
-
-	const escapedQuery = query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-	const searchRegex = new RegExp(escapedQuery, flags);
-	logger.debug("TOOL", "Escaped query compiled to regex", {
-		originalQuery: query,
-		flags,
-	});
-
-	return searchRegex;
-}
-
-async function performGrepSearch(
-	filesToSearch: string[],
-	searchRegex: RegExp,
-	maxResults: number,
-): Promise<Array<{ path: string; matches: Array<{ line: number; text: string; column: number; match: RegExpExecArray | null }> }>> {
-	const results: Array<{ path: string; matches: Array<{ line: number; text: string; column: number; match: RegExpExecArray | null }> }> = [];
-	let totalMatches = 0;
-
-	for (const file of filesToSearch) {
-		if (totalMatches >= maxResults) {
-			break;
-		}
-		if (await isTextFile(file)) {
-			try {
-				const content = await readFileContent(file);
-				const fileMatches = searchFileContent(content, searchRegex);
-				if (fileMatches.length > 0) {
-					const limitedMatches = fileMatches.slice(
-						0,
-						maxResults - totalMatches,
-					);
-					results.push({ path: file, matches: limitedMatches });
-					totalMatches += limitedMatches.length;
-				}
-			} catch {
-				// Silent fail for unreadable files
-			}
-		}
-	}
-
-	return results;
+  const foundFiles: string[] = [];
+  for (const pattern of patterns) {
+    const effectivePattern =
+      recursive && !pattern.includes("/") && !pattern.includes("**")
+        ? `**/${pattern}`
+        : pattern;
+
+    const glob = new Glob(effectivePattern);
+    for await (const file of glob.scan({
+      cwd: dirPath,
+      onlyFiles: true,
+      dot: includeHidden,
+    })) {
+      const fullPath = path.resolve(dirPath, file);
+      if (!foundFiles.includes(fullPath)) {
+        foundFiles.push(fullPath);
+      }
+    }
+  }
+  return foundFiles;
 }
 
-function formatGrepResults(
-	results: Array<{ path: string; matches: Array<{ line: number; text: string; column: number; match: RegExpExecArray | null }> }>,
-	query: string,
-): string {
-	if (results.length === 0) {
-		return `No matches found for query "${query}" in the searched files`;
-	}
-
-	const totalMatches = results.reduce((sum, r) => sum + r.matches.length, 0);
-	let output = `Found ${totalMatches} matches for query "${query}" in ${results.length} files:\n\n`;
-	for (const result of results) {
-		output += `File: ${result.path}\n`;
-		for (const match of result.matches) {
-			output += `  Line ${match.line}, Col ${match.column}: ${match.text}\n`;
-		}
-		output += "\n";
-	}
-
-	return output;
+function compileSearchRegex(
+  query: string,
+  regex: boolean,
+  caseSensitive: boolean
+): RegExp {
+  const flags = caseSensitive ? "gm" : "gim";
+
+  if (regex) {
+    try {
+      return new RegExp(query, flags);
+    } catch (e) {
+      throw new Error(`Invalid regex pattern: ${(e as Error).message}`);
+    }
+  }
+
+  const escapedQuery = query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  return new RegExp(escapedQuery, flags);
 }
 
 // Create the modernized tool using the tool() function
-export const grepContentTool = tool(
-	async (
-		{
-			searchPath = ".",
-			query,
-			patterns = ["*"],
-			caseSensitive = false,
-			regex = false,
-			recursive = true,
-			maxResults = 100,
-		},
-		config,
-	) => {
-		const timingId = logger.timingStart("grepContent");
-
-		logger.info("TOOL", "grep_content called", {
-			searchPath,
-			queryLength: query.length,
-			patterns,
-			caseSensitive,
-			regex,
-			recursive,
-			maxResults,
-		});
-
-		try {
-			const workingDir = config?.context?.workingDir;
-			if (!workingDir) {
-				throw new Error(
-					"Context with workingDir is required for file operations",
-				);
-			}
-			logger.debug("TOOL", "Working directory", {
-				workingDir: workingDir.replace(Bun.env.HOME || "", "~"),
-			});
-
-			if (!query) {
-				logger.error("TOOL", "Query parameter missing", undefined, {});
-				throw new Error('The "query" parameter is required');
-			}
-
-			const resolvedPath = await validateAndResolvePath(workingDir, searchPath);
-			const filesToSearch = await findFilesToSearch(resolvedPath, patterns, recursive);
-
-			const searchRegex = compileSearchRegex(query, regex, caseSensitive);
-			const results = await performGrepSearch(filesToSearch, searchRegex, maxResults);
-
-			logger.timingEnd(timingId, "TOOL", "grep_content completed");
-			logger.debug("TOOL", "Search completed", {
-				filesSearched: filesToSearch.length,
-				filesWithMatches: results.length,
-				totalMatches: results.reduce((sum, r) => sum + r.matches.length, 0),
-			});
-
-			return formatGrepResults(results, query);
-		} catch (error) {
-			logger.error(
-				"TOOL",
-				"grep_content failed",
-				error instanceof Error ? error : new Error(String(error)),
-				{ searchPath, query },
-			);
-			return `Error searching content: ${(error as Error).message}`;
-		}
-	},
-	{
-		name: "grep_content",
-		description: `A powerful search tool built on ripgrep
+export const grepTool = tool(
+  async (
+    {
+      searchPath = ".",
+      query,
+      patterns = ["*"],
+      caseSensitive = false,
+      regex = false,
+      recursive = true,
+      maxResults = 100,
+      contextBefore = 0,
+      contextAfter = 0,
+      exclude = [],
+      includeHidden = false,
+    },
+    config
+  ) => {
+    const timingId = logger.timingStart("grep");
+
+    logger.info("TOOL", "grep called", {
+      searchPath,
+      query,
+      patterns,
+      caseSensitive,
+      regex,
+      recursive,
+      maxResults,
+      contextBefore,
+      contextAfter,
+      exclude,
+      includeHidden,
+    });
+
+    try {
+      const workingDir = config?.context?.workingDir;
+      if (!workingDir) {
+        throw new Error(
+          "Context with workingDir is required for file operations"
+        );
+      }
+
+      if (!query) {
+        throw new Error('The "query" parameter is required');
+      }
+
+      // Validate path
+      const resolvedPath = path.resolve(workingDir, searchPath);
+      const resolvedWorkingDir = path.resolve(workingDir);
+      const relativePath = path.relative(resolvedWorkingDir, resolvedPath);
+
+      if (relativePath.startsWith("..")) {
+        throw new Error(
+          `Search path "${searchPath}" attempts to escape the working directory sandbox`
+        );
+      }
+
+      let stats: import("node:fs").Stats;
+      try {
+        stats = await stat(resolvedPath);
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+          throw new Error(`Search path "${searchPath}" does not exist`);
+        }
+        throw error;
+      }
+
+      if (!stats.isDirectory()) {
+        throw new Error(`Search path "${searchPath}" is not a directory`);
+      }
+
+      // Initialize GitIgnoreService
+      const gitignore = new GitIgnoreService(workingDir);
+      await gitignore.initialize();
+
+      const filesToSearch = await findFilesToSearch(
+        resolvedPath,
+        patterns,
+        recursive,
+        includeHidden
+      );
+
+      const searchRegex = compileSearchRegex(query, regex, caseSensitive);
+      const excludeGlobs = exclude.map((pattern) => new Glob(pattern));
+      const results: Array<{ path: string; matches: SearchMatch[] }> = [];
+      let totalMatches = 0;
+
+      for (const file of filesToSearch) {
+        if (totalMatches >= maxResults) {
+          break;
+        }
+
+        const relativeFileToWorkingDir = path.relative(workingDir, file);
+
+        // 1. Check if file should be ignored by .gitignore
+        if (gitignore.shouldIgnore(file, false)) {
+          continue;
+        }
+
+        // 2. Check if file should be excluded by manual exclude patterns
+        const isExcluded = excludeGlobs.some((eg) =>
+          eg.match(relativeFileToWorkingDir)
+        );
+        if (isExcluded) {
+          continue;
+        }
+
+        if (await isTextFile(file)) {
+          const fileMatches = await searchFileWithContext(
+            file,
+            searchRegex,
+            contextBefore,
+            contextAfter
+          );
+
+          if (fileMatches.length > 0) {
+            const remainingSlot = maxResults - totalMatches;
+            const limitedMatches = fileMatches.slice(0, remainingSlot);
+
+            results.push({
+              path: path.relative(workingDir, file),
+              matches: limitedMatches,
+            });
+            totalMatches += limitedMatches.length;
+          }
+        }
+      }
+
+      logger.timingEnd(timingId, "TOOL", "grep completed");
+
+      if (results.length === 0) {
+        return `No matches found for query "${query}" in the searched files`;
+      }
+
+      return formatSearchResults(results);
+    } catch (error) {
+      logger.error(
+        "TOOL",
+        "grep failed",
+        error instanceof Error ? error : new Error(String(error)),
+        { searchPath, query }
+      );
+
+      return formatToolError({
+        operation: "grep",
+        path: searchPath,
+        cause: error,
+        suggestion: getToolSuggestion("grep", searchPath),
+      });
+    }
+  },
+  {
+    name: "grep",
+    description: `A powerful search tool for finding text patterns in files.
 
 Usage:
-- ALWAYS use grep_content for search tasks.
+- ALWAYS use grep for search tasks.
 - Supports full regex syntax (e.g., "log.*Error", "function\\s+\\w+")
-- Filter files with glob parameter (e.g., "*.js", "**/*.tsx") or type parameter (e.g., "js", "py", "rust")
-- Pattern syntax: Uses ripgrep (not grep) - literal braces need escaping (use \`interface{}\` to find \`interface{}\` in Go code)
+- Filter files with glob parameter (e.g., "*.js", "**/*.tsx")
+- Pattern syntax: Uses JavaScript regex - literal braces need escaping (use \`interface{}\` to find \`interface{}\` in Go code)
 `,
-		schema: z.object({
-			searchPath: z.string().describe("The directory path to search in"),
-			query: z
-				.string()
-				.describe(
-					"The search query - the text or pattern to look for in files",
-				),
-			patterns: z
-				.array(z.string())
-				.optional()
-				.default(["*"])
-				.describe("File patterns to search in (e.g., ['*.js', '*.ts'])"),
-			caseSensitive: z
-				.boolean()
-				.optional()
-				.default(false)
-				.describe(
-					"Whether the search should be case-sensitive. Defaults to `false`",
-				),
-			regex: z
-				.boolean()
-				.optional()
-				.default(false)
-				.describe(
-					"Whether the query should be treated as a regular expression. Defaults to `false`",
-				),
-			recursive: z
-				.boolean()
-				.optional()
-				.default(true)
-				.describe(
-					"Whether to search recursively in subdirectories. Defaults to `true`",
-				),
-			maxResults: z
-				.number()
-				.optional()
-				.default(100)
-				.describe("Maximum number of matches to return. Defaults to 100"),
-		}),
-	},
+    schema: z.object({
+      searchPath: z.string().describe("The directory path to search in"),
+      query: z
+        .string()
+        .describe(
+          "The search query - the text or pattern to look for in files"
+        ),
+      patterns: z
+        .array(z.string())
+        .optional()
+        .default(["*"])
+        .describe("File patterns to search in (e.g., ['*.js', '*.ts'])"),
+      caseSensitive: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe(
+          "Whether the search should be case-sensitive. Defaults to `false`"
+        ),
+      regex: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe(
+          "Whether the query should be treated as a regular expression. Defaults to `false`"
+        ),
+      recursive: z
+        .boolean()
+        .optional()
+        .default(true)
+        .describe(
+          "Whether to search recursively in subdirectories. Defaults to `true`"
+        ),
+      maxResults: z
+        .number()
+        .optional()
+        .default(100)
+        .describe("Maximum number of matches to return. Defaults to 100"),
+      contextBefore: z
+        .number()
+        .optional()
+        .default(0)
+        .describe(
+          "Number of lines of context to show before each match. Defaults to 0"
+        ),
+      contextAfter: z
+        .number()
+        .optional()
+        .default(0)
+        .describe(
+          "Number of lines of context to show after each match. Defaults to 0"
+        ),
+      exclude: z
+        .array(z.string())
+        .optional()
+        .default([])
+        .describe(
+          "Array of glob patterns to exclude from results (e.g., ['dist/**', 'node_modules/**'])"
+        ),
+      includeHidden: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe(
+          "Whether to include hidden files and directories in the search. Defaults to `false`"
+        ),
+    }),
+  }
 );