@skroyc/librarian 0.1.0

package/src/tools/file-finding.tool.ts

@@ -0,0 +1,324 @@
+import { tool } from "langchain";
+import { z } from "zod";
+import fs from "node:fs/promises";
+import type { Dirent } from "node:fs";
+import path from "node:path";
+import { logger } from "../utils/logger.js";
+
+// Find files matching glob patterns in a directory
+async function findFiles(
+	searchPath: string,
+	patterns: string[],
+	options: {
+		exclude?: string[];
+		recursive?: boolean;
+		maxResults?: number;
+		includeHidden?: boolean;
+	} = {},
+): Promise<string[]> {
+	const {
+		exclude = [],
+		recursive = true,
+		maxResults = 1000,
+		includeHidden = false,
+	} = options;
+
+	const foundFiles: string[] = [];
+	const processedPaths = new Set<string>();
+
+	// Process each pattern
+	for (const pattern of patterns) {
+		// Only add **/ prefix if recursive and pattern doesn't already contain path separators
+		const effectivePattern =
+			recursive && !pattern.includes("**") && !pattern.includes("/")
+				? `**/${pattern}`
+				: pattern;
+
+		// Simple glob matching implementation
+		const matchingFiles = await simpleGlobSearch(searchPath, effectivePattern, {
+			recursive,
+			includeHidden,
+		});
+
+		for (const file of matchingFiles) {
+			if (foundFiles.length >= maxResults) {
+				break;
+			}
+
+			if (processedPaths.has(file)) {
+				continue;
+			}
+
+			processedPaths.add(file);
+
+			// Check if file should be excluded
+			const relativePath = path.relative(searchPath, file);
+			if (exclude.some((excl) => simpleMatch(relativePath, excl))) {
+				continue;
+			}
+
+			foundFiles.push(file);
+		}
+
+		if (foundFiles.length >= maxResults) {
+			break;
+		}
+	}
+
+	return foundFiles.slice(0, maxResults);
+}
+
+async function handleDirectoryEntry(
+	entry: Dirent,
+	basePath: string,
+	pattern: string,
+	options: { recursive: boolean; includeHidden: boolean },
+	results: string[],
+): Promise<void> {
+	const fullPath = path.join(basePath, entry.name);
+
+	if (pattern === "**" || pattern.includes("**") || options.recursive) {
+		const basePattern = pattern.split("/")[0] || pattern;
+		if (simpleMatch(entry.name, basePattern)) {
+			results.push(fullPath);
+		}
+
+		if (options.recursive) {
+			const subDirResults = await simpleGlobSearch(
+				fullPath,
+				pattern,
+				options,
+			);
+			results.push(...subDirResults);
+		}
+	}
+}
+
+function handleFileEntry(
+	entry: Dirent,
+	basePath: string,
+	pattern: string,
+	results: string[],
+): void {
+	const fullPath = path.join(basePath, entry.name);
+	const relativePath = path.relative(basePath, fullPath);
+	const basePattern = pattern.includes("**/")
+		? pattern.split("**/")[1] || ""
+		: pattern;
+	if (
+		simpleMatch(entry.name, basePattern) ||
+		simpleMatch(relativePath, basePattern)
+	) {
+		results.push(fullPath);
+	}
+}
+
+// Simple glob search implementation (basic pattern matching)
+async function simpleGlobSearch(
+	basePath: string,
+	pattern: string,
+	options: {
+		recursive: boolean;
+		includeHidden: boolean;
+	},
+): Promise<string[]> {
+	const results: string[] = [];
+
+	try {
+		const entries = await fs.readdir(basePath, { withFileTypes: true });
+
+		for (const entry of entries) {
+			if (!options.includeHidden && entry.name.startsWith(".")) {
+				continue;
+			}
+
+			if (entry.isDirectory()) {
+				await handleDirectoryEntry(entry, basePath, pattern, options, results);
+			} else if (entry.isFile()) {
+				handleFileEntry(entry, basePath, pattern, results);
+			}
+		}
+	} catch {
+		return [];
+	}
+
+	return results;
+}
+
+// Simple pattern matching (supports * and ? wildcards)
+function simpleMatch(str: string, pattern: string): boolean {
+	// Handle exact match first
+	if (pattern === str) {
+		return true;
+	}
+
+	// Convert glob pattern to regex
+	const regexPattern = pattern
+		.replace(/\./g, "\\.") // Escape dots
+		.replace(/\*/g, ".*") // Replace * with .*
+		.replace(/\?/g, "."); // Replace ? with .
+
+	const regex = new RegExp(`^${regexPattern}$`);
+	return regex.test(str);
+}
+
+// Create the modernized tool using the tool() function
+export const fileFindTool = tool(
+	async (
+		{
+			searchPath = ".",
+			patterns,
+			exclude = [],
+			recursive = true,
+			maxResults = 1000,
+			includeHidden = false,
+		},
+		config,
+	) => {
+		const timingId = logger.timingStart("fileFind");
+
+		logger.info("TOOL", "file_find called", {
+			searchPath,
+			patterns,
+			exclude,
+			recursive,
+			maxResults,
+			includeHidden,
+		});
+
+		try {
+			// Get working directory from config context - required for security
+			const workingDir = config?.context?.workingDir;
+			if (!workingDir) {
+				throw new Error(
+					"Context with workingDir is required for file operations",
+				);
+			}
+			logger.debug("TOOL", "Working directory", {
+				workingDir: workingDir.replace(Bun.env.HOME || "", "~"),
+			});
+
+			// Validate the path to prevent directory traversal
+			const resolvedPath = path.resolve(workingDir, searchPath);
+			const resolvedWorkingDir = path.resolve(workingDir);
+			const relativePath = path.relative(resolvedWorkingDir, resolvedPath);
+
+			logger.debug("TOOL", "Path validation", {
+				resolvedPath: resolvedPath.replace(Bun.env.HOME || "", "~"),
+				resolvedWorkingDir: resolvedWorkingDir.replace(Bun.env.HOME || "", "~"),
+				relativePath,
+				validated: !relativePath.startsWith(".."),
+			});
+
+			if (relativePath.startsWith("..")) {
+				logger.error(
+					"PATH",
+					"Search path escapes working directory sandbox",
+					undefined,
+					{ searchPath, relativePath },
+				);
+				throw new Error(
+					`Search path "${searchPath}" attempts to escape the working directory sandbox`,
+				);
+			}
+
+			// Validate that the search path exists and is a directory
+			const stats = await fs.stat(resolvedPath);
+			if (!stats.isDirectory()) {
+				logger.error("TOOL", "Search path is not a directory", undefined, {
+					searchPath,
+				});
+				throw new Error(`Search path "${searchPath}" is not a directory`);
+			}
+
+			// Find matching files
+			const foundFiles = await findFiles(resolvedPath, patterns || ["*"], {
+				exclude,
+				recursive,
+				maxResults,
+				includeHidden,
+			});
+
+			logger.timingEnd(timingId, "TOOL", "file_find completed");
+			logger.debug("TOOL", "File search successful", {
+				searchPath,
+				foundCount: foundFiles.length,
+				patterns,
+			});
+
+			if (foundFiles.length === 0) {
+				return `No files found matching patterns: ${patterns?.join(", ") || "*"}`;
+			}
+
+			// Format results
+			let output = `Found ${foundFiles.length} files matching patterns [${patterns?.join(", ") || "*"}]:\n\n`;
+
+			for (const file of foundFiles) {
+				const relativePath = path.relative(resolvedPath, file);
+				output += `${relativePath}\n`;
+			}
+
+			return output;
+		} catch (error) {
+			logger.error(
+				"TOOL",
+				"file_find failed",
+				error instanceof Error ? error : new Error(String(error)),
+				{ searchPath, patterns },
+			);
+			return `Error finding files: ${(error as Error).message}`;
+		}
+	},
+	{
+		name: "find_files",
+		description: `Discovers files using glob patterns. Respects .gitignore.
+Usage
+- Fast file pattern matching command that works with any codebase size
+- Supports glob patterns like "**/*.js" or "src/**/*.ts"
+- Returns matching file paths sorted
+- Use this command when you need to find files by name patterns
+- When you are doing an open ended search that may require multiple rounds of globbing and grepping, use the Task tool instead
+- You can call multiple commands in a single response. It is always better to speculatively perform multiple searches in parallel if they are potentially useful.
+`,
+		schema: z.object({
+			searchPath: z
+				.string()
+				.describe(
+					"The directory path to search in, relative to the working directory",
+				),
+			patterns: z
+				.array(z.string())
+				.describe(
+					"Array of glob patterns to match files (e.g., ['*.js', '*.ts'])",
+				),
+			exclude: z
+				.array(z.string())
+				.optional()
+				.default([])
+				.describe(
+					"Array of patterns to exclude from results. Defaults to none",
+				),
+			recursive: z
+				.boolean()
+				.optional()
+				.default(true)
+				.describe(
+					"Whether to search recursively in subdirectories. Defaults to `true`",
+				),
+			maxResults: z
+				.number()
+				.optional()
+				.default(100)
+				.describe(
+					"Maximum number of files to return. Maximum of 100 by default",
+				),
+			includeHidden: z
+				.boolean()
+				.optional()
+				.default(false)
+				.describe(
+					"Whether to include hidden files and directories. Defaults to `false`",
+				),
+		}),
+	},
+);

package/src/tools/file-listing.tool.ts

@@ -0,0 +1,212 @@
+import { tool } from "langchain";
+import { z } from "zod";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { logger } from "../utils/logger.js";
+
+// Define types for file system operations
+interface FileSystemEntry {
+	name: string;
+	path: string;
+	isDirectory: boolean;
+	size?: number | undefined;
+	modified?: Date | undefined;
+	permissions?: string | undefined;
+	lineCount?: number | undefined;
+}
+
+interface DirectoryListing {
+	entries: FileSystemEntry[];
+	totalCount: number;
+	filteredCount?: number;
+}
+
+// Simplified function to check if a file should be ignored (basic .gitignore handling)
+async function shouldIgnoreFile(filePath: string): Promise<boolean> {
+	const basename = path.basename(filePath);
+
+	// Basic checks for common files/directories to ignore
+	if (basename.startsWith(".") && basename !== ".env") {
+		// Check for common gitignored items
+		const ignoredItems = [
+			"node_modules",
+			".git",
+			".DS_Store",
+			".idea",
+			".vscode",
+			".pytest_cache",
+		];
+		if (ignoredItems.includes(basename)) {
+			return true;
+		}
+	}
+
+	// Check for common build/output directories
+	const commonBuildDirs = ["dist", "build", "out", "target", "coverage"];
+	if (
+		commonBuildDirs.includes(basename) &&
+		(await fs
+			.stat(filePath)
+			.then((s) => s.isDirectory())
+			.catch(() => false))
+	) {
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * List directory contents with metadata
+ */
+async function listDirectory(
+	dirPath: string,
+	includeHidden = false,
+): Promise<DirectoryListing> {
+	const entries = await fs.readdir(dirPath, { withFileTypes: true });
+	const fileEntries: FileSystemEntry[] = [];
+
+	for (const entry of entries) {
+		if (!includeHidden && entry.name.startsWith(".")) {
+			continue;
+		}
+
+		const fullPath = path.join(dirPath, entry.name);
+
+		try {
+			// Check if file should be ignored
+			if (!includeHidden && (await shouldIgnoreFile(fullPath))) {
+				continue;
+			}
+
+			const stats = await fs.stat(fullPath);
+			const name = path.basename(fullPath);
+
+			logger.debug("TOOL", "Processing file entry", {
+				name,
+				isDirectory: stats.isDirectory(),
+			});
+
+			const metadata: FileSystemEntry = {
+				name,
+				path: fullPath,
+				isDirectory: stats.isDirectory(),
+				size: stats.isFile() ? stats.size : undefined,
+				modified: stats.mtime,
+				permissions: stats.mode?.toString(8),
+			};
+
+			fileEntries.push(metadata);
+		} catch {
+			// Skip files that can't be accessed or have metadata errors
+		}
+	}
+
+	// Sort: directories first, then by name
+	fileEntries.sort((a, b) => {
+		if (a.isDirectory !== b.isDirectory) {
+			return a.isDirectory ? -1 : 1;
+		}
+		return a.name.localeCompare(b.name);
+	});
+
+	return {
+		entries: fileEntries,
+		totalCount: fileEntries.length,
+		filteredCount: fileEntries.length,
+	};
+}
+
+// Create the modernized tool using the tool() function
+export const fileListTool = tool(
+	async ({ directoryPath = ".", includeHidden = false }, config) => {
+		const timingId = logger.timingStart("fileList");
+
+		logger.info("TOOL", "file_list called", { directoryPath, includeHidden });
+
+		try {
+			// Get working directory from config context - required for security
+			const workingDir = config?.context?.workingDir;
+			if (!workingDir) {
+				throw new Error(
+					"Context with workingDir is required for file operations",
+				);
+			}
+			logger.debug("TOOL", "Working directory", {
+				workingDir: workingDir.replace(Bun.env.HOME || "", "~"),
+			});
+
+			// Validate path to prevent directory traversal
+			const resolvedPath = path.resolve(workingDir, directoryPath);
+			const resolvedWorkingDir = path.resolve(workingDir);
+			const relativePath = path.relative(resolvedWorkingDir, resolvedPath);
+
+			logger.debug("TOOL", "Path validation", {
+				resolvedPath: resolvedPath.replace(Bun.env.HOME || "", "~"),
+				resolvedWorkingDir: resolvedWorkingDir.replace(Bun.env.HOME || "", "~"),
+				relativePath,
+				validated: !relativePath.startsWith(".."),
+			});
+
+			// Check if resolved path escapes working directory
+			if (relativePath.startsWith("..")) {
+				logger.error(
+					"PATH",
+					"Directory path escapes working directory sandbox",
+					undefined,
+					{ directoryPath, relativePath },
+				);
+				throw new Error(
+					`Directory path "${directoryPath}" attempts to escape working directory sandbox`,
+				);
+			}
+
+			// List the directory
+			const listing = await listDirectory(resolvedPath, includeHidden);
+
+			// Format the result for the AI
+			let result = `Contents of directory: ${resolvedPath}\n\n`;
+			result += `Total entries: ${listing.totalCount}\n\n`;
+
+			for (const entry of listing.entries) {
+				const type = entry.isDirectory ? "DIR" : "FILE";
+				const size = entry.size ? ` (${entry.size} bytes)` : "";
+				const lineCount = entry.lineCount ? ` (${entry.lineCount} lines)` : "";
+				result += `${type}  ${entry.name}${size}${lineCount}\n`;
+			}
+
+			logger.timingEnd(timingId, "TOOL", "file_list completed");
+			logger.debug("TOOL", "Directory listing successful", {
+				directoryPath,
+				entryCount: listing.totalCount,
+				dirCount: listing.entries.filter((e) => e.isDirectory).length,
+				fileCount: listing.entries.filter((e) => !e.isDirectory).length,
+			});
+
+			return result;
+		} catch (error) {
+			logger.error(
+				"TOOL",
+				"list failed",
+				error instanceof Error ? error : new Error(String(error)),
+				{ directoryPath },
+			);
+			return `Error listing directory: ${(error as Error).message}`;
+		}
+	},
+	{
+		name: "list",
+		description:
+			"List the contents of a directory with metadata. Use this to understand the structure of a repository or directory.",
+		schema: z.object({
+			directoryPath: z.string().describe("The directory path to list"),
+			includeHidden: z
+				.boolean()
+				.optional()
+				.default(false)
+				.describe(
+					"Whether to include hidden files and directories. Defaults to `false`",
+				),
+		}),
+	},
+);

package/src/tools/file-reading.tool.ts

@@ -0,0 +1,154 @@
+import { tool } from "langchain";
+import { z } from "zod";
+import path from "node:path";
+import { logger } from "../utils/logger.js";
+import { isTextFile } from "../utils/file-utils.js";
+
+// Check if file is an image file
+function isImageFile(filePath: string): boolean {
+	const imageExtensions = new Set([
+		".png",
+		".jpg",
+		".jpeg",
+		".gif",
+		".bmp",
+		".webp",
+		".svg",
+		".ico",
+		".tiff",
+		".tif",
+	]);
+	const ext = path.extname(filePath).toLowerCase();
+	return imageExtensions.has(ext);
+}
+
+// Check if file is an audio file
+function isAudioFile(filePath: string): boolean {
+	const audioExtensions = new Set([
+		".mp3",
+		".wav",
+		".ogg",
+		".flac",
+		".m4a",
+		".aac",
+		".wma",
+		".opus",
+		".webm",
+	]);
+	const ext = path.extname(filePath).toLowerCase();
+	return audioExtensions.has(ext);
+}
+
+// Safe file read with encoding detection
+async function readFileContent(filePath: string): Promise<string> {
+	try {
+		return await Bun.file(filePath).text();
+	} catch (error: unknown) {
+		const errorMessage =
+			error instanceof Error ? error.message : "Unknown error";
+		throw new Error(`Failed to read file: ${errorMessage}`);
+	}
+}
+
+// Create the modernized tool using the tool() function
+export const fileReadTool = tool(
+	async ({ filePath }, config) => {
+		const timingId = logger.timingStart("fileRead");
+
+		logger.info("TOOL", "file_read called", { filePath });
+
+		try {
+			// Get working directory from config context - required for security
+			const workingDir = config?.context?.workingDir;
+			if (!workingDir) {
+				throw new Error(
+					"Context with workingDir is required for file operations",
+				);
+			}
+			logger.debug("TOOL", "Working directory", {
+				workingDir: workingDir.replace(Bun.env.HOME || "", "~"),
+			});
+
+			// Validate the path to prevent directory traversal
+			const resolvedPath = path.resolve(workingDir, filePath);
+			const resolvedWorkingDir = path.resolve(workingDir);
+			const relativePath = path.relative(resolvedWorkingDir, resolvedPath);
+
+			logger.debug("TOOL", "Path validation", {
+				resolvedPath: resolvedPath.replace(Bun.env.HOME || "", "~"),
+				resolvedWorkingDir: resolvedWorkingDir.replace(Bun.env.HOME || "", "~"),
+				relativePath,
+				validated: !relativePath.startsWith(".."),
+			});
+
+			if (relativePath.startsWith("..")) {
+				logger.error(
+					"PATH",
+					"File path escapes working directory sandbox",
+					undefined,
+					{ filePath, relativePath },
+				);
+				throw new Error(
+					`File path "${filePath}" attempts to escape the working directory sandbox`,
+				);
+			}
+
+			// Check if it's a media file
+			if (isImageFile(resolvedPath) || isAudioFile(resolvedPath)) {
+				logger.debug("TOOL", "File is media file", {
+					filePath,
+					fileType: isImageFile(resolvedPath) ? "image" : "audio",
+				});
+				return `This is a media file (${isImageFile(resolvedPath) ? "image" : "audio"}). Media files cannot be read as text. Path: ${filePath}`;
+			}
+
+			// Check if it's a text file
+			const isText = await isTextFile(resolvedPath);
+			if (!isText) {
+				logger.debug("TOOL", "File is not a text file", { filePath });
+				return `This file is not a text file and cannot be read as text. Path: ${filePath}`;
+			}
+
+			logger.debug("TOOL", "File type validated as text", { filePath });
+
+			// Read file content
+			const content = await readFileContent(resolvedPath);
+
+			// Limit content size to avoid overwhelming the AI
+			const maxContentLength = 50_000; // 50KB limit
+			if (content.length > maxContentLength) {
+				logger.debug("TOOL", "File content truncated", {
+					filePath,
+					originalSize: content.length,
+					maxSize: maxContentLength,
+				});
+				logger.timingEnd(timingId, "TOOL", "file_read completed (truncated)");
+				return `File content is too large (${content.length} characters). First ${maxContentLength} characters:\n\n${content.substring(0, maxContentLength)}\n\n[Content truncated due to length]`;
+			}
+
+			logger.timingEnd(timingId, "TOOL", "file_read completed");
+			logger.debug("TOOL", "File read successful", {
+				filePath,
+				contentLength: content.length,
+			});
+
+			return `Content of file: ${filePath}\n\n${content}`;
+		} catch (error) {
+			logger.error(
+				"TOOL",
+				"file_read failed",
+				error instanceof Error ? error : new Error(String(error)),
+				{ filePath },
+			);
+			return `Error reading file: ${(error as Error).message}`;
+		}
+	},
+	{
+		name: "file_read",
+		description:
+			"Read the contents of a file. Use this to examine the content of a specific file.",
+		schema: z.object({
+			filePath: z.string().describe("The path to the file to read"),
+		}),
+	},
+);