@skroyc/librarian 0.1.0

package/src/tools/grep-content.tool.ts

@@ -0,0 +1,325 @@
+import { tool } from "langchain";
+import { z } from "zod";
+import { readdir, stat } from "node:fs/promises";
+import path from "node:path";
+import { logger } from "../utils/logger.js";
+import { isTextFile } from "../utils/file-utils.js";
+
+// Safe file read with encoding detection
+async function readFileContent(filePath: string): Promise<string> {
+	try {
+		return await Bun.file(filePath).text();
+	} catch (error: unknown) {
+		const errorMessage =
+			error instanceof Error ? error.message : "Unknown error";
+		throw new Error(`Failed to read file: ${errorMessage}`);
+	}
+}
+
+// Search for content in a single file
+function searchFileContent(content: string, regex: RegExp) {
+	const lines = content.split("\n");
+	const matches: Array<{ line: number; text: string; column: number; match: RegExpExecArray | null }> = [];
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i];
+		if (line === undefined) {
+			continue;
+		}
+
+		const match = regex.exec(line);
+
+		if (match) {
+			matches.push({
+				line: i + 1,
+				column: match.index + 1,
+				text: line,
+				match,
+			});
+			regex.lastIndex = 0;
+		}
+	}
+
+	return matches;
+}
+
+// Find files matching a pattern in a directory
+async function findFiles(
+	dirPath: string,
+	pattern: string,
+	recursive = true,
+): Promise<string[]> {
+	const foundFiles: string[] = [];
+
+	const entries = await readdir(dirPath, { withFileTypes: true });
+
+	for (const entry of entries) {
+		const fullPath = path.join(dirPath, entry.name);
+
+		if (entry.isDirectory()) {
+			if (recursive) {
+				const subDirFiles = await findFiles(fullPath, pattern, recursive);
+				foundFiles.push(...subDirFiles);
+			}
+		} else if (entry.isFile() && (pattern === "*" || entry.name.includes(pattern.replace(/\*/g, "")))) {
+			foundFiles.push(fullPath);
+		}
+	}
+
+	return foundFiles;
+}
+
+async function validateAndResolvePath(workingDir: string, searchPath: string): Promise<string> {
+	const resolvedPath = path.resolve(workingDir, searchPath);
+	const resolvedWorkingDir = path.resolve(workingDir);
+	const relativePath = path.relative(resolvedWorkingDir, resolvedPath);
+
+	logger.debug("TOOL", "Path validation", {
+		resolvedPath: resolvedPath.replace(Bun.env.HOME || "", "~"),
+		resolvedWorkingDir: resolvedWorkingDir.replace(Bun.env.HOME || "", "~"),
+		relativePath,
+		validated: !relativePath.startsWith(".."),
+	});
+
+	if (relativePath.startsWith("..")) {
+		logger.error(
+			"PATH",
+			"Search path escapes working directory sandbox",
+			undefined,
+			{ searchPath, relativePath },
+		);
+		throw new Error(
+			`Search path "${searchPath}" attempts to escape the working directory sandbox`,
+		);
+	}
+
+	const stats = await stat(resolvedPath);
+	if (!stats.isDirectory()) {
+		logger.error("TOOL", "Search path is not a directory", undefined, {
+			searchPath,
+		});
+		throw new Error(`Search path "${searchPath}" is not a directory`);
+	}
+
+	return resolvedPath;
+}
+
+async function findFilesToSearch(
+	resolvedPath: string,
+	patterns: string[],
+	recursive: boolean,
+): Promise<string[]> {
+	let filesToSearch: string[] = [];
+	for (const pattern of patterns) {
+		const foundFiles = await findFiles(resolvedPath, pattern, recursive);
+		filesToSearch = [...filesToSearch, ...foundFiles];
+	}
+
+	logger.debug("TOOL", "Files to search", {
+		count: filesToSearch.length,
+		patterns,
+	});
+
+	return filesToSearch;
+}
+
+function compileSearchRegex(query: string, regex: boolean, caseSensitive: boolean): RegExp {
+	const flags = caseSensitive ? "gm" : "gim";
+
+	if (regex) {
+		try {
+			const searchRegex = new RegExp(query, flags);
+			logger.debug("TOOL", "Regex pattern compiled", { query, flags });
+			return searchRegex;
+		} catch (e) {
+			logger.error(
+				"TOOL",
+				"Invalid regex pattern",
+				e instanceof Error ? e : new Error(String(e)),
+				{ query },
+			);
+			throw new Error(`Invalid regex pattern: ${(e as Error).message}`);
+		}
+	}
+
+	const escapedQuery = query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	const searchRegex = new RegExp(escapedQuery, flags);
+	logger.debug("TOOL", "Escaped query compiled to regex", {
+		originalQuery: query,
+		flags,
+	});
+
+	return searchRegex;
+}
+
+async function performGrepSearch(
+	filesToSearch: string[],
+	searchRegex: RegExp,
+	maxResults: number,
+): Promise<Array<{ path: string; matches: Array<{ line: number; text: string; column: number; match: RegExpExecArray | null }> }>> {
+	const results: Array<{ path: string; matches: Array<{ line: number; text: string; column: number; match: RegExpExecArray | null }> }> = [];
+	let totalMatches = 0;
+
+	for (const file of filesToSearch) {
+		if (totalMatches >= maxResults) {
+			break;
+		}
+		if (await isTextFile(file)) {
+			try {
+				const content = await readFileContent(file);
+				const fileMatches = searchFileContent(content, searchRegex);
+				if (fileMatches.length > 0) {
+					const limitedMatches = fileMatches.slice(
+						0,
+						maxResults - totalMatches,
+					);
+					results.push({ path: file, matches: limitedMatches });
+					totalMatches += limitedMatches.length;
+				}
+			} catch {
+				// Silent fail for unreadable files
+			}
+		}
+	}
+
+	return results;
+}
+
+function formatGrepResults(
+	results: Array<{ path: string; matches: Array<{ line: number; text: string; column: number; match: RegExpExecArray | null }> }>,
+	query: string,
+): string {
+	if (results.length === 0) {
+		return `No matches found for query "${query}" in the searched files`;
+	}
+
+	const totalMatches = results.reduce((sum, r) => sum + r.matches.length, 0);
+	let output = `Found ${totalMatches} matches for query "${query}" in ${results.length} files:\n\n`;
+	for (const result of results) {
+		output += `File: ${result.path}\n`;
+		for (const match of result.matches) {
+			output += `  Line ${match.line}, Col ${match.column}: ${match.text}\n`;
+		}
+		output += "\n";
+	}
+
+	return output;
+}
+
+// Create the modernized tool using the tool() function
+export const grepContentTool = tool(
+	async (
+		{
+			searchPath = ".",
+			query,
+			patterns = ["*"],
+			caseSensitive = false,
+			regex = false,
+			recursive = true,
+			maxResults = 100,
+		},
+		config,
+	) => {
+		const timingId = logger.timingStart("grepContent");
+
+		logger.info("TOOL", "grep_content called", {
+			searchPath,
+			queryLength: query.length,
+			patterns,
+			caseSensitive,
+			regex,
+			recursive,
+			maxResults,
+		});
+
+		try {
+			const workingDir = config?.context?.workingDir;
+			if (!workingDir) {
+				throw new Error(
+					"Context with workingDir is required for file operations",
+				);
+			}
+			logger.debug("TOOL", "Working directory", {
+				workingDir: workingDir.replace(Bun.env.HOME || "", "~"),
+			});
+
+			if (!query) {
+				logger.error("TOOL", "Query parameter missing", undefined, {});
+				throw new Error('The "query" parameter is required');
+			}
+
+			const resolvedPath = await validateAndResolvePath(workingDir, searchPath);
+			const filesToSearch = await findFilesToSearch(resolvedPath, patterns, recursive);
+
+			const searchRegex = compileSearchRegex(query, regex, caseSensitive);
+			const results = await performGrepSearch(filesToSearch, searchRegex, maxResults);
+
+			logger.timingEnd(timingId, "TOOL", "grep_content completed");
+			logger.debug("TOOL", "Search completed", {
+				filesSearched: filesToSearch.length,
+				filesWithMatches: results.length,
+				totalMatches: results.reduce((sum, r) => sum + r.matches.length, 0),
+			});
+
+			return formatGrepResults(results, query);
+		} catch (error) {
+			logger.error(
+				"TOOL",
+				"grep_content failed",
+				error instanceof Error ? error : new Error(String(error)),
+				{ searchPath, query },
+			);
+			return `Error searching content: ${(error as Error).message}`;
+		}
+	},
+	{
+		name: "grep_content",
+		description: `A powerful search tool built on ripgrep
+
+Usage:
+- ALWAYS use grep_content for search tasks.
+- Supports full regex syntax (e.g., "log.*Error", "function\\s+\\w+")
+- Filter files with glob parameter (e.g., "*.js", "**/*.tsx") or type parameter (e.g., "js", "py", "rust")
+- Pattern syntax: Uses ripgrep (not grep) - literal braces need escaping (use \`interface{}\` to find \`interface{}\` in Go code)
+`,
+		schema: z.object({
+			searchPath: z.string().describe("The directory path to search in"),
+			query: z
+				.string()
+				.describe(
+					"The search query - the text or pattern to look for in files",
+				),
+			patterns: z
+				.array(z.string())
+				.optional()
+				.default(["*"])
+				.describe("File patterns to search in (e.g., ['*.js', '*.ts'])"),
+			caseSensitive: z
+				.boolean()
+				.optional()
+				.default(false)
+				.describe(
+					"Whether the search should be case-sensitive. Defaults to `false`",
+				),
+			regex: z
+				.boolean()
+				.optional()
+				.default(false)
+				.describe(
+					"Whether the query should be treated as a regular expression. Defaults to `false`",
+				),
+			recursive: z
+				.boolean()
+				.optional()
+				.default(true)
+				.describe(
+					"Whether to search recursively in subdirectories. Defaults to `true`",
+				),
+			maxResults: z
+				.number()
+				.optional()
+				.default(100)
+				.describe("Maximum number of matches to return. Defaults to 100"),
+		}),
+	},
+);

package/src/utils/file-utils.ts

@@ -0,0 +1,39 @@
+/**
+ * File utility functions
+ * Shared utilities for file type detection and file operations
+ */
+
+import path from 'node:path';
+
+/**
+ * Check if a file is text-based
+ * Uses extension detection and binary content detection
+ */
+export async function isTextFile(filePath: string): Promise<boolean> {
+  const textExtensions = new Set([
+    '.txt', '.js', '.ts', '.jsx', '.tsx', '.json', '.yaml', '.yml', '.md',
+    '.html', '.htm', '.css', '.scss', '.sass', '.less', '.py', '.rb', '.java',
+    '.cpp', '.c', '.h', '.hpp', '.go', '.rs', '.php', '.sql', '.xml', '.csv',
+    '.toml', '.lock', '.sh', '.bash', '.zsh', '.env', '.dockerfile', 'dockerfile',
+    '.gitignore', '.npmrc', '.prettierrc', '.eslintrc', '.editorconfig', '.jsonc'
+  ]);
+
+  const ext = path.extname(filePath).toLowerCase();
+  if (textExtensions.has(ext)) {
+    return true;
+  }
+
+   // For files without extensions or unknown extensions, check for null bytes
+   try {
+     const buffer = await Bun.file(filePath).arrayBuffer();
+     const uint8Array = new Uint8Array(buffer);
+     for (let i = 0; i < Math.min(512, uint8Array.length); i++) {
+       if (uint8Array[i] === 0) {
+         return false;
+       }
+     }
+     return true;
+   } catch {
+     return false;
+   }
+}

package/src/utils/logger.ts

@@ -0,0 +1,295 @@
+/**
+ * Logger utility for librarian CLI
+ * Writes all logs to timestamped files in ~/.config/librarian/logs/
+ * Silent failure on write errors
+ * Metadata-only logging (no sensitive data)
+ */
+
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import type { FileSink } from "bun";
+
+export type LogLevel = 'INFO' | 'DEBUG' | 'WARN' | 'ERROR';
+export type LogComponent = 'CLI' | 'CONFIG' | 'LIBRARIAN' | 'AGENT' | 'TOOL' | 'LLM' | 'GIT' | 'PATH' | 'LOGGER' | 'TIMING';
+
+interface LogMetadata {
+  [key: string]: unknown;
+}
+
+interface TimingOperation {
+  operation: string;
+  startTime: number;
+}
+
+class Logger {
+  private static instance: Logger;
+  private writer: FileSink | null = null;
+  private debugMode = false;
+  private readonly timingOperations: Map<string, TimingOperation> = new Map();
+
+  private constructor() {
+    this.initializeLogFile().catch(() => {
+      // Silent - do nothing on initialization errors
+    });
+  }
+
+  static getInstance(): Logger {
+    if (!Logger.instance) {
+      Logger.instance = new Logger();
+    }
+    return Logger.instance;
+  }
+
+  setDebugMode(enabled: boolean): void {
+    this.debugMode = enabled;
+  }
+
+  /**
+    * Initialize the log file with timestamp
+    * Format: ~/.config/librarian/logs/YYYY-MM-DD_HH-MM-SS_mmm-librarian.log
+    */
+  private async initializeLogFile(): Promise<void> {
+    try {
+      // Create timestamp for filename
+      const now = new Date();
+      const isoString = now.toISOString();
+
+      if (!isoString) {
+        throw new Error('Failed to generate timestamp');
+      }
+
+      // Create timestamp: YYYY-MM-DD_HH-MM-SS
+      const year = now.getFullYear();
+      const month = String(now.getMonth() + 1).padStart(2, '0');
+      const day = String(now.getDate()).padStart(2, '0');
+      const hours = String(now.getHours()).padStart(2, '0');
+      const minutes = String(now.getMinutes()).padStart(2, '0');
+      const seconds = String(now.getSeconds()).padStart(2, '0');
+      const ms = String(now.getMilliseconds()).padStart(3, '0');
+
+      const timestamp = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}_${ms}`;
+
+      // Log directory
+      const logDir = path.join(os.homedir(), '.config', 'librarian', 'logs');
+
+       // Ensure directory exists
+       if (!(await Bun.file(logDir).exists())) {
+         await mkdir(logDir, { recursive: true });
+       }
+
+      // Log filename
+      const logFilename = `${timestamp}-librarian.log`;
+      const logPath = path.join(logDir, logFilename);
+
+       // Create write stream
+       const logFile = Bun.file(logPath);
+       this.writer = logFile.writer();
+
+       // Log initialization
+       this.info('LOGGER', `Logging initialized: ${logPath}`);
+
+    } catch {
+      // Silent - do nothing on initialization errors
+    }
+  }
+
+  /**
+   * Format timestamp: YYYY-MM-DD HH:MM:SS.mmm
+   */
+  private formatTimestamp(): string {
+    const now = new Date();
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, '0');
+    const day = String(now.getDate()).padStart(2, '0');
+    const hours = String(now.getHours()).padStart(2, '0');
+    const minutes = String(now.getMinutes()).padStart(2, '0');
+    const seconds = String(now.getSeconds()).padStart(2, '0');
+    const ms = String(now.getMilliseconds()).padStart(3, '0');
+
+    return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${ms}`;
+  }
+
+  /**
+   * Redact sensitive data from metadata
+   */
+  private redactMetadata(metadata: LogMetadata): LogMetadata {
+    const redacted: LogMetadata = {};
+
+    for (const [key, value] of Object.entries(metadata)) {
+      redacted[key] = this.getRedactedValue(key, value);
+    }
+
+    return redacted;
+  }
+
+  private getRedactedValue(key: string, value: unknown): unknown {
+    const lowerKey = key.toLowerCase();
+
+    if (['apiKey', 'token', 'secret', 'password'].includes(lowerKey)) {
+      return '[REDACTED]';
+    }
+
+    if ((key === 'query' || key === 'content') && typeof value === 'string') {
+      return value.length;
+    }
+
+    if ((key === 'repoUrl' || key === 'baseURL') && typeof value === 'string') {
+      return this.redactUrl(value);
+    }
+
+    if (key === 'workingDir' && typeof value === 'string') {
+      return value.replace(os.homedir(), '~');
+    }
+
+    if (typeof value === 'string' && (value.includes(os.homedir()) || value.includes('/home/'))) {
+      return value.replace(os.homedir(), '~');
+    }
+
+    return value;
+  }
+
+  private redactUrl(value: string): string {
+    try {
+      const url = new URL(value);
+      return url.hostname;
+    } catch {
+      return '[INVALID_URL]';
+    }
+  }
+
+  /**
+   * Format log entry
+   */
+  private formatLogEntry(level: LogLevel, component: LogComponent, message: string, metadata?: LogMetadata): string {
+    const timestamp = this.formatTimestamp();
+    let entry = `[${timestamp}] [${level}] [${component}] ${message}`;
+
+    if (metadata && Object.keys(metadata).length > 0) {
+      const redactedMetadata = this.redactMetadata(metadata);
+      entry += ` | ${JSON.stringify(redactedMetadata)}`;
+    }
+
+    return entry;
+  }
+
+   /**
+    * Write log entry to file
+    */
+   private writeLog(entry: string): void {
+     try {
+       if (this.writer) {
+         this.writer.write(`${entry}\n`);
+       }
+     } catch {
+       // Silent - do nothing on write errors
+     }
+   }
+
+  /**
+   * Internal logging method
+   */
+  private log(level: LogLevel, component: LogComponent, message: string, metadata?: LogMetadata): void {
+    try {
+      const entry = this.formatLogEntry(level, component, message, metadata);
+      this.writeLog(entry);
+    } catch {
+      // Silent - do nothing on log errors
+    }
+  }
+
+  /**
+   * INFO level - Always logged
+   */
+  info(component: LogComponent, message: string, metadata?: LogMetadata): void {
+    this.log('INFO', component, message, metadata);
+  }
+
+  /**
+   * DEBUG level - Only logged when debug mode is enabled
+   */
+  debug(component: LogComponent, message: string, metadata?: LogMetadata): void {
+    if (this.debugMode) {
+      this.log('DEBUG', component, message, metadata);
+    }
+  }
+
+  /**
+   * WARN level - Always logged
+   */
+  warn(component: LogComponent, message: string, metadata?: LogMetadata): void {
+    this.log('WARN', component, message, metadata);
+  }
+
+  /**
+   * ERROR level - Always logged with stack trace
+   */
+  error(component: LogComponent, message: string, error?: Error, metadata?: LogMetadata): void {
+    const errorMetadata: LogMetadata = metadata ? { ...metadata } : {};
+
+    if (error) {
+      errorMetadata.errorName = error.name;
+      errorMetadata.errorMessage = error.message;
+
+      // Include stack trace in metadata
+      if (error.stack) {
+        errorMetadata.stack = error.stack;
+      }
+    }
+
+    this.log('ERROR', component, message, errorMetadata);
+  }
+
+  /**
+   * Start timing an operation
+   * Returns an operation ID for later timingEnd call
+   */
+  timingStart(operation: string): string {
+    const operationId = `${operation}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+    this.timingOperations.set(operationId, {
+      operation,
+      startTime: performance.now()
+    });
+
+    this.debug('TIMING', `Started: ${operation}`);
+
+    return operationId;
+  }
+
+  /**
+   * End timing an operation
+   */
+  timingEnd(operationId: string, component: LogComponent, message?: string): void {
+    const timing = this.timingOperations.get(operationId);
+
+    if (timing) {
+      const duration = performance.now() - timing.startTime;
+      const durationMs = Math.round(duration);
+
+      this.timingOperations.delete(operationId);
+
+      const logMessage = message || `Completed: ${timing.operation}`;
+      this.info(component, logMessage, { duration: `${durationMs}ms` });
+
+      if (this.debugMode) {
+        this.debug('TIMING', `Ended: ${timing.operation}`, { durationMs });
+      }
+    }
+  }
+
+   /**
+    * Clean up resources
+    */
+   close(): void {
+     try {
+       if (this.writer) {
+         this.writer.end();
+       }
+     } catch {
+       // Silent
+     }
+   }
+}
+
+// Export singleton instance
+export const logger = Logger.getInstance();

package/src/utils/path-utils.ts

@@ -0,0 +1,17 @@
+/**
+ * Path utility functions
+ * Shared utilities for path manipulation
+ */
+
+import path from 'node:path';
+import os from 'node:os';
+
+/**
+ * Expand tilde (~) in file paths to home directory
+ */
+export function expandTilde(filePath: string): string {
+  if (filePath.startsWith('~/')) {
+    return path.join(os.homedir(), filePath.slice(2));
+  }
+  return filePath;
+}

package/tsconfig.json

@@ -0,0 +1,37 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "removeComments": true,
+    "noImplicitAny": true,
+    "strictNullChecks": true,
+    "strictFunctionTypes": true,
+    "noImplicitThis": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedIndexedAccess": true,
+    "noImplicitOverride": true,
+    "allowUnreachableCode": false,
+    "allowUnusedLabels": false,
+    "exactOptionalPropertyTypes": true
+  },
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "tests"
+  ]
+}