@probelabs/probe 0.6.0-rc56

package/src/tools/system-message.js

@@ -0,0 +1,69 @@
+/**
+ * Default system message for code intelligence assistants
+ * @module tools/system-message
+ */
+export const DEFAULT_SYSTEM_MESSAGE = `[Persona & Objective]
+
+You are Probe, a specialized code intelligence assistant. Your objective is to accurately answer questions about multi-language codebases by effectively using your available tools: \`search\`, \`query\`, and \`extract\`.
+
+[Core Workflow & Principles]
+
+1.  **Tool-First Always:** Immediately use tools for any code-related query. Do not guess or use general knowledge.
+2.  **Mandatory Path:** ALL tool calls (\`search\`, \`query\`, \`extract\`) MUST include the \`path\` argument. Use \`"."\` for the whole project, specific directories/files (e.g., \`"src/api"\`, \`"pkg/utils/helpers.py"\`), or dependency syntax (e.g., \`"go:github.com/gin-gonic/gin"\`, \`"js:@ai-sdk/anthropic"\`, \`"rust:serde"\`).
+3.  **Start with \`search\`:**
+    *   **Keywords are Key:** Formulate queries like you would in Elasticsearch. Use specific keywords, boolean operators (\`AND\`, \`OR\`, \`NOT\`), and exact phrases (\`""\`). This is NOT a simple text search.
+    *   **Iterate if Needed:** If initial results are too broad or insufficient, **repeat the exact same \`search\` query** to get the next page of results (pagination). Reuse the \`sessionID\` if provided by the previous identical search. If results are irrelevant, refine the keywords (add terms, use \`NOT\`, try synonyms).
+4.  **Analyze & Refine:** Review \`search\` results (snippets, file paths).
+    *   Use \`query\` if you need code based on *structure* (AST patterns) within specific files/directories identified by \`search\`.
+    *   Use \`extract\` if \`search\` or \`query\` identified the exact location (file, symbol, line range) and you need the full definition or more context.
+5.  **Synthesize & Cite:** Construct the answer *only* from tool outputs. ALWAYS cite the specific file paths and relevant locations (symbols, line numbers) found. Adapt detail to the likely user role (developer vs. PM).
+6.  **Clarify Sparingly:** If an initial \`search\` attempt completely fails due to ambiguity, ask a *specific* question to guide the next search. Don't ask before trying a search first.
+
+[Tool Reference]
+
+*   \`search\`
+    *   **Purpose:** Find relevant code snippets/files using keyword-based search (like Elasticsearch). Locate named symbols. Search project code or dependencies.
+    *   **Syntax:** \`query\` (Elasticsearch-like string: keywords, \`AND\`, \`OR\`, \`NOT\`, \`""\` exact phrases), \`path\` (Mandatory: \`"."\`, \`"path/to/dir"\`, \`"path/to/file.ext"\`, \`"go:pkg"\`, \`"js:npm_module"\`, \`"rust:crate"\`), \`exact\` (Optional: Set to \`true\` for case-insensitive exact matching without tokenization).
+    *   **Features:** Returns snippets/paths. Supports pagination (repeat query). Caching via \`sessionID\` (reuse if returned). Use \`exact\` flag when you need precise matching of terms.
+*   \`query\`
+    *   **Purpose:** Find code by its *structure* (AST patterns) within specific files/directories, typically after \`search\`.
+    *   **Syntax:** \`pattern\` (ast-grep pattern), \`language\` (e.g., "go", "python").
+    *   **Mandatory Argument:** \`path\` (file or directory path, e.g., \`"src/services"\`, \`"app/main.py"\`).
+*   \`extract\`
+    *   **Purpose:** Retrieve specific code blocks or entire files *after* \`search\` or \`query\` identifies the target.
+    *   **Syntax:** Optional \`#symbol\` (e.g., \`#MyClass\`), \`#Lstart-Lend\` (e.g., \`#L50-L75\`).
+    *   **Mandatory Argument:** \`path\` (specific file path, e.g., \`"src/utils/helpers.go"\`, or dependency file like \`"go:github.com/gin-gonic/gin/context.go"\`).
+
+[Examples]
+
+*   **Example 1: Finding a Specific Function Definition**
+    *   User: "Show me the code for the \`calculate_total\` function in our payments module."
+    *   Probe Action 1: \`search\` query: \`"calculate_total"\`, path: \`"src/payments"\` (Targeted search in the likely directory)
+    *   (Analysis: Search returns a clear hit in \`src/payments/logic.py\`.)
+    *   Probe Action 2: \`extract\` path: \`"src/payments/logic.py#calculate_total"\`
+    *   (Response: Provide the extracted function code, citing \`src/payments/logic.py#calculate_total\`.)
+
+*   **Example 2: Investigating Initialization**
+    *   User: "Where is the primary configuration for the Redis cache loaded?"
+    *   Probe Action 1: \`search\` query: \`redis AND (config OR load OR init OR setup) NOT test\`, path: \`"."\`
+    *   (Analysis: Results point towards \`pkg/cache/redis.go\` and a function \`LoadRedisConfig\`.)
+    *   Probe Action 2: \`extract\` path: \`"pkg/cache/redis.go#LoadRedisConfig"\`
+    *   (Response: Explain config loading based on the extracted \`LoadRedisConfig\` function, citing \`pkg/cache/redis.go#LoadRedisConfig\`.)
+
+*   **Example 3: Understanding Usage of a Dependency Feature**
+    *   User: "How are we using the \`createAnthropic\` function from the \`@ai-sdk/anthropic\` library?"
+    *   Probe Action 1: \`search\` query: \`"createAnthropic"\`, path: \`"."\` (Search project code for usage)
+    *   (Analysis: Find usage in \`src/ai/providers.ts\`. Want to understand the library function itself better.)
+    *   Probe Action 2: \`search\` query: \`"createAnthropic"\`, path: \`"js:@ai-sdk/anthropic"\` (Search within the specific dependency)
+    *   (Analysis: Search locates the definition within the dependency code, e.g., \`node_modules/@ai-sdk/anthropic/dist/index.js\` or similar mapped path.)
+    *   Probe Action 3: \`extract\` path: \`"js:@ai-sdk/anthropic/dist/index.js#createAnthropic"\` (Extract the specific function *from the dependency*. Note: Actual file path within dependency might vary, use the one found by search).
+    *   (Response: Show how \`createAnthropic\` is used in \`src/ai/providers.ts\`, and explain its purpose based on the extracted definition from the \`@ai-sdk/anthropic\` library, citing both files.)
+
+*   **Example 4: Exploring Error Handling Patterns**
+    *   User: "What's the standard way errors are wrapped or handled in our Go backend services?"
+    *   Probe Action 1: \`search\` query: \`error AND (wrap OR handle OR new) AND lang:go NOT test\`, path: \`"service/"\` (Focus on service directories)
+    *   (Analysis: Many results. See frequent use of \`fmt.Errorf\` and a custom \`errors.Wrap\` in several files like \`service/user/handler.go\`.)
+    *   Probe Action 2: \`search\` query: \`import AND "pkg/errors"\`, path: \`"service/"\` (Check where a potential custom error package is used)
+    *   (Analysis: Confirms \`pkg/errors\` is widely used.)
+    *   Probe Action 3: \`query\` language: \`go\`, pattern: \`errors.Wrap($$$)\`, path: \`"service/"\` (Find structural usage of the custom wrapper)
+    *   (Response: Summarize error handling: Mention standard \`fmt.Errorf\` and the prevalent use of a custom \`errors.Wrap\` function from \`pkg/errors\`, providing examples from locations found by search/query like \`service/user/handler.go\`.)`

package/src/tools/vercel.js

@@ -0,0 +1,215 @@
+/**
+ * Tools for Vercel AI SDK
+ * @module tools/vercel
+ */
+
+import { tool } from 'ai';
+import { search } from '../search.js';
+import { query } from '../query.js';
+import { extract } from '../extract.js';
+import { searchSchema, querySchema, extractSchema, searchDescription, queryDescription, extractDescription } from './common.js';
+
+/**
+ * Search tool generator
+ * 
+ * @param {Object} [options] - Configuration options
+ * @param {string} [options.sessionId] - Session ID for caching search results
+ * @param {number} [options.maxTokens=10000] - Default max tokens
+ * @param {boolean} [options.debug=false] - Enable debug logging
+ * @returns {Object} Configured search tool
+ */
+export const searchTool = (options = {}) => {
+	const { sessionId, maxTokens = 10000, debug = false } = options;
+
+	return tool({
+		name: 'search',
+		description: searchDescription,
+		parameters: searchSchema,
+		execute: async ({ query: searchQuery, path, allow_tests, exact, maxTokens: paramMaxTokens, language }) => {
+			try {
+				// Use parameter maxTokens if provided, otherwise use the default
+				const effectiveMaxTokens = paramMaxTokens || maxTokens;
+
+				// Use the path from parameters if provided, otherwise use defaultPath from config
+				let searchPath = path || options.defaultPath || '.';
+
+				// If path is "." or "./", use the defaultPath if available
+				if ((searchPath === "." || searchPath === "./") && options.defaultPath) {
+					if (debug) {
+						console.error(`Using default path "${options.defaultPath}" instead of "${searchPath}"`);
+					}
+					searchPath = options.defaultPath;
+				}
+
+				if (debug) {
+					console.error(`Executing search with query: "${searchQuery}", path: "${searchPath}", exact: ${exact ? 'true' : 'false'}, language: ${language || 'all'}, session: ${sessionId || 'none'}`);
+				}
+
+				const results = await search({
+					query: searchQuery,
+					path: searchPath,
+					allow_tests,
+					exact,
+					json: false,
+					maxTokens: effectiveMaxTokens,
+					session: sessionId, // Pass session ID if provided
+					language // Pass language parameter if provided
+				});
+
+				return results;
+			} catch (error) {
+				console.error('Error executing search command:', error);
+				return `Error executing search command: ${error.message}`;
+			}
+		}
+	});
+};
+
+/**
+ * Query tool generator
+ * 
+ * @param {Object} [options] - Configuration options
+ * @param {boolean} [options.debug=false] - Enable debug logging
+ * @returns {Object} Configured query tool
+ */
+export const queryTool = (options = {}) => {
+	const { debug = false } = options;
+
+	return tool({
+		name: 'query',
+		description: queryDescription,
+		parameters: querySchema,
+		execute: async ({ pattern, path, language, allow_tests }) => {
+			try {
+				// Use the path from parameters if provided, otherwise use defaultPath from config
+				let queryPath = path || options.defaultPath || '.';
+
+				// If path is "." or "./", use the defaultPath if available
+				if ((queryPath === "." || queryPath === "./") && options.defaultPath) {
+					if (debug) {
+						console.error(`Using default path "${options.defaultPath}" instead of "${queryPath}"`);
+					}
+					queryPath = options.defaultPath;
+				}
+
+				if (debug) {
+					console.error(`Executing query with pattern: "${pattern}", path: "${queryPath}", language: ${language || 'auto'}`);
+				}
+
+				const results = await query({
+					pattern,
+					path: queryPath,
+					language,
+					allow_tests,
+					json: false
+				});
+
+				return results;
+			} catch (error) {
+				console.error('Error executing query command:', error);
+				return `Error executing query command: ${error.message}`;
+			}
+		}
+	});
+};
+
+/**
+ * Extract tool generator
+ * 
+ * @param {Object} [options] - Configuration options
+ * @param {boolean} [options.debug=false] - Enable debug logging
+ * @returns {Object} Configured extract tool
+ */
+export const extractTool = (options = {}) => {
+	const { debug = false } = options;
+
+	return tool({
+		name: 'extract',
+		description: extractDescription,
+		parameters: extractSchema,
+		execute: async ({ file_path, input_content, line, end_line, allow_tests, context_lines, format }) => {
+			try {
+				// Use the defaultPath from config for context
+				let extractPath = options.defaultPath || '.';
+
+				// If path is "." or "./", use the defaultPath if available
+				if ((extractPath === "." || extractPath === "./") && options.defaultPath) {
+					if (debug) {
+						console.error(`Using default path "${options.defaultPath}" instead of "${extractPath}"`);
+					}
+					extractPath = options.defaultPath;
+				}
+
+				if (debug) {
+					if (file_path) {
+						console.error(`Executing extract with file: "${file_path}", path: "${extractPath}", context lines: ${context_lines || 10}`);
+					} else if (input_content) {
+						console.error(`Executing extract with input content, path: "${extractPath}", context lines: ${context_lines || 10}`);
+					}
+				}
+
+				// Create a temporary file for input content if provided
+				let tempFilePath = null;
+				let extractOptions = { path: extractPath };
+
+				if (input_content) {
+					// Import required modules
+					const { writeFileSync, unlinkSync } = await import('fs');
+					const { join } = await import('path');
+					const { tmpdir } = await import('os');
+					const { randomUUID } = await import('crypto');
+
+					// Create a temporary file with the input content
+					tempFilePath = join(tmpdir(), `probe-extract-${randomUUID()}.txt`);
+					writeFileSync(tempFilePath, input_content);
+
+					if (debug) {
+						console.error(`Created temporary file for input content: ${tempFilePath}`);
+					}
+
+					// Set up extract options with input file
+					extractOptions = {
+						inputFile: tempFilePath,
+						allowTests: allow_tests,
+						contextLines: context_lines,
+						format
+					};
+				} else if (file_path) {
+					// Parse file_path to handle line numbers and symbol names
+					const files = [file_path];
+
+					// Set up extract options with files
+					extractOptions = {
+						files,
+						allowTests: allow_tests,
+						contextLines: context_lines,
+						format
+					};
+				} else {
+					throw new Error('Either file_path or input_content must be provided');
+				}
+
+				// Execute the extract command
+				const results = await extract(extractOptions);
+
+				// Clean up temporary file if created
+				if (tempFilePath) {
+					const { unlinkSync } = await import('fs');
+					try {
+						unlinkSync(tempFilePath);
+						if (debug) {
+							console.error(`Removed temporary file: ${tempFilePath}`);
+						}
+					} catch (cleanupError) {
+						console.error(`Warning: Failed to remove temporary file: ${cleanupError.message}`);
+					}
+				}
+
+				return results;
+			} catch (error) {
+				console.error('Error executing extract command:', error);
+				return `Error executing extract command: ${error.message}`;
+			}
+		}
+	});
+};

package/src/utils/file-lister.js

@@ -0,0 +1,193 @@
+/**
+ * File listing utility for the probe package
+ * @module utils/file-lister
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { promisify } from 'util';
+import { exec } from 'child_process';
+
+const execAsync = promisify(exec);
+
+/**
+ * List files in a directory by nesting level, respecting .gitignore
+ * 
+ * @param {Object} options - Options for listing files
+ * @param {string} options.directory - Directory to list files from
+ * @param {number} [options.maxFiles=100] - Maximum number of files to return
+ * @param {boolean} [options.respectGitignore=true] - Whether to respect .gitignore
+ * @returns {Promise<string[]>} - Array of file paths
+ */
+export async function listFilesByLevel(options) {
+	const {
+		directory,
+		maxFiles = 100,
+		respectGitignore = true
+	} = options;
+
+	// Check if directory exists
+	if (!fs.existsSync(directory)) {
+		throw new Error(`Directory does not exist: ${directory}`);
+	}
+
+	// Use git ls-files if .git directory exists and respectGitignore is true
+	const gitDirExists = fs.existsSync(path.join(directory, '.git'));
+	if (gitDirExists && respectGitignore) {
+		try {
+			return await listFilesUsingGit(directory, maxFiles);
+		} catch (error) {
+			console.error(`Warning: Failed to use git ls-files: ${error.message}`);
+			console.error('Falling back to manual file listing');
+		}
+	}
+
+	// Fall back to manual file listing
+	return await listFilesByLevelManually(directory, maxFiles, respectGitignore);
+}
+
+/**
+ * List files using git ls-files (respects .gitignore by default)
+ * 
+ * @param {string} directory - Directory to list files from
+ * @param {number} maxFiles - Maximum number of files to return
+ * @returns {Promise<string[]>} - Array of file paths
+ */
+async function listFilesUsingGit(directory, maxFiles) {
+	// Use git ls-files to get tracked files (respects .gitignore)
+	const { stdout } = await execAsync('git ls-files', { cwd: directory });
+
+	// Split output into lines and filter out empty lines
+	const files = stdout.split('\n').filter(Boolean);
+
+	// Sort files by directory depth (breadth-first)
+	const sortedFiles = files.sort((a, b) => {
+		const depthA = a.split(path.sep).length;
+		const depthB = b.split(path.sep).length;
+		return depthA - depthB;
+	});
+
+	// Limit to maxFiles
+	return sortedFiles.slice(0, maxFiles);
+}
+
+/**
+ * List files manually by nesting level
+ * 
+ * @param {string} directory - Directory to list files from
+ * @param {number} maxFiles - Maximum number of files to return
+ * @param {boolean} respectGitignore - Whether to respect .gitignore
+ * @returns {Promise<string[]>} - Array of file paths
+ */
+async function listFilesByLevelManually(directory, maxFiles, respectGitignore) {
+	// Load .gitignore patterns if needed
+	let ignorePatterns = [];
+	if (respectGitignore) {
+		ignorePatterns = loadGitignorePatterns(directory);
+	}
+
+	// Initialize result array
+	const result = [];
+
+	// Initialize queue with root directory
+	const queue = [{ dir: directory, level: 0 }];
+
+	// Process queue (breadth-first)
+	while (queue.length > 0 && result.length < maxFiles) {
+		const { dir, level } = queue.shift();
+
+		try {
+			// Read directory contents
+			const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+			// Process files first (at current level)
+			const files = entries.filter(entry => entry.isFile());
+			for (const file of files) {
+				if (result.length >= maxFiles) break;
+
+				const filePath = path.join(dir, file.name);
+				const relativePath = path.relative(directory, filePath);
+
+				// Skip if file matches any ignore pattern
+				if (shouldIgnore(relativePath, ignorePatterns)) continue;
+
+				result.push(relativePath);
+			}
+
+			// Then add directories to queue for next level
+			const dirs = entries.filter(entry => entry.isDirectory());
+			for (const subdir of dirs) {
+				const subdirPath = path.join(dir, subdir.name);
+				const relativeSubdirPath = path.relative(directory, subdirPath);
+
+				// Skip if directory matches any ignore pattern
+				if (shouldIgnore(relativeSubdirPath, ignorePatterns)) continue;
+
+				// Skip node_modules and .git directories
+				if (subdir.name === 'node_modules' || subdir.name === '.git') continue;
+
+				queue.push({ dir: subdirPath, level: level + 1 });
+			}
+		} catch (error) {
+			console.error(`Warning: Could not read directory ${dir}: ${error.message}`);
+		}
+	}
+
+	return result;
+}
+
+/**
+ * Load .gitignore patterns from a directory
+ * 
+ * @param {string} directory - Directory to load .gitignore from
+ * @returns {string[]} - Array of ignore patterns
+ */
+function loadGitignorePatterns(directory) {
+	const gitignorePath = path.join(directory, '.gitignore');
+
+	if (!fs.existsSync(gitignorePath)) {
+		return [];
+	}
+
+	try {
+		const content = fs.readFileSync(gitignorePath, 'utf8');
+		return content
+			.split('\n')
+			.map(line => line.trim())
+			.filter(line => line && !line.startsWith('#'));
+	} catch (error) {
+		console.error(`Warning: Could not read .gitignore: ${error.message}`);
+		return [];
+	}
+}
+
+/**
+ * Check if a file path should be ignored based on ignore patterns
+ * 
+ * @param {string} filePath - File path to check
+ * @param {string[]} ignorePatterns - Array of ignore patterns
+ * @returns {boolean} - Whether the file should be ignored
+ */
+function shouldIgnore(filePath, ignorePatterns) {
+	if (!ignorePatterns.length) return false;
+
+	// Simple pattern matching (could be improved with minimatch or similar)
+	for (const pattern of ignorePatterns) {
+		// Exact match
+		if (pattern === filePath) return true;
+
+		// Directory match (pattern ends with /)
+		if (pattern.endsWith('/') && filePath.startsWith(pattern)) return true;
+
+		// File extension match (pattern starts with *.)
+		if (pattern.startsWith('*.') && filePath.endsWith(pattern.substring(1))) return true;
+
+		// Wildcard at start (pattern starts with *)
+		if (pattern.startsWith('*') && filePath.endsWith(pattern.substring(1))) return true;
+
+		// Wildcard at end (pattern ends with *)
+		if (pattern.endsWith('*') && filePath.startsWith(pattern.substring(0, pattern.length - 1))) return true;
+	}
+
+	return false;
+}

package/src/utils.js

@@ -0,0 +1,120 @@
+/**
+ * Utility functions for the probe package
+ * @module utils
+ */
+
+import path from 'path';
+import fs from 'fs-extra';
+import { fileURLToPath } from 'url';
+import { downloadProbeBinary } from './downloader.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const binDir = path.resolve(__dirname, '..', 'bin');
+
+// Store the binary path
+let probeBinaryPath = '';
+
+/**
+ * Get the path to the probe binary, downloading it if necessary
+ * @param {Object} options - Options for getting the binary
+ * @param {boolean} [options.forceDownload=false] - Force download even if binary exists
+ * @param {string} [options.version] - Specific version to download
+ * @returns {Promise<string>} - Path to the binary
+ */
+export async function getBinaryPath(options = {}) {
+	const { forceDownload = false, version } = options;
+
+	// Return cached path if available and not forcing download
+	if (probeBinaryPath && !forceDownload && fs.existsSync(probeBinaryPath)) {
+		return probeBinaryPath;
+	}
+
+	// Check environment variable
+	if (process.env.PROBE_PATH && fs.existsSync(process.env.PROBE_PATH) && !forceDownload) {
+		probeBinaryPath = process.env.PROBE_PATH;
+		return probeBinaryPath;
+	}
+
+	// Check bin directory
+	const isWindows = process.platform === 'win32';
+	const binaryName = isWindows ? 'probe.exe' : 'probe';
+	const binaryPath = path.join(binDir, binaryName);
+
+	if (fs.existsSync(binaryPath) && !forceDownload) {
+		probeBinaryPath = binaryPath;
+		return probeBinaryPath;
+	}
+
+	// Download if not found or force download
+	console.log(`${forceDownload ? 'Force downloading' : 'Binary not found. Downloading'} probe binary...`);
+	probeBinaryPath = await downloadProbeBinary(version);
+	return probeBinaryPath;
+}
+
+/**
+ * Manually set the path to the probe binary
+ * @param {string} binaryPath - Path to the probe binary
+ * @throws {Error} If the binary doesn't exist at the specified path
+ */
+export function setBinaryPath(binaryPath) {
+	if (!fs.existsSync(binaryPath)) {
+		throw new Error(`No binary found at path: ${binaryPath}`);
+	}
+
+	probeBinaryPath = binaryPath;
+}
+
+/**
+ * Ensure the bin directory exists
+ * @returns {Promise<void>}
+ */
+export async function ensureBinDirectory() {
+	await fs.ensureDir(binDir);
+}
+
+/**
+ * Build command-line arguments from an options object
+ * @param {Object} options - Options object
+ * @param {Array<string>} flagMap - Map of option keys to command-line flags
+ * @returns {Array<string>} - Array of command-line arguments
+ */
+export function buildCliArgs(options, flagMap) {
+	const cliArgs = [];
+
+	for (const [key, flag] of Object.entries(flagMap)) {
+		if (key in options) {
+			const value = options[key];
+
+			if (typeof value === 'boolean') {
+				if (value) {
+					cliArgs.push(flag);
+				}
+			} else if (Array.isArray(value)) {
+				for (const item of value) {
+					cliArgs.push(flag, item);
+				}
+			} else if (value !== undefined && value !== null) {
+				cliArgs.push(flag, value.toString());
+			}
+		}
+	}
+
+	return cliArgs;
+}
+
+/**
+ * Escape a string for use in a command line
+ * @param {string} str - String to escape
+ * @returns {string} - Escaped string
+ */
+export function escapeString(str) {
+  if (process.platform === 'win32') {
+    // For Windows PowerShell, escape double quotes and wrap with double quotes
+    return `"${str.replace(/"/g, '\\"')}"`;
+  } else {
+    // Use single quotes for POSIX shells
+    // Escape single quotes in the string by replacing ' with '\''
+    return `'${str.replace(/'/g, "'\\''")}'`;
+  }
+}