@probelabs/probe 0.6.0-rc56

package/src/downloader.js

@@ -0,0 +1,571 @@
+/**
+ * Binary downloader for the probe package
+ * @module downloader
+ */
+
+import axios from 'axios';
+import fs from 'fs-extra';
+import path from 'path';
+import { createHash } from 'crypto';
+import { promisify } from 'util';
+import { exec as execCallback } from 'child_process';
+import tar from 'tar';
+import os from 'os';
+import { fileURLToPath } from 'url';
+import { ensureBinDirectory } from './utils.js';
+
+const exec = promisify(execCallback);
+
+// GitHub repository information
+const REPO_OWNER = "buger";
+const REPO_NAME = "probe";
+const BINARY_NAME = "probe";
+
+// Get the directory of the current module
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Local storage directory for downloaded binaries
+const LOCAL_DIR = path.resolve(__dirname, '..', 'bin');
+
+// Version info file path
+const VERSION_INFO_PATH = path.join(LOCAL_DIR, 'version-info.json');
+
+/**
+ * Detects the current OS and architecture
+ * @returns {Object} Object containing OS and architecture information
+ */
+function detectOsArch() {
+	const osType = os.platform();
+	const archType = os.arch();
+
+	let osInfo;
+	let archInfo;
+
+	// Detect OS
+	switch (osType) {
+		case 'linux':
+			osInfo = {
+				type: 'linux',
+				keywords: ['linux', 'Linux', 'gnu']
+			};
+			break;
+		case 'darwin':
+			osInfo = {
+				type: 'darwin',
+				keywords: ['darwin', 'Darwin', 'mac', 'Mac', 'apple', 'Apple', 'osx', 'OSX']
+			};
+			break;
+	case 'win32':
+		osInfo = {
+			type: 'windows',
+			keywords: ['windows', 'Windows', 'msvc', 'pc-windows']
+		};
+		break;
+		default:
+			throw new Error(`Unsupported operating system: ${osType}`);
+	}
+
+	// Detect architecture
+	switch (archType) {
+		case 'x64':
+			archInfo = {
+				type: 'x86_64',
+				keywords: ['x86_64', 'amd64', 'x64', '64bit', '64-bit']
+			};
+			break;
+		case 'arm64':
+			archInfo = {
+				type: 'aarch64',
+				keywords: ['arm64', 'aarch64', 'arm', 'ARM']
+			};
+			break;
+		default:
+			throw new Error(`Unsupported architecture: ${archType}`);
+	}
+
+	console.log(`Detected OS: ${osInfo.type}, Architecture: ${archInfo.type}`);
+	return { os: osInfo, arch: archInfo };
+}
+
+/**
+ * Gets the latest release information from GitHub
+ * @param {string} [version] - Specific version to get
+ * @returns {Promise<Object>} Release information
+ */
+async function getLatestRelease(version) {
+	console.log('Fetching release information...');
+
+	try {
+		let releaseUrl;
+		if (version) {
+			// Always use the specified version
+			releaseUrl = `https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/releases/tags/v${version}`;
+		} else {
+			// Use the latest release only if no version is specified
+			releaseUrl = `https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/releases/latest`;
+		}
+
+		const response = await axios.get(releaseUrl);
+
+		if (response.status !== 200) {
+			throw new Error(`Failed to fetch release information: ${response.statusText}`);
+		}
+
+		const tag = response.data.tag_name;
+		const assets = response.data.assets.map(asset => ({
+			name: asset.name,
+			url: asset.browser_download_url
+		}));
+
+		console.log(`Found release: ${tag} with ${assets.length} assets`);
+		return { tag, assets };
+	} catch (error) {
+		if (axios.isAxiosError(error) && error.response?.status === 404) {
+			// If the specific version is not found, try to get all releases
+			console.log(`Release v${version} not found, trying to fetch all releases...`);
+
+			const response = await axios.get(`https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/releases`);
+
+			if (response.data.length === 0) {
+				throw new Error('No releases found');
+			}
+
+			// Try to find a release that matches the version prefix
+			let bestRelease = response.data[0]; // Default to first release
+
+			if (version && version !== '0.0.0') {
+				// Try to find a release that starts with the same version prefix
+				const versionParts = version.split('.');
+				const versionPrefix = versionParts.slice(0, 2).join('.'); // e.g., "0.2" from "0.2.2-rc7"
+
+				console.log(`Looking for releases matching prefix: ${versionPrefix}`);
+
+				for (const release of response.data) {
+					const releaseTag = release.tag_name.startsWith('v') ?
+						release.tag_name.substring(1) : release.tag_name;
+
+					if (releaseTag.startsWith(versionPrefix)) {
+						console.log(`Found matching release: ${release.tag_name}`);
+						bestRelease = release;
+						break;
+					}
+				}
+			}
+
+			const tag = bestRelease.tag_name;
+			const assets = bestRelease.assets.map(asset => ({
+				name: asset.name,
+				url: asset.browser_download_url
+			}));
+
+			console.log(`Using release: ${tag} with ${assets.length} assets`);
+			return { tag, assets };
+		}
+
+		throw error;
+	}
+}
+
+/**
+ * Finds the best matching asset for the current OS and architecture
+ * @param {Array} assets - List of assets
+ * @param {Object} osInfo - OS information
+ * @param {Object} archInfo - Architecture information
+ * @returns {Object} Best matching asset
+ */
+function findBestAsset(assets, osInfo, archInfo) {
+	console.log(`Finding appropriate binary for ${osInfo.type} ${archInfo.type}...`);
+
+	let bestAsset = null;
+	let bestScore = 0;
+
+	for (const asset of assets) {
+		// Skip checksum files
+		if (asset.name.endsWith('.sha256') || asset.name.endsWith('.md5') || asset.name.endsWith('.asc')) {
+			continue;
+		}
+
+		if (osInfo.type === 'windows' && asset.name.match(/darwin|linux/)) {
+			console.log(`Skipping non-Windows binary: ${asset.name}`);
+			continue;
+		} else if (osInfo.type === 'darwin' && asset.name.match(/windows|msvc|linux/)) {
+			console.log(`Skipping non-macOS binary: ${asset.name}`);
+			continue;
+		} else if (osInfo.type === 'linux' && asset.name.match(/darwin|windows|msvc/)) {
+			console.log(`Skipping non-Linux binary: ${asset.name}`);
+			continue;
+		}
+
+		let score = 0;
+		console.log(`Evaluating asset: ${asset.name}`);
+
+		// Check for OS match - give higher priority to exact OS matches
+		let osMatched = false;
+		for (const keyword of osInfo.keywords) {
+			if (asset.name.includes(keyword)) {
+				score += 10;
+				osMatched = true;
+				console.log(`  OS match found (${keyword}): +10, score = ${score}`);
+				break;
+			}
+		}
+
+		// Check for architecture match
+		for (const keyword of archInfo.keywords) {
+			if (asset.name.includes(keyword)) {
+				score += 5;
+				console.log(`  Arch match found (${keyword}): +5, score = ${score}`);
+				break;
+			}
+		}
+
+		// Prefer exact matches for binary name
+		if (asset.name.startsWith(`${BINARY_NAME}-`)) {
+			score += 3;
+			console.log(`  Binary name match: +3, score = ${score}`);
+		}
+
+		if (osMatched && score >= 15) {
+			score += 5;
+			console.log(`  OS+Arch bonus: +5, score = ${score}`);
+		}
+
+		console.log(`  Final score for ${asset.name}: ${score}`);
+
+		// If we have a perfect match, use it immediately
+		if (score === 23) {
+			console.log(`Found perfect match: ${asset.name}`);
+			return asset;
+		}
+
+		// Otherwise, keep track of the best match so far
+		if (score > bestScore) {
+			bestScore = score;
+			bestAsset = asset;
+			console.log(`  New best asset: ${asset.name} (score: ${score})`);
+		}
+	}
+
+	if (!bestAsset) {
+		throw new Error(`Could not find a suitable binary for ${osInfo.type} ${archInfo.type}`);
+	}
+
+	console.log(`Selected asset: ${bestAsset.name} (score: ${bestScore})`);
+	return bestAsset;
+}
+
+/**
+ * Downloads the asset and its checksum
+ * @param {Object} asset - Asset to download
+ * @param {string} outputDir - Directory to save to
+ * @returns {Promise<Object>} Paths to the asset and checksum
+ */
+async function downloadAsset(asset, outputDir) {
+	await fs.ensureDir(outputDir);
+
+	const assetPath = path.join(outputDir, asset.name);
+	console.log(`Downloading ${asset.name}...`);
+
+	// Download the asset
+	const assetResponse = await axios.get(asset.url, { responseType: 'arraybuffer' });
+	await fs.writeFile(assetPath, Buffer.from(assetResponse.data));
+
+	// Try to download the checksum
+	const checksumUrl = `${asset.url}.sha256`;
+	let checksumPath = null;
+
+	try {
+		console.log(`Downloading checksum...`);
+		const checksumResponse = await axios.get(checksumUrl);
+		checksumPath = path.join(outputDir, `${asset.name}.sha256`);
+		await fs.writeFile(checksumPath, checksumResponse.data);
+	} catch (error) {
+		console.log('No checksum file found, skipping verification');
+	}
+
+	return { assetPath, checksumPath };
+}
+
+/**
+ * Verifies the checksum of the downloaded asset
+ * @param {string} assetPath - Path to the asset
+ * @param {string|null} checksumPath - Path to the checksum file
+ * @returns {Promise<boolean>} Whether verification succeeded
+ */
+async function verifyChecksum(assetPath, checksumPath) {
+	if (!checksumPath) {
+		return true;
+	}
+
+	console.log(`Verifying checksum...`);
+
+	// Read the expected checksum
+	const checksumContent = await fs.readFile(checksumPath, 'utf-8');
+	const expectedChecksum = checksumContent.trim().split(' ')[0];
+
+	// Calculate the actual checksum
+	const fileBuffer = await fs.readFile(assetPath);
+	const actualChecksum = createHash('sha256').update(fileBuffer).digest('hex');
+
+	if (expectedChecksum !== actualChecksum) {
+		console.error(`Checksum verification failed!`);
+		console.error(`Expected: ${expectedChecksum}`);
+		console.error(`Actual: ${actualChecksum}`);
+		return false;
+	}
+
+	console.log(`Checksum verified successfully`);
+	return true;
+}
+
+/**
+ * Extracts and installs the binary
+ * @param {string} assetPath - Path to the asset
+ * @param {string} outputDir - Directory to extract to
+ * @returns {Promise<string>} Path to the extracted binary
+ */
+async function extractBinary(assetPath, outputDir) {
+	console.log(`Extracting ${path.basename(assetPath)}...`);
+
+	const assetName = path.basename(assetPath);
+	const isWindows = os.platform() === 'win32';
+	const binaryName = isWindows ? `${BINARY_NAME}.exe` : BINARY_NAME;
+	const binaryPath = path.join(outputDir, binaryName);
+
+	try {
+		// Create a temporary extraction directory
+		const extractDir = path.join(outputDir, 'temp_extract');
+		await fs.ensureDir(extractDir);
+
+		// Determine file type and extract accordingly
+		if (assetName.endsWith('.tar.gz') || assetName.endsWith('.tgz')) {
+			console.log(`Extracting tar.gz to ${extractDir}...`);
+			await tar.extract({
+				file: assetPath,
+				cwd: extractDir
+			});
+		} else if (assetName.endsWith('.zip')) {
+			console.log(`Extracting zip to ${extractDir}...`);
+			await exec(`unzip -q "${assetPath}" -d "${extractDir}"`);
+		} else {
+			// Assume it's a direct binary
+			console.log(`Copying binary directly to ${binaryPath}`);
+			await fs.copyFile(assetPath, binaryPath);
+
+			// Make the binary executable
+			if (!isWindows) {
+				await fs.chmod(binaryPath, 0o755);
+			}
+
+			// Clean up the extraction directory
+			await fs.remove(extractDir);
+			console.log(`Binary installed to ${binaryPath}`);
+			return binaryPath;
+		}
+
+		// Find the binary in the extracted files
+		console.log(`Searching for binary in extracted files...`);
+		const findBinary = async (dir) => {
+			const entries = await fs.readdir(dir, { withFileTypes: true });
+
+			for (const entry of entries) {
+				const fullPath = path.join(dir, entry.name);
+
+				if (entry.isDirectory()) {
+					const result = await findBinary(fullPath);
+					if (result) return result;
+				} else if (entry.isFile()) {
+					// Check if this is the binary we're looking for
+					if (entry.name === binaryName ||
+						entry.name === BINARY_NAME ||
+						(isWindows && entry.name.endsWith('.exe'))) {
+						return fullPath;
+					}
+				}
+			}
+
+			return null;
+		};
+
+		const binaryFilePath = await findBinary(extractDir);
+
+		if (!binaryFilePath) {
+			// List all extracted files for debugging
+			const allFiles = await fs.readdir(extractDir, { recursive: true });
+			console.error(`Binary not found in extracted files. Found: ${allFiles.join(', ')}`);
+			throw new Error(`Binary not found in the archive.`);
+		}
+
+		// Copy the binary directly to the final location
+		console.log(`Found binary at ${binaryFilePath}`);
+		console.log(`Copying binary to ${binaryPath}`);
+		await fs.copyFile(binaryFilePath, binaryPath);
+
+		// Make the binary executable
+		if (!isWindows) {
+			await fs.chmod(binaryPath, 0o755);
+		}
+
+		// Clean up
+		await fs.remove(extractDir);
+
+		console.log(`Binary successfully installed to ${binaryPath}`);
+		return binaryPath;
+	} catch (error) {
+		console.error(`Error extracting binary: ${error instanceof Error ? error.message : String(error)}`);
+		throw error;
+	}
+}
+
+/**
+ * Gets version info from the version file
+ * @returns {Promise<Object|null>} Version information
+ */
+async function getVersionInfo() {
+	try {
+		if (await fs.pathExists(VERSION_INFO_PATH)) {
+			const content = await fs.readFile(VERSION_INFO_PATH, 'utf-8');
+			return JSON.parse(content);
+		}
+		return null;
+	} catch (error) {
+		console.warn(`Warning: Could not read version info: ${error}`);
+		return null;
+	}
+}
+
+/**
+ * Saves version info to the version file
+ * @param {string} version - Version to save
+ * @returns {Promise<void>}
+ */
+async function saveVersionInfo(version) {
+	const versionInfo = {
+		version,
+		lastUpdated: new Date().toISOString()
+	};
+
+	await fs.writeFile(VERSION_INFO_PATH, JSON.stringify(versionInfo, null, 2));
+	console.log(`Version info saved: ${version}`);
+}
+
+/**
+ * Gets the package version from package.json
+ * @returns {Promise<string>} Package version
+ */
+async function getPackageVersion() {
+	try {
+		// Try multiple possible locations for package.json
+		const possiblePaths = [
+			path.resolve(__dirname, '..', 'package.json'),      // When installed from npm: src/../package.json
+			path.resolve(__dirname, '..', '..', 'package.json') // In development: src/../../package.json
+		];
+
+		for (const packageJsonPath of possiblePaths) {
+			try {
+				if (fs.existsSync(packageJsonPath)) {
+					console.log(`Found package.json at: ${packageJsonPath}`);
+					const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
+					if (packageJson.version) {
+						console.log(`Using version from package.json: ${packageJson.version}`);
+						return packageJson.version;
+					}
+				}
+			} catch (err) {
+				console.error(`Error reading package.json at ${packageJsonPath}:`, err);
+			}
+		}
+
+		// If we can't find the version in package.json, return a default version
+		return '0.0.0';
+	} catch (error) {
+		console.error('Error getting package version:', error);
+		return '0.0.0';
+	}
+}
+
+/**
+ * Downloads the probe binary
+ * @param {string} [version] - Specific version to download
+ * @returns {Promise<string>} Path to the downloaded binary
+ */
+export async function downloadProbeBinary(version) {
+	try {
+		// Create the bin directory if it doesn't exist
+		await ensureBinDirectory();
+
+		// If no version is specified, use the package version
+		if (!version || version === '0.0.0') {
+			version = await getPackageVersion();
+		}
+
+		console.log(`Downloading probe binary (version: ${version || 'latest'})...`);
+
+		const isWindows = os.platform() === 'win32';
+		const binaryName = isWindows ? `${BINARY_NAME}.exe` : BINARY_NAME;
+		const binaryPath = path.join(LOCAL_DIR, binaryName);
+
+		// Check if the binary already exists and version matches
+		if (await fs.pathExists(binaryPath)) {
+			const versionInfo = await getVersionInfo();
+
+			// If versions match, use existing binary
+			if (versionInfo && versionInfo.version === version) {
+				console.log(`Using existing binary at ${binaryPath} (version: ${versionInfo.version})`);
+				return binaryPath;
+			}
+
+			console.log(`Existing binary version (${versionInfo?.version || 'unknown'}) doesn't match requested version (${version}). Downloading new version...`);
+		}
+
+		// Get OS and architecture information
+		const { os: osInfo, arch: archInfo } = detectOsArch();
+
+		// Determine which version to download
+		let versionToUse = version;
+		if (!versionToUse || versionToUse === '0.0.0') {
+			console.log('No specific version requested, will use the latest release');
+			versionToUse = undefined;
+		} else {
+			console.log(`Looking for release with version: ${versionToUse}`);
+		}
+
+		// Get release information
+		const { tag, assets } = await getLatestRelease(versionToUse);
+		const tagVersion = tag.startsWith('v') ? tag.substring(1) : tag;
+		console.log(`Found release version: ${tagVersion}`);
+
+		// Find and download the appropriate asset
+		const bestAsset = findBestAsset(assets, osInfo, archInfo);
+		const { assetPath, checksumPath } = await downloadAsset(bestAsset, LOCAL_DIR);
+
+		// Verify checksum if available
+		const checksumValid = await verifyChecksum(assetPath, checksumPath);
+		if (!checksumValid) {
+			throw new Error('Checksum verification failed');
+		}
+
+		// Extract the binary
+		const extractedBinaryPath = await extractBinary(assetPath, LOCAL_DIR);
+
+		// Save the version information
+		await saveVersionInfo(tagVersion);
+
+		// Clean up the downloaded archive
+		try {
+			await fs.remove(assetPath);
+			if (checksumPath) {
+				await fs.remove(checksumPath);
+			}
+		} catch (err) {
+			console.log(`Warning: Could not clean up temporary files: ${err}`);
+		}
+
+		console.log(`Binary successfully installed at ${extractedBinaryPath} (version: ${tagVersion})`);
+		return extractedBinaryPath;
+	} catch (error) {
+		console.error('Error downloading probe binary:', error);
+		throw error;
+	}
+}

package/src/extract.js

@@ -0,0 +1,147 @@
+/**
+ * Extract functionality for the probe package
+ * @module extract
+ */
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { getBinaryPath, buildCliArgs, escapeString } from './utils.js';
+
+const execAsync = promisify(exec);
+
+/**
+ * Flag mapping for extract options
+ * Maps option keys to command-line flags
+ */
+const EXTRACT_FLAG_MAP = {
+	allowTests: '--allow-tests',
+	contextLines: '--context',
+	format: '--format',
+	inputFile: '--input-file'
+};
+
+/**
+ * Extract code blocks from files
+ * 
+ * @param {Object} options - Extract options
+ * @param {string[]} [options.files] - Files to extract from (can include line numbers with colon, e.g., "/path/to/file.rs:10")
+ * @param {string} [options.inputFile] - Path to a file containing unstructured text to extract file paths from
+ * @param {boolean} [options.allowTests] - Include test files
+ * @param {number} [options.contextLines] - Number of context lines to include
+ * @param {string} [options.format] - Output format ('markdown', 'plain', 'json')
+ * @param {Object} [options.binaryOptions] - Options for getting the binary
+ * @param {boolean} [options.binaryOptions.forceDownload] - Force download even if binary exists
+ * @param {string} [options.binaryOptions.version] - Specific version to download
+ * @param {boolean} [options.json] - Return results as parsed JSON instead of string
+ * @returns {Promise<string|Object>} - Extracted code as string or parsed JSON
+ * @throws {Error} If the extraction fails
+ */
+export async function extract(options) {
+	if (!options) {
+		throw new Error('Options object is required');
+	}
+
+	// Either files or inputFile must be provided
+	if ((!options.files || !Array.isArray(options.files) || options.files.length === 0) && !options.inputFile) {
+		throw new Error('Either files array or inputFile must be provided');
+	}
+
+	// Get the binary path
+	const binaryPath = await getBinaryPath(options.binaryOptions || {});
+
+	// Build CLI arguments from options
+	const cliArgs = buildCliArgs(options, EXTRACT_FLAG_MAP);
+
+	// If json option is true, override format to json
+	if (options.json && !options.format) {
+		cliArgs.push('--format', 'json');
+	}
+
+	// Add files as positional arguments if provided
+	if (options.files && Array.isArray(options.files) && options.files.length > 0) {
+		for (const file of options.files) {
+			cliArgs.push(escapeString(file));
+		}
+	}
+
+	// Create a single log record with all extract parameters
+	let logMessage = `\nExtract:`;
+	if (options.files && options.files.length > 0) {
+		logMessage += ` files="${options.files.join(', ')}"`;
+	}
+	if (options.inputFile) logMessage += ` inputFile="${options.inputFile}"`;
+	if (options.allowTests) logMessage += " allowTests=true";
+	if (options.contextLines) logMessage += ` contextLines=${options.contextLines}`;
+	if (options.format) logMessage += ` format=${options.format}`;
+	if (options.json) logMessage += " json=true";
+	console.error(logMessage);
+
+	// Execute command
+	const command = `${binaryPath} extract ${cliArgs.join(' ')}`;
+
+	try {
+		const { stdout, stderr } = await execAsync(command);
+
+		if (stderr) {
+			console.error(`stderr: ${stderr}`);
+		}
+
+		// Parse the output to extract token usage information
+		let tokenUsage = {
+			requestTokens: 0,
+			responseTokens: 0,
+			totalTokens: 0
+		};
+
+		// Calculate approximate request tokens
+		if (options.files && Array.isArray(options.files)) {
+			tokenUsage.requestTokens = options.files.join(' ').length / 4;
+		} else if (options.inputFile) {
+			tokenUsage.requestTokens = options.inputFile.length / 4;
+		}
+
+		// Try to extract token information from the output
+		if (stdout.includes('Total tokens returned:')) {
+			const tokenMatch = stdout.match(/Total tokens returned: (\d+)/);
+			if (tokenMatch && tokenMatch[1]) {
+				tokenUsage.responseTokens = parseInt(tokenMatch[1], 10);
+				tokenUsage.totalTokens = tokenUsage.requestTokens + tokenUsage.responseTokens;
+			}
+		}
+
+		// Add token usage information to the output
+		let output = stdout;
+
+		// Add token usage information at the end if not already present
+		if (!output.includes('Token Usage:')) {
+			output += `\nToken Usage:\n  Request tokens: ${tokenUsage.requestTokens}\n  Response tokens: ${tokenUsage.responseTokens}\n  Total tokens: ${tokenUsage.totalTokens}\n`;
+		}
+
+		// Parse JSON if requested or if format is json
+		if (options.json || options.format === 'json') {
+			try {
+				const jsonOutput = JSON.parse(stdout);
+
+				// Add token usage to JSON output
+				if (!jsonOutput.token_usage) {
+					jsonOutput.token_usage = {
+						request_tokens: tokenUsage.requestTokens,
+						response_tokens: tokenUsage.responseTokens,
+						total_tokens: tokenUsage.totalTokens
+					};
+				}
+
+				return jsonOutput;
+			} catch (error) {
+				console.error('Error parsing JSON output:', error);
+				return output; // Fall back to string output with token usage
+			}
+		}
+
+		return output;
+	} catch (error) {
+		// Enhance error message with command details
+		const errorMessage = `Error executing extract command: ${error.message}\nCommand: ${command}`;
+		throw new Error(errorMessage);
+	}
+}