@lincoln504/pi-research 0.1.13 → 1.0.1

package/src/tools/grep.ts

@@ -1,335 +1,57 @@
 /**
- * rg_grep Tool (Ripgrep)
+ * Grep Tool
  *
- * Standalone grep tool using ripgrep (rg) or fallback to grep.
+ * Thin wrapper around the SDK's built-in grep tool that adds
+ * ToolUsageTracker budget enforcement before delegating execution.
  */
 
-import { spawn } from 'node:child_process';
-import * as nodePath from 'node:path';
-import type { ToolDefinition, AgentToolResult, ExtensionContext } from '@mariozechner/pi-coding-agent';
-import { Type } from '@sinclair/typebox';
+import { createGrepToolDefinition } from '@earendil-works/pi-coding-agent';
+import { Value } from 'typebox/value';
+import type { ToolDefinition } from '@earendil-works/pi-coding-agent';
 import type { ToolUsageTracker } from '../utils/tool-usage-tracker.ts';
-
-const DEFAULT_MAX_BYTES = 100 * 1024; // 100KB
-const DEFAULT_MAX_LINES = 200;
-const STREAM_MAX_BYTES = 10 * 1024 * 1024; // 10MB max in stream (prevent RangeError)
-
-interface RgGrepParams {
-  pattern: string;
-  path?: string;
-  flags?: string;
-}
-
-function truncateHead(content: string, maxBytes: number, maxLines: number): string {
-  const lines = content.split('\n');
-
-  if (lines.length <= maxLines && Buffer.byteLength(content, 'utf-8') <= maxBytes) {
-    return content;
-  }
-
-  const truncatedLines = lines.slice(0, maxLines);
-  let result = truncatedLines.join('\n');
-
-  if (Buffer.byteLength(result, 'utf-8') > maxBytes) {
-    let byteCount = 0;
-    const charArray: string[] = [];
-
-    for (const char of result) {
-      byteCount += Buffer.byteLength(char, 'utf-8');
-      if (byteCount > maxBytes) break;
-      charArray.push(char);
-    }
-
-    result = charArray.join('');
-  }
-
-  return result;
-}
-
-async function execCommand(
-  command: string,
-  args: string[],
-): Promise<{ stdout: string; stderr: string; exitCode: number | null; wasTruncated: boolean }> {
-  return new Promise((resolve, reject) => {
-    const child = spawn(command, args);
-    let stdout = '';
-    let stderr = '';
-    let wasTruncated = false;
-
-    child.stdout?.on('data', (data) => {
-      // Check if adding this chunk would exceed stream limit
-      const chunk = data.toString();
-      if (stdout.length + chunk.length > STREAM_MAX_BYTES) {
-        // Stop collecting data to prevent RangeError
-        wasTruncated = true;
-        child.kill();
-        return;
-      }
-      stdout += chunk;
-    });
-
-    child.stderr?.on('data', (data) => {
-      stderr += data.toString();
-    });
-
-    child.on('close', (code) => {
-      resolve({ stdout, stderr, exitCode: code, wasTruncated });
-    });
-
-    child.on('error', (error) => {
-      reject(error);
-    });
-  });
-}
-
-function resolveWorkspacePath(input: string): string {
-  const root = process.cwd();
-  const resolved = nodePath.resolve(root, input);
-  if (resolved !== root && !resolved.startsWith(root + nodePath.sep)) {
-    throw new Error(`Path outside workspace: ${input}`);
-  }
-  return resolved;
-}
-
-function sanitizeInput(input: string): string[] {
-  // Simple sanitization: split on whitespace, preserve quoted strings
-  const result: string[] = [];
-  let current = '';
-  let inQuotes = false;
-
-  for (const char of input) {
-    if (char === '"' || char === "'") {
-      inQuotes = !inQuotes;
-    } else if (char === ' ' && !inQuotes) {
-      if (current) {
-        result.push(current);
-        current = '';
-      }
-    } else {
-      current += char;
-    }
-  }
-
-  if (current) {
-    result.push(current);
-  }
-
-  return result;
-}
-
-function normalizeSearchOutput(output: string): string {
-  return process.platform === 'win32' ? output.replace(/\\/g, '/') : output;
-}
+import { MAX_GREP_CALLS } from '../constants.ts';
 
 export function createGrepTool(options: {
   tracker: ToolUsageTracker;
+  cwd?: string;
 }): ToolDefinition {
+  const cwd = options.cwd ?? process.cwd();
+  const sdkGrep = createGrepToolDefinition(cwd);
+  const { execute: sdkExecute } = sdkGrep;
+
   return {
-    name: 'grep',
+    ...sdkGrep,
     label: 'Code Search',
-    description: 'Search codebase using ripgrep (rg) or grep fallback. Fast recursive text search.',
-    promptSnippet: 'Search codebase using ripgrep/grep',
     promptGuidelines: [
-      'Use grep for fast recursive text search in codebases.',
-      'Pattern supports regex. Path and flags are optional.',
-      'Falls back to grep if rg is not available.',
-      'CRITICAL: You are allowed a maximum of 4 gathering calls total across ALL tools. Use them for breadth.',
+      'Available for fast recursive text search in codebases.',
+      'Pattern supports regex. Use glob to scope to file types (e.g., "**/*.ts").',
+      'Options: ignoreCase, literal (disable regex), context (lines around match), limit (max matches).',
+      `Local code search — it does NOT consume your web-gathering budget (limit: ${MAX_GREP_CALLS} calls).`,
     ],
-    parameters: Type.Object({
-      pattern: Type.String({ description: 'Search pattern (regex supported)' }),
-      path: Type.Optional(Type.String({ description: 'Directory to search (default: current directory)' })),
-      flags: Type.Optional(Type.String({ description: 'Additional flags (e.g., "-i" for case-insensitive)' })),
-    }),
-    async execute(
-      _toolCallId: string,
-      params: unknown,
-      _signal: unknown,
-      _onUpdate: unknown,
-      _ctx: ExtensionContext,
-    ): Promise<AgentToolResult<unknown>> {
-      // Record call in tracker - returns false if limit reached
-      const allowed = options.tracker.recordCall('grep');
-      if (!allowed) {
-        // THROW to prevent researcher from calling again
-        throw new Error(options.tracker.getLimitMessage('grep'));
+    execute: async (toolCallId, params, signal, onUpdate, ctx) => {
+      if (!Value.Check(sdkGrep.parameters, params)) {
+        return {
+          content: [{ type: 'text', text: 'Invalid parameters for grep tool.' }],
+          details: { error: 'invalid_parameters' },
+        };
       }
 
-      const record = params as RgGrepParams;
-      const { pattern } = record;
-      const path = resolveWorkspacePath(record.path ?? '.');
-      const flags = record.flags ?? '';
-
-      if (!pattern) {
+      if (!options.tracker.recordCall('grep')) {
         return {
-          content: [{ type: 'text', text: 'Error: pattern is required' }],
-          details: {},
+          content: [{ type: 'text', text: options.tracker.getLimitMessage('grep') }],
+          details: { blocked: true, reason: 'limit_reached' },
         };
       }
 
-      // Sanitize inputs to prevent shell injection
-      const flagParts = flags ? sanitizeInput(flags) : [];
-
-      // Try rg first
       try {
-        const rgArgs = ['--no-heading', '-n', ...flagParts, pattern, path];
-        const { stdout, stderr, exitCode, wasTruncated } = await execCommand('rg', rgArgs);
-
-        const truncated = truncateHead(normalizeSearchOutput(stdout), DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES);
-
-        let markdown = '# Search Results (rg)\n\n';
-        markdown += `**Pattern:** \`${pattern}\`\n`;
-        markdown += `**Path:** \`${path}\`\n`;
-        if (flags) markdown += `**Flags:** \`${flags}\`\n`;
-        markdown += `**Exit Code:** ${exitCode}\n\n`;
-
-        if (wasTruncated) {
-          markdown += '⚠️ **Output was truncated to prevent memory overflow (>10MB)**\n\n';
-        }
-
-        if (truncated) {
-          markdown += '```\n' + truncated + '\n```';
-        } else {
-          markdown += 'No matches found.';
-        }
-
-        if (stderr) {
-          markdown += '\n\n**Stderr:**\n```\n' + stderr + '\n```';
-        }
-
+        return await sdkExecute(toolCallId, params as any, signal, onUpdate, ctx);
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
         return {
-          content: [{ type: 'text', text: markdown }],
-          details: {
-            command: 'rg',
-            args: rgArgs,
-            exitCode,
-            wasTruncated,
-          },
+          content: [{ type: 'text', text: `Error: ${msg}` }],
+          details: { error: 'execution_error', message: msg },
         };
-      } catch (rgError) {
-        // rg not found, try grep
-        try {
-          const grepArgs = ['-rn', ...flagParts, pattern, path];
-          const { stdout, stderr, exitCode, wasTruncated } = await execCommand('grep', grepArgs);
-
-          const truncated = truncateHead(normalizeSearchOutput(stdout), DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES);
-
-          let markdown = '# Search Results (grep)\n\n';
-          markdown += `**Pattern:** \`${pattern}\`\n`;
-          markdown += `**Path:** \`${path}\`\n`;
-          if (flags) markdown += `**Flags:** \`${flags}\`\n`;
-          markdown += `**Exit Code:** ${exitCode}\n\n`;
-          markdown += '*Note: Using grep fallback (rg not available)*\n\n';
-
-          if (wasTruncated) {
-            markdown += '⚠️ **Output was truncated to prevent memory overflow (>10MB)**\n\n';
-          }
-
-          if (truncated) {
-            markdown += '```\n' + truncated + '\n```';
-          } else {
-            markdown += 'No matches found.';
-          }
-
-          if (stderr) {
-            markdown += '\n\n**Stderr:**\n```\n' + stderr + '\n```';
-          }
-
-          return {
-            content: [{ type: 'text', text: markdown }],
-            details: {
-              command: 'grep',
-              args: grepArgs,
-              exitCode,
-              wasTruncated,
-            },
-          };
-        } catch (grepError) {
-          return {
-            content: [
-              {
-                type: 'text',
-                text: `Error: Neither rg nor grep is available.\n\nrg error: ${rgError}\ngrep error: ${grepError}`,
-              },
-            ],
-            details: {},
-          };
-        }
       }
     },
-  };
-}
-
-/**
- * Standalone function to execute rg/grep search
- */
-export async function grep(pattern: string, path: string = '.', flags: string = ''): Promise<string> {
-  if (!pattern) {
-    throw new Error('Pattern is required');
-  }
-
-  const resolvedPath = resolveWorkspacePath(path);
-  const flagParts = flags ? sanitizeInput(flags) : [];
-
-  // Try rg first
-  try {
-    const rgArgs = ['--no-heading', '-n', ...flagParts, pattern, resolvedPath];
-    const { stdout, stderr, exitCode, wasTruncated } = await execCommand('rg', rgArgs);
-
-    let markdown = '# Search Results (rg)\n\n';
-    markdown += `**Pattern:** \`${pattern}\`\n`;
-    markdown += `**Path:** \`${resolvedPath}\`\n`;
-    if (flags) markdown += `**Flags:** \`${flags}\`\n`;
-    markdown += `**Exit Code:** ${exitCode}\n\n`;
-
-    const truncated = truncateHead(normalizeSearchOutput(stdout), DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES);
-
-    if (wasTruncated) {
-      markdown += '⚠️ **Output was truncated to prevent memory overflow (>10MB)**\n\n';
-    }
-
-    if (truncated) {
-      markdown += '```\n' + truncated + '\n```';
-    } else {
-      markdown += 'No matches found.';
-    }
-
-    if (stderr) {
-      markdown += '\n\n**Stderr:**\n```\n' + stderr + '\n```';
-    }
-
-    return markdown;
-  } catch (rgError) {
-    // rg not found, try grep
-    try {
-      const grepArgs = ['-rn', ...flagParts, pattern, resolvedPath];
-      const { stdout, stderr, exitCode, wasTruncated } = await execCommand('grep', grepArgs);
-
-      let markdown = '# Search Results (grep)\n\n';
-      markdown += `**Pattern:** \`${pattern}\`\n`;
-      markdown += `**Path:** \`${resolvedPath}\`\n`;
-      if (flags) markdown += `**Flags:** \`${flags}\`\n`;
-      markdown += `**Exit Code:** ${exitCode}\n\n`;
-      markdown += '*Note: Using grep fallback (rg not available)*\n\n';
-
-      const truncated = truncateHead(normalizeSearchOutput(stdout), DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES);
-
-      if (wasTruncated) {
-        markdown += '⚠️ **Output was truncated to prevent memory overflow (>10MB)**\n\n';
-      }
-
-      if (truncated) {
-        markdown += '```\n' + truncated + '\n```';
-      } else {
-        markdown += 'No matches found.';
-      }
-
-      if (stderr) {
-        markdown += '\n\n**Stderr:**\n```\n' + stderr + '\n```';
-      }
-
-      return markdown;
-    } catch (grepError) {
-      return `Error: Neither rg nor grep is available.\n\nrg error: ${rgError}\ngrep error: ${grepError}`;
-    }
-  }
+  } as ToolDefinition;
 }

package/src/tools/health-tool-definition.ts

@@ -0,0 +1,101 @@
+/**
+ * Health Tool Definition
+ *
+ * Provides health check functionality for the research system:
+ * - Check system health status across all components (browser pool, knowledge store, GPU lock)
+ */
+
+import type {
+  ToolDefinition,
+  AgentToolResult,
+  ExtensionContext,
+} from '@earendil-works/pi-coding-agent';
+import { Type, type Static } from 'typebox';
+import { healthRegistry } from '../healthcheck/index.ts';
+
+/**
+ * Create the health check tool definition
+ */
+export function createHealthTool(): ToolDefinition {
+  const parameters = Type.Object({
+    verbose: Type.Optional(Type.Boolean({
+      description: 'Show detailed diagnostic information.',
+      default: true,
+    })),
+    probe: Type.Optional(Type.Boolean({
+      description: 'Force liveness checks (browser/GPU models).',
+      default: false,
+    })),
+  });
+
+  type HealthParams = Static<typeof parameters>;
+
+  return {
+    name: 'health',
+    label: 'Health Check',
+    description: 'Verify system status (browser pool, knowledge store, GPU lock).',
+    promptSnippet: 'Run system health checks',
+    parameters,
+    renderShell: 'self',
+    executionMode: 'parallel',
+    async execute(
+      _toolCallId: string,
+      params: unknown,
+      _signal: AbortSignal | undefined,
+      _onUpdate: unknown,
+      _ctx: ExtensionContext,
+    ): Promise<AgentToolResult<unknown>> {
+      const { verbose = true, probe = false } = params as HealthParams;
+
+      const outputLines: string[] = [];
+
+      // Run health checks
+      try {
+        const systemHealth = await healthRegistry.runAll({ force: probe });
+
+        outputLines.push('## System Health Status');
+        outputLines.push('');
+
+        const statusIcon = systemHealth.status === 'healthy' ? '[OK]' :
+                          systemHealth.status === 'degraded' ? '[WARN]' : '[ERROR]';
+        const statusText = systemHealth.status === 'healthy' ? 'All systems operational' :
+                          systemHealth.status === 'degraded' ? 'System degraded (non-critical issues)' :
+                          'System unhealthy (critical failures)';
+
+        outputLines.push(`**${statusIcon} ${statusText}**`);
+        outputLines.push('');
+
+        for (const component of systemHealth.components) {
+          const icon = component.healthy ? '[OK]' : '[FAIL]';
+          const criticalMark = healthRegistry.isCritical(component.component) ? ' [CRITICAL]' : '';
+          outputLines.push(`${icon} **${component.component}**${criticalMark}`);
+          
+          if (component.error) {
+            outputLines.push(`  - Error: ${component.error}`);
+          }
+          
+          if (verbose && component.diagnostic) {
+            const diagnostics = Object.entries(component.diagnostic)
+              .map(([k, v]) => `${k}: ${v}`)
+              .join(', ');
+            if (diagnostics) {
+              outputLines.push(`  - Diagnostic: ${diagnostics}`);
+            }
+          }
+          
+          if (verbose) {
+            outputLines.push(`  - Duration: ${component.durationMs.toFixed(1)}ms`);
+          }
+          outputLines.push('');
+        }
+
+        outputLines.push(`Checked at: ${new Date(systemHealth.timestamp).toLocaleString()}`);
+
+        return { content: [{ type: 'text', text: outputLines.join('\n') }], details: { health: systemHealth } };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return { content: [{ type: 'text', text: `**Health check failed**\n\n${message}` }], details: { error: message } };
+      }
+    },
+  };
+}

package/src/tools/index.ts

@@ -1,39 +1,53 @@
 /**
  * Tool Factory
  *
- * Creates all research tools with properly configured SearXNG URL.
+ * Creates all research tools with properly configured browser manager.
  * These tools are used by both direct tool calls and researcher agents.
+ *
+ * NOTE: `research_knowledge_search` is NOT included here. It is a top-level tool
+ * registered via pi.registerTool() in src/index.ts. Researcher sub-agents
+ * already receive knowledge store data via the store_section prompt injection.
  */
 
-import type { ToolDefinition, ExtensionContext } from '@mariozechner/pi-coding-agent';
+import type { ToolDefinition, ExtensionContext } from '@earendil-works/pi-coding-agent';
 import type { ToolUsageTracker } from '../utils/tool-usage-tracker.ts';
 import type { SystemResearchState } from '../orchestration/deep-research-types.ts';
+import { createReadToolDefinition } from '@earendil-works/pi-coding-agent';
 import { createSearchTool } from './search.ts';
 import { createScrapeTool } from './scrape.ts';
 import { createSecuritySearchTool } from './security.ts';
 import { createStackexchangeTool } from './stackexchange.ts';
 import { createGrepTool } from './grep.ts';
+import { createYoutubeTranscriptTool } from './youtube-transcript.ts';
+import type { Config } from '../config.ts';
 
 interface CreateToolsOptions {
-  searxngUrl: string;
+  cwd: string;
   ctx: ExtensionContext;
   tracker: ToolUsageTracker;
   getGlobalState?: () => SystemResearchState;
   updateGlobalLinks?: (links: string[]) => void;
-  /** Returns tokens consumed by this researcher session so far (for context-aware scrape gating). */
+  /** Optional researcher ID for per-researcher scrape tracking. */
+  researcherId?: string;
+  /** Callback invoked during search with cumulative link count found so far */
+  onSearchProgress?: (links: number) => void;
+  /** Fires for each individual URL as it completes (success or failure). Used for per-URL TUI flash. */
+  onUrlScrapeResult?: (url: string, success: boolean) => void;
+  /** Returns total tokens used so far in this researcher's session (for context gating). */
   getTokensUsed?: () => number;
-  /** Model context window size in tokens; defaults to DEFAULT_MODEL_CONTEXT_WINDOW. */
+  /** Context window size in tokens. */
   contextWindowSize?: number;
+  config?: Config;
 }
 
 /**
- * Create all research tools
+ * Create all research tools for researcher sub-agents.
  */
 export function createResearchTools(options: CreateToolsOptions): ToolDefinition[] {
   // Create a fresh fallback state per tool instance to avoid state leakage between research runs
-  // This is only used when getGlobalState is not provided (e.g., quick mode with single researcher)
   const createFallbackState = (): SystemResearchState => ({
     version: 1,
+    researchId: 'fallback',
     rootQuery: '',
     complexity: 1,
     currentRound: 1,
@@ -55,15 +69,29 @@ export function createResearchTools(options: CreateToolsOptions): ToolDefinition
   };
 
   return [
-    createSearchTool(resolvedOptions),
+    createReadToolDefinition(options.cwd) as ToolDefinition,
+    createSearchTool({
+      ...resolvedOptions,
+      onProgress: options.onSearchProgress,
+    }),
     createScrapeTool({
       ...resolvedOptions,
+      onUrlScrapeResult: options.onUrlScrapeResult,
       getTokensUsed: options.getTokensUsed,
       contextWindowSize: options.contextWindowSize,
     }),
     createSecuritySearchTool(resolvedOptions),
     createStackexchangeTool(resolvedOptions),
-    createGrepTool(resolvedOptions),
+    createGrepTool({ tracker: resolvedOptions.tracker, cwd: resolvedOptions.cwd }),
+    createYoutubeTranscriptTool({
+      ctx: resolvedOptions.ctx,
+      tracker: resolvedOptions.tracker,
+      onUrlScrapeResult: options.onUrlScrapeResult,
+      getGlobalState: resolvedOptions.getGlobalState,
+      updateGlobalLinks: resolvedOptions.updateGlobalLinks,
+      researcherId: resolvedOptions.researcherId,
+      config: resolvedOptions.config,
+    }),
   ];
 }
 
@@ -75,3 +103,4 @@ export { createScrapeTool } from './scrape.ts';
 export { createSecuritySearchTool } from './security.ts';
 export { createStackexchangeTool } from './stackexchange.ts';
 export { createGrepTool } from './grep.ts';
+export { createYoutubeTranscriptTool } from './youtube-transcript.ts';