@lincoln504/pi-research 0.0.0

package/src/healthcheck/index.ts

@@ -0,0 +1,309 @@
+/**
+ * Health Check Module
+ *
+ * QUICK connectivity validation for search and scrape tools.
+ * Makes real requests to verify network is alive.
+ * Fast timeouts, simple checks (just verify tools respond with content).
+ */
+
+import { logger } from '../logger.ts';
+import { getConfig } from '../config.ts';
+import { getManager } from '../infrastructure/searxng-lifecycle.ts';
+import { search } from '../web-research/search.ts';
+import { scrapeSingle } from '../web-research/scrapers.ts';
+import { setSearxngManager } from '../web-research/utils.ts';
+import { getActiveSearxngEngines } from '../utils/searxng-config.ts';
+import { withTimeout } from '../web-research/retry-utils.ts';
+import type { SearXNGResult } from '../web-research/types.ts';
+
+// Health check cache to avoid running multiple health checks concurrently
+interface CachedHealthCheck {
+  result: HealthCheckResult;
+  timestamp: number;
+}
+
+let cachedHealthCheck: CachedHealthCheck | null = null;
+const HEALTH_CHECK_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
+let healthCheckPromise: Promise<HealthCheckResult> | null = null;
+
+// Get timeout from config or use defaults
+const config = getConfig();
+const SEARCH_TIMEOUT_MS = config.HEALTH_CHECK_TIMEOUT_MS || 15000;  // 15s max for search
+const SCRAPE_TIMEOUT_MS = config.HEALTH_CHECK_TIMEOUT_MS ? config.HEALTH_CHECK_TIMEOUT_MS + 5000 : 20000; // 5s longer than search
+
+export interface HealthCheckResult {
+  success: boolean;
+  searchOk: boolean;
+  scrapeOk: boolean;
+  error?: string;
+  details: {
+    searchQuery?: string;
+    searchResultCount?: number;
+    searchDurationMs?: number;
+    workingEngines?: string[];
+    workingEngineCount?: number;
+    scrapedUrl?: string;
+    scrapedContentLength?: number;
+    scrapedDurationMs?: number;
+  };
+}
+
+/**
+ * QUICK scrape validation: just check content exists and is readable
+ */
+function validateScrapeOutput(markdown: string): { valid: boolean; error?: string } {
+  if (typeof markdown !== 'string') {
+    return { valid: false, error: 'Content not a string' };
+  }
+
+  const trimmed = markdown.trim();
+  if (trimmed.length === 0) {
+    return { valid: false, error: 'Content is empty' };
+  }
+
+  // Just check it has some readable text (not garbage)
+  if (!/[a-zA-Z0-9]{10,}/.test(markdown)) {
+    return { valid: false, error: 'Content not readable' };
+  }
+
+  return { valid: true };
+}
+
+/**
+ * Healthcheck uses the same web-research search/scrape helpers as the main tool.
+ * Ensure they are wired to the current lifecycle manager instead of relying on
+ * an external caller to have registered it beforehand.
+ */
+function ensureHealthcheckManager(): void {
+  const manager = getManager();
+  if (!manager) {
+    throw new Error('SearXNG manager not initialized. Call initLifecycle() before runHealthCheck().');
+  }
+
+  setSearxngManager(manager);
+}
+
+/**
+ * Internal health check implementation
+ * Runs the actual health check tests
+ */
+async function performHealthCheck(): Promise<HealthCheckResult> {
+  logger.log('[healthcheck] Starting network connectivity health check...');
+
+  const result: HealthCheckResult = {
+    success: false,
+    searchOk: false,
+    scrapeOk: false,
+    error: undefined,
+    details: {},
+  };
+
+  try {
+    // Skip health check if explicitly requested via env
+    if (process.env['PI_RESEARCH_SKIP_HEALTHCHECK'] === '1') {
+      logger.log('[healthcheck] Skipping health check (PI_RESEARCH_SKIP_HEALTHCHECK=1)');
+      result.success = true;
+      result.searchOk = true;
+      result.scrapeOk = true;
+      return result;
+    }
+
+    ensureHealthcheckManager();
+
+    // PHASE 1: Test general web search engines (Bing, DuckDuckGo, Brave — NOT Wikipedia)
+    logger.log('[healthcheck] Phase 1: Testing general web search engines (timeout: ' + SEARCH_TIMEOUT_MS + 'ms)...');
+    const searchQuery = 'open source software';
+    const searchStartTime = Date.now();
+
+    let queryResults;
+    try {
+      queryResults = await withTimeout(
+        search([searchQuery]),
+        SEARCH_TIMEOUT_MS,
+        'Search'
+      );
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      result.error = `Search request failed: ${msg}. Network may be unreachable or SearXNG not responding.`;
+      logger.error('[healthcheck] Search error:', result.error);
+      return result;
+    }
+
+    const searchDurationMs = Date.now() - searchStartTime;
+
+    if (queryResults.length === 0 || !queryResults[0]) {
+      result.error = 'Search returned empty result object (SearXNG not initialized?)';
+      logger.error('[healthcheck]', result.error);
+      return result;
+    }
+
+    const qr = queryResults[0];
+    if (qr.error) {
+      const errMsg = qr.error.message || String(qr.error);
+      result.error = `Search request failed: ${errMsg}. SearXNG or engines are not responding.`;
+      logger.error('[healthcheck]', result.error);
+      return result;
+    }
+
+    // Check for at least 2 working general search engines (not Wikipedia)
+    // Wikipedia is an encyclopedic engine, not suitable for general web research
+    // Load the list of active engines from the actual SearXNG config (not hardcoded)
+    const generalEngines = getActiveSearxngEngines();
+    if (generalEngines.length === 0) {
+      result.error = 'No active general search engines found in SearXNG configuration';
+      logger.error('[healthcheck]', result.error);
+      return result;
+    }
+    logger.log(`[healthcheck] Checking for results from engines: [${generalEngines.join(', ')}]`);
+
+    const generalResults = qr.results.filter((r: SearXNGResult) =>
+      generalEngines.includes((r.engine || '').toLowerCase())
+    );
+
+    // Count how many engines returned at least one result
+    const engineCounts = new Map<string, number>();
+    generalEngines.forEach(e => engineCounts.set(e, 0));
+    generalResults.forEach((r: SearXNGResult) => {
+      const engine = (r.engine || '').toLowerCase();
+      if (engineCounts.has(engine)) {
+        engineCounts.set(engine, (engineCounts.get(engine) || 0) + 1);
+      }
+    });
+
+    const workingEngines = Array.from(engineCounts.entries()).filter(([_, count]) => count > 0);
+    const workingEngineCount = workingEngines.length;
+
+    // Require at least 1 engine to return results (local use may have rate limits)
+    if (workingEngineCount < 1) {
+      const engineReport = Array.from(engineCounts.entries())
+        .map(([e, c]) => `${e}: ${c}`)
+        .join(', ');
+      result.error = `No working engines found: 0/${generalEngines.length} returned results. Need at least 1 engine. Engine results: ${engineReport}. Your network or search engines may be blocked.`;
+      logger.error('[healthcheck] Search validation failed:', result.error);
+      return result;
+    }
+
+    result.searchOk = true;
+    result.details.searchQuery = searchQuery;
+    result.details.searchResultCount = generalResults.length;
+    result.details.searchDurationMs = searchDurationMs;
+    result.details.workingEngines = workingEngines.map(([e, _]) => e);
+    result.details.workingEngineCount = workingEngineCount;
+    const workingEnginesList = workingEngines.map(([e, _]) => e).join(', ');
+    logger.log(`[healthcheck] ✓ Search OK: ${generalResults.length} results from ${workingEngineCount} engines [${workingEnginesList}] in ${searchDurationMs}ms`);
+
+    // PHASE 2: Test scrape with timeout enforcement
+    logger.log('[healthcheck] Phase 2: Testing scrape tool (timeout: ' + SCRAPE_TIMEOUT_MS + 'ms)...');
+    const scrapeUrl = 'https://en.wikipedia.org/wiki/Python_(programming_language)';
+    const scrapeStartTime = Date.now();
+
+    let scrapeResult;
+    try {
+      scrapeResult = await withTimeout(
+        scrapeSingle(scrapeUrl),
+        SCRAPE_TIMEOUT_MS,
+        'Scrape'
+      );
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      result.error = `Scrape request failed: ${msg}. Network timeout or target unreachable.`;
+      logger.error('[healthcheck]', result.error);
+      return result;
+    }
+
+    const scrapeDurationMs = Date.now() - scrapeStartTime;
+
+    if (!scrapeResult) {
+      result.error = 'Scrape returned null result (scraper crashed?)';
+      logger.error('[healthcheck]', result.error);
+      return result;
+    }
+
+    if (scrapeResult.source === 'failed') {
+      result.error = `Scrape failed: ${scrapeResult.error || 'unknown error'}. Check if target URL is accessible.`;
+      logger.error('[healthcheck]', result.error);
+      return result;
+    }
+
+    // ROBUST validation of scraped content
+    const markdown = scrapeResult.markdown || '';
+    const scrapeValidation = validateScrapeOutput(markdown);
+    if (!scrapeValidation.valid) {
+      result.error = `Scrape validation failed: ${scrapeValidation.error}`;
+      logger.error('[healthcheck]', result.error);
+      return result;
+    }
+
+    result.scrapeOk = true;
+    result.details.scrapedUrl = scrapeUrl;
+    result.details.scrapedContentLength = markdown.length;
+    result.details.scrapedDurationMs = scrapeDurationMs;
+    logger.log(`[healthcheck] ✓ Scrape OK: ${markdown.length} bytes in ${scrapeDurationMs}ms`);
+
+    // All checks passed
+    result.success = true;
+    const totalMs = searchDurationMs + scrapeDurationMs;
+    logger.log(`[healthcheck] ✓ HEALTH CHECK PASSED in ${totalMs}ms - all systems ready`);
+    return result;
+  } catch (err) {
+    result.error = `Unexpected error during health check: ${err instanceof Error ? err.message : String(err)}`;
+    logger.error('[healthcheck] Unexpected error:', err);
+    return result;
+  }
+}
+
+/**
+ * Run health check with caching to avoid concurrent executions
+ * If a health check is already in progress, returns the same promise
+ * If a recent successful health check is cached, returns the cached result
+ */
+export async function runHealthCheck(): Promise<HealthCheckResult> {
+  const now = Date.now();
+
+  // Check if we have a recent cached result
+  if (cachedHealthCheck) {
+    const age = now - cachedHealthCheck.timestamp;
+    if (age < HEALTH_CHECK_CACHE_TTL_MS) {
+      logger.log(`[healthcheck] Using cached health check result (${Math.round(age / 1000)}s old)`);
+      return cachedHealthCheck.result;
+    } else {
+      logger.log(`[healthcheck] Cached health check result expired (${Math.round(age / 1000)}s old)`);
+      cachedHealthCheck = null;
+    }
+  }
+
+  // If a health check is already running, return the existing promise
+  if (healthCheckPromise) {
+    logger.log('[healthcheck] Health check already in progress, reusing existing promise');
+    return healthCheckPromise;
+  }
+
+  // Start a new health check
+  healthCheckPromise = (async () => {
+    try {
+      const result = await performHealthCheck();
+      // Cache successful results
+      if (result.success) {
+        cachedHealthCheck = {
+          result,
+          timestamp: now,
+        };
+      }
+      return result;
+    } finally {
+      // Clear the promise so a new health check can be started
+      healthCheckPromise = null;
+    }
+  })();
+
+  return healthCheckPromise;
+}
+
+/**
+ * Clear the health check cache
+ * Useful for testing or forcing a recheck
+ */
+export function clearHealthCheckCache(): void {
+  cachedHealthCheck = null;
+  logger.log('[healthcheck] Health check cache cleared');
+}

package/src/index.ts

@@ -0,0 +1,71 @@
+/**
+ * pi-research Extension
+ *
+ * Orchestrates multi-agent research using a state-driven deep mode architecture.
+ * Manages SearXNG as singleton (shared across all agents).
+ * SearXNG is initialized on first research() call, NOT on session_start.
+ * Extension-owned cleanup is delegated to pi's session lifecycle.
+ */
+
+import type { ExtensionAPI } from '@mariozechner/pi-coding-agent';
+import { createResearchTool } from './tool.ts';
+import { logger, isVerboseFromEnv, getDefaultDebugLogPathTemplate } from './logger.ts';
+import { checkDockerAvailability } from './infrastructure/searxng-lifecycle.ts';
+import { shutdownManager } from './utils/shutdown-manager.ts';
+import { readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+/**
+ * Load prompt from markdown file
+ */
+function loadPrompt(name: string): string {
+  const path = join(__dirname, 'prompts', `${name}.md`);
+  try {
+    return readFileSync(path, 'utf-8');
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    logger.error(`[pi-research] Failed to load prompt from ${path}: ${msg}`);
+    return '';
+  }
+}
+
+export default function (pi: ExtensionAPI) {
+  logger.log('[pi-research] Extension loading...');
+
+  // NOTE: SearXNG is still initialized lazily on first research() call in tool.ts.
+  // Cleanup runs on pi's session lifecycle instead of owning host process signals.
+
+  pi.on('session_shutdown', async (_event, _ctx) => {
+    await shutdownManager.runCleanup('session_shutdown');
+  });
+
+  // Notify user of Docker status and log file location when starting a new session
+  pi.on('session_start', async (_event, ctx) => {
+    // Check Docker availability on startup
+    const dockerCheck = await checkDockerAvailability();
+    if (!dockerCheck.running) {
+      ctx.ui.notify(`pi-research: ${dockerCheck.error}`, 'warning');
+    }
+
+    if (isVerboseFromEnv()) {
+      ctx.ui.notify(`pi-research: debug log → ${getDefaultDebugLogPathTemplate()}`, 'info');
+    }
+  });
+
+  // Register research tool
+  pi.registerTool(createResearchTool());
+
+  // Append research tool usage instructions to the system prompt
+  pi.on('before_agent_start', async (event) => {
+    const researchPrompt = loadPrompt('research-tool-usage');
+    return {
+      systemPrompt: event.systemPrompt + '\n\n' + researchPrompt
+    };
+  });
+
+  logger.log('[pi-research] Extension loaded');
+}