@lincoln504/pi-research 0.1.12 → 0.1.13

package/README.md

@@ -4,6 +4,13 @@
 
 Multi-agent web research extension for [pi](https://github.com/badlogic/pi-mono). Runs parallel AI researchers against a self-hosted SearXNG search container, scrapes pages, queries security databases, and tracks everything in a real-time terminal UI.
 
+There are other ways to connect a `pi` session to the internet. Here is why this one is designed the way it is:
+
+- **SearXNG-based** — free, no API keys required by default. A single Docker container is shared across all concurrent research calls on the same machine; no duplicate instances, no per-session overhead. Point `SEARXNG_URL` at an existing instance to skip container management entirely.
+- **Unified subagent system** — web access, scraping, and synthesis are handled by isolated researcher agents. Research context stays out of the main chat session while still enabling thorough multi-source exploration.
+- **Safety** — researcher agents have no access to file writes, file edits, or shell commands. All web tools are scoped to the subagent system and executed in a structured, tool-limited sequence to keep agents focused and prevent runaway use.
+- **Minimal integration** — a system prompt hook provides light guidance to the host LLM on when and how to invoke the tool. No manual prompt engineering required. The extension does not conflict with other subagent or tool systems, and the unified toolset means no need to wire up or tune multiple separate tools.
+
 ---
 
 ## Contents
@@ -47,6 +54,7 @@ Multi-agent web research extension for [pi](https://github.com/badlogic/pi-mono)
 - The `pi` CLI installed and configured
 - Docker daemon running (required for SearXNG)
 - Internet access
+- LLM in pi with minimum 100k context window (optimized for 100k+)  
 
 ---
 
@@ -56,10 +64,21 @@ Multi-agent web research extension for [pi](https://github.com/badlogic/pi-mono)
 pi install npm:@lincoln504/pi-research
 ```
 
+The installation will automatically download and install:
+- Node.js dependencies
+- Playwright browser binaries (Chromium)
+
+This process may take a few minutes on first install. If browser installation fails during postinstall, chromium will be installed automatically when you first use the research tool.
+
+**Manual browser installation (if needed)**
+
+```bash
+npm run install:browsers
+```
+
 **From a checkout**
 
 ```bash
-npm install
 pi install .
 ```
 
@@ -69,11 +88,8 @@ pi install .
 
 **macOS** — Install Docker Desktop and start it from Applications.
 
-**Linux** — Install Docker and add your user to the docker group:
+**Linux** — Install Docker for your distribution ([docs.docker.com/engine/install](https://docs.docker.com/engine/install)), then:
 ```bash
-# Install Docker
-sudo apt-get update && sudo apt-get install -y docker.io
-
 # Start Docker
 sudo systemctl start docker
 sudo systemctl enable docker
@@ -110,6 +126,7 @@ Do a deep dive on CVE-2024-3094 at depth 2
 
 | Variable | Default | Description |
 |----------|---------|-------------|
+| `SEARXNG_URL` | — | External SearXNG base URL (e.g. `http://localhost:8080`). When set, Docker container management is skipped entirely and this URL is used directly |
 | `BRAVE_SEARCH_API_KEY` | — | Enables `braveapi` engine (official REST API) |
 | `PI_RESEARCH_VERBOSE` | — | Set to `1` to write diagnostic JSONL logs to a temp file (see below) |
 | `PI_RESEARCH_SKIP_HEALTHCHECK` | — | Skip startup health check (set to `1` if engines are blocked) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lincoln504/pi-research",
-  "version": "0.1.12",
+  "version": "0.1.13",
   "description": "A set of web research tools (search, scrape, and others) and research agent system that uses them safely and efficiently.",
   "type": "module",
   "private": false,
@@ -50,10 +50,10 @@
     "test:integration": "vitest run --config config/tooling/vitest.config.integration.ts",
     "test:watch": "vitest --config config/tooling/vitest.config.unit.ts",
     "test:coverage": "vitest run --config config/tooling/vitest.config.ts --coverage",
-    "postinstall": "node scripts/setup.js",
+    "postinstall": "node scripts/setup.js || exit 0",
     "install:browsers": "npx playwright install chromium",
-    "install:deps": "npx playwright install-deps",
-    "setup": "npm install && npm run install:browsers"
+    "install:deps": "npx playwright install-deps chromium",
+    "setup": "npm run install:browsers"
   },
   "pi": {
     "extensions": [

package/scripts/setup.js

@@ -4,13 +4,19 @@
  * pi-research Postinstall Setup Script
  *
  * This script runs automatically after `npm install` and ensures:
- * 1. Playwright browser binaries are installed (Chromium, Firefox, WebKit)
+ * 1. Playwright browser binaries are installed (Chromium)
  * 2. System dependencies are available (if run with --system-deps flag)
  *
+ * Key Design Principles:
+ * - NEVER exits with code 1 - this would block npm install from completing
+ * - Uses npx playwright install, which works if playwright is in node_modules
+ * - Provides clear feedback about installation status
+ * - Gracefully handles missing dependencies (warns but doesn't fail)
+ *
  * Usage:
- *   npm run setup          # Full setup: install deps + browsers
- *   npm run install:browsers # Install only browser binaries
- *   npm run install:deps    # Install system dependencies only
+ *   npm run setup              # Full setup: install deps + browsers
+ *   npm run install:browsers   # Install only browser binaries
+ *   npm run install:deps       # Install system dependencies only
  *
  * Environment Variables:
  *   PLAYWRIGHT_BROWSERS_PATH    - Custom path for browser cache
@@ -19,7 +25,7 @@
 
 import { execSync } from 'child_process';
 import { existsSync, readdirSync, statSync } from 'fs';
-import { join, dirname } from 'path';
+import { dirname, join } from 'path';
 import { fileURLToPath } from 'url';
 import { homedir, platform } from 'os';
 
@@ -50,17 +56,6 @@ if (major < requiredVersions.min) {
 
 console.log(`   Required: >= ${requiredVersions.min}.0.0`);
 
-// Check if playwright is installed
-const playwrightPath = join(process.cwd(), 'node_modules', 'playwright');
-
-if (!existsSync(playwrightPath)) {
-  console.error('\n❌ ERROR: playwright is not installed.');
-  console.error('   Run `npm install` first, then `npm run setup`.');
-  process.exit(1);
-}
-
-console.log('✅ Playwright package installed');
-
 // Detect OS for browser installation
 const isLinux = platform() === 'linux';
 const isDarwin = platform() === 'darwin';
@@ -68,88 +63,130 @@ const isWindows = platform() === 'win32';
 
 console.log(`\n📦 Detected platform: ${platform()}`);
 
-// Install browsers
-console.log('\n🌐 Installing Playwright browsers...');
+// Track installation success
+let playwrightInstalled = false;
+let browsersInstalled = false;
+let systemDepsInstalled = false;
 
-try {
-  // Check if user wants to install system dependencies
-  const installDeps = process.argv.includes('--system-deps') ||
-                      process.env.PLAYWRIGHT_INSTALL_DEPS === 'true';
-
-  if (installDeps && isLinux) {
-    console.log('⚙️  Installing Linux system dependencies for Playwright...');
-    try {
-      execSync('npx playwright install-deps', {
-        stdio: 'inherit',
-        env: { ...process.env, PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '0' }
-      });
-      console.log('✅ System dependencies installed');
-    } catch (error) {
-      console.warn('⚠️  Could not install system dependencies automatically.');
-      console.warn('   Try running: sudo apt-get install $(npx playwright install-deps --print-deps)');
-    }
-  }
+// Check if we should skip browser installation
+const skipBrowserDownload = process.env.PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD === '1';
 
-  // Install browser binaries
-  // Only Chromium is needed for scraping (as per scrapers.ts implementation)
-  console.log('   Installing Chromium browser...');
-  execSync('npx playwright install chromium', {
-    stdio: 'inherit',
-    env: {
-      ...process.env,
-      PLAYWRIGHT_BROWSERS_PATH: process.env.PLAYWRIGHT_BROWSERS_PATH || ''
-    }
-  });
-
-  console.log('✅ Browsers installed successfully');
-
-  // Verify installation
-  console.log('\n📋 Verifying browser installation...');
-  const browsers = ['chromium']; // Only Chromium is required for scraping
-  const cacheDir = process.env['PLAYWRIGHT_BROWSERS_PATH'] ||
-    (process.platform === 'win32'
-      ? join(process.env['LOCALAPPDATA'] || join(homedir(), 'AppData', 'Local'), 'ms-playwright')
-      : join(homedir(), '.cache', 'ms-playwright'));
-
-  browsers.forEach(browser => {
-    let browserDirs = [];
-    try {
-      browserDirs = readdirSync(cacheDir).filter(f =>
-        f.startsWith(browser) && statSync(join(cacheDir, f)).isDirectory()
-      );
-    } catch {
-      // cache dir may not exist yet or may be at a different path
+if (skipBrowserDownload) {
+  console.log('\n⏭️  Skipping browser installation (PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1)');
+} else {
+  // Install browsers using npx playwright
+  console.log('\n🌐 Installing Playwright browsers...');
+
+  try {
+    // Check if user wants to install system dependencies
+    const installDeps = process.argv.includes('--system-deps') ||
+                        process.env.PLAYWRIGHT_INSTALL_DEPS === 'true';
+
+    if (installDeps && isLinux) {
+      console.log('⚙️  Installing Linux system dependencies for Playwright...');
+      try {
+        execSync('npx playwright install-deps', {
+          stdio: 'inherit',
+          env: { ...process.env, PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '0' }
+        });
+        console.log('✅ System dependencies installed');
+        systemDepsInstalled = true;
+      } catch (error) {
+        console.warn('⚠️  Could not install system dependencies automatically.');
+        console.warn('   Try running: sudo apt-get install $(npx playwright install-deps --print-deps)');
+        // Don't fail - continue with browser installation
+      }
     }
 
-    if (browserDirs.length > 0) {
-      console.log(`   ✅ ${browser}: ${browserDirs.sort().join(', ')}`);
+    // Install browser binaries
+    // Only Chromium is needed for scraping (as per scrapers.ts implementation)
+    console.log('   Installing Chromium browser...');
+    execSync('npx playwright install chromium', {
+      stdio: 'inherit',
+      env: {
+        ...process.env,
+        PLAYWRIGHT_BROWSERS_PATH: process.env.PLAYWRIGHT_BROWSERS_PATH || ''
+      }
+    });
+
+    console.log('✅ Browsers installed successfully');
+    browsersInstalled = true;
+
+  } catch (error) {
+    // Parse error message to provide better guidance
+    const errorMsg = error instanceof Error ? error.message : String(error);
+
+    if (errorMsg.includes('npx: command not found')) {
+      console.warn('\n⚠️  npx not available - skipping browser installation');
+      console.warn('   Chromium can be installed later with: npm run install:browsers');
+    } else if (errorMsg.includes('playwright') || errorMsg.includes('ENOENT')) {
+      console.warn('\n⚠️  Playwright package not yet available - skipping browser installation');
+      console.warn('   This is normal during initial npm install.');
+      console.warn('   Chromium will be installed automatically when you run the research tool.');
+      console.warn('   Or manually with: npm run install:browsers');
     } else {
-      console.warn(`   ⚠️  ${browser}: Not found in cache`);
+      console.warn('\n⚠️  Browser installation failed:');
+      console.warn(`   ${errorMsg}`);
+      console.warn('\n💡 You can install Chromium later with:');
+      console.warn('   npm run install:browsers');
     }
-  });
 
-  // Show browser versions
-  console.log('\n🔍 Browser versions:');
+    // NEVER exit with code 1 - let npm install complete
+    console.log('\n📦 pi-research installation will continue...');
+    console.log('='.repeat(60));
+    console.log('✅ pi-research installed (browsers can be installed later)');
+    console.log('='.repeat(60) + '\n');
+    process.exit(0);
+  }
+}
+
+// Verify installation
+console.log('\n📋 Verifying browser installation...');
+
+const browsers = ['chromium']; // Only Chromium is required for scraping
+const cacheDir = process.env['PLAYWRIGHT_BROWSERS_PATH'] ||
+  (process.platform === 'win32'
+    ? join(process.env['LOCALAPPDATA'] || join(homedir(), 'AppData', 'Local'), 'ms-playwright')
+    : join(homedir(), '.cache', 'ms-playwright'));
+
+browsers.forEach(browser => {
+  let browserDirs = [];
   try {
-    const playwrightVersion = execSync('npx playwright --version', { encoding: 'utf8' }).trim();
-    console.log(`   Playwright: ${playwrightVersion}`);
-  } catch (error) {
-    console.warn('   Could not determine Playwright version');
+    browserDirs = readdirSync(cacheDir).filter(f =>
+      f.startsWith(browser) && statSync(join(cacheDir, f)).isDirectory()
+    );
+  } catch {
+    // cache dir may not exist yet or may be at a different path
+  }
+
+  if (browserDirs.length > 0) {
+    console.log(`   ✅ ${browser}: ${browserDirs.sort().join(', ')}`);
+  } else {
+    console.warn(`   ⚠️  ${browser}: Not found in cache`);
   }
+});
 
+// Show browser versions
+console.log('\n🔍 Browser versions:');
+try {
+  const playwrightVersion = execSync('npx playwright --version', { encoding: 'utf8' }).trim();
+  console.log(`   Playwright: ${playwrightVersion}`);
+  playwrightInstalled = true;
 } catch (error) {
-  console.error('\n❌ ERROR: Failed to install browsers');
-  console.error('   Error:', error.message);
-  console.error('\n💡 Troubleshooting:');
-  console.error('   1. Ensure you have internet connectivity');
-  console.error('   2. Try running: npm run install:browsers');
-  console.error('   3. Check Node.js version: node --version');
-  console.error('   4. For Linux: sudo apt-get install -y $(npx playwright install-deps --print-deps)');
-  process.exit(1);
+  console.warn('   Could not determine Playwright version');
 }
 
+// Final status message
+const setupComplete = browsersInstalled && playwrightInstalled;
+
 console.log('\n' + '='.repeat(60));
-console.log('🎉 Setup complete! Chromium browser installed. pi-research is ready to use.');
+if (setupComplete) {
+  console.log('🎉 Setup complete! Chromium browser installed. pi-research is ready to use.');
+} else {
+  console.log('📦 pi-research installed successfully!');
+  console.log('   Chromium browser installation may be in progress or was skipped.');
+  console.log('   The browser will be installed automatically when you first use the research tool.');
+}
 console.log('='.repeat(60));
 
 // Additional info
@@ -158,6 +195,7 @@ console.log('   • npm test              - Run unit tests');
 console.log('   • npm run type-check    - Type check TypeScript');
 console.log('   • npm run lint          - Run ESLint');
 console.log('   • npm run setup         - Full setup with system deps');
+console.log('   • npm run install:browsers - Install Chromium browser only');
 
 // Platform-specific tips
 if (isLinux) {

package/src/config.ts

@@ -27,6 +27,8 @@ export interface Config {
   CONSOLE_RESTORE_DELAY_MS: number;
   /** BCP 47 language tag sent to SearXNG to filter search results (default: 'en-US') */
   SEARCH_LANGUAGE: string;
+  /** External SearXNG base URL — bypasses Docker container management entirely when set */
+  SEARXNG_URL?: string;
 }
 
 /**
@@ -41,6 +43,7 @@ const DEFAULTS: Config = {
   TUI_REFRESH_DEBOUNCE_MS: 10,
   CONSOLE_RESTORE_DELAY_MS: 15000,
   SEARCH_LANGUAGE: 'en-US',
+  SEARXNG_URL: undefined,
 };
 
 /**
@@ -110,6 +113,7 @@ export function createConfig(env: Record<string, string | undefined> = process.e
       DEFAULTS.CONSOLE_RESTORE_DELAY_MS
     ),
     SEARCH_LANGUAGE: parseEnvString(env, 'PI_RESEARCH_SEARCH_LANGUAGE') ?? DEFAULTS.SEARCH_LANGUAGE,
+    SEARXNG_URL: parseEnvString(env, 'SEARXNG_URL'),
   };
 }
 
@@ -200,7 +204,15 @@ export function getBraveSearchApiKey(): string | undefined {
   return getConfig().BRAVE_SEARCH_API_KEY;
 }
 
+/**
+ * @deprecated Use getConfig().SEARXNG_URL instead
+ */
+export function getExternalSearxngUrl(): string | undefined {
+  return getConfig().SEARXNG_URL;
+}
+
 // Legacy variable exports (for backward compatibility, but reflect module load time values)
 export const RESEARCHER_TIMEOUT_MS = getResearcherTimeoutMs();
 export const PROXY_URL = getProxyUrl();
 export const BRAVE_SEARCH_API_KEY = getBraveSearchApiKey();
+export const SEARXNG_URL = getExternalSearxngUrl();

package/src/index.ts

@@ -47,7 +47,7 @@ export default function (pi: ExtensionAPI) {
   pi.on('session_start', async (_event, ctx) => {
     // Skip Docker check during install to avoid errors on fresh systems
     // Docker will be checked when research tool is first used
-    if (!process.env['PI_INSTALL_MODE']) {
+    if (!process.env['PI_INSTALL_MODE'] && !process.env['SEARXNG_URL']) {
       const dockerCheck = await checkDockerAvailability();
       if (!dockerCheck.running) {
         ctx.ui.notify(`pi-research: ${dockerCheck.error}`, 'warning');

package/src/infrastructure/searxng-lifecycle.ts

@@ -24,7 +24,7 @@ import * as path from 'node:path';
 import * as fs from 'node:fs';
 import * as os from 'node:os';
 import { fileURLToPath } from 'node:url';
-import { PROXY_URL, BRAVE_SEARCH_API_KEY } from '../config.ts';
+import { PROXY_URL, BRAVE_SEARCH_API_KEY, SEARXNG_URL } from '../config.ts';
 import { shutdownManager } from '../utils/shutdown-manager.ts';
 
 /**
@@ -98,6 +98,8 @@ export interface SearxngLifecycleConfig {
   manager?: DockerSearxngManager | null;
   proxyUrl?: string;
   braveApiKey?: string;
+  /** External SearXNG base URL. When set, Docker container management is skipped entirely. */
+  searxngUrl?: string;
 }
 
 type ResolvedConfig = {
@@ -106,6 +108,7 @@ type ResolvedConfig = {
   manager: DockerSearxngManager | null;
   proxyUrl: string | undefined;
   braveApiKey: string | undefined;
+  searxngUrl: string | undefined;
 };
 
 /**
@@ -132,6 +135,7 @@ const DEFAULT_CONFIG: ResolvedConfig = {
   manager: null,
   proxyUrl: PROXY_URL,
   braveApiKey: BRAVE_SEARCH_API_KEY,
+  searxngUrl: SEARXNG_URL,
 };
 
 /**
@@ -152,6 +156,10 @@ export class SearxngLifecycleManager implements ISearxngLifecycleManager {
   private statusCallbacks: StatusCallback[] = [];
   private readonly config: ResolvedConfig;
   private proxySettingsPath: string | null = null;
+  /** Coalesces concurrent init() calls onto a single in-flight promise. */
+  private initializationPromise: Promise<void> | null = null;
+  /** Set when SEARXNG_URL is configured; signals that Docker management is bypassed. */
+  private externalSearxngUrl: string | null = null;
 
   constructor(config: Partial<SearxngLifecycleConfig> = {}) {
     this.config = { ...DEFAULT_CONFIG, ...config } as ResolvedConfig;
@@ -162,9 +170,15 @@ export class SearxngLifecycleManager implements ISearxngLifecycleManager {
   }
 
   /**
-   * Get manager instance (for passing to web-research module)
+   * Get manager instance (for passing to web-research module).
+   * In external URL mode, returns a minimal shim that satisfies the getSearxngUrl()
+   * interface used by web-research tools and the health check.
    */
   getManager(): DockerSearxngManager | null {
+    if (this.externalSearxngUrl) {
+      const url = this.externalSearxngUrl;
+      return { getSearxngUrl: () => url } as unknown as DockerSearxngManager;
+    }
     return this.manager;
   }
 
@@ -279,6 +293,46 @@ export class SearxngLifecycleManager implements ISearxngLifecycleManager {
       return;
     }
 
+    // Coalesce concurrent init() calls: only the first proceeds; latecomers
+    // await the in-flight promise and return once it resolves.
+    // This assignment and check are both synchronous (no await between them),
+    // so JavaScript's single-threaded event loop guarantees only one caller
+    // can set initializationPromise before any other caller checks it.
+    if (this.initializationPromise) {
+      logger.log('[pi-research] SearXNG init already in progress, awaiting');
+      await this.initializationPromise;
+      return;
+    }
+
+    let resolveInit!: () => void;
+    let rejectInit!: (err: unknown) => void;
+    this.initializationPromise = new Promise<void>((resolve, reject) => {
+      resolveInit = resolve;
+      rejectInit = reject;
+    });
+
+    try {
+      await this._performInit(ctx);
+      resolveInit();
+    } catch (err) {
+      rejectInit(err);
+      throw err;
+    } finally {
+      this.initializationPromise = null;
+    }
+  }
+
+  private async _performInit(ctx: ExtensionContext): Promise<void> {
+    // External SearXNG URL: skip all Docker management and use the URL directly.
+    if (this.config.searxngUrl) {
+      this.externalSearxngUrl = this.config.searxngUrl;
+      this.currentStatus = { state: 'active', url: this.config.searxngUrl, isFunctional: false };
+      this.initialized = true;
+      logger.log('[pi-research] External SEARXNG_URL configured — Docker management skipped:', this.config.searxngUrl);
+      this.notifyStatusChange();
+      return;
+    }
+
     // Skip real Docker verification when a test/dummy manager is injected.
     if (!this.config.manager) {
       const dockerCheck = await verifyDockerInstalled();
@@ -396,6 +450,10 @@ export class SearxngLifecycleManager implements ISearxngLifecycleManager {
   }
 
   async ensureRunning(): Promise<string> {
+    if (this.externalSearxngUrl) {
+      return this.externalSearxngUrl;
+    }
+
     if (!this.manager) {
       throw new Error('SearXNG not initialized. Call initLifecycle() first.');
     }
@@ -411,6 +469,15 @@ export class SearxngLifecycleManager implements ISearxngLifecycleManager {
   }
 
   async shutdown(): Promise<void> {
+    if (this.externalSearxngUrl) {
+      // External URL mode — no container to tear down
+      this.externalSearxngUrl = null;
+      this.currentStatus = { state: 'inactive', url: '', isFunctional: false };
+      this.initialized = false;
+      this.notifyStatusChange();
+      return;
+    }
+
     if (!this.manager || !this.initialized) {
       logger.log('[pi-research] SearXNG not active, skipping shutdown');
       return;
@@ -454,7 +521,7 @@ export class SearxngLifecycleManager implements ISearxngLifecycleManager {
   }
 
   isInitialized(): boolean {
-    return this.initialized && this.manager !== null;
+    return this.initialized && (this.manager !== null || this.externalSearxngUrl !== null);
   }
 }
 

package/src/utils/text-utils.ts

@@ -79,5 +79,15 @@ export function ensureAssistantResponse(session: AgentSession, label: string): s
     throw new Error(`${label}: Provider error - ${msg}`);
   }
 
-  return extractText(last);
+  const text = extractText(last);
+  if (!text.trim()) {
+    // No text content blocks in the final assistant message. This typically means
+    // all tool calls failed (e.g. SearXNG unreachable) and the model produced only
+    // thinking blocks, or the session ended without a visible response.
+    throw new Error(
+      `${label}: Researcher produced no text output. ` +
+      `This usually means SearXNG was unavailable during the run — check Docker status and retry.`
+    );
+  }
+  return text;
 }