@lincoln504/pi-research 0.0.0

package/package.json

@@ -0,0 +1,92 @@
+{
+  "name": "@lincoln504/pi-research",
+  "version": "0.0.0",
+  "description": "Multi-agent research Deep Mode orchestration for pi. State-driven parallel researchers with web search, scraping, security databases, and Stack Exchange integration.",
+  "type": "module",
+  "private": false,
+  "main": "src/index.ts",
+  "files": [
+    "src/",
+    "config/",
+    "docs/",
+    "scripts/",
+    "LICENSE",
+    "README.md",
+    "tsconfig.json"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Lincoln504/pi-research.git"
+  },
+  "homepage": "https://github.com/Lincoln504/pi-research#readme",
+  "bugs": {
+    "url": "https://github.com/Lincoln504/pi-research/issues"
+  },
+  "keywords": [
+    "pi",
+    "pi-package",
+    "research",
+    "agent",
+    "orchestration",
+    "web-search",
+    "scraping",
+    "security",
+    "stack-exchange"
+  ],
+  "author": "Lincolndeen",
+  "contributors": [
+    "MuratAitov"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": "^22.13.0 || >=24.0.0"
+  },
+  "scripts": {
+    "lint": "npx eslint --config config/tooling/eslint.config.js src/",
+    "lint:fix": "npx eslint --config config/tooling/eslint.config.js src/ --fix",
+    "type-check": "tsc --noEmit",
+    "test": "vitest run --config config/tooling/vitest.config.unit.ts",
+    "test:unit": "vitest run --config config/tooling/vitest.config.unit.ts",
+    "test:integration": "vitest run --config config/tooling/vitest.config.integration.ts",
+    "test:watch": "vitest --config config/tooling/vitest.config.unit.ts",
+    "test:coverage": "vitest run --config config/tooling/vitest.config.ts --coverage",
+    "postinstall": "node scripts/setup.js",
+    "install:browsers": "npx playwright install chromium",
+    "install:deps": "npx playwright install-deps",
+    "setup": "npm install && npm run install:browsers"
+  },
+  "pi": {
+    "extensions": [
+      "src/index.ts"
+    ]
+  },
+  "dependencies": {
+    "@kreuzberg/html-to-markdown-node": "^2.30.0",
+    "dockerode": "^4.0.10",
+    "js-yaml": "^4.1.1",
+    "node-html-markdown": "^2.0.0",
+    "playwright": "^1.52.0"
+  },
+  "optionalDependencies": {
+    "@kreuzberg/html-to-markdown-node-darwin-arm64": "^2.30.0",
+    "@kreuzberg/html-to-markdown-node-darwin-x64": "^2.30.0",
+    "@kreuzberg/html-to-markdown-node-win32-x64-msvc": "^2.30.0"
+  },
+  "peerDependencies": {
+    "@mariozechner/pi-coding-agent": ">=0.65.0",
+    "@sinclair/typebox": ">=0.34.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@types/dockerode": "^4.0.1",
+    "@types/js-yaml": "^4.0.9",
+    "@typescript-eslint/eslint-plugin": "^8.58.0",
+    "@typescript-eslint/parser": "^8.58.0",
+    "@vitest/coverage-v8": "^4.1.2",
+    "eslint": "^10.1.0",
+    "eslint-config-prettier": "^10.1.0",
+    "testcontainers": "^11.13.0",
+    "typescript": "^6.0.2",
+    "vitest": "^4.1.2"
+  }
+}

package/scripts/setup.js

@@ -0,0 +1,174 @@
+#!/usr/bin/env node
+
+/**
+ * pi-research Postinstall Setup Script
+ *
+ * This script runs automatically after `npm install` and ensures:
+ * 1. Playwright browser binaries are installed (Chromium, Firefox, WebKit)
+ * 2. System dependencies are available (if run with --system-deps flag)
+ *
+ * Usage:
+ *   npm run setup          # Full setup: install deps + browsers
+ *   npm run install:browsers # Install only browser binaries
+ *   npm run install:deps    # Install system dependencies only
+ *
+ * Environment Variables:
+ *   PLAYWRIGHT_BROWSERS_PATH    - Custom path for browser cache
+ *   PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1  - Skip browser download
+ */
+
+import { execSync } from 'child_process';
+import { existsSync, readdirSync, statSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { homedir, platform } from 'os';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+console.log('='.repeat(60));
+console.log('🔧 pi-research Setup');
+console.log('='.repeat(60));
+
+// Check Node.js version
+const nodeVersion = process.version.replace('v', '');
+const [major] = nodeVersion.split('.').map(Number);
+
+console.log(`\nℹ️  Node.js version: v${nodeVersion}`);
+
+const requiredVersions = {
+  min: 22,
+  preferred: [22, 24]
+};
+
+if (major < requiredVersions.min) {
+  console.warn(
+    `⚠️  WARNING: Node.js ${nodeVersion} is below the minimum required version ${requiredVersions.min}.`
+  );
+  console.warn('   Please upgrade to Node.js 22.13.0+ or 24.x+');
+}
+
+console.log(`   Required: >= ${requiredVersions.min}.0.0`);
+
+// Check if playwright is installed
+const playwrightPath = join(process.cwd(), 'node_modules', 'playwright');
+
+if (!existsSync(playwrightPath)) {
+  console.error('\n❌ ERROR: playwright is not installed.');
+  console.error('   Run `npm install` first, then `npm run setup`.');
+  process.exit(1);
+}
+
+console.log('✅ Playwright package installed');
+
+// Detect OS for browser installation
+const isLinux = platform() === 'linux';
+const isDarwin = platform() === 'darwin';
+const isWindows = platform() === 'win32';
+
+console.log(`\n📦 Detected platform: ${platform()}`);
+
+// Install browsers
+console.log('\n🌐 Installing Playwright browsers...');
+
+try {
+  // Check if user wants to install system dependencies
+  const installDeps = process.argv.includes('--system-deps') ||
+                      process.env.PLAYWRIGHT_INSTALL_DEPS === 'true';
+
+  if (installDeps && isLinux) {
+    console.log('⚙️  Installing Linux system dependencies for Playwright...');
+    try {
+      execSync('npx playwright install-deps', {
+        stdio: 'inherit',
+        env: { ...process.env, PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '0' }
+      });
+      console.log('✅ System dependencies installed');
+    } catch (error) {
+      console.warn('⚠️  Could not install system dependencies automatically.');
+      console.warn('   Try running: sudo apt-get install $(npx playwright install-deps --print-deps)');
+    }
+  }
+
+  // Install browser binaries
+  // Only Chromium is needed for scraping (as per scrapers.ts implementation)
+  console.log('   Installing Chromium browser...');
+  execSync('npx playwright install chromium', {
+    stdio: 'inherit',
+    env: {
+      ...process.env,
+      PLAYWRIGHT_BROWSERS_PATH: process.env.PLAYWRIGHT_BROWSERS_PATH || ''
+    }
+  });
+
+  console.log('✅ Browsers installed successfully');
+
+  // Verify installation
+  console.log('\n📋 Verifying browser installation...');
+  const browsers = ['chromium']; // Only Chromium is required for scraping
+  const cacheDir = process.env['PLAYWRIGHT_BROWSERS_PATH'] ||
+    (process.platform === 'win32'
+      ? join(process.env['LOCALAPPDATA'] || join(homedir(), 'AppData', 'Local'), 'ms-playwright')
+      : join(homedir(), '.cache', 'ms-playwright'));
+
+  browsers.forEach(browser => {
+    let browserDirs = [];
+    try {
+      browserDirs = readdirSync(cacheDir).filter(f =>
+        f.startsWith(browser) && statSync(join(cacheDir, f)).isDirectory()
+      );
+    } catch {
+      // cache dir may not exist yet or may be at a different path
+    }
+
+    if (browserDirs.length > 0) {
+      console.log(`   ✅ ${browser}: ${browserDirs.sort().join(', ')}`);
+    } else {
+      console.warn(`   ⚠️  ${browser}: Not found in cache`);
+    }
+  });
+
+  // Show browser versions
+  console.log('\n🔍 Browser versions:');
+  try {
+    const playwrightVersion = execSync('npx playwright --version', { encoding: 'utf8' }).trim();
+    console.log(`   Playwright: ${playwrightVersion}`);
+  } catch (error) {
+    console.warn('   Could not determine Playwright version');
+  }
+
+} catch (error) {
+  console.error('\n❌ ERROR: Failed to install browsers');
+  console.error('   Error:', error.message);
+  console.error('\n💡 Troubleshooting:');
+  console.error('   1. Ensure you have internet connectivity');
+  console.error('   2. Try running: npm run install:browsers');
+  console.error('   3. Check Node.js version: node --version');
+  console.error('   4. For Linux: sudo apt-get install -y $(npx playwright install-deps --print-deps)');
+  process.exit(1);
+}
+
+console.log('\n' + '='.repeat(60));
+console.log('🎉 Setup complete! Chromium browser installed. pi-research is ready to use.');
+console.log('='.repeat(60));
+
+// Additional info
+console.log('\n📚 Quick start:');
+console.log('   • npm test              - Run unit tests');
+console.log('   • npm run type-check    - Type check TypeScript');
+console.log('   • npm run lint          - Run ESLint');
+console.log('   • npm run setup         - Full setup with system deps');
+
+// Platform-specific tips
+if (isLinux) {
+  console.log('\n💡 Linux tip: If you encounter issues, install system deps:');
+  console.log('   sudo apt-get install -y libgbm1 libnss3 libatk1.0-0');
+  console.log('   libatk-bridge2.0-0 libcups2 libxkbcommon0 libxcomposite1');
+} else if (isDarwin) {
+  console.log('\n💡 macOS: Ensure Xcode Command Line Tools are installed:');
+  console.log('   xcode-select --install');
+} else if (isWindows) {
+  console.log('\n💡 Windows: Ensure you have the latest Visual C++ redistributables');
+}
+
+console.log('\n');

package/src/config.ts

@@ -0,0 +1,206 @@
+/**
+ * Configuration Module (Refactored for Testability)
+ *
+ * Uses factory pattern instead of module-level caching.
+ * Allows for different configurations in tests.
+ */
+
+import { logger } from './logger.ts';
+
+/**
+ * Configuration values
+ */
+export interface Config {
+  /** Per-researcher timeout in milliseconds (default: 240000) */
+  RESEARCHER_TIMEOUT_MS: number;
+  /** Maximum researchers allowed to run simultaneously (default: 3) */
+  MAX_CONCURRENT_RESEARCHERS: number;
+  /** Proxy URL for SearXNG searches (optional) */
+  PROXY_URL?: string;
+  /** Brave Search API key — enables the braveapi engine in SearXNG when set (optional) */
+  BRAVE_SEARCH_API_KEY?: string;
+  /** Health check timeout in milliseconds (default: 15000) */
+  HEALTH_CHECK_TIMEOUT_MS?: number;
+  /** Global TUI refresh debounce in milliseconds (default: 10) */
+  TUI_REFRESH_DEBOUNCE_MS: number;
+  /** Console restore delay after research in milliseconds (default: 15000) */
+  CONSOLE_RESTORE_DELAY_MS: number;
+  /** BCP 47 language tag sent to SearXNG to filter search results (default: 'en-US') */
+  SEARCH_LANGUAGE: string;
+}
+
+/**
+ * Configuration defaults
+ */
+const DEFAULTS: Config = {
+  RESEARCHER_TIMEOUT_MS: 240000,
+  MAX_CONCURRENT_RESEARCHERS: 3,
+  PROXY_URL: undefined,
+  BRAVE_SEARCH_API_KEY: undefined,
+  HEALTH_CHECK_TIMEOUT_MS: 15000,
+  TUI_REFRESH_DEBOUNCE_MS: 10,
+  CONSOLE_RESTORE_DELAY_MS: 15000,
+  SEARCH_LANGUAGE: 'en-US',
+};
+
+/**
+ * Parse environment variable to number with default
+ */
+function parseEnvNumber(
+  env: Record<string, string | undefined>,
+  key: string,
+  defaultValue: number,
+  base: number = 10
+): number {
+  const value = env[key];
+  if (value === undefined) {
+    return defaultValue;
+  }
+  const parsed = parseInt(value, base);
+  if (isNaN(parsed)) {
+    logger.warn(`[config] Invalid value for ${key}, using default: ${defaultValue}`);
+    return defaultValue;
+  }
+  return parsed;
+}
+
+/**
+ * Get optional string environment variable
+ */
+function parseEnvString(
+  env: Record<string, string | undefined>,
+  key: string
+): string | undefined {
+  const value = env[key];
+  return value === undefined ? undefined : value;
+}
+
+/**
+ * Create configuration from environment variables
+ * @param env Optional environment object (for testing)
+ * @returns Configuration object
+ */
+export function createConfig(env: Record<string, string | undefined> = process.env): Config {
+  return {
+    RESEARCHER_TIMEOUT_MS: parseEnvNumber(
+      env,
+      'PI_RESEARCH_RESEARCHER_TIMEOUT_MS',
+      DEFAULTS.RESEARCHER_TIMEOUT_MS
+    ),
+    MAX_CONCURRENT_RESEARCHERS: parseEnvNumber(
+      env,
+      'PI_RESEARCH_MAX_CONCURRENT_RESEARCHERS',
+      DEFAULTS.MAX_CONCURRENT_RESEARCHERS
+    ),
+    PROXY_URL: parseEnvString(env, 'PROXY_URL'),
+    BRAVE_SEARCH_API_KEY: parseEnvString(env, 'BRAVE_SEARCH_API_KEY'),
+    HEALTH_CHECK_TIMEOUT_MS: parseEnvNumber(
+      env,
+      'PI_RESEARCH_HEALTH_CHECK_TIMEOUT_MS',
+      DEFAULTS.HEALTH_CHECK_TIMEOUT_MS as number,
+    ),
+    TUI_REFRESH_DEBOUNCE_MS: parseEnvNumber(
+      env,
+      'PI_RESEARCH_TUI_REFRESH_DEBOUNCE_MS',
+      DEFAULTS.TUI_REFRESH_DEBOUNCE_MS
+    ),
+    CONSOLE_RESTORE_DELAY_MS: parseEnvNumber(
+      env,
+      'PI_RESEARCH_CONSOLE_RESTORE_DELAY_MS',
+      DEFAULTS.CONSOLE_RESTORE_DELAY_MS
+    ),
+    SEARCH_LANGUAGE: parseEnvString(env, 'PI_RESEARCH_SEARCH_LANGUAGE') ?? DEFAULTS.SEARCH_LANGUAGE,
+  };
+}
+
+/**
+ * Global configuration instance (for backward compatibility)
+ */
+let globalConfig: Config | null = null;
+
+/**
+ * Get global configuration
+ * Creates from environment if not yet initialized
+ */
+export function getConfig(): Config {
+  if (!globalConfig) {
+    globalConfig = createConfig();
+  }
+  return globalConfig;
+}
+
+/**
+ * Set global configuration (for testing or manual override)
+ */
+export function setConfig(config: Config): void {
+  globalConfig = config;
+}
+
+/**
+ * Reset global configuration to environment defaults
+ */
+export function resetConfig(): void {
+  globalConfig = null;
+}
+
+/**
+ * Validate configuration values
+ * @param config Configuration to validate
+ * @throws {Error} if configuration is invalid
+ */
+export function validateConfig(config: Config = getConfig()): void {
+  if (config.RESEARCHER_TIMEOUT_MS < 30000 || config.RESEARCHER_TIMEOUT_MS > 600000) {
+    throw new Error(
+      `PI_RESEARCH_RESEARCHER_TIMEOUT_MS must be between 30000ms (30s) and 600000ms (10m), got ${config.RESEARCHER_TIMEOUT_MS}ms`
+    );
+  }
+
+  if (config.MAX_CONCURRENT_RESEARCHERS < 1 || config.MAX_CONCURRENT_RESEARCHERS > 10) {
+    throw new Error(
+      `PI_RESEARCH_MAX_CONCURRENT_RESEARCHERS must be between 1 and 10, got ${config.MAX_CONCURRENT_RESEARCHERS}`
+    );
+  }
+
+  // Warn if proxy is configured
+  if (config.PROXY_URL) {
+    logger.warn('[config] Proxy configured - searches will be routed through:', config.PROXY_URL);
+  }
+
+  logger.debug('[config] Configuration validated:', {
+    RESEARCHER_TIMEOUT_MS: config.RESEARCHER_TIMEOUT_MS,
+    PROXY_URL: config.PROXY_URL,
+    HEALTH_CHECK_TIMEOUT_MS: config.HEALTH_CHECK_TIMEOUT_MS,
+  });
+}
+
+// ============================================================================
+// Backward Compatibility Exports
+// ============================================================================
+
+/**
+ * @deprecated Use getConfig().RESEARCHER_TIMEOUT_MS instead
+ * Returns current config value - reflects environment changes if config is reset
+ */
+export function getResearcherTimeoutMs(): number {
+  return getConfig().RESEARCHER_TIMEOUT_MS;
+}
+
+/**
+ * @deprecated Use getConfig().PROXY_URL instead
+ * Returns current config value - reflects environment changes if config is reset
+ */
+export function getProxyUrl(): string | undefined {
+  return getConfig().PROXY_URL;
+}
+
+/**
+ * @deprecated Use getConfig().BRAVE_SEARCH_API_KEY instead
+ */
+export function getBraveSearchApiKey(): string | undefined {
+  return getConfig().BRAVE_SEARCH_API_KEY;
+}
+
+// Legacy variable exports (for backward compatibility, but reflect module load time values)
+export const RESEARCHER_TIMEOUT_MS = getResearcherTimeoutMs();
+export const PROXY_URL = getProxyUrl();
+export const BRAVE_SEARCH_API_KEY = getBraveSearchApiKey();

package/src/constants.ts

@@ -0,0 +1,148 @@
+/**
+ * Project Constants
+ *
+ * Centralized constants for magic numbers and configuration values.
+ */
+
+// ==================== Time Constants ====================
+
+/** Flash duration for UI slice animations in milliseconds */
+export const FLASH_DURATION_MS = 500;
+
+/** Delay between API requests in milliseconds (e.g., NVD rate limit) */
+export const REQUEST_DELAY_MS = 6500;
+
+// ==================== Research Constraints ====================
+
+/** Maximum gathering (tool) calls per researcher */
+export const MAX_GATHERING_CALLS = 4;
+
+/** Maximum scrape tool calls per researcher (1 handshake + up to 3 execution batches) */
+export const MAX_SCRAPE_CALLS = 4;
+
+/** Maximum URLs to scrape per batch */
+export const MAX_SCRAPE_URLS = 3;
+
+/** Maximum siblings (researchers) allowed to run simultaneously across all rounds */
+export const MAX_CONCURRENT_RESEARCHERS = 3;
+
+
+// ==================== Complexity Levels ====================
+
+/**
+ * Complexity 0: Quick mode (no orchestrator, 1 researcher, 1 round)
+ * Complexity 1: Normal (2 researchers, 2 rounds)
+ * Complexity 2: Deep (3 researchers, 3 rounds)
+ * Complexity 3: Ultra (5 researchers, 5 rounds)
+ */
+
+/** Initial researcher count for complexity 1 (Normal) */
+export const INITIAL_RESEARCHERS_LEVEL_1 = 2;
+
+/** Initial researcher count for complexity 2 (Deep) */
+export const INITIAL_RESEARCHERS_LEVEL_2 = 3;
+
+/** Initial researcher count for complexity 3 (Ultra) */
+export const INITIAL_RESEARCHERS_LEVEL_3 = 5;
+
+
+/** Maximum rounds for complexity 1 (Normal) */
+export const MAX_ROUNDS_LEVEL_1 = 2;
+
+/** Maximum rounds for complexity 2 (Deep) */
+export const MAX_ROUNDS_LEVEL_2 = 3;
+
+/** Maximum rounds for complexity 3 (Ultra) */
+export const MAX_ROUNDS_LEVEL_3 = 5;
+
+/**
+ * Extra rounds the lead evaluator may earn beyond targetRounds when it judges
+ * critical gaps still exist. Rounds targetRounds+1 through targetRounds+MAX_EXTRA_ROUNDS
+ * are "bonus" territory; round targetRounds+MAX_EXTRA_ROUNDS+1 is unreachable (hard limit).
+ */
+export const MAX_EXTRA_ROUNDS = 2;
+
+
+// ==================== Timeout Constants ====================
+
+/** Default timeout for SearXNG requests in milliseconds */
+export const SEARXNG_TIMEOUT_MS = 30000;
+
+/** Default timeout for OSV API requests in milliseconds */
+export const OSV_TIMEOUT_MS = 30000;
+
+/** Default timeout for NVD API requests in milliseconds */
+export const NVD_TIMEOUT_MS = 30000;
+
+// ==================== Export Constants ====================
+
+/** Maximum query length for filename sanitization */
+export const MAX_QUERY_LENGTH = 50;
+
+/** Maximum retry attempts for export file collision */
+export const MAX_EXPORT_RETRIES = 3;
+
+// ==================== Retry Constants ====================
+
+/** Default maximum retry attempts for transient errors */
+export const DEFAULT_MAX_RETRIES = 3;
+
+/** Default initial delay for exponential backoff in milliseconds */
+export const DEFAULT_INITIAL_DELAY_MS = 1000;
+
+/** Default maximum delay for exponential backoff in milliseconds */
+export const DEFAULT_MAX_DELAY_MS = 10000;
+
+// ==================== Context-Aware Scraping Constants ====================
+
+/**
+ * Fraction of context window consumed beyond which all scraping is blocked.
+ * At or above this threshold, researchers skip batches and move to synthesis.
+ */
+export const MAX_CONTEXT_FRACTION_FOR_SCRAPING = 0.55;
+
+/**
+ * Estimated tokens consumed per scrape call (handshake + content).
+ * Used to project whether another batch is feasible given remaining context.
+ */
+export const AVG_TOKENS_PER_SCRAPE = 15000;
+
+/**
+ * Fallback context window size (tokens) when the model does not advertise one.
+ * Corresponds to typical large-context models (Sonnet, Gemini 1.5 Pro, etc.).
+ */
+export const DEFAULT_MODEL_CONTEXT_WINDOW = 200000;
+
+/**
+ * Maximum URLs per second-scrape batch (more targeted than batch 1).
+ * Kept lower to preserve context for synthesis after focused follow-up.
+ */
+export const BATCH_2_MAX_SCRAPE_URLS = 2;
+
+/**
+ * Default maxConcurrency for the second scrape batch.
+ * Higher than batch 1 because batch 2 URLs are already prioritized.
+ */
+export const BATCH_2_DEFAULT_CONCURRENCY = 15;
+
+// ==================== UI Constants ====================
+
+/** Maximum characters for final report before truncation */
+export const MAX_REPORT_LENGTH = 200000;
+
+/** Visual width of the ====------ progress bar in the TUI header */
+export const PROGRESS_BAR_WIDTH = 18;
+
+// ==================== Orchestrator Constants ====================
+
+/** Maximum characters per researcher report when sent to lead evaluator (truncated with link preservation) */
+export const MAX_EVALUATOR_REPORT_LENGTH = 50000;
+
+/** Delay in milliseconds between launching concurrent researchers to stagger startup */
+export const RESEARCHER_LAUNCH_DELAY_MS = 1500;
+
+/** Delay in milliseconds after state update to ensure propagation before next action */
+export const STATE_PROPAGATION_DELAY_MS = 50;
+
+/** Minimum streaming text length before updating UI with estimated tokens */
+export const STREAMING_UPDATE_THRESHOLD_CHARS = 200;