@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/constants.ts

@@ -4,80 +4,126 @@
  * Centralized constants for magic numbers and configuration values.
  */
 
+import { getConfig, type Config } from './config.ts';
+
 // ==================== Time Constants ====================
 
-/** Flash duration for UI slice animations in milliseconds */
-export const FLASH_DURATION_MS = 500;
+/** Delay between API requests in milliseconds */
+export const REQUEST_DELAY_MS_NVD = 6000;
+export const REQUEST_DELAY_MS_OTHER = 1000;
 
-/** Delay between API requests in milliseconds (e.g., NVD rate limit) */
-export const REQUEST_DELAY_MS = 6500;
+/** Duration (ms) for a green (success) flash on a researcher column */
+export const FLASH_GREEN_DURATION_MS = 400;
 
-// ==================== Research Constraints ====================
+/** Duration (ms) for a red (failure) flash on a researcher column */
+export const FLASH_RED_DURATION_MS = 700;
 
-/** Maximum gathering (tool) calls per researcher */
-export const MAX_GATHERING_CALLS = 4;
+/** Duration (ms) for a tool status 'pop' (word flash) in a researcher column */
+export const FLASH_STATUS_DURATION_MS = 800;
 
-/** Maximum scrape tool calls per researcher (1 handshake + up to 3 execution batches) */
-export const MAX_SCRAPE_CALLS = 4;
+/** Minimum gap (ms) between queued flashes for the same researcher column */
+export const FLASH_QUEUE_GAP_MS = 150;
 
-/** Maximum URLs to scrape per batch */
-export const MAX_SCRAPE_URLS = 3;
+// ==================== Research Constraints ====================
 
-/** Maximum siblings (researchers) allowed to run simultaneously across all rounds */
-export const MAX_CONCURRENT_RESEARCHERS = 3;
+/** Maximum gathering (tool) calls per researcher (search, security_search, stackexchange, youtube_transcript — shared web-API budget).
+ *  This is the schema default; use getMaxGatheringCalls(config) for the configured value. */
+export const MAX_GATHERING_CALLS = 12;
 
+/** Maximum local grep (ripgrep) calls per researcher. grep is local/free and does not hit
+ *  rate-limited web APIs, so it has its own generous budget instead of sharing the gathering
+ *  pool — but it is still bounded to prevent a pathological search loop. */
+export const MAX_GREP_CALLS = 30;
 
-// ==================== Complexity Levels ====================
+/**
+ * Get the maximum gathering calls from config (env/config-file driven, mirrors getMaxScrapeBatches).
+ * Not surfaced in the TUI — advanced knob only.
+ */
+export function getMaxGatheringCalls(config?: Config): number {
+  try {
+    return (config || getConfig()).MAX_GATHERING_CALLS;
+  } catch {
+    return MAX_GATHERING_CALLS; // Fallback to default
+  }
+}
+
+/**
+ * Get the maximum scrape batches from config.
+ * This function should be used instead of MAX_SCRAPE_CALLS to support dynamic configuration.
+ */
+export function getMaxScrapeBatches(config?: Config): number {
+  try {
+    const batches = (config || getConfig()).MAX_SCRAPE_BATCHES;
+    return batches === 0 || batches > 99 ? 999999 : batches;
+  } catch {
+    return 2; // Fallback to default
+  }
+}
+
+/** Maximum URLs to scrape per batch */
+export const MAX_SCRAPE_URLS = 6;
 
 /**
- * Complexity 0: Quick mode (no orchestrator, 1 researcher, 1 round)
- * Complexity 1: Normal (2 researchers, 2 rounds)
- * Complexity 2: Deep (3 researchers, 3 rounds)
- * Complexity 3: Ultra (5 researchers, 5 rounds)
+ * Get the units per researcher for the progress bar.
+ * Units = 1 (for start/search) + number of scrape batches.
+ * When batches are unlimited (MAX_SCRAPE_BATCHES=0), caps at 4 for a sane progress estimate.
  */
+export function getUnitsPerResearcher(config?: Config): number {
+  const batches = getMaxScrapeBatches(config);
+  return 1 + Math.min(batches, 4);
+}
 
-/** Initial researcher count for complexity 1 (Normal) */
-export const INITIAL_RESEARCHERS_LEVEL_1 = 2;
+/** Progress bar weight per lead evaluator round */
+export const LEAD_EVAL_UNITS = 2;
 
-/** Initial researcher count for complexity 2 (Deep) */
-export const INITIAL_RESEARCHERS_LEVEL_2 = 3;
 
-/** Initial researcher count for complexity 3 (Ultra) */
-export const INITIAL_RESEARCHERS_LEVEL_3 = 5;
+// ==================== Complexity Levels ====================
 
+/** Maximum researchers (siblings) in any single round per complexity level */
+export const MAX_TEAM_SIZE_LEVEL_1 = 2;
+export const MAX_TEAM_SIZE_LEVEL_2 = 3;
+export const MAX_TEAM_SIZE_LEVEL_3 = 5;
 
-/** Maximum rounds for complexity 1 (Normal) */
+/** Maximum research rounds per complexity level */
 export const MAX_ROUNDS_LEVEL_1 = 2;
-
-/** Maximum rounds for complexity 2 (Deep) */
 export const MAX_ROUNDS_LEVEL_2 = 3;
-
-/** Maximum rounds for complexity 3 (Ultra) */
-export const MAX_ROUNDS_LEVEL_3 = 5;
+export const MAX_ROUNDS_LEVEL_3 = 3;
 
 /**
- * Extra rounds the lead evaluator may earn beyond targetRounds when it judges
- * critical gaps still exist. Rounds targetRounds+1 through targetRounds+MAX_EXTRA_ROUNDS
- * are "bonus" territory; round targetRounds+MAX_EXTRA_ROUNDS+1 is unreachable (hard limit).
+ * Additional research rounds allowed past MAX_ROUNDS_LEVEL_n when queued
+ * steering messages exist at the end of the last planned round. Each
+ * consumed steering message unlocks one more round, up to this cap per
+ * session. This lets the user push research deeper via Alt+Enter even
+ * after the normal round budget is exhausted, without letting steering
+ * spin research forever.
  */
-export const MAX_EXTRA_ROUNDS = 2;
+export const MAX_EXTRA_ROUNDS_WITH_STEERING = 2;
 
 
 // ==================== Timeout Constants ====================
 
-/** Default timeout for SearXNG requests in milliseconds */
-export const SEARXNG_TIMEOUT_MS = 30000;
+export const OSV_TIMEOUT_MS = 10000;
 
-/** Default timeout for OSV API requests in milliseconds */
-export const OSV_TIMEOUT_MS = 30000;
-
-/** Default timeout for NVD API requests in milliseconds */
-export const NVD_TIMEOUT_MS = 30000;
+/**
+ * Hard cap for the out-of-process WebGPU viability probe when the embedding model
+ * is ALREADY cached on disk. A real WebGPU init on a capable host completes in
+ * ~1-3s; this bounds the fall-back-to-CPU on a GPU-less host (headless VM/CI/
+ * software-Vulkan) to seconds instead of the multi-minute model-init budget. The
+ * first-ever run (model not yet downloaded) keeps the full init budget — see
+ * resolveEmbeddingDevice. Mirrors the cached-pipeline 30s reuse in embedder.ts.
+ */
+export const WEBGPU_PROBE_TIMEOUT_MS = 30_000;
 
 // ==================== Export Constants ====================
 
+/** Maximum query length for validation */
+export const MAX_QUERY_LENGTH = 100000;
+
 /** Maximum query length for filename sanitization */
-export const MAX_QUERY_LENGTH = 50;
+export const MAX_FILENAME_QUERY_LENGTH = 150;
+
+/** Minimum query length */
+export const MIN_QUERY_LENGTH = 3;
 
 /** Maximum retry attempts for export file collision */
 export const MAX_EXPORT_RETRIES = 3;
@@ -93,56 +139,41 @@ export const DEFAULT_INITIAL_DELAY_MS = 1000;
 /** Default maximum delay for exponential backoff in milliseconds */
 export const DEFAULT_MAX_DELAY_MS = 10000;
 
-// ==================== Context-Aware Scraping Constants ====================
+// ==================== Scraping Constants ====================
 
-/**
- * Fraction of context window consumed beyond which all scraping is blocked.
- * At or above this threshold, researchers skip batches and move to synthesis.
- */
-export const MAX_CONTEXT_FRACTION_FOR_SCRAPING = 0.55;
-
-/**
- * Estimated tokens consumed per scrape call (handshake + content).
- * Used to project whether another batch is feasible given remaining context.
- */
-export const AVG_TOKENS_PER_SCRAPE = 15000;
-
-/**
- * Fallback context window size (tokens) when the model does not advertise one.
- * Corresponds to typical large-context models (Sonnet, Gemini 1.5 Pro, etc.).
- */
 export const DEFAULT_MODEL_CONTEXT_WINDOW = 200000;
 
-/**
- * Maximum URLs per second-scrape batch (more targeted than batch 1).
- * Kept lower to preserve context for synthesis after focused follow-up.
- */
-export const BATCH_2_MAX_SCRAPE_URLS = 2;
+/** Upper bound on batch-2+ scrape concurrency; the caller additionally caps this
+ *  to the real browser-pool capacity (WORKER_THREADS × WORKER_CONCURRENCY). */
+export const BATCH_2_DEFAULT_CONCURRENCY = 15;
 
 /**
- * Default maxConcurrency for the second scrape batch.
- * Higher than batch 1 because batch 2 URLs are already prioritized.
+ * Per-document cap (in characters) on scraped content that flows into LLM
+ * context — the scrape tool result and the session scraped-content cache
+ * (shared-links), whose entries are re-served into researcher sessions and
+ * embedded by the knowledge store.
+ *
+ * The scraper itself caps only RAW bytes (25MB HTML / 100MB PDF — see
+ * scraper-types.ts), and the pre-scrape context gate bills a flat
+ * AVG_TOKENS_PER_SCRAPE (~2500 tokens) per URL, so without this cap a single
+ * huge PDF/page could inject millions of tokens into a researcher session,
+ * overflow its context window, and kill the run.
+ *
+ * 500k chars ≈ 125k tokens: a deliberately permissive ceiling — real articles
+ * and long-form docs are never cut; only pathological pages (multi-hundred-page
+ * PDFs, dumps) hit it. This is a backstop against the multi-million-token
+ * blowout, not a tuning knob: the post-batch context gate handles ordinary
+ * budget pressure. Truncation appends an explicit "[content truncated: …]"
+ * marker (see truncateWithMarker).
  */
-export const BATCH_2_DEFAULT_CONCURRENCY = 15;
-
-// ==================== UI Constants ====================
-
-/** Maximum characters for final report before truncation */
-export const MAX_REPORT_LENGTH = 200000;
-
-/** Visual width of the ====------ progress bar in the TUI header */
-export const PROGRESS_BAR_WIDTH = 18;
+export const MAX_SCRAPE_CONTENT_CHARS_PER_DOC = 500_000;
 
 // ==================== Orchestrator Constants ====================
 
-/** Maximum characters per researcher report when sent to lead evaluator (truncated with link preservation) */
-export const MAX_EVALUATOR_REPORT_LENGTH = 50000;
-
-/** Delay in milliseconds between launching concurrent researchers to stagger startup */
+/** Delay in milliseconds between launching concurrent researchers to stagger browser pool startup */
 export const RESEARCHER_LAUNCH_DELAY_MS = 1500;
 
-/** Delay in milliseconds after state update to ensure propagation before next action */
-export const STATE_PROPAGATION_DELAY_MS = 50;
-
-/** Minimum streaming text length before updating UI with estimated tokens */
-export const STREAMING_UPDATE_THRESHOLD_CHARS = 200;
+/** Hard cap on search queries per researcher, enforced after LLM planning */
+export const MAX_QUERIES_PER_RESEARCHER_LEVEL_1 = 10;
+export const MAX_QUERIES_PER_RESEARCHER_LEVEL_2 = 15;
+export const MAX_QUERIES_PER_RESEARCHER_LEVEL_3 = 20;

package/src/core/health-check-service.ts

@@ -0,0 +1,265 @@
+/**
+ * Health Check Service
+ *
+ * Manages health check cache state and backoff logic.
+ * Provides thread-safe access to health check state and prevents
+ * thundering herd problems during health checks.
+ */
+
+import type { IHealthCheckService, HealthCheckResult } from './service-interfaces.ts';
+import { ServiceNames } from './service-interfaces.ts';
+import { ServiceLifecycle } from './service-registry.ts';
+import { logger } from '../logger.ts';
+
+/**
+ * Default backoff configuration
+ */
+const DEFAULT_BACKOFF_BASE_MS = 1000; // 1 second
+const DEFAULT_BACKOFF_MAX_MS = 60000; // 1 minute
+const DEFAULT_BACKOFF_MULTIPLIER = 2;
+
+/**
+ * Health Check Service Implementation
+ */
+export class HealthCheckService implements IHealthCheckService {
+  readonly name = ServiceNames.HEALTH_CHECK_CACHE;
+  lifecycle = ServiceLifecycle.UNINITIALIZED;
+
+  // Thread-safe state management using atomic operations
+  private _pendingCheck: Promise<HealthCheckResult> | null = null;
+  private _failureCount: number = 0;
+  private _backoffUntil: number = 0;
+  
+  // Backoff configuration
+  private readonly _backoffBaseMs: number;
+  private readonly _backoffMaxMs: number;
+  private readonly _backoffMultiplier: number;
+
+  constructor(options?: {
+    backoffBaseMs?: number;
+    backoffMaxMs?: number;
+    backoffMultiplier?: number;
+  }) {
+    this._backoffBaseMs = options?.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
+    this._backoffMaxMs = options?.backoffMaxMs ?? DEFAULT_BACKOFF_MAX_MS;
+    this._backoffMultiplier = options?.backoffMultiplier ?? DEFAULT_BACKOFF_MULTIPLIER;
+  }
+
+  async initialize(): Promise<void> {
+    if (this.lifecycle === ServiceLifecycle.INITIALIZED) {
+      return;
+    }
+
+    this.lifecycle = ServiceLifecycle.INITIALIZING;
+    logger.debug('[HealthCheckService] Initializing...');
+
+    // Initialize with default state
+    this._pendingCheck = null;
+    this._failureCount = 0;
+    this._backoffUntil = 0;
+
+    this.lifecycle = ServiceLifecycle.INITIALIZED;
+    logger.debug('[HealthCheckService] Initialized');
+  }
+
+  async dispose(): Promise<void> {
+    if (this.lifecycle === ServiceLifecycle.DISPOSED) {
+      return;
+    }
+
+    this.lifecycle = ServiceLifecycle.DISPOSING;
+    logger.debug('[HealthCheckService] Disposing...');
+
+    // Clear state
+    this.clear();
+
+    this.lifecycle = ServiceLifecycle.DISPOSED;
+    logger.debug('[HealthCheckService] Disposed');
+  }
+
+  /**
+   * Get the cached pending health check promise
+   */
+  getPendingCheck(): Promise<HealthCheckResult> | null {
+    return this._pendingCheck;
+  }
+
+  /**
+   * Set a pending health check promise
+   * This should only be called from orchestration code that coordinates health checks
+   */
+  setPendingCheck(promise: Promise<HealthCheckResult> | null): void {
+    this._pendingCheck = promise;
+
+    // Clean up the promise when it settles to avoid memory leaks
+    if (promise) {
+      promise.finally(() => {
+        // Only clear if this is still the same promise
+        if (this._pendingCheck === promise) {
+          this._pendingCheck = null;
+        }
+      });
+    }
+  }
+
+  /**
+   * Get the failure count
+   */
+  getFailureCount(): number {
+    return this._failureCount;
+  }
+
+  /**
+   * Increment the failure count
+   */
+  incrementFailureCount(): void {
+    this._failureCount++;
+    logger.debug(`[HealthCheckService] Failure count incremented to ${this._failureCount}`);
+  }
+
+  /**
+   * Reset the failure count
+   */
+  resetFailureCount(): void {
+    if (this._failureCount > 0) {
+      logger.debug(`[HealthCheckService] Failure count reset from ${this._failureCount} to 0`);
+    }
+    this._failureCount = 0;
+  }
+
+  /**
+   * Get the backoff timestamp
+   */
+  getBackoffUntil(): number {
+    return this._backoffUntil;
+  }
+
+  /**
+   * Set the backoff timestamp
+   */
+  setBackoffUntil(timestamp: number): void {
+    this._backoffUntil = timestamp;
+    const remainingMs = Math.max(0, timestamp - Date.now());
+    if (remainingMs > 0) {
+      logger.debug(`[HealthCheckService] Backoff set: ${remainingMs}ms remaining`);
+    }
+  }
+
+  /**
+   * Clear all cache state
+   */
+  clear(): void {
+    this._pendingCheck = null;
+    this._failureCount = 0;
+    this._backoffUntil = 0;
+    logger.debug('[HealthCheckService] Cache cleared');
+  }
+
+  /**
+   * Check if a backoff is currently active
+   */
+  isBackoffActive(): boolean {
+    return Date.now() < this._backoffUntil;
+  }
+
+  /**
+   * Get remaining backoff time in milliseconds
+   */
+  getBackoffRemainingMs(): number {
+    return Math.max(0, this._backoffUntil - Date.now());
+  }
+
+  /**
+   * Calculate the next backoff duration based on failure count
+   * Uses exponential backoff with jitter
+   */
+  calculateNextBackoffMs(): number {
+    // Exponential backoff: base * 2^failures, capped at max
+    const backoffMs = Math.min(
+      this._backoffBaseMs * Math.pow(this._backoffMultiplier, this._failureCount),
+      this._backoffMaxMs
+    );
+
+    // Add jitter (±25%) to prevent thundering herd
+    const jitter = backoffMs * 0.25 * (Math.random() * 2 - 1);
+
+    return Math.floor(backoffMs + jitter);
+  }
+
+  /**
+   * Set backoff based on current failure count
+   */
+  setBackoff(): void {
+    const backoffMs = this.calculateNextBackoffMs();
+    this._backoffUntil = Date.now() + backoffMs;
+    logger.debug(`[HealthCheckService] Backoff set: ${backoffMs}ms (${this._failureCount} failures)`);
+  }
+
+  /**
+   * Record a health check failure
+   * Increments failure count and sets backoff
+   */
+  recordFailure(): void {
+    this.incrementFailureCount();
+    this.setBackoff();
+  }
+
+  /**
+   * Record a health check success
+   * Resets failure count and clears backoff
+   */
+  recordSuccess(): void {
+    this.resetFailureCount();
+    this._backoffUntil = 0;
+    logger.debug('[HealthCheckService] Health check success recorded');
+  }
+
+  /**
+   * Get the current backoff state as an object
+   */
+  getBackoffState(): {
+    isActive: boolean;
+    remainingMs: number;
+    failureCount: number;
+    nextBackoffMs: number;
+  } {
+    return {
+      isActive: this.isBackoffActive(),
+      remainingMs: this.getBackoffRemainingMs(),
+      failureCount: this._failureCount,
+      nextBackoffMs: this.calculateNextBackoffMs(),
+    };
+  }
+
+  /**
+   * Wait for backoff to complete if active
+   * Returns immediately if no backoff is active
+   */
+  async waitForBackoff(): Promise<void> {
+    const remainingMs = this.getBackoffRemainingMs();
+    if (remainingMs > 0) {
+      logger.debug(`[HealthCheckService] Waiting for backoff: ${remainingMs}ms`);
+      await new Promise(resolve => setTimeout(resolve, remainingMs));
+      logger.debug('[HealthCheckService] Backoff complete');
+    }
+  }
+
+  /**
+   * Check if a new health check should be allowed
+   * Returns false if backoff is active or a check is already pending
+   */
+  shouldAllowCheck(): boolean {
+    if (this._pendingCheck !== null) {
+      logger.debug('[HealthCheckService] Check not allowed: already pending');
+      return false;
+    }
+
+    if (this.isBackoffActive()) {
+      const remainingMs = this.getBackoffRemainingMs();
+      logger.debug(`[HealthCheckService] Check not allowed: backoff active (${remainingMs}ms remaining)`);
+      return false;
+    }
+
+    return true;
+  }
+}

package/src/core/interfaces/health-check-interfaces.ts

@@ -0,0 +1,50 @@
+/**
+ * Health Check Service Interfaces
+ */
+
+import type { IService } from '../service-registry.ts';
+
+/**
+ * Health check result
+ */
+export interface HealthCheckResult {
+  success: boolean;
+  searchOk: boolean;
+  scrapeOk: boolean;
+  error?: string;
+  timestamp: string;
+}
+
+/**
+ * Health check cache state
+ */
+export interface HealthCheckCache {
+  pending: Promise<HealthCheckResult> | null;
+  failureCount: number;
+  backoffUntil: number;
+}
+
+/**
+ * Health check service interface
+ */
+export interface IHealthCheckService extends IService {
+  getPendingCheck(): Promise<HealthCheckResult> | null;
+  setPendingCheck(promise: Promise<HealthCheckResult> | null): void;
+  getFailureCount(): number;
+  incrementFailureCount(): void;
+  resetFailureCount(): void;
+  getBackoffUntil(): number;
+  setBackoffUntil(timestamp: number): void;
+  clear(): void;
+  isBackoffActive(): boolean;
+  getBackoffRemainingMs(): number;
+  }
+
+  /**
+  * Health check registry interface
+  */
+  export interface IHealthRegistryService extends IService {
+  register(name: string, check: (options?: { force?: boolean }) => Promise<{ healthy: boolean; error?: string; diagnostic?: Record<string, any> }>, options?: { timeoutMs?: number; critical?: boolean }): void;
+  isCritical(componentName: string): boolean;
+  runAll(options?: { force?: boolean }): Promise<{ status: 'healthy' | 'degraded' | 'unhealthy'; components: any[]; timestamp: string }>;
+  }