@lincoln504/pi-research 0.0.0

package/src/utils/session-state.ts

@@ -0,0 +1,295 @@
+/**
+ * Research Session State Management
+ *
+ * Tracks failures and state for multiple simultaneous research sessions,
+ * scoped by parent Pi session to prevent cross-context interference.
+ */
+
+import { generateSessionId as generateUniqueSessionId } from './shared-links.ts';
+import { logger } from '../logger.ts';
+import { getConfig } from '../config.ts';
+import type { ResearchPanelState } from '../tui/research-panel.ts';
+
+/**
+ * State container for a single Pi session
+ */
+interface PiSessionState {
+  /** Map of research run ID → array of failed researcher IDs */
+  failures: Map<string, string[]>;
+  /** Ordered list of research run IDs for TUI stacking (Index 0 = oldest/bottom-most) */
+  order: string[];
+  /** Registry of panel states for each research run */
+  panels: Map<string, ResearchPanelState>;
+  /** Subscribers for order changes in this Pi session */
+  subscribers: Array<() => void>;
+  /** Global debounce timer for this specific Pi session */
+  refreshTimeout: NodeJS.Timeout | null;
+  /** Single update function for the Master Widget of this Pi session */
+  masterUpdate: (() => void) | null;
+}
+
+/**
+ * Global map of Pi session ID → PiSessionState
+ */
+const piSessions = new Map<string, PiSessionState>();
+
+/**
+ * Get or create state for a specific Pi session
+ */
+function getPiState(piSessionId: string): PiSessionState {
+  let state = piSessions.get(piSessionId);
+  if (!state) {
+    state = {
+      failures: new Map(),
+      order: [],
+      panels: new Map(),
+      subscribers: [],
+      refreshTimeout: null,
+      masterUpdate: null,
+    };
+    piSessions.set(piSessionId, state);
+  }
+  return state;
+}
+
+/**
+ * Maximum allowed unique failed researchers before stopping research
+ */
+const MAX_FAILED_RESEARCHERS = 2;
+
+/**
+ * Subscribe to session order changes for a specific Pi session
+ */
+export function onSessionOrderChange(piSessionId: string, callback: () => void): () => void {
+  const state = getPiState(piSessionId);
+  state.subscribers.push(callback);
+  return () => {
+    const index = state.subscribers.indexOf(callback);
+    if (index !== -1) {
+      state.subscribers.splice(index, 1);
+    }
+  };
+}
+
+/**
+ * Notify subscribers of session order change
+ */
+function notifyOrderChange(piSessionId: string): void {
+  const state = getPiState(piSessionId);
+  for (const subscriber of state.subscribers) {
+    try {
+      subscriber();
+    } catch (error) {
+      logger.error(`[session-state] Error in subscriber for ${piSessionId}:`, error);
+    }
+  }
+}
+
+/**
+ * Register a panel state for a research run
+ */
+export function registerSessionPanel(piSessionId: string, researchId: string, panel: ResearchPanelState): void {
+  const state = getPiState(piSessionId);
+  state.panels.set(researchId, panel);
+  
+  if (!state.order.includes(researchId)) {
+    state.order.push(researchId);
+    notifyOrderChange(piSessionId);
+  }
+}
+
+/**
+ * Unregister a panel state
+ */
+export function unregisterSessionPanel(piSessionId: string, researchId: string): void {
+  const state = getPiState(piSessionId);
+  state.panels.delete(researchId);
+}
+
+/**
+ * Register the update function for a Pi session's Master Widget
+ */
+export function registerMasterUpdate(piSessionId: string, update: () => void): void {
+  const state = getPiState(piSessionId);
+  state.masterUpdate = update;
+}
+
+/**
+ * Refresh the Master Widget for a Pi session
+ */
+export function refreshAllSessions(piSessionId: string): void {
+  const state = getPiState(piSessionId);
+
+  // Clear existing timeout for this specific Pi session
+  if (state.refreshTimeout) {
+    clearTimeout(state.refreshTimeout);
+  }
+
+  const debounceMs = getConfig().TUI_REFRESH_DEBOUNCE_MS;
+  state.refreshTimeout = setTimeout(() => {
+    try {
+      // Validate order integrity
+      const validIds = state.order.filter(id =>
+        state.panels.has(id) && state.failures.has(id)
+      );
+
+      if (validIds.length !== state.order.length) {
+        for (const id of state.order) {
+          if (!validIds.includes(id)) {
+            state.failures.delete(id);
+            state.panels.delete(id);
+          }
+        }
+        state.order.length = 0;
+        state.order.push(...validIds);
+      }
+
+      // Trigger the single Master Update for this Pi session
+      if (state.masterUpdate) {
+        try {
+          state.masterUpdate();
+        } catch (error) {
+          logger.error(`[session-state] Error updating Master Widget for ${piSessionId}:`, error);
+        }
+      }
+    } finally {
+      state.refreshTimeout = null;
+    }
+  }, debounceMs);
+}
+
+/**
+ * Start a new research run within a Pi session
+ */
+export function startResearchSession(piSessionId: string): string {
+  const researchId = generateUniqueSessionId('research');
+  const state = getPiState(piSessionId);
+  state.failures.set(researchId, []);
+  return researchId;
+}
+
+/**
+ * Clear pending refreshes for a Pi session
+ */
+export function clearPendingRefresh(piSessionId: string): void {
+  const state = piSessions.get(piSessionId);
+  if (state?.refreshTimeout) {
+    clearTimeout(state.refreshTimeout);
+    state.refreshTimeout = null;
+  }
+}
+
+/**
+ * End a research run
+ */
+export function endResearchSession(piSessionId: string, researchId: string): void {
+  const state = piSessions.get(piSessionId);
+  if (!state) return;
+
+  state.failures.delete(researchId);
+  state.panels.delete(researchId);
+
+  const index = state.order.indexOf(researchId);
+  if (index !== -1) {
+    state.order.splice(index, 1);
+    notifyOrderChange(piSessionId);
+  }
+
+  // If this was the last research run in the Pi session, clean up the state
+  if (state.order.length === 0 && state.panels.size === 0 && state.subscribers.length === 0) {
+    clearPendingRefresh(piSessionId);
+    piSessions.delete(piSessionId);
+  }
+}
+
+/**
+ * Check if a research run is the bottom-most in its Pi session
+ */
+export function isBottomMostSession(piSessionId: string, researchId: string): boolean {
+  const state = piSessions.get(piSessionId);
+  if (!state || state.order.length === 0) return false;
+  return state.order[0] === researchId;
+}
+
+/**
+ * Record a researcher failure
+ */
+export function recordResearcherFailure(piSessionId: string, researchId: string, researcherId: string): void {
+  const state = getPiState(piSessionId);
+  const failures = state.failures.get(researchId) || [];
+  failures.push(researcherId);
+  state.failures.set(researchId, failures);
+}
+
+/**
+ * Get list of unique failed researchers in a research run
+ */
+export function getFailedResearchers(piSessionId: string, researchId: string): string[] {
+  const state = getPiState(piSessionId);
+  const failures = state.failures.get(researchId) || [];
+  return [...new Set(failures)];
+}
+
+/**
+ * Check if research should stop due to too many unique failures
+ */
+export function shouldStopResearch(piSessionId: string, researchId: string): boolean {
+  return getFailedResearchers(piSessionId, researchId).length >= MAX_FAILED_RESEARCHERS;
+}
+
+/**
+ * Get formatted error message for research stoppage
+ */
+export function getResearchStopMessage(piSessionId: string, researchId: string): string {
+  const failed = getFailedResearchers(piSessionId, researchId);
+  const count = failed.length;
+
+  return [
+    `Research stopped: ${count} researcher(s) failed: ${failed.join(', ')}.`,
+    '',
+    'This indicates infrastructure failure — multiple researchers could not complete research.',
+    'Possible causes: network unavailable, SearXNG container not running, search engines returning errors.',
+    '',
+    '▎ If the health check passed (search and scrape verified), this failure is at the AI session layer —',
+    '   check model availability, API key, and context settings.',
+    '',
+    'Troubleshooting:',
+    '• Verify network connection is active',
+    '• Check SearXNG container: `docker ps | grep searxng`',
+    '• View SearXNG logs: `docker logs pi-searxng` (check for engine HTTP errors like 503, 400)',
+    '• Verify PROXY_URL if configured (optional): should be socks5://host:port or http://host:port',
+    '• Check PI_RESEARCH_RESEARCHER_TIMEOUT_MS if set (default: 4 minutes)',
+    '',
+    'Partial results may be available below.',
+  ].join('\n');
+}
+
+/**
+ * Get all active research panels in a Pi session, in display order (newest first)
+ */
+export function getPiActivePanels(piSessionId: string): ResearchPanelState[] {
+  const state = piSessions.get(piSessionId);
+  if (!state) return [];
+  // Return in reverse order (newest first) for top-to-bottom stacking in a single widget
+  return [...state.order].reverse().map(id => state.panels.get(id)!).filter(Boolean);
+}
+
+/**
+ * Get ordered list of active research runs in a Pi session
+ */
+export function getPiActiveSessionOrder(piSessionId: string): string[] {
+  const state = piSessions.get(piSessionId);
+  return state ? [...state.order] : [];
+}
+
+/**
+ * Reset all state (for testing only)
+ */
+export function resetAllPiSessions(): void {
+  for (const state of piSessions.values()) {
+    if (state.refreshTimeout) {
+      clearTimeout(state.refreshTimeout);
+    }
+  }
+  piSessions.clear();
+}

package/src/utils/shared-links.ts

@@ -0,0 +1,123 @@
+/**
+ * Shared Links Pool Management
+ *
+ * Centralized deterministic link state management system.
+ * Single source of truth: global allScrapedLinks array.
+ * No per-session state - everything flows through global state.
+ */
+
+import * as crypto from 'node:crypto';
+
+/**
+ * Generate a 4-character alphanumeric hash
+ */
+function generate4CharHash(): string {
+  // Use randomBytes for Node.js environments, fallback to Math.random for test environments
+  let hash: string;
+  try {
+    const bytes = crypto.randomBytes(2);
+    hash = bytes.toString('hex').substring(0, 4);
+  } catch {
+    // Fallback for test environments where crypto.randomBytes might not be available
+    // Generate a valid 4-character hex string
+    const randomHex = Math.floor(Math.random() * 65536).toString(16).padStart(4, '0');
+    hash = randomHex;
+  }
+  return hash;
+}
+
+/**
+ * Generate a unique session ID with 4-char hash
+ * @param baseId Base identifier (e.g., request ID)
+ * @returns Unique session ID (e.g., "abc123-x9k2")
+ */
+export function generateSessionId(baseId: string): string {
+  const hash = generate4CharHash();
+  return `${baseId}-${hash}`;
+}
+
+/**
+ * Normalize a URL for comparison (lowercase, remove trailing slash)
+ */
+export function normalizeUrl(url: string): string {
+  try {
+    const normalized = url.toLowerCase().trim();
+    // Remove trailing slash except for root
+    if (normalized.length > 1 && normalized.endsWith('/')) {
+      return normalized.slice(0, -1);
+    }
+    return normalized;
+  } catch {
+    return url.toLowerCase().trim();
+  }
+}
+
+/**
+ * Deduplicate a list of URLs against already-scraped URLs.
+ * Uses normalized comparison for accuracy.
+ * 
+ * @param urls - URLs to check
+ * @param alreadyScraped - Array of already-scraped URLs
+ * @returns Object with kept URLs and removed duplicates
+ */
+export function deduplicateUrls(urls: string[], alreadyScraped: string[]): {
+  kept: string[];
+  duplicates: string[];
+} {
+  const normalizedAlreadyScraped = new Set(
+    alreadyScraped.map(normalizeUrl)
+  );
+
+  const kept: string[] = [];
+  const duplicates: string[] = [];
+
+  for (const url of urls) {
+    const normalized = normalizeUrl(url);
+    if (normalizedAlreadyScraped.has(normalized)) {
+      duplicates.push(url);
+    } else {
+      kept.push(url);
+    }
+  }
+
+  return { kept, duplicates };
+}
+
+/**
+ * Format all completed reports into a shared links section for researcher prompts.
+ * Extracts links from CITED LINKS and SCRAPE CANDIDATES sections.
+ */
+export function formatSharedLinksFromState(aspects: Record<string, { id: string; query: string; report?: string }>): string {
+  const completed = Object.values(aspects).filter(a => a.report);
+  if (completed.length === 0) return '';
+
+  let output = '## Shared Links from Previous Research\n\n';
+  output += 'The following links have already been examined by your siblings:\n\n';
+
+  for (const aspect of completed) {
+    const response = aspect.report!;
+    output += `### Aspect: ${aspect.query} (ID: ${aspect.id})\n\n`;
+
+    // Extract CITED LINKS
+    const citedMatch = response.match(/###?\s*CITED\s*LINKS?\s*\n([\s\S]*?)(?=\n###|$)/i);
+    if (citedMatch) {
+      output += '#### CITED LINKS\n';
+      output += citedMatch[1]!.trim() + '\n\n';
+    }
+
+    // Extract SCRAPE CANDIDATES
+    const candidatesMatch = response.match(/###?\s*SCRAPE\s*CANDIDATES?\s*\n([\s\S]*?)(?=\n###|$)/i);
+    if (candidatesMatch) {
+      output += '#### SCRAPE CANDIDATES\n';
+      output += candidatesMatch[1]!.trim() + '\n\n';
+    }
+  }
+
+  output += '**Review these links before scraping to avoid redundancy.**\n';
+  return output;
+}
+
+/**
+ * Legacy stub for compatibility
+ */
+export function cleanupSharedLinks(_sessionId: string): void {}

package/src/utils/shutdown-manager.ts

@@ -0,0 +1,54 @@
+import { logger } from '../logger.ts';
+
+export type CleanupTask = () => Promise<void> | void;
+
+export class ShutdownManager {
+  private tasks: CleanupTask[] = [];
+  private cleanupPromise: Promise<void> | null = null;
+
+  register(task: CleanupTask) {
+    if (this.tasks.includes(task)) {
+      return;
+    }
+
+    this.tasks.push(task);
+  }
+
+  async runCleanup(reason: string): Promise<void> {
+    if (this.cleanupPromise !== null) {
+      return this.cleanupPromise;
+    }
+
+    const tasksToRun = [...this.tasks].reverse();
+    this.cleanupPromise = (async () => {
+      if (tasksToRun.length === 0) {
+        logger.log(`[ShutdownManager] No cleanup tasks registered (${reason}).`);
+        return;
+      }
+
+      logger.log(`[ShutdownManager] Running cleanup tasks (${reason})...`);
+
+      // Detach the current task set so repeated cleanup calls are idempotent,
+      // while future sessions can register a fresh cleanup set.
+      this.tasks = [];
+
+      for (const task of tasksToRun) {
+        try {
+          await task();
+        } catch (error) {
+          logger.error('[ShutdownManager] Error during cleanup task:', error);
+        }
+      }
+
+      logger.log('[ShutdownManager] Cleanup complete.');
+    })();
+
+    try {
+      await this.cleanupPromise;
+    } finally {
+      this.cleanupPromise = null;
+    }
+  }
+}
+
+export const shutdownManager = new ShutdownManager();

package/src/utils/text-utils.ts

@@ -0,0 +1,83 @@
+import type { AgentSession } from '@mariozechner/pi-coding-agent';
+import type { AssistantMessage } from '@mariozechner/pi-ai';
+
+/**
+ * Text Utilities
+ *
+ * Shared text extraction and formatting functions.
+ */
+
+/**
+ * Extract text content from a message
+ *
+ * Handles various message content formats:
+ * - String content
+ * - Array of content blocks (TextContent, ThinkingContent, ToolCall, etc.)
+ * - Accepts any object with a content field for robustness
+ *
+ * @param message - Message object with content field, or undefined/null
+ * @returns Extracted text string, or empty string if invalid
+ */
+export function extractText(message: unknown): string {
+  // Validate message exists and is an object
+  if (!message || typeof message !== 'object') {
+    return '';
+  }
+
+  const msg = message as Record<string, unknown>;
+  const { content } = msg;
+
+  // Handle string content
+  if (typeof content === 'string') {
+    return content;
+  }
+
+  // Handle array of content blocks
+  if (Array.isArray(content)) {
+    try {
+      return (content as unknown[])
+        .filter((b) => b && typeof b === 'object' && (b as Record<string, unknown>)['type'] === 'text')
+        .map((b) => {
+          const blockObj = b as Record<string, unknown>;
+          const text = blockObj['text'];
+          return typeof text === 'string' ? text : '';
+        })
+        .filter((t: string) => t.length > 0) // Filter empty strings
+        .join('\n');
+    } catch (_error) {
+      // If array processing fails, return empty string
+    }
+  }
+
+  // Unknown content format
+  return '';
+}
+
+/**
+ * Ensures the assistant completed successfully and returns the text content.
+ * Throws an error if the assistant failed or stopped with an error.
+ * 
+ * @param session - The agent session to check
+ * @param label - Label for error reporting
+ * @returns The extracted text content
+ * @throws Error if assistant failed
+ */
+export function ensureAssistantResponse(session: AgentSession, label: string): string {
+  const msgs = session.messages;
+  const last = [...msgs].reverse().find((m) => m.role === 'assistant') as AssistantMessage | undefined;
+
+  if (!last) {
+    throw new Error(`${label}: No assistant response found`);
+  }
+
+  // Check for explicit error stop reason or error message
+  if (last.stopReason === 'error' || (last.errorMessage && last.stopReason !== 'aborted')) {
+    const msg = last.errorMessage || 'Unknown error';
+    if (msg.includes('429')) {
+      throw new Error(`Model API rate limit (429) — wait a moment and retry. Details: ${msg}`);
+    }
+    throw new Error(`${label}: Provider error - ${msg}`);
+  }
+
+  return extractText(last);
+}

package/src/utils/tool-usage-tracker.ts

@@ -0,0 +1,135 @@
+/**
+ * Tool Usage Tracker
+ *
+ * Tracks and enforces limits on researcher tool usage.
+ * Prevents excessive calls to prevent rate limiting and control costs.
+ * Enforcement is code-based only — when a limit is hit, the tool throws.
+ */
+
+import { logger } from '../logger.ts';
+import { MAX_GATHERING_CALLS, MAX_SCRAPE_CALLS } from '../constants.ts';
+
+export interface ToolLimits {
+  // Combined gathering limit (search, security_search, stackexchange, grep)
+  gathering?: number;
+  // Scrape limit
+  scrape?: number;
+  // read tool (default pi file read) has no limit
+  read?: number;
+}
+
+export interface ToolUsage {
+  category: string;
+  callCount: number;
+  limit?: number;
+}
+
+export class ToolUsageTracker {
+  private usage: Map<string, ToolUsage> = new Map();
+  private limits: ToolLimits;
+
+  constructor(limits: ToolLimits) {
+    this.limits = limits;
+  }
+
+  /**
+   * Get category for a tool
+   */
+  private getCategory(toolName: string): string {
+    const gatheringTools = ['search', 'security_search', 'stackexchange', 'grep'];
+    if (gatheringTools.includes(toolName)) {
+      return 'gathering';
+    }
+    return toolName;
+  }
+
+  /**
+   * Record a tool call and enforce limits gracefully
+   * Returns true if call is allowed and recorded, false if limit is reached.
+   * On block: counter is NOT incremented, tool should return a limit message.
+   */
+  recordCall(toolName: string): boolean {
+    const category = this.getCategory(toolName);
+    const limit = this.limits[category as keyof ToolLimits];
+    const usage = this.getUsage(category);
+
+    // Check limit — if reached, return false without incrementing
+    if (limit !== undefined && usage.callCount >= limit) {
+      logger.debug(
+        `[tool-usage] category=${category} blocked tool=${toolName} (limit=${limit})`
+      );
+      return false;
+    }
+
+    // Allowed — increment and log
+    usage.callCount++;
+    logger.debug(
+      `[tool-usage] category=${category} calls=${usage.callCount}/${limit ?? 'unlimited'} tool=${toolName}`
+    );
+    return true;
+  }
+
+  /**
+   * Get current call count for a tool
+   */
+  getCallCount(toolName: string): number {
+    const category = this.getCategory(toolName);
+    return this.getUsage(category).callCount;
+  }
+
+  /**
+   * Get limit-reached message for a blocked tool
+   */
+  getLimitMessage(toolName: string): string {
+    const category = this.getCategory(toolName);
+    const usage = this.getUsage(category);
+    const limit = usage.limit;
+
+    if (category === 'scrape') {
+      return `SCRAPE PROTOCOL COMPLETE: You have completed all ${limit} scrape calls (handshake + 3 batches). ` +
+        `This tool cannot be used again. Proceed immediately to Phase 3: synthesize your findings and submit your report.`;
+    }
+    return `GATHERING LIMIT REACHED: All ${limit} gathering calls have been used. ` +
+      `This tool and all other gathering tools cannot be used again. ` +
+      `Proceed to Phase 2: call scrape with your collected URLs now.`;
+  }
+
+  /**
+   * Get current usage for a category
+   */
+  getUsage(category: string): ToolUsage {
+    if (!this.usage.has(category)) {
+      this.usage.set(category, {
+        category,
+        callCount: 0,
+        limit: this.limits[category as keyof ToolLimits],
+      });
+    }
+    return this.usage.get(category)!;
+  }
+
+  /**
+   * Get usage statistics for all categories
+   */
+  getStats(): Map<string, ToolUsage> {
+    return new Map(this.usage);
+  }
+
+  /**
+   * Reset all usage
+   */
+  reset(): void {
+    this.usage.clear();
+  }
+}
+
+/**
+ * Create default tool limits for a researcher
+ */
+export function createDefaultToolLimits(): ToolLimits {
+  return {
+    gathering: MAX_GATHERING_CALLS,
+    scrape: MAX_SCRAPE_CALLS,
+    read: undefined,
+  };
+}

package/src/web-research/dependencies.ts

@@ -0,0 +1,20 @@
+/**
+ * Web Research Extension - Dependencies
+ *
+ * Dependencies are managed by npm via package.json.
+ * No custom install/uninstall commands needed.
+ */
+
+import { getDependencyStatus } from './scrapers.ts';
+
+/**
+ * Check all dependencies and return detailed status
+ */
+export function checkAllDependencies(): {
+  playwrightAvailable: boolean;
+  } {
+  const scrapersStatus = getDependencyStatus();
+  return {
+    playwrightAvailable: scrapersStatus.playwrightAvailable,
+  };
+}