@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/tools/stackexchange.ts

@@ -4,10 +4,12 @@
  * Search and retrieve data from Stack Exchange network via REST API v2.3.
  */
 
-import type { ToolDefinition, AgentToolResult, ExtensionContext } from '@mariozechner/pi-coding-agent';
-import { Type } from '@sinclair/typebox';
+import type { ToolDefinition, AgentToolResult, ExtensionContext } from '@earendil-works/pi-coding-agent';
+import { Type, type Static } from 'typebox';
+import { Value } from 'typebox/value';
 import { stackexchangeCommand } from '../stackexchange/index.ts';
 import type { ToolUsageTracker } from '../utils/tool-usage-tracker.ts';
+import { getMaxGatheringCalls } from '../constants.ts';
 
 export function createStackexchangeTool(options: {
   ctx: ExtensionContext;
@@ -15,44 +17,56 @@ export function createStackexchangeTool(options: {
 }): ToolDefinition {
   const { tracker } = options;
 
+  const StackExchangeParamsSchema = Type.Object({
+    command: Type.String({
+      description: 'Command: search, get, user, or site',
+    }),
+    query: Type.Optional(Type.String({
+      description: 'Search query (for search command)',
+    })),
+    id: Type.Optional(Type.Union([Type.String(), Type.Number()])),
+    site: Type.Optional(Type.String({
+      description: 'Stack Exchange site (default: stackoverflow.com)',
+    })),
+    limit: Type.Optional(Type.Number({
+      description: 'Results count (1-100, default: 10)',
+      default: 10,
+      minimum: 1,
+      maximum: 100,
+    })),
+    maxPages: Type.Optional(Type.Number({
+      description: 'Maximum pages to fetch for pagination (1-10, default: 5)',
+      default: 5,
+      minimum: 1,
+      maximum: 10,
+    })),
+    format: Type.Optional(Type.String({
+      description: 'Output format: table, json, or compact (default: table)',
+      default: 'table',
+    })),
+    tags: Type.Optional(Type.String({
+      description: 'Filter by tags (comma-separated)',
+    })),
+  });
+
+  type StackExchangeParams = Static<typeof StackExchangeParamsSchema>;
+
   return {
     name: 'stackexchange',
     label: 'Stack Exchange Search',
-    description: 'Search and retrieve data from Stack Exchange network via REST API v2.3 (anonymous: 300 requests/day, with key: 10,000 requests/day)',
+    description: 'Search and retrieve data from Stack Exchange network via REST API v2.3 with pagination support (anonymous: 300 requests/day, with key: 10,000 requests/day)',
     promptSnippet: 'Search Stack Overflow and other Stack Exchange sites for questions, answers, and user information',
     promptGuidelines: [
-      'Use stackexchange to find technical answers on Stack Overflow',
+      'Available for finding technical answers on Stack Overflow',
       'Great for finding code solutions, debugging help, and best practices',
       'Works with any Stack Exchange site (Stack Overflow, SuperUser, AskUbuntu, etc.)',
       'Anonymous access: 300 requests/day. Set STACKEXCHANGE_API_KEY env var for 10,000/day.',
       'Use tags to filter by specific topics.',
-      'CRITICAL: You are allowed a maximum of 4 gathering calls total across ALL tools. Use them for breadth.',
+      'Use maxPages parameter to control pagination for search results (default: 5 pages).',
+      `CRITICAL: You are allowed a maximum of ${getMaxGatheringCalls()} gathering calls across your web tools (search, security_search, stackexchange). Use them for breadth.`,
     ],
-    parameters: Type.Object({
-      command: Type.String({
-        description: 'Command: search, get, user, or site',
-      }),
-      query: Type.Optional(Type.String({
-        description: 'Search query (for search command)',
-      })),
-      id: Type.Optional(Type.Union([Type.String(), Type.Number()])),
-      site: Type.Optional(Type.String({
-        description: 'Stack Exchange site (default: stackoverflow.com)',
-      })),
-      limit: Type.Optional(Type.Number({
-        description: 'Results count (1-100, default: 10)',
-        default: 10,
-        minimum: 1,
-        maximum: 100,
-      })),
-      format: Type.Optional(Type.String({
-        description: 'Output format: table, json, or compact (default: table)',
-        default: 'table',
-      })),
-      tags: Type.Optional(Type.String({
-        description: 'Filter by tags (comma-separated)',
-      })),
-    }),
+    parameters: StackExchangeParamsSchema,
+    executionMode: 'parallel',
     async execute(
       _toolCallId,
       params,
@@ -63,12 +77,21 @@ export function createStackexchangeTool(options: {
       // Record call in tracker - returns false if limit reached
       const allowed = tracker.recordCall('stackexchange');
       if (!allowed) {
-        // THROW to prevent researcher from calling again
-        throw new Error(tracker.getLimitMessage('stackexchange'));
+          return {
+            content: [{ type: 'text', text: tracker.getLimitMessage('stackexchange') }],
+            details: { blocked: true, reason: 'limit_reached' },
+          };
+      }
+
+      if (!Value.Check(StackExchangeParamsSchema, params)) {
+          return {
+            content: [{ type: 'text', text: 'Invalid parameters for stackexchange tool.' }],
+            details: { error: 'invalid_parameters' },
+          };
       }
 
-      const paramsRecord = params as Record<string, unknown>;
-      const command = paramsRecord['command'] as string;
+      const p = params as StackExchangeParams;
+      const command = p.command;
 
       if (!command || typeof command !== 'string') {
         throw new Error('Stack Exchange command is required and must be a string');
@@ -77,7 +100,7 @@ export function createStackexchangeTool(options: {
       try {
         return await stackexchangeCommand({
           command,
-          params: paramsRecord,
+          params: p as Record<string, unknown>,
           ctx: extensionCtx,
           signal,
         });

package/src/tools/youtube-transcript.ts

@@ -0,0 +1,163 @@
+/**
+ * youtube_transcript Tool
+ *
+ * Fetches transcripts for YouTube videos the researcher selected from its link
+ * pool. Used like the scrape tool: the agent picks which YouTube links to read,
+ * and this tool reads up to a few of them in one batch.
+ *
+ * Constraints (enforced here):
+ *  - One call per researcher (hard limit youtube_transcript: 1 in the tracker).
+ *  - Up to YOUTUBE_TRANSCRIPT_MAX_VIDEOS (default 3) videos per call.
+ *
+ * Robustness comes from the backend (PoToken minting, empty-body guard, per-video
+ * isolation, circuit breaker, timeouts); this wrapper handles validation, rate
+ * limiting, metrics, and never throws to the agent.
+ */
+
+import type { ToolDefinition, AgentToolResult, ExtensionContext, AgentToolUpdateCallback } from '@earendil-works/pi-coding-agent';
+import { Type, type Static } from 'typebox';
+import { Value } from 'typebox/value';
+import { youtubeTranscriptCommand } from '../youtube/index.ts';
+import type { ToolUsageTracker } from '../utils/tool-usage-tracker.ts';
+import type { SystemResearchState } from '../orchestration/deep-research-types.ts';
+import { cacheScrapedContent, registerScrapedLinks, registerResearcherScrapes } from '../utils/shared-links.ts';
+import { type Config, getConfig } from '../config.ts';
+import { logger } from '../logger.ts';
+import { metrics } from '../utils/metrics.ts';
+
+export function createYoutubeTranscriptTool(options: {
+  ctx: ExtensionContext;
+  tracker?: ToolUsageTracker;
+  /** Fires for each video as it completes (success or failure) — used for TUI flash. */
+  onUrlScrapeResult?: (url: string, success: boolean) => void;
+  /** Provides the research session id for knowledge-store ingestion of transcripts. */
+  getGlobalState?: () => SystemResearchState;
+  /** Registers transcribed URLs in the global pool (mirrors scrape). */
+  updateGlobalLinks?: (links: string[]) => void;
+  /** Researcher id for per-researcher source tracking. */
+  researcherId?: string;
+  config?: Config;
+}): ToolDefinition {
+  const { tracker } = options;
+  const config = options.config || getConfig(options.ctx.cwd);
+  const maxVideos = config.YOUTUBE_TRANSCRIPT_MAX_VIDEOS;
+
+  const YoutubeParamsSchema = Type.Object({
+    urls: Type.Array(
+      Type.String({ description: 'YouTube watch/share/shorts/embed links (or bare 11-char video IDs)' }),
+      { minItems: 1, maxItems: maxVideos, description: `Up to ${maxVideos} YouTube video links to transcribe` },
+    ),
+  });
+
+  type YoutubeParams = Static<typeof YoutubeParamsSchema>;
+
+  return {
+    name: 'youtube_transcript',
+    label: 'YouTube Transcripts',
+    description:
+      `Fetch the transcript (captions) of up to ${maxVideos} YouTube videos in one batch. ` +
+      `Use it like the scrape tool, but for YouTube links from your search results: select the ` +
+      `most relevant videos and read what they actually say. One call per researcher.`,
+    promptSnippet: `Read transcripts of up to ${maxVideos} YouTube videos from your link pool (one call only)`,
+    promptGuidelines: [
+      `Select up to ${maxVideos} of the most relevant YouTube links from your search results — be judicious, exactly like choosing what to scrape.`,
+      'You may call this tool only ONCE, so choose the videos carefully before calling.',
+      'Pass full watch URLs, youtu.be links, or shorts/embed/live links. Non-YouTube links are ignored.',
+      'Videos without captions, or that are private/age-restricted, are reported as unavailable — the rest still succeed.',
+      "When you cite a YouTube source in CITED LINKS, use the video's ACTUAL title and channel (both shown in this tool's output) as the description — e.g. \"'<exact title>' by <channel>\". Do NOT paraphrase or invent a title; the watch URL alone is not human-identifiable.",
+    ],
+    parameters: YoutubeParamsSchema,
+    executionMode: 'parallel',
+    async execute(
+      _callId: string,
+      params: unknown,
+      signal: AbortSignal | undefined,
+      _onUpdate: AgentToolUpdateCallback<unknown>,
+      extensionCtx: ExtensionContext,
+    ): Promise<AgentToolResult<unknown>> {
+      // Rate-limit enforcement (one call per researcher).
+      if (tracker) {
+        const allowed = tracker.recordCall('youtube_transcript');
+        if (!allowed) {
+          metrics.increment('tool_youtube_transcript_calls_total', 1, { status: 'rate_limited' });
+          return {
+            content: [{ type: 'text', text: tracker.getLimitMessage('youtube_transcript') }],
+            details: { blocked: true, reason: 'limit_reached' },
+          };
+        }
+      }
+
+      if (!Value.Check(YoutubeParamsSchema, params)) {
+        metrics.increment('tool_youtube_transcript_calls_total', 1, { status: 'invalid_params' });
+        return {
+          content: [{ type: 'text', text: `Invalid parameters for youtube_transcript tool. Expected up to ${maxVideos} YouTube URLs.` }],
+          details: { error: 'invalid_params' },
+        };
+      }
+
+      const p = params as YoutubeParams;
+      const urls = p.urls.map((u) => u.trim()).filter((u) => u.length > 0);
+      if (urls.length === 0) {
+        metrics.increment('tool_youtube_transcript_calls_total', 1, { status: 'no_urls' });
+        return {
+          content: [{ type: 'text', text: 'No YouTube links provided.' }],
+          details: { error: 'no_urls' },
+        };
+      }
+
+      // Resolve config against the live cwd for timeout/lang. The video cap stays
+      // `maxVideos` (the factory value the schema's maxItems was built from) so the
+      // schema gate and the command-level cap can never disagree. The PoToken
+      // request key is intentionally not a tool parameter — it is read from the
+      // env var by the minter (SDK callers may pass it through the command).
+      const liveConfig = options.config || getConfig(extensionCtx.cwd);
+
+      // Session id for knowledge-store ingestion: caching each transcript under
+      // its watch URL lets the orchestrator's post-round writer pick it up as the
+      // document `content` and embed the FULL transcript (gated by the writer
+      // queue, i.e. only when the knowledge store is enabled). Also feeds in-run
+      // dedup so a sibling never re-fetches the same video.
+      const researchId = options.getGlobalState?.().researchId;
+      const cachedUrls: string[] = [];
+
+      try {
+        const result = await youtubeTranscriptCommand({
+          urls,
+          maxVideos,
+          timeoutMs: liveConfig.YOUTUBE_TRANSCRIPT_TIMEOUT_MS,
+          lang: liveConfig.YOUTUBE_TRANSCRIPT_LANG,
+          signal,
+          onVideoComplete: (videoId, success) =>
+            options.onUrlScrapeResult?.(`https://www.youtube.com/watch?v=${videoId}`, success),
+          onTranscriptDocument: (url, document) => {
+            if (researchId) {
+              cacheScrapedContent(researchId, url, document);
+              cachedUrls.push(url);
+            }
+          },
+        });
+
+        if (researchId && cachedUrls.length > 0) {
+          registerScrapedLinks(researchId, cachedUrls);
+          options.updateGlobalLinks?.(cachedUrls);
+          if (options.researcherId) {
+            registerResearcherScrapes(researchId, options.researcherId, cachedUrls);
+          }
+        }
+
+        return result;
+      } catch (error) {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        logger.error(`[youtube_transcript tool] Failed: ${errorMsg}`);
+        metrics.increment('tool_youtube_transcript_calls_total', 1, { status: 'error' });
+        return {
+          content: [{
+            type: 'text',
+            text: `# YouTube Transcript Failed\n\n**Error:** ${errorMsg}\n\nThis may be due to network issues or YouTube bot-protection. Continue with the rest of your research.`,
+          }],
+          details: { error: errorMsg },
+        };
+      }
+    },
+  };
+}

package/src/tui/knowledge-search-panel.ts

@@ -0,0 +1,86 @@
+/**
+ * Knowledge Search TUI Panel
+ *
+ * A minimal, static bordered box displayed while the knowledge store
+ * search tool is executing. One line of text inside a box, no header,
+ * no animations — just "searching knowledge store" in accent-colored borders.
+ *
+ * Visual layout (width = terminal width):
+ *
+ * ┌──────────────────────────────────────────┐
+ * │         searching knowledge store        │
+ * └──────────────────────────────────────────┘
+ *
+ * Design principles:
+ * - Same architectural pattern as the research panel (widget factory,
+ *   Component interface, theme usage)
+ * - No header line above the box (unlike the research panel)
+ * - 3 rows total (top border + content + bottom border) vs research panel's 4+
+ * - Static text, no wave animation or progress tracking
+ * - Message is horizontally centered within the box
+ * - Narrow terminals are handled the same way as the rest of the project:
+ *   the message is fit with truncateToWidth (ellipsis) and measured with
+ *   visibleWidth, so multibyte/zero-width content can never overflow the border
+ * - Safe rendering: never crashes, graceful fallback on error
+ */
+
+import { type Component, visibleWidth, truncateToWidth } from '@earendil-works/pi-tui';
+import type { Theme } from '../types/research-panel-types.ts';
+
+/**
+ * The static message displayed while the knowledge store is being searched.
+ */
+const SEARCHING_MESSAGE = 'searching knowledge store';
+
+/**
+ * Create the knowledge search panel component.
+ *
+ * Returns a Component factory compatible with pi's setWidget() API.
+ * The component renders a simple bordered box with the searching message.
+ */
+export function createKnowledgeSearchPanel(): (tui: unknown, theme: Theme) => Component {
+  return (_tui: unknown, theme: Theme) => {
+    const component: Component = {
+      render(width: number): string[] {
+        if (width < 4) return [];
+
+        const innerWidth = Math.max(0, width - 2); // subtract left + right border
+
+        // Fit the message to the inner width, adding an ellipsis when too
+        // narrow — the same truncation helper the research panel uses. Measure
+        // with visibleWidth so the padding maths is correct even if the message
+        // ever contains wide/zero-width characters.
+        const displayText = truncateToWidth(SEARCHING_MESSAGE, innerWidth);
+        const textWidth = visibleWidth(displayText);
+
+        // Center horizontally within the box.
+        const totalPad = Math.max(0, innerWidth - textWidth);
+        const leftPad = Math.floor(totalPad / 2);
+        const rightPad = totalPad - leftPad;
+        const paddedContent = ' '.repeat(leftPad) + displayText + ' '.repeat(rightPad);
+
+        // Row 0: top border ┌───...───┐
+        const topBorder = theme.fg('accent', '┌' + '─'.repeat(innerWidth) + '┐');
+
+        // Row 1: content │ text │
+        const contentLine = theme.fg('accent', '│') +
+          theme.fg('muted', paddedContent) +
+          theme.fg('accent', '│');
+
+        // Row 2: bottom border └───...───┘
+        const bottomBorder = theme.fg('accent', '└' + '─'.repeat(innerWidth) + '┘');
+
+        return [topBorder, contentLine, bottomBorder];
+      },
+
+      invalidate(): void {
+        const comp = component as Component & { parent?: { invalidate: () => void } };
+        if (comp.parent && typeof comp.parent.invalidate === 'function') {
+          comp.parent.invalidate();
+        }
+      },
+    };
+
+    return component;
+  };
+}

package/src/tui/pulse.ts

@@ -0,0 +1,86 @@
+/**
+ * TUI Animation Pulse
+ * 
+ * Centralized clock for TUI animations to ensure visual synchronization
+ * across multiple widgets and panels.
+ */
+
+import { logger } from '../logger.ts';
+
+// ~30 FPS. Deliberately kept ABOVE pi-tui's render cap
+// (MIN_RENDER_INTERVAL_MS = 16). Matching the cap (16ms) makes the pulse and
+// the renderer beat against each other — render delay is max(0, 16 - elapsed),
+// so two timers of the same period produce irregular spacing (visible jitter)
+// for no gain: the wave head is one discrete cell wide, so it only steps a cell
+// roughly every other 60fps frame anyway. 33ms gives a steady, even cadence
+// where every pulse renders once. The real smoothness fix is renderImmediate()
+// in the observer (bypassing the 100ms state-refresh debounce), not raw FPS.
+const FRAME_INTERVAL_MS = 33;
+
+interface PulseSubscriber {
+  (frame: number): void;
+}
+
+class TuiPulse {
+  private frame = 0;
+  private timer: NodeJS.Timeout | null = null;
+  private subscribers = new Set<PulseSubscriber>();
+
+  /**
+   * Start the pulse if not already running
+   */
+  public start(): void {
+    if (this.timer) return;
+    
+    logger.debug('[Pulse] Starting global TUI pulse');
+    this.timer = setInterval(() => {
+      this.frame++;
+      for (const subscriber of this.subscribers) {
+        try {
+          subscriber(this.frame);
+        } catch (err) {
+          logger.error('[Pulse] Error in subscriber:', err);
+        }
+      }
+    }, FRAME_INTERVAL_MS);
+    
+    if (this.timer.unref) {
+      this.timer.unref();
+    }
+  }
+
+  /**
+   * Stop the pulse if no subscribers remain
+   */
+  public stop(): void {
+    if (this.timer && this.subscribers.size === 0) {
+      logger.debug('[Pulse] Stopping global TUI pulse (no subscribers)');
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+  }
+
+  /**
+   * Subscribe to the pulse. Returns an unsubscribe function.
+   */
+  public subscribe(callback: PulseSubscriber): () => void {
+    this.subscribers.add(callback);
+    this.start();
+    
+    return () => {
+      this.subscribers.delete(callback);
+      if (this.subscribers.size === 0) {
+        this.stop();
+      }
+    };
+  }
+
+  /**
+   * Get the current global frame count
+   */
+  public getFrame(): number {
+    return this.frame;
+  }
+}
+
+export const tuiPulse = new TuiPulse();

package/src/tui/research-health.ts

@@ -0,0 +1,124 @@
+/**
+ * Research Health Helpers
+ *
+ * Helper functions for health checking during research
+ */
+
+import { runHealthCheck, healthRegistry, isBusyPoolHealthFailure } from '../healthcheck/index.ts';
+import { logger } from '../logger.ts';
+import { safeUnref } from '../utils/safe-unref.ts';
+import {
+  addSlice,
+  activateSlice,
+  removeSlice,
+  type ResearchPanelState,
+} from './research-panel.ts';
+
+export interface HealthCheckContext {
+  panelState: ResearchPanelState;
+  onUpdate: () => void;
+}
+
+export interface HealthCheckResult {
+  success: boolean;
+  error?: string;
+}
+
+/**
+ * Ensure functional health before starting research
+ * Displays a health check slice and runs verification
+ */
+export async function ensureFunctionalHealth(
+  ctx: HealthCheckContext
+): Promise<void> {
+  const { panelState, onUpdate } = ctx;
+
+  const sliceLabel = 'health check ...';
+  addSlice(panelState, sliceLabel, sliceLabel, false);
+  activateSlice(panelState, sliceLabel);
+  onUpdate();
+
+  try {
+    const health = await runHealthCheck();
+    if (!health.success) {
+      // Distinguish a DEAD/uninitialized pool (must abort) from one that is merely
+      // BUSY: under concurrent load the BrowserRuntime readiness probe queues behind
+      // in-flight scrapes and times out even though the pool is operational. Mirror
+      // the quick orchestrator and proceed-with-warning rather than aborting — the
+      // per-task timeouts bound the work. Without this the TUI aborts the exact
+      // contention case formatHealthError now (correctly) describes as non-fatal.
+      if (isBusyPoolHealthFailure(health)) {
+        logger.warn('[ResearchHealth] Browser pool busy — readiness probe queued out under load, but the pool is operational. Proceeding; per-task timeouts bound the work.');
+      } else {
+        const raw = health.error || '';
+        const msg = formatHealthError(raw);
+        throw new Error(msg);
+      }
+    }
+  } finally {
+    removeSlice(panelState, sliceLabel);
+    onUpdate();
+  }
+}
+
+
+/**
+ * Format health check error into user-friendly message
+ */
+export function formatHealthError(raw: string): string {
+  if (raw.includes('Xvfb') || raw.includes('xvfb') || raw.includes('virtual display') || raw.includes('display server') || raw.includes('DISPLAY not set')) {
+    return 'No display server found on Linux. Install Xvfb for TTY/Wayland use: sudo apt install xvfb';
+  } else if (raw.includes('not found') || raw.includes('not installed') || raw.includes('binaries')) {
+    return 'Browser engine not installed. Run `npx camoufox-js fetch` to install it.';
+  } else if (raw.includes('Timeout') || raw.includes('timeout') || raw.includes('timed out')) {
+    // The readiness probes check browser/state/knowledge availability — they do NOT perform a
+    // web search, so a timeout here is NOT the search engine blocking us. It is almost always
+    // slow browser startup, resource contention (several research runs sharing the pool/GPU at
+    // once, which makes these probes time out together), or a teardown race when a run is
+    // cancelled mid-flight. Keep this message honest so a contention timeout is not misread as
+    // bot-blocking.
+    return 'A readiness check timed out before the browser/services were ready. This is usually slow browser startup, resource contention (e.g. multiple research runs at once sharing the pool), or a cancelled run. It is not a sign the search engine is refusing automated traffic — these readiness probes do not run a search. Retry; if it persists, reduce concurrency and check system resources.';
+  } else if (raw.includes('net::ERR') || raw.includes('ECONNREFUSED') || raw.includes('ENOTFOUND')) {
+    return 'The browser hit a network error reaching the web. Check your internet connection.';
+  } else {
+    return `Browser readiness check failed. Enable debug logging (PI_RESEARCH_DEBUG=true) for details. (${raw || 'unknown error'})`;
+  }
+}
+
+/**
+ * Periodic health monitoring for long-running research
+ */
+export interface HealthMonitor {
+  start: () => void;
+  stop: () => void;
+}
+
+export function createHealthMonitor(): HealthMonitor {
+  let timer: NodeJS.Timeout | null = null;
+
+  const start = () => {
+    if (timer) return;
+    
+    timer = setInterval(async () => {
+      try {
+        const health = await healthRegistry.runAll();
+        const failedNonCritical = health.components.filter(c => !c.healthy && !healthRegistry.isCritical(c.component));
+        if (failedNonCritical.length > 0) {
+          logger.warn(`[research] Periodic health check: non-critical components degraded: ${failedNonCritical.map(c => c.component).join(', ')}`);
+        }
+      } catch (error) {
+        logger.debug('[research] Periodic health check failed (non-blocking):', error);
+      }
+    }, 30000); // Every 30 seconds
+    safeUnref(timer);
+  };
+
+  const stop = () => {
+    if (timer) {
+      clearInterval(timer);
+      timer = null;
+    }
+  };
+
+  return { start, stop };
+}