@lincoln504/pi-research 0.0.0

package/src/orchestration/deep-research-reducer.ts

@@ -0,0 +1,129 @@
+/**
+ * Deep Research Reducer
+ *
+ * PURE FUNCTION logic for state transitions.
+ * Contains 0 side effects. 100% testable without mocks.
+ */
+
+import type { SystemResearchState, DeepResearchEvent } from './deep-research-types.ts';
+
+export function deepResearchReducer(state: SystemResearchState, event: DeepResearchEvent): SystemResearchState {
+  // Deep clone state to ensure true purity
+  const newState: SystemResearchState = {
+    ...state,
+    lastUpdated: Date.now(),
+    initialAgenda: [...state.initialAgenda],
+    allScrapedLinks: [...state.allScrapedLinks],
+    aspects: { ...state.aspects }
+  };
+
+  switch (event.type) {
+    case 'PLANNING_COMPLETE': {
+      newState.initialAgenda = event.agenda;
+      newState.status = 'researching';
+      // Assign first batch to aspects
+      event.agenda.slice(0, event.initialCount).forEach((q: string, i: number) => {
+        const id = `1.${i + 1}`;
+        newState.aspects[id] = { id, query: q, status: 'pending' };
+      });
+      break;
+    }
+
+    case 'SIBLING_STARTED': {
+      const aspect = newState.aspects[event.id];
+      if (aspect) {
+        newState.aspects[event.id] = {
+          ...aspect,
+          status: 'running'
+        };
+      }
+      break;
+    }
+
+    case 'SIBLING_COMPLETED': {
+      const aspect = newState.aspects[event.id];
+      if (aspect) {
+        newState.aspects[event.id] = {
+          ...aspect,
+          status: 'completed',
+          report: event.report
+        };
+      }
+      break;
+    }
+
+    case 'SIBLING_FAILED': {
+      const aspect = newState.aspects[event.id];
+      if (aspect) {
+        newState.aspects[event.id] = {
+          ...aspect,
+          status: 'failed',
+          error: event.error
+        };
+      }
+      break;
+    }
+
+    case 'SIBLING_TOKENS': {
+      const aspect = newState.aspects[event.id];
+      if (aspect) {
+        // Accumulate tokens and cost for this sibling
+        const currentTokens = aspect.tokens || 0;
+        const currentCost = aspect.cost || 0;
+        newState.aspects[event.id] = {
+          ...aspect,
+          tokens: currentTokens + event.tokens,
+          cost: currentCost + event.cost
+        };
+      }
+      break;
+    }
+
+    case 'LINKS_SCRAPED': {
+      newState.allScrapedLinks = [...new Set([...newState.allScrapedLinks, ...event.links])];
+      break;
+    }
+
+    case 'PROMOTION_STARTED': {
+      newState.promotedId = event.id;
+      break;
+    }
+
+    case 'PROMOTION_DECISION': {
+      if (event.finalSynthesis !== undefined) {
+        // Case 1: Lead Evaluator chose to synthesize - complete with synthesis
+        newState.finalSynthesis = event.finalSynthesis;
+        newState.status = 'completed';
+        newState.promotedId = undefined;
+      } else if (event.nextQueries.length > 0 && newState.currentRound < event.maxRounds) {
+        // Case 2: Lead Evaluator chose to continue AND we're under max rounds
+        newState.currentRound++;
+        newState.status = 'researching';
+        newState.promotedId = undefined; // Reset for new round
+        event.nextQueries.forEach((q: string, i: number) => {
+          const id = `${newState.currentRound}.${i + 1}`;
+          newState.aspects[id] = { id, query: q, status: 'pending' };
+        });
+      } else {
+        // Case 3: Max rounds reached OR no next queries
+        // Note: Per lead-evaluator prompt, when max rounds are reached, the evaluator
+        // should synthesize. If nextQueries are provided at max rounds, this is a
+        // protocol violation, and we enforce the constraint by completing anyway.
+        newState.status = 'completed';
+        newState.promotedId = undefined;
+      }
+      break;
+    }
+  }
+
+  return newState;
+}
+
+/**
+ * Utility to check if a round is finished
+ */
+export function isRoundComplete(state: SystemResearchState, round: number): boolean {
+  const roundAspects = Object.values(state.aspects).filter(a => a.id.startsWith(`${round}.`));
+  if (roundAspects.length === 0) return false;
+  return roundAspects.every(a => a.status === 'completed' || a.status === 'failed');
+}

package/src/orchestration/deep-research-types.ts

@@ -0,0 +1,54 @@
+/**
+ * Deep Research Types
+ *
+ * Defines the schema for the monolithic ResearchState persisted in the session tree.
+ */
+
+export type ResearchStatus = 'planning' | 'researching' | 'evaluating' | 'synthesizing' | 'completed' | 'failed';
+export type SiblingStatus = 'pending' | 'running' | 'completed' | 'failed';
+
+export type DeepResearchEvent =
+  | { type: 'PLANNING_COMPLETE'; agenda: string[]; initialCount: number }
+  | { type: 'SIBLING_STARTED'; id: string }
+  | { type: 'SIBLING_COMPLETED'; id: string; report: string }
+  | { type: 'SIBLING_FAILED'; id: string; error: string }
+  | { type: 'LINKS_SCRAPED'; links: string[] }
+  | { type: 'PROMOTION_STARTED'; id: string }
+  | { type: 'PROMOTION_DECISION'; nextQueries: string[]; finalSynthesis?: string; maxRounds: number }
+  | { type: 'SIBLING_TOKENS'; id: string; tokens: number; cost: number };
+
+export interface ResearchSibling {
+  id: string; // e.g. "1.1", "2.1"
+  query: string;
+  status: SiblingStatus;
+  report?: string;
+  error?: string;
+  tokens?: number;
+  cost?: number;
+}
+
+export interface SystemResearchState {
+  version: number;
+  rootQuery: string;
+  complexity: 1 | 2 | 3; // 1: Normal, 2: Deep, 3: Ultra
+  currentRound: number;
+  status: ResearchStatus;
+  lastUpdated: number;
+  
+  // The full exhaustive list of aspects planned by the initial coordinator
+  initialAgenda: string[];
+  
+  // Global pool updated IMMEDIATELY by the scrape tool
+  allScrapedLinks: string[];
+  
+  // Sequence of research aspects actively being worked or completed
+  aspects: {
+    [id: string]: ResearchSibling;
+  };
+  
+  // Track who was promoted to Lead Evaluator in this round to prevent race conditions
+  promotedId?: string;
+
+  // Final result if completed
+  finalSynthesis?: string;
+}

package/src/orchestration/id-utils.ts

@@ -0,0 +1,63 @@
+/**
+ * Researcher ID Utilities
+ *
+ * Abstracts hierarchical IDs (1.1, 1.2, 2.1) to sequential display numbers (1, 2, 3)
+ * for cleaner UI while maintaining internal structure.
+ */
+
+import type { SystemResearchState } from './deep-research-types.ts';
+
+/**
+ * Get sequential display number for a researcher ID
+ *
+ * Maps internal hierarchical IDs to sequential display numbers:
+ * - Round 1: 1.1 → 1, 1.2 → 2, 1.3 → 3
+ * - Round 2: 2.1 → 4, 2.2 → 5, 2.3 → 6
+ * - Continue sequentially across all rounds
+ */
+export function getDisplayNumber(state: SystemResearchState, internalId: string): string {
+  const aspects = Object.values(state.aspects).sort((a, b) => {
+    const aParts = a.id.split('.');
+    const bParts = b.id.split('.');
+    const aRound = parseInt(aParts[0] ?? '0');
+    const aNum = parseInt(aParts[1] ?? '0');
+    const bRound = parseInt(bParts[0] ?? '0');
+    const bNum = parseInt(bParts[1] ?? '0');
+
+    if (aRound !== bRound) return aRound - bRound;
+    return aNum - bNum;
+  });
+
+  const index = aspects.findIndex(a => a.id === internalId);
+  return index >= 0 ? (index + 1).toString() : internalId;
+}
+
+/**
+ * Get a researcher's role context
+ *
+ * Returns role information for a researcher:
+ * - Number in this round (1, 2, 3)
+ * - Total siblings in this round
+ * - Current round
+ */
+export function getResearcherRoleContext(
+  state: SystemResearchState,
+  internalId: string
+): {
+  displayNumber: string;
+  roundNumber: number;
+  totalInRound: number;
+  isLastInRound: boolean;
+} {
+  const parts = internalId.split('.');
+  const round = parseInt(parts[0] ?? '0');
+  const sibling = parseInt(parts[1] ?? '0');
+  const roundAspects = Object.values(state.aspects).filter(a => a.id.startsWith(`${round}.`));
+
+  return {
+    displayNumber: getDisplayNumber(state, internalId),
+    roundNumber: round,
+    totalInRound: roundAspects.length,
+    isLastInRound: sibling === roundAspects.length,
+  };
+}

package/src/orchestration/researcher.ts

@@ -0,0 +1,104 @@
+/**
+ * Researcher Session Factory
+ *
+ * Responsible for instantiating individual researcher agent sessions. 
+ * Each researcher is an autonomous agent equipped with a suite of specialized tools:
+ * - Web search via SearXNG
+ * - Context-aware URL scraping via Playwright
+ * - Security vulnerability database queries (NVD, CISA, OSV)
+ * - Technical Q&A retrieval from Stack Exchange
+ * - Local code search via Ripgrep
+ */
+
+import type { AgentSession, ModelRegistry, SettingsManager, ExtensionContext } from '@mariozechner/pi-coding-agent';
+import { createAgentSession, createReadTool, SessionManager } from '@mariozechner/pi-coding-agent';
+import type { Model } from '@mariozechner/pi-ai';
+import { createResearchTools } from '../tools/index.ts';
+import { makeResourceLoader } from '../utils/make-resource-loader.ts';
+import { ToolUsageTracker, createDefaultToolLimits } from '../utils/tool-usage-tracker.ts';
+import type { SystemResearchState } from './deep-research-types.ts';
+
+export interface CreateResearcherSessionOptions {
+  cwd: string;
+  ctxModel: Model<any> | undefined;
+  modelRegistry: ModelRegistry;
+  settingsManager: SettingsManager;
+  systemPrompt: string;
+  searxngUrl: string;
+  extensionCtx: ExtensionContext;
+  // Optional: real closures for global state management
+  getGlobalState?: () => SystemResearchState;
+  updateGlobalLinks?: (links: string[]) => void;
+  /** Returns tokens consumed by this researcher session so far (for context-aware scrape gating). */
+  getTokensUsed?: () => number;
+  /** Model context window size in tokens. */
+  contextWindowSize?: number;
+}
+
+export async function createResearcherSession(options: CreateResearcherSessionOptions): Promise<AgentSession> {
+  const {
+    cwd,
+    ctxModel,
+    modelRegistry,
+    settingsManager,
+    systemPrompt,
+    searxngUrl,
+    extensionCtx,
+    getGlobalState,
+    updateGlobalLinks,
+    getTokensUsed,
+    contextWindowSize,
+  } = options;
+
+  // Validate required parameters
+  if (!ctxModel) {
+    throw new Error('No model selected. Please select a model before using the research tool.');
+  }
+
+  if (!systemPrompt || typeof systemPrompt !== 'string') {
+    throw new Error('Invalid system prompt: must be a non-empty string');
+  }
+
+  if (!searxngUrl || typeof searxngUrl !== 'string') {
+    throw new Error('Invalid SearXNG URL: must be a non-empty string');
+  }
+
+  // Create tool usage tracker for this researcher
+  const tracker = new ToolUsageTracker(createDefaultToolLimits());
+
+  // Use provided closures or fallback to safe dummies
+  const globalState = getGlobalState || (() => ({} as any));
+  const globalLinks = updateGlobalLinks || (() => {});
+
+  try {
+    const result = await createAgentSession({
+      cwd,
+      tools: [createReadTool(cwd)],
+      customTools: createResearchTools({
+        searxngUrl,
+        ctx: extensionCtx,
+        tracker,
+        getGlobalState: globalState,
+        updateGlobalLinks: globalLinks,
+        getTokensUsed,
+        contextWindowSize,
+      }),
+      sessionManager: SessionManager.inMemory(), // Each researcher gets its own isolated session
+      settingsManager,
+      model: ctxModel,
+      modelRegistry,
+      resourceLoader: makeResourceLoader(systemPrompt),
+    });
+
+    if (!result || !result.session) {
+      throw new Error('Session creation returned invalid result');
+    }
+
+    return result.session;
+  } catch (error) {
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    throw new Error(`Failed to create researcher session: ${errorMsg}`, {
+      cause: error,
+    });
+  }
+}

package/src/orchestration/session-context.ts

@@ -0,0 +1,43 @@
+/**
+ * Session Context Formatter
+ *
+ * Formats the full parent session context for the initial coordinator.
+ * Returns the uncompacted history of the current conversation branch.
+ */
+
+import type { ExtensionContext } from '@mariozechner/pi-coding-agent';
+import { 
+  buildSessionContext, 
+  convertToLlm, 
+  serializeConversation 
+} from '@mariozechner/pi-coding-agent';
+
+/**
+ * Formats parent context for the initial coordinator.
+ * Provides the full linear history of the current branch.
+ */
+export async function formatParentContext(ctx: ExtensionContext): Promise<string> {
+  const branch = ctx.sessionManager.getBranch();
+  const sessionContext = buildSessionContext(branch);
+  const allMessages = sessionContext.messages;
+
+  if (allMessages.length === 0) {
+    return 'No previous context available.';
+  }
+
+  // Limit context to prevent overflow for the initial coordinator
+  const MAX_CONTEXT_MESSAGES = 15;
+  const recentMessages = allMessages.length > MAX_CONTEXT_MESSAGES 
+    ? allMessages.slice(-MAX_CONTEXT_MESSAGES) 
+    : allMessages;
+
+  const llmMessages = convertToLlm(recentMessages);
+  const serialized = serializeConversation(llmMessages);
+    
+  return [
+    '## Parent Conversation History',
+    'The following is the recent history of the conversation branch for your reference:',
+    '',
+    serialized
+  ].join('\n');
+}

package/src/prompts/research-tool-usage.md

@@ -0,0 +1,45 @@
+### 🔍 RESEARCH TOOL USAGE
+
+**For any web/internet research questions, use the `research` tool.**
+
+The `research` tool (from pi-research extension) is your tool for web/internet research.
+
+---
+
+#### What counts as web research?
+
+- Questions requiring current information (news, trends, latest developments)
+- Questions about products, services, companies
+- Questions requiring statistics or data
+- Questions about people, places, events, or topics external to this project
+- "What is X?", "How does X work?", "Tell me about X" questions
+
+#### What is NOT web research (use other tools for these)
+
+- Reading files in the project → use `read` tool
+- Running commands or tests → use `bash` tool
+- Analyzing code in this repository
+- Questions about the project itself
+
+---
+
+#### DEPTH PARAMETER — Controls research intensity
+
+**DEFAULT:** Omit depth parameter (uses `depth: 0` - Quick mode)
+- 1 researcher, 1 round
+- Fastest for most queries
+
+**Higher depths** (only when user requests thoroughness):
+- `depth: 1` — Normal: 2 researchers, up to 2 rounds
+- `depth: 2` — Deep: 3 researchers, up to 3 rounds
+- `depth: 3` — Ultra: 5 researchers, up to 5 rounds
+
+**Key rule:** Default to depth 0. Use higher depths when user explicitly asks for "exhaustive", "deep-dive", "ultra", or "comprehensive" research.
+
+---
+
+#### MULTIPLE RESEARCH REQUESTS — Parallel vs. merged
+
+**Different scopes** (unrelated topics): call `research` multiple times simultaneously in a single response — one tool call per distinct topic, run in parallel.
+
+**Similar or overlapping scope**: merge into a single `research` call and consider raising depth one level to capture the additional breadth.

package/src/prompts/researcher.md

@@ -0,0 +1,166 @@
+# Researcher
+
+You are a research agent. Investigate your assigned topic thoroughly.
+
+## Your Job (One Complete Cycle)
+
+Complete ONE research cycle with strict tool limits:
+1. **Phase 1**: Up to 4 gathering tool calls (search/security/stackexchange/grep combined)
+2. **Phase 2**: Up to 4 scrape tool calls (handshake + up to 3 batches)
+3. **Phase 3**: Synthesize findings and report
+4. **STOP**: Research complete
+
+**CRITICAL**: If you are the last researcher alive in your round, complete your own research first. Do not cut corners.
+
+One complete cycle per researcher. Search → scrape → report → done.
+
+---
+
+## Phase 1: Information Gathering (Max 4 Calls Total)
+
+You have **4 tool calls total** for gathering across ALL tools (search, security_search, stackexchange, grep combined).
+
+**Strategy**:
+- Each tool call can include **multiple queries**
+- Use these 4 calls wisely — one well-crafted `search()` with 5 queries beats 5 separate calls
+- After your 4th gathering call, move to Phase 2
+
+**Search Disambiguation**:
+- **Detect Ambiguity Early**: Watch for patterns like:
+  - Results from single domain (e.g., only the official website, only Stack Overflow, only dictionaries)
+  - Programming results when researching non-technical subjects
+  - Dictionary definitions for complex topics
+  - Results about different topics with shared keywords
+
+- **Query Reformulation**: When ambiguity or single-domain domination detected, vary word order, add domain context, use synonyms, target `site:en.wikipedia.org` for encyclopedic context, or use `sourceType: "news"` to bypass official site content.
+
+- **Exhaustive Breadth**: Use 3-5 queries per gathering call. If Tool Call 1 fails, use ALL remaining for reformulation.
+- **Technical Sources**: For technical topics, prioritize official documentation and the latest version; avoid outdated tutorials or unofficial summaries.
+
+**Available Tools**:
+- `search`: General web search (Bing, Google, etc. via SearXNG)
+  - `sourceType: "news"` — current events, recent releases
+  - `sourceType: "github"` — repos, packages, open-source libraries
+  - `freshness: "day" | "week" | "month" | "year"` — time filter
+- `security_search`: Security databases (CVE, NVD, OSV, GitHub, CISA KEV)
+  - **USE FOR**: vulnerabilities, CVE IDs, package security
+  - Filters: severity, CVE ID, package name, ecosystem, actively exploited
+- `stackexchange`: Stack Overflow and Stack Exchange network
+  - **USE FOR**: technical questions, code solutions, debugging, best practices
+  - Works with any Stack Exchange site, supports tags
+- `grep`: Search patterns in local code
+- `read`: Read local files identified by grep
+
+**Rules**:
+- Maximum of 4 gathering tool calls total
+- Do NOT scrape during this phase
+- Collect promising URLs from results
+- After 4th gathering tool call, move to Phase 2
+
+---
+
+## Phase 2: Scrape Protocol (Context-Aware, Up to 4 Calls)
+
+ONLY after Phase 1 is complete:
+
+1. **Handshake** (Call 1): Call `scrape` with intended URLs. Tool returns already-scraped links and available batches.
+2. **Batch 1** (Call 2): Filtered list (max 3 URLs). Primary broad scraping. Provide `excludeLinks` for deprioritized URLs.
+3. **Batch 2** (Call 3): Targeted follow-up (max 2 URLs). Use for gaps, retries, or second angle. Auto-deduplicated.
+4. **Batch 3** (Call 4): Deep-dive sub-topic (max 3 URLs).
+
+**Rules**:
+- Must call `scrape` at least twice (Handshake + Batch 1)
+- If tool responds with "Context Budget Reached", skip batch and move to Phase 3
+- Batch 1: max 3 URLs — wide net
+- Batch 2: max 2 URLs — targeted
+- Batch 3: max 3 URLs — deep-dive
+- Provide `excludeLinks` in Batch 1 for considered but deprioritized links
+- Synthesize findings from all batches before reporting
+
+---
+
+## Phase 3: Detailed Report and STOP
+
+IMMEDIATELY after Phase 2 completes, you are DONE.
+
+Do this:
+- ✅ **Synthesize Everything**: Compile findings from Phase 1, Phase 2, and injected sibling reports
+- ✅ **Be Verbose and Deep**: Provide full breadth, depth, nuance. Don't just list facts — provide context, analysis, conflicting information
+- ✅ **Evidence-Based**: Every major claim has a [citation](URL)
+- ✅ **Report Cited Links**: What you actually scraped and used
+- ✅ **Report Scrape Candidates**: What you found but didn't scrape, with reasons
+- ✅ **STOP IMMEDIATELY**: Submit report and wait
+
+The last researcher to finish inherits **Lead Evaluator** role.
+
+## Response Format
+
+```markdown
+## [Detailed Topic Title]
+
+### Executive Summary
+[High-level synthesis of findings, incorporating sibling insights]
+
+### Comprehensive Findings
+#### [Theme/Area 1]
+- **Detailed Finding**: [Exhaustive explanation with nuanced context and multiple [citations](URL)]
+- **Analysis**: [Your analytical take, identifying patterns or gaps]
+
+#### [Theme/Area 2]
+- **Detailed Finding**: [Continue for all major areas...]
+
+### Nuance & Conflicting Data
+[Identify areas where sources disagree or topic is nuanced/ambiguous]
+
+### CITED LINKS
+* [URL] - Detailed description of value added
+* [URL] - Detailed description of value added
+
+### SCRAPE CANDIDATES
+* [URL] - Why this remains a high-value target
+* [URL] - Why this was deprioritized
+```
+
+## Detailed Findings and Nuance
+
+Your research is a deep investigation:
+- Provide full breadth, depth, nuance from sources
+- Build upon and integrate sibling reports
+- Accuracy and depth matter more than matching original label
+- Explicitly identify conflicting data or nuanced areas
+
+**Submit your report and STOP IMMEDIATELY.** Do not add anything else.
+
+---
+
+## Tool Failures & Error Handling
+
+**Partial Failures (Continue)**:
+- Continue with tools that work
+- Document what failed briefly
+- Use alternative approaches
+- Do NOT use "ERROR:" prefix
+
+**Complete Failures (Use ERROR)**:
+- Exhausted all 4 gathering tool calls with NO useful results
+- ALL scraping attempts failed or skipped
+- NO findings to report
+- Signals systemic problem requiring coordinator intervention
+
+---
+
+## Research Lifecycle
+
+1. Receive assignment
+2. Execute Phase 1 (up to 4 gathering calls)
+3. Execute Phase 2 (up to 4 scrape calls)
+4. Execute Phase 3 (report)
+5. **STOP**
+
+You do NOT:
+- Wait for feedback
+- Suggest next steps
+- Provide additional context
+- Attempt to improve findings
+
+The coordinator receives your report and decides next steps.

package/src/prompts/system-coordinator.md

@@ -0,0 +1,24 @@
+# Research Coordinator
+
+You are an expert research coordinator. Review the chat history and create a list of research tasks.
+
+## Your Job
+
+1. **Review** the context provided.
+2. **Decompose** the query into broad research tasks. Each task should explore a significant dimension of the topic. Slight overlaps are encouraged for cross-validation.
+3. **Delegate** tasks as a JSON array for parallel researchers.
+
+## Output Format
+
+Return ONLY a JSON array of strings:
+
+```json
+["task 1", "task 2", "task 3"]
+```
+
+**CRITICAL REQUIREMENTS**:
+- Output must be a valid JSON array of strings
+- Each task must be a plain string (not an object like `{"query": "text"}`)
+- Each task string must be non-empty after trimming
+- Do not include any explanation, preamble, or markdown formatting
+- No code blocks — just the raw JSON array