@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/stackexchange/index.ts

@@ -18,12 +18,13 @@ import {
   formatUsersTable,
   formatSitesTable,
   formatCompactQuestions,
-} from './output/index.ts';
+} from './output/table.ts';
 import {
   formatUsersCompact,
   formatSitesCompact,
 } from './output/compact.ts';
-import type { ExtensionContext, AgentToolResult } from '@mariozechner/pi-coding-agent';
+import type { ExtensionContext, AgentToolResult } from '@earendil-works/pi-coding-agent';
+import { logger } from '../logger.ts';
 
 function notify(
   ctx: ExtensionContext,
@@ -40,7 +41,7 @@ function loadConfig(): StackExchangeConfig {
   return {
     defaultSite: 'stackoverflow.com',
     apiKey: process.env['STACKEXCHANGE_API_KEY'] ?? null,
-    requestTimeout: 30000, // 30 seconds
+    requestTimeout: 10000, // 10 seconds
   };
 }
 
@@ -75,7 +76,7 @@ export async function stackexchangeCommand(options: {
         'error',
       );
       return {
-        content: [{ type: 'text', text: `❌ Stack Exchange API quota exhausted (${quota.remaining}/${quota.max} remaining). Please wait until the quota resets.` }],
+        content: [{ type: 'text', text: `[Error] Stack Exchange API quota exhausted (${quota.remaining}/${quota.max} remaining). Please wait until the quota resets.` }],
         details: {
           quota,
           command,
@@ -124,22 +125,28 @@ export async function stackexchangeCommand(options: {
       output = formatTable(result);
     }
 
-    // Add quota info
+    // Add source and quota info
     const quota = client.getQuotaInfo();
-    output += `\n---\n**API Quota:** ${quota.remaining}/${quota.max} remaining\n`;
+    output += `\n---\n**Source: Stack Exchange**\n**API Quota:** ${quota.remaining}/${quota.max} remaining\n`;
+
+    // Grounding signal for the researcher-executor grounding gate: how many real items this
+    // command returned. search yields a Question[] (0 = nothing found = NOT grounding); get/
+    // user/site yield a single populated object. See researcher-executor.ts grounding accumulation.
+    const groundingHits = Array.isArray(result) ? result.length : (result ? 1 : 0);
 
     return {
       content: [{ type: 'text', text: output }],
       details: {
         quota,
         command,
+        groundingHits,
       },
     };
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
     notify(ctx, `Stack Exchange error: ${message}`, 'error');
     return {
-      content: [{ type: 'text', text: `❌ Stack Exchange error: ${message}` }],
+      content: [{ type: 'text', text: `[Error] Stack Exchange error: ${message}` }],
       details: {
         quota: client.getQuotaInfo(),
         command,
@@ -149,7 +156,7 @@ export async function stackexchangeCommand(options: {
 }
 
 /**
- * Execute search command
+ * Execute search command with pagination support
  */
 async function executeSearch(
   params: Record<string, unknown>,
@@ -160,29 +167,73 @@ async function executeSearch(
   const query = params['query'] as string | undefined;
   const site = (params['site'] as string | undefined) ?? config.defaultSite;
   const limit = Math.min((params['limit'] as number | undefined) ?? 10, 100);
+  // Cap the LLM-supplied page count so a large value can't fan out into a quota-burning
+  // paging storm against the rate-limited API (mirrors the `limit` clamp). Only the upper
+  // bound is imposed — a 0/negative value still means "no pages" (loop runs zero times),
+  // preserving the pre-cap behavior.
+  const maxPages = Math.min(10, (params['maxPages'] as number | undefined) ?? 5);
   const tagsInput = params['tags'] as string | null;
   // Convert tags: "tag1,tag2" is converted to semicolon-separated for API
   const tags = tagsInput ? tagsInput.split(',').map(t => t.trim()).filter(t => t.length > 0).join(';') : undefined;
 
-  // Build query parameters
-  const queryParams = {
-    order: 'desc' as const,
-    sort: 'relevance' as const,
-    q: query ?? undefined,
-    tagged: tags ?? undefined,
-    pagesize: limit,
-    site,
-  };
+  const allQuestions: Question[] = [];
+  // Fetch up to `limit` per page (API max 30) so the common case needs a single
+  // request. maxPages is only a safety cap for when a page returns short of the
+  // limit — it is NOT a target, so we must not divide the limit across it (that
+  // made the default limit=10 spend 5 calls of 2 items each, burning quota and
+  // raising the odds of a mid-search backoff).
+  const pageSize = Math.min(30, limit); // Up to 30 per page
+  
+  for (let page = 1; page <= maxPages && allQuestions.length < limit; page++) {
+    // Stop paging once cancelled: the API is rate-limited (a shared per-window quota),
+    // so continuing to fetch pages for an abandoned run burns quota that a live run
+    // needs. Return whatever earlier pages already produced (graceful, matches the
+    // per-page-failure behavior below).
+    if (signal?.aborted) break;
+
+    // Build query parameters with pagination
+    const queryParams = {
+      order: 'desc' as const,
+      sort: 'relevance' as const,
+      q: query ?? undefined,
+      tagged: tags ?? undefined,
+      pagesize: pageSize,
+      page,
+      site,
+      // Include the question body inline. SE quota is charged per request, not per
+      // filter, so this enriches each result at zero extra quota cost and the table
+      // renderer already displays q.body when present.
+      filter: Filters.WITH_BODY,
+    };
 
-  const searchParams = buildSearchQuery(queryParams);
+    const searchParams = buildSearchQuery(queryParams);
 
-  // Make request
-  const response = await client.request<Question>(
-    { method: 'GET', endpoint: '/search/advanced', params: searchParams },
-    signal,
-  );
+    // Make request. If a later page fails (rate limit, HTTP error) but we already
+    // have results from earlier pages, return those rather than discarding the whole
+    // search — only rethrow when we have nothing to show.
+    let response;
+    try {
+      response = await client.request<Question>(
+        { method: 'GET', endpoint: '/search/advanced', params: searchParams },
+        signal,
+      );
+    } catch (err) {
+      if (allQuestions.length > 0) {
+        logger.warn(`[stackexchange] Pagination stopped at page ${page} after error; returning ${allQuestions.length} partial result(s):`, err);
+        break;
+      }
+      throw err;
+    }
 
-  return response.items;
+    if (response.items.length === 0) {
+      // No more results
+      break;
+    }
+
+    allQuestions.push(...response.items);
+  }
+
+  return allQuestions.slice(0, limit);
 }
 
 /**
@@ -209,7 +260,7 @@ async function executeGet(
   });
 
   const response = await client.request<Question>(
-    { method: 'GET', endpoint: `/questions/${id}`, params: questionParams },
+    { method: 'GET', endpoint: `/questions/${encodeURIComponent(id)}`, params: questionParams },
     signal,
   );
 
@@ -230,7 +281,7 @@ async function executeGet(
   });
 
   const answersResponse = await client.request<Answer>(
-    { method: 'GET', endpoint: `/questions/${id}/answers`, params: answersParams },
+    { method: 'GET', endpoint: `/questions/${encodeURIComponent(id)}/answers`, params: answersParams },
     signal,
   );
 
@@ -261,7 +312,10 @@ async function executeUser(
   });
 
   const response = await client.request<User>(
-    { method: 'GET', endpoint: `/users/${id}`, params: userParams },
+    // Escape the LLM-supplied id in the path (consistent with executeGet) — it is
+    // host-confined to the StackExchange API base, but an unescaped id could otherwise
+    // inject extra path/query segments.
+    { method: 'GET', endpoint: `/users/${encodeURIComponent(String(id))}`, params: userParams },
     signal,
   );
 

package/src/stackexchange/output/compact.ts

@@ -2,7 +2,7 @@
  * Compact output formatter for Stack Exchange results
  */
 
-import type { Question, Answer, User, Site } from '../types';
+import type { Question, Answer, User, Site } from '../types.ts';
 
 export function formatQuestionsCompact(questions: Question[]): string {
   if (questions.length === 0) {
@@ -12,7 +12,7 @@ export function formatQuestionsCompact(questions: Question[]): string {
   const lines: string[] = [];
   let index = 1;
   for (const q of questions) {
-    const accepted = q.accepted_answer_id ? '✓' : ' ';
+    const accepted = q.accepted_answer_id ? '[accepted]' : '         ';
     lines.push(`${index}. ${accepted} [${q.title}](${q.link}) (${q.score} pts, ${q.answer_count} ans, ${q.view_count} views)`);
     index++;
   }
@@ -27,9 +27,9 @@ export function formatAnswersCompact(answers: Answer[]): string {
   const lines: string[] = [];
   let index = 1;
   for (const a of answers) {
-    const accepted = a.is_accepted ? '✓' : ' ';
-    const authorName = a.owner?.display_name ?? 'Unknown';
-    lines.push(`${index}. ${accepted} by ${authorName} (${a.score} pts)`);
+    const accepted = a.is_accepted ? '[accepted]' : '         ';
+    const displayName = a.owner?.display_name ?? 'Unknown';
+    lines.push(`${index}. ${accepted} by ${displayName} (${a.score} pts)`);
     index++;
   }
   return lines.join('\n');
@@ -43,7 +43,8 @@ export function formatUsersCompact(users: User[]): string {
   const lines: string[] = [];
   let index = 1;
   for (const u of users) {
-    lines.push(`${index}. ${u.display_name} (rep: ${u.reputation}, 🥇${u.badge_counts.gold} 🥈${u.badge_counts.silver} 🥉${u.badge_counts.bronze})`);
+    const bc = u.badge_counts ?? { gold: 0, silver: 0, bronze: 0 };
+    lines.push(`${index}. ${u.display_name} (rep: ${u.reputation}, gold:${bc.gold} silver:${bc.silver} bronze:${bc.bronze})`);
     index++;
   }
   return lines.join('\n');

package/src/stackexchange/output/table.ts

@@ -2,7 +2,7 @@
  * Table output formatter for Stack Exchange results
  */
 
-import type { Question, Answer, User, Site } from '../types';
+import type { Question, Answer, User, Site } from '../types.ts';
 
 export function formatQuestionsTable(questions: Question[]): string {
   if (questions.length === 0) {
@@ -16,7 +16,7 @@ export function formatQuestionsTable(questions: Question[]): string {
     output += `## ${q.title}\n\n`;
     output += `- **Score:** ${q.score}\n`;
     output += `- **Views:** ${q.view_count}\n`;
-    output += `- **Answers:** ${q.answer_count}${q.accepted_answer_id ? ' ✓' : ''}\n`;
+    output += `- **Answers:** ${q.answer_count}${q.accepted_answer_id ? ' [accepted]' : ''}\n`;
     output += `- **Tags:** ${q.tags.join(', ')}\n`;
     if (q.owner) {
       output += `- **Author:** ${q.owner.display_name} (rep: ${q.owner.reputation})\n`;
@@ -24,8 +24,12 @@ export function formatQuestionsTable(questions: Question[]): string {
     output += `- **Link:** ${q.link}\n`;
     output += `- **Created:** ${new Date(q.creation_date * 1000).toLocaleString()}\n`;
 
-    if (q.body && q.body.length < 1000) {
-      output += `- **Body:** ${q.body.substring(0, 500)}...\n`;
+    if (q.body) {
+      // Truncate long bodies to a preview rather than dropping them entirely — the
+      // previous `length < 1000` guard omitted the Body line for the longest
+      // questions (the ones most in need of truncation).
+      const body = q.body.length > 500 ? `${q.body.substring(0, 500)}...` : q.body;
+      output += `- **Body:** ${body}\n`;
     }
 
     output += '\n---\n\n';
@@ -45,7 +49,7 @@ export function formatAnswersTable(answers: Answer[]): string {
   for (const a of answers) {
     const authorName = a.owner?.display_name ?? 'Unknown';
     output += `## Answer by ${authorName}\n\n`;
-    output += `- **Score:** ${a.score}${a.is_accepted ? ' ✓ (Accepted)' : ''}\n`;
+    output += `- **Score:** ${a.score}${a.is_accepted ? ' [Accepted]' : ''}\n`;
     if (a.owner) {
       output += `- **Author:** ${a.owner.display_name} (rep: ${a.owner.reputation})\n`;
     }
@@ -73,7 +77,7 @@ export function formatUsersTable(users: User[]): string {
   for (const u of users) {
     output += `## ${u.display_name}\n\n`;
     output += `- **Reputation:** ${u.reputation}\n`;
-    output += `- **Badges:** 🥇 ${u.badge_counts.gold} 🥈 ${u.badge_counts.silver} 🥉 ${u.badge_counts.bronze}\n`;
+    output += `- **Badges:** gold:${u.badge_counts.gold} silver:${u.badge_counts.silver} bronze:${u.badge_counts.bronze}\n`;
     output += `- **User ID:** ${u.user_id}\n`;
     output += `- **Member since:** ${new Date(u.creation_date * 1000).toLocaleDateString()}\n`;
     if (u.location) {
@@ -115,7 +119,7 @@ export function formatCompactQuestions(questions: Question[]): string {
   let output = '';
   let index = 1;
   for (const q of questions) {
-    const accepted = q.accepted_answer_id ? '✓' : ' ';
+    const accepted = q.accepted_answer_id ? '[accepted]' : '';
     output += `${index}. [${q.title}](${q.link}) ${accepted} (score: ${q.score}, answers: ${q.answer_count})\n`;
     index++;
   }

package/src/stackexchange/queries.ts

@@ -156,13 +156,3 @@ export const Filters = {
   // Get answers with body
   ANSWERS_WITH_BODY: '!9_bDE(fI5',
 };
-
-/**
- * Parse IDs from query string (handle both single IDs and comma/semicolon separated)
- */
-export function parseIds(ids: string | number | (string | number)[]): string {
-  if (Array.isArray(ids)) {
-    return ids.join(';');
-  }
-  return String(ids);
-}

package/src/stackexchange/rest-client.ts

@@ -5,6 +5,8 @@
 
 import type { StackExchangeWrapper } from './types.ts';
 import { logger } from '../logger.ts';
+import { CircuitBreaker } from '../utils/circuit-breaker.ts';
+import { metrics } from '../utils/metrics.ts';
 
 const API_BASE = 'https://api.stackexchange.com/2.3';
 
@@ -21,6 +23,7 @@ export class StackExchangeClient {
   private quotaMax = 300;
   private requestCount = 0;
   private lastBackoff: number | null = null;
+  private circuitBreaker: CircuitBreaker;
 
   constructor(
     apiKey: string | null,
@@ -28,93 +31,165 @@ export class StackExchangeClient {
   ) {
     this._apiKey = apiKey;
     this._timeout = timeout;
+    this.circuitBreaker = new CircuitBreaker({
+      failureThreshold: 5,
+      resetTimeoutMs: 10000,
+      name: 'StackExchange API',
+      isTransientError: (err) => {
+        if (err instanceof Error) {
+            // Count network errors and 5xx errors, but not 4xx client errors (except 429).
+            // Match status codes on a word boundary so an incidental "50"/"429" inside a
+            // larger number (e.g. a token count) does not spuriously look transient — same
+            // class as the word-boundary fix in isTransientSynthesisError.
+            const msg = err.message.toLowerCase();
+            return msg.includes('timeout') || msg.includes('network') || msg.includes('econn') || /\b5\d\d\b/.test(msg) || /\b429\b/.test(msg);
+        }
+        return true;
+      }
+    });
   }
 
   async request<T>(options: RequestOptions, signal?: AbortSignal): Promise<StackExchangeWrapper<T>> {
-    // Check for backoff from previous requests
-    if (this.lastBackoff && this.lastBackoff > Date.now()) {
-      const waitTime = Math.ceil((this.lastBackoff - Date.now()) / 1000);
-      throw new Error(
-        `Rate limited. Please wait ${waitTime} seconds before making more requests.`,
-      );
-    }
-
-    const url = new URL(`${API_BASE}${options.endpoint}`);
-    url.search = options.params.toString();
-
-    // Add API key if provided
-    if (this._apiKey) {
-      url.searchParams.set('key', this._apiKey);
-    }
-
-    // Add site parameter if not already present (for methods that need it)
-    if (!url.searchParams.has('site') && !options.endpoint.startsWith('/sites')) {
-      url.searchParams.set('site', 'stackoverflow.com');
-    }
-
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => {
-      controller.abort();
-    }, this._timeout);
-
-    // Chain the signal if provided
-    if (signal) {
-      signal.addEventListener('abort', () => {
-        clearTimeout(timeoutId);
-        controller.abort();
-      });
-    }
-
-    try {
-      const response = await fetch(url.toString(), {
-        method: options.method,
-        signal: controller.signal,
-        headers: {
-          'Accept': 'application/json',
-        },
-      });
-
-      clearTimeout(timeoutId);
-
-      const data = await response.json() as StackExchangeWrapper<T>;
-
-      // Handle API errors
-      if (data.error_id) {
+    const startTime = Date.now();
+    return this.circuitBreaker.execute(async () => {
+      // Check for backoff from previous requests
+      if (this.lastBackoff && this.lastBackoff > Date.now()) {
+        const waitTime = Math.ceil((this.lastBackoff - Date.now()) / 1000);
+        metrics.increment('stackexchange_backoff_wait_total', 1);
         throw new Error(
-          `Stack Exchange API Error (${data.error_id} - ${data.error_name}): ${data.error_message}`,
+          `Rate limited. Please wait ${waitTime} seconds before making more requests.`,
         );
       }
 
-      // Update quota tracking
-      this.quotaRemaining = data.quota_remaining;
-      this.quotaMax = data.quota_max;
-      this.requestCount++;
+      const url = new URL(`${API_BASE}${options.endpoint}`);
+      url.search = options.params.toString();
 
-      // Handle backoff
-      if (data.backoff) {
-        this.lastBackoff = Date.now() + (data.backoff * 1000);
-        logger.warn(`[StackExchange] Backoff required: ${data.backoff} seconds`);
+      // Add API key if provided
+      if (this._apiKey) {
+        url.searchParams.set('key', this._apiKey);
       }
 
-      return data;
-    } catch (error) {
-      clearTimeout(timeoutId);
+      // Add site parameter if not already present (for methods that need it)
+      if (!url.searchParams.has('site') && !options.endpoint.startsWith('/sites')) {
+        url.searchParams.set('site', 'stackoverflow.com');
+      }
 
-      if (error instanceof Error && error.name === 'AbortError') {
-        throw new Error(`Request timeout after ${this._timeout}ms`, { cause: error });
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => {
+        controller.abort();
+      }, this._timeout);
+
+      // Chain the signal if provided
+      const abortHandler = () => {
+        clearTimeout(timeoutId);
+        controller.abort();
+      };
+      if (signal) {
+        signal.addEventListener('abort', abortHandler, { once: true });
       }
 
-      throw error;
-    }
+      try {
+        const response = await fetch(url.toString(), {
+          method: options.method,
+          signal: controller.signal,
+          headers: {
+            'Accept': 'application/json',
+          },
+        });
+
+        // NB: the timeout is cleared in `finally`, not here. Clearing it now —
+        // when only the headers have arrived — would leave the body read
+        // (response.json() below) with no timeout, so a stalled body could hang
+        // for the whole process lifetime.
+
+        // The SE API returns a structured JSON wrapper (with error_id) even for
+        // logical errors, so we still parse on non-2xx. But a maintenance page /
+        // CDN interstitial is non-JSON: surface it as an HTTP status error (which
+        // the circuit breaker recognizes as transient) instead of an opaque
+        // "Unexpected token <" SyntaxError that hides the real cause.
+        let data: StackExchangeWrapper<T>;
+        try {
+          data = await response.json() as StackExchangeWrapper<T>;
+        } catch (parseErr) {
+          if (!response.ok) throw new Error(`HTTP ${response.status} from Stack Exchange API`, { cause: parseErr });
+          throw parseErr;
+        }
+
+        // Handle API errors
+        if (data.error_id) {
+          const errorName = data.error_name ?? 'unknown';
+          metrics.increment('stackexchange_errors_total', 1, { 
+            endpoint: options.endpoint, 
+            error_id: data.error_id.toString(),
+            error_name: errorName 
+          });
+          throw new Error(
+            `Stack Exchange API Error (${data.error_id} - ${data.error_name}): ${data.error_message}`,
+          );
+        }
+
+        // Update quota tracking
+        this.quotaRemaining = data.quota_remaining;
+        this.quotaMax = data.quota_max;
+        this.requestCount++;
+        
+        // Track quota metrics
+        metrics.setGauge('stackexchange_quota_remaining', this.quotaRemaining);
+        metrics.setGauge('stackexchange_quota_max', this.quotaMax);
+        metrics.setGauge('stackexchange_quota_used', this.quotaMax - this.quotaRemaining);
+
+        // Handle backoff
+        if (data.backoff) {
+          this.lastBackoff = Date.now() + (data.backoff * 1000);
+          metrics.increment('stackexchange_backoff_total', 1, { seconds: data.backoff.toString() });
+          metrics.observe('stackexchange_backoff_duration_seconds', data.backoff);
+          logger.warn(`[StackExchange] Backoff required: ${data.backoff} seconds`);
+        }
+        
+        // Track successful requests
+        metrics.increment('stackexchange_requests_total', 1, { endpoint: options.endpoint, status: 'success' });
+        
+        return data;
+      } catch (error) {
+        const duration = Date.now() - startTime;
+        metrics.observe('stackexchange_request_duration_ms', duration, { endpoint: options.endpoint, status: 'error' });
+        metrics.increment('stackexchange_requests_total', 1, { endpoint: options.endpoint, status: 'error' });
+        
+        if (error instanceof Error && error.name === 'AbortError') {
+          metrics.increment('stackexchange_timeouts_total', 1, { endpoint: options.endpoint });
+          throw new Error(`Request timeout after ${this._timeout}ms`, { cause: error });
+        }
+
+        throw error;
+      } finally {
+        // Single clear point: covers both the success and error paths and keeps
+        // the timeout armed across the body read above.
+        clearTimeout(timeoutId);
+        if (signal) {
+          signal.removeEventListener('abort', abortHandler);
+        }
+      }
+    });
   }
 
   getQuotaInfo(): { remaining: number; max: number; requestCount: number; lastBackoff: number | null } {
-    return {
+    const info = {
       remaining: this.quotaRemaining,
       max: this.quotaMax,
       requestCount: this.requestCount,
       lastBackoff: this.lastBackoff,
     };
+    
+    // Track quota exhaustion events
+    if (this.isQuotaExhausted()) {
+      metrics.increment('stackexchange_quota_exhausted_total', 1);
+    }
+    
+    if (this.isQuotaLow()) {
+      metrics.increment('stackexchange_quota_low_total', 1);
+    }
+    
+    return info;
   }
 
   isQuotaExhausted(): boolean {

package/src/stackexchange/types.ts

@@ -51,10 +51,6 @@ export interface Question {
   body_markdown?: string;
 }
 
-export interface QuestionWithAnswers extends Question {
-  answers?: Answer[];
-}
-
 export interface Answer {
   owner?: {
     account_id: number;
@@ -114,16 +110,3 @@ export interface Site {
   icon_url?: string;
 }
 
-// Command parameters
-export interface CommandParams {
-  command: 'search' | 'get' | 'user' | 'site';
-  query?: string;
-  id?: string | number;
-  site: string;
-  limit: number;
-  format: 'table' | 'json' | 'compact';
-  tags?: string | null;
-  full: boolean;
-  minimal: boolean;
-}
-