@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/utils/text-utils.ts

@@ -1,5 +1,7 @@
-import type { AgentSession } from '@mariozechner/pi-coding-agent';
-import type { AssistantMessage } from '@mariozechner/pi-ai';
+import type { AgentSession } from '@earendil-works/pi-coding-agent';
+import type { AssistantMessage } from '@earendil-works/pi-ai';
+import { logger } from '../logger.ts';
+import { stripTrailingLlmPunctuation } from './url-utils.ts';
 
 /**
  * Text Utilities
@@ -11,7 +13,7 @@ import type { AssistantMessage } from '@mariozechner/pi-ai';
  * Extract text content from a message
  *
  * Handles various message content formats:
- * - String content
+ * - String content (with tag-based thinking removal)
  * - Array of content blocks (TextContent, ThinkingContent, ToolCall, etc.)
  * - Accepts any object with a content field for robustness
  *
@@ -29,7 +31,7 @@ export function extractText(message: unknown): string {
 
   // Handle string content
   if (typeof content === 'string') {
-    return content;
+    return stripThinkingTags(content);
   }
 
   // Handle array of content blocks
@@ -40,7 +42,8 @@ export function extractText(message: unknown): string {
         .map((b) => {
           const blockObj = b as Record<string, unknown>;
           const text = blockObj['text'];
-          return typeof text === 'string' ? text : '';
+          // Ensure we also strip tags if the text block itself contains them (rare but possible)
+          return typeof text === 'string' ? stripThinkingTags(text) : '';
         })
         .filter((t: string) => t.length > 0) // Filter empty strings
         .join('\n');
@@ -53,6 +56,51 @@ export function extractText(message: unknown): string {
   return '';
 }
 
+/**
+ * Truncate content to at most `maxChars` characters, appending an explicit
+ * `[content truncated: showing N of M chars]` marker so an LLM consuming the
+ * text knows it is incomplete.
+ *
+ * Guarantees:
+ * - Content at or under the cap is returned unchanged (no marker).
+ * - Truncated output (body + marker) never exceeds `maxChars`, which makes the
+ *   function idempotent — re-applying the same cap to already-truncated output
+ *   is a no-op (no nested markers).
+ * - Cuts at a clean boundary (last newline/space) when one exists in the final
+ *   20% of the kept text, so words/lines are not split mid-token.
+ */
+export function truncateWithMarker(content: string, maxChars: number): string {
+  if (content.length <= maxChars) return content;
+
+  const total = content.length;
+  const makeMarker = (kept: number) => `\n\n[content truncated: showing ${kept} of ${total} chars]`;
+  // Reserve marker room using the longest possible marker (kept ≤ total, so the
+  // digit count of `kept` never exceeds that of `total`).
+  const keep = Math.max(0, maxChars - makeMarker(total).length);
+  let body = content.slice(0, keep);
+  // Prefer a clean boundary near the cut point.
+  const boundary = Math.max(body.lastIndexOf('\n'), body.lastIndexOf(' '));
+  if (boundary > keep * 0.8) body = body.slice(0, boundary);
+  return body + makeMarker(body.length);
+}
+
+/**
+ * Robustly remove thinking/reasoning tags from a string.
+ * Handles <thought>, <thinking>, and <reasoning> tags (case-insensitive).
+ */
+export function stripThinkingTags(text: string): string {
+  if (!text) return '';
+  
+  // Replace tags and their content: <thought>...</thought>, etc.
+  // Using dotAll (s flag) to handle multi-line thinking blocks.
+  // Also handles unclosed tags at the end of the string (truncated responses).
+  return text
+    .replace(/<thought>[\s\S]*?(?:<\/thought>|$)/gi, '')
+    .replace(/<thinking>[\s\S]*?(?:<\/thinking>|$)/gi, '')
+    .replace(/<reasoning>[\s\S]*?(?:<\/reasoning>|$)/gi, '')
+    .trim();
+}
+
 /**
  * Ensures the assistant completed successfully and returns the text content.
  * Throws an error if the assistant failed or stopped with an error.
@@ -70,7 +118,9 @@ export function ensureAssistantResponse(session: AgentSession, label: string): s
     throw new Error(`${label}: No assistant response found`);
   }
 
-  // Check for explicit error stop reason or error message
+  // Check for explicit error stop reason or error message. An aborted session with an
+  // errorMessage is an interruption, not a provider error — fall through so any partial
+  // text it produced is salvaged below (and, if empty, reported as an abort not a model fault).
   if (last.stopReason === 'error' || (last.errorMessage && last.stopReason !== 'aborted')) {
     const msg = last.errorMessage || 'Unknown error';
     if (msg.includes('429')) {
@@ -79,5 +129,212 @@ export function ensureAssistantResponse(session: AgentSession, label: string): s
     throw new Error(`${label}: Provider error - ${msg}`);
   }
 
-  return extractText(last);
+  // Warn if the response was truncated due to token limit
+  if (last.stopReason === 'length') {
+    const text = extractText(last);
+    if (text.trim()) {
+      logger.warn(`${label}: Response truncated by token limit — returning partial result`);
+      return text;
+    }
+    throw new Error(`${label}: Response was truncated by token limit and produced no usable text.`);
+  }
+
+  const text = extractText(last);
+  if (!text.trim()) {
+    // An aborted session with no salvageable text was interrupted mid-turn (quit/SIGTERM
+    // during research, or a timeout abort) before the model emitted its final answer. That
+    // is NOT a model-capability failure — surface an explicit "Aborted" so callers skip
+    // retries (see researcher-executor) instead of blaming the model with the message below.
+    if (last.stopReason === 'aborted') {
+      throw new Error(`${label}: Aborted`);
+    }
+    // zero text content blocks in the final assistant message. This typically means
+    // all tool calls failed, or the session ended without a visible response.
+    throw new Error(
+      `${label}: produced no text output — the model returned no final text block after its tool calls. ` +
+      `This is usually a model-capability issue (a very small or thinking-only model that doesn't emit a final ` +
+      `answer after using tools), or every tool call in the turn failed. Try a more capable model or retry.`
+    );
+  }
+  return text;
+}
+
+/**
+ * Format a timestamp as a human-readable relative time string.
+ */
+export function formatTimeAgo(isoTimestamp: string): string {
+  const parsed = new Date(isoTimestamp).getTime();
+  if (Number.isNaN(parsed)) return 'unknown';
+  const diffMs = Date.now() - parsed;
+  const diffMinutes = Math.floor(diffMs / 60_000);
+  if (diffMinutes < 1) return 'just now';
+  if (diffMinutes < 60) return `${diffMinutes}m ago`;
+  const diffHours = Math.floor(diffMinutes / 60);
+  if (diffHours < 24) return `${diffHours}h ago`;
+  return `${Math.floor(diffHours / 24)}d ago`;
+}
+
+/**
+ * Format a duration in ms as a human-readable string (e.g., "12.3s", "1m 23s", "2h 15m").
+ */
+export function formatDuration(ms: number): string {
+  if (ms < 1000) return `${ms}ms`;
+  const seconds = ms / 1000;
+  if (seconds < 60) return `${seconds.toFixed(1)}s`;
+  const minutes = Math.floor(seconds / 60);
+  const remainingSeconds = Math.round(seconds % 60);
+  if (minutes < 60) return `${minutes}m ${remainingSeconds}s`;
+  const hours = Math.floor(minutes / 60);
+  const remainingMinutes = minutes % 60;
+  return `${hours}h ${remainingMinutes}m`;
+}
+
+import * as path from 'node:path';
+
+/**
+ * Consistently normalize a workspace path for database storage and filtering.
+ * Resolves to absolute path and removes trailing separators.
+ */
+export function normalizeWorkspacePath(wsPath: string): string {
+  if (!wsPath) return '';
+  const resolved = path.resolve(wsPath);
+  // Normalize trailing slash: /path/to/dir/ -> /path/to/dir
+  return (resolved.endsWith(path.sep) && resolved.length > 1) 
+    ? resolved.slice(0, -1) 
+    : resolved;
+}
+
+export interface Citation {
+  url: string;
+  description: string;
+  source?: string;
+  /** The number the report actually wrote for this entry (e.g. 3 for `[3]`). Used to remap
+   *  inline `[N]` by the written label, not list position, so non-sequential numbering maps right. */
+  number?: number;
+}
+
+/**
+ * A citation URL is only plausible if it parses and has a real host (a dot in
+ * the hostname). This rejects fragments like `https://www` that occur when an
+ * LLM soft-wraps or truncates a URL in its CITED LINKS — without this guard such
+ * a fragment becomes a garbage citation AND shifts every downstream global ID.
+ */
+function isPlausibleCitationUrl(url: string): boolean {
+  try {
+    const u = new URL(url);
+    return (u.protocol === 'http:' || u.protocol === 'https:') && u.hostname.includes('.');
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Offset of the LAST line-leading "CITED LINKS" section header in a report, or -1
+ * when none is present.
+ *
+ * ONLY an all-caps "CITED LINKS" at the start of a line (optionally after markdown
+ * heading `#` / blockquote `>` / bold `**` markers and whitespace) counts as the
+ * section marker — NOT an incidental lowercase "...the cited links below..." in
+ * prose. An unanchored, case-insensitive /CITED LINKS/i match would fire on such a
+ * mention and, with a trailing `[\s\S]*$`, silently truncate (or mis-parse) the
+ * whole report body from that point. Matching a header line and taking the LAST
+ * occurrence (the real trailing section) avoids that while staying idempotent
+ * against the `CITED LINKS\n...` block that formatCitedLinks() emits.
+ */
+export function lastCitedLinksHeaderIndex(text: string): number {
+  // Case-insensitive but LINE-LEADING: a mid-prose "...the cited links below..." has
+  // non-whitespace before it on the line, so it cannot match — only a header at the
+  // start of a line (after optional markdown markers) does.
+  const re = /(^|\n)([ \t]*(?:#{1,6}[ \t]*|>[ \t]*|\*\*)?)CITED LINKS\b/gi;
+  let last = -1;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(text)) !== null) {
+    // Offset of the 'C' in CITED LINKS = match start + leading newline + markers.
+    last = m.index + m[1]!.length + m[2]!.length;
+  }
+  return last;
+}
+
+/**
+ * Parses the CITED LINKS section from a researcher report, extracting URLs, sources, and their descriptions.
+ * Handles both single-line 'URL - description' and multi-line 'Source:'/'Description:' formats.
+ */
+export function parseCitations(report: string): Citation[] {
+  const headerIdx = lastCitedLinksHeaderIndex(report);
+  if (headerIdx < 0) return [];
+  const section = report.slice(headerIdx);
+
+  const citations: Citation[] = [];
+  // Match a list marker [N] or N. (optionally **-wrapped) ONLY at the start of a line.
+  // Entries in the CITED LINKS section are always line-leading, so anchoring to line
+  // start (^ with the `m` flag) is loss-free — and critically it stops the bare "N."
+  // form from matching a digits-then-dot run INSIDE a URL (e.g. `report-2024.pdf`,
+  // `/v2.0/`, or an IP literal `192.168.1.10`), which previously truncated or dropped
+  // those citations. The split and the parallel number-capture regex MUST stay
+  // structurally identical so blocks[i] and writtenNumbers[i] remain index-aligned.
+  const blocks = section.split(/^[^\S\n]*(?:\*\*)?[^\S\n]*(?:\[\d+\]|\d+\.)[^\S\n]*(?:\*\*)?/m).slice(1);
+  // Capture the written number for each delimiter in parallel, using a SEPARATE global+capturing
+  // regex (adding a capture group to the split pattern above would inject the captures into the
+  // split output and break block indexing). blocks[i] is the content after delimiter i, so
+  // writtenNumbers[i] is blocks[i]'s label — used to remap inline [N] by label, not position.
+  const writtenNumbers = [...section.matchAll(/^[^\S\n]*(?:\*\*)?[^\S\n]*(?:\[(\d+)\]|(\d+)\.)[^\S\n]*(?:\*\*)?/gm)]
+    .map(m => parseInt(m[1] ?? m[2] ?? '', 10));
+
+  for (let bi = 0; bi < blocks.length; bi++) {
+    const block = blocks[bi]!;
+    const writtenNumber = Number.isFinite(writtenNumbers[bi]) ? writtenNumbers[bi] : undefined;
+    const lines = block.trim().split('\n');
+    if (lines.length === 0) continue;
+    
+    const firstLine = lines[0]!.trim();
+    let url: string;
+    let desc = '';
+    let source = '';
+    
+    // Handles both plain inline format (URL — desc) and the format produced by
+    // the researcher prompt template: URL [Source: Scrape] — desc.
+    // Group 1: URL, Group 2: optional source tag contents, Group 3: description.
+    const inlineMatch = /^(https?:\/\/[^\s\n]+)(?:\s+\[Source:\s*([^\]]*)\])?(?:\s*[—–-]\s*([^\n]*))?/.exec(firstLine);
+    if (inlineMatch) {
+      // Balance-aware trailing-punctuation strip (shared with normalizeUrl): a BLIND
+      // ')' strip here truncated balanced-paren URLs — e.g. Wikipedia's
+      // "…/Python_(programming_language)" — before normalizeUrl ever saw them, shipping
+      // a broken 404 in the CITED LINKS section. stripTrailingLlmPunctuation removes a
+      // ')' only when unbalanced and preserves unreserved trailing '_'/'~'.
+      url = stripTrailingLlmPunctuation(inlineMatch[1]!.trim());
+      source = (inlineMatch[2]?.trim() || '');
+      desc = (inlineMatch[3]?.trim() || '').replace(/[*_~`]+$/, '');
+    } else {
+      const candidateUrl = stripTrailingLlmPunctuation(firstLine.split(/\s+/)[0]!.trim());
+      if (!candidateUrl.startsWith('http')) continue;
+      url = candidateUrl;
+    }
+    
+    const descLines: string[] = [];
+    let foundDescTag = false;
+    for (let i = 1; i < lines.length; i++) {
+      const line = lines[i]!.trim();
+      if (line.match(/^Source:/i)) {
+        source = line.replace(/^Source:\s*/i, '').trim();
+        continue;
+      }
+      if (!foundDescTag) {
+        if (line.match(/^Description:/i)) {
+          foundDescTag = true;
+          descLines.push(line.replace(/^Description:\s*/i, ''));
+        }
+      } else {
+        descLines.push(line);
+      }
+    }
+    
+    if (descLines.length > 0) {
+      desc = descLines.join('\n').trim();
+    }
+    
+    if (url && isPlausibleCitationUrl(url)) {
+      citations.push({ url, description: desc, source, number: writtenNumber });
+    }
+  }
+  return citations;
 }

package/src/utils/tool-usage-tracker.ts

@@ -7,13 +7,20 @@
  */
 
 import { logger } from '../logger.ts';
-import { MAX_GATHERING_CALLS, MAX_SCRAPE_CALLS } from '../constants.ts';
+import { MAX_GATHERING_CALLS, MAX_GREP_CALLS, getMaxGatheringCalls, getMaxScrapeBatches } from '../constants.ts';
+import type { Config } from '../config.ts';
 
 export interface ToolLimits {
-  // Combined gathering limit (search, security_search, stackexchange, grep)
+  // Combined gathering limit (search, security_search, stackexchange, youtube_transcript)
   gathering?: number;
   // Scrape limit
   scrape?: number;
+  // Specific tool limits (overrides category limit if stricter)
+  search?: number;
+  // youtube_transcript is capped at one call per researcher
+  youtube_transcript?: number;
+  // grep is local/free ripgrep — its own generous budget, not the shared gathering pool
+  grep?: number;
   // read tool (default pi file read) has no limit
   read?: number;
 }
@@ -22,6 +29,7 @@ export interface ToolUsage {
   category: string;
   callCount: number;
   limit?: number;
+  toolCounts: Map<string, number>;
 }
 
 export class ToolUsageTracker {
@@ -36,7 +44,9 @@ export class ToolUsageTracker {
    * Get category for a tool
    */
   private getCategory(toolName: string): string {
-    const gatheringTools = ['search', 'security_search', 'stackexchange', 'grep'];
+    // grep is intentionally NOT here: it is local/free ripgrep, not a rate-limited web
+    // API, so it gets its own dedicated budget instead of competing for the gathering pool.
+    const gatheringTools = ['search', 'security_search', 'stackexchange', 'youtube_transcript'];
     if (gatheringTools.includes(toolName)) {
       return 'gathering';
     }
@@ -50,48 +60,101 @@ export class ToolUsageTracker {
    */
   recordCall(toolName: string): boolean {
     const category = this.getCategory(toolName);
-    const limit = this.limits[category as keyof ToolLimits];
+    const catLimit = this.limits[category as keyof ToolLimits];
+    const toolLimit = this.limits[toolName as keyof ToolLimits];
+    
     const usage = this.getUsage(category);
 
-    // Check limit — if reached, return false without incrementing
-    if (limit !== undefined && usage.callCount >= limit) {
+    // Check category limit
+    if (catLimit !== undefined && usage.callCount >= catLimit) {
       logger.debug(
-        `[tool-usage] category=${category} blocked tool=${toolName} (limit=${limit})`
+        `[tool-usage] category=${category} blocked tool=${toolName} (limit=${catLimit})`
       );
       return false;
     }
 
+    // Check specific tool limit
+    const toolCount = usage.toolCounts.get(toolName) || 0;
+    if (toolLimit !== undefined && toolCount >= toolLimit) {
+        logger.debug(
+            `[tool-usage] tool=${toolName} blocked (limit=${toolLimit})`
+        );
+        return false;
+    }
+
     // Allowed — increment and log
     usage.callCount++;
+    usage.toolCounts.set(toolName, toolCount + 1);
+    
     logger.debug(
-      `[tool-usage] category=${category} calls=${usage.callCount}/${limit ?? 'unlimited'} tool=${toolName}`
+      `[tool-usage] category=${category} calls=${usage.callCount}/${catLimit ?? 'unlimited'} tool=${toolName} count=${toolCount + 1}/${toolLimit ?? 'unlimited'}`
     );
     return true;
   }
 
   /**
-   * Get current call count for a tool
+   * Get current call count for a tool's category
    */
   getCallCount(toolName: string): number {
     const category = this.getCategory(toolName);
     return this.getUsage(category).callCount;
   }
 
+  /**
+   * Get current call count for a specific tool
+   */
+  getToolCallCount(toolName: string): number {
+    const category = this.getCategory(toolName);
+    const usage = this.getUsage(category);
+    return usage.toolCounts.get(toolName) || 0;
+  }
+
+  /**
+   * Get the specific limit configured for a tool
+   */
+  getToolLimit(toolName: string): number | undefined {
+    return this.limits[toolName as keyof ToolLimits];
+  }
+
   /**
    * Get limit-reached message for a blocked tool
    */
   getLimitMessage(toolName: string): string {
     const category = this.getCategory(toolName);
     const usage = this.getUsage(category);
-    const limit = usage.limit;
+    
+    const toolLimit = this.limits[toolName as keyof ToolLimits];
+    const catLimit = usage.limit;
+
+    if (toolName === 'search' && toolLimit === 1) {
+        return `SEARCH LIMIT REACHED: You have already used your one search call. ` +
+            `Proceed to scraping: use the scrape tool for full deep-dives into your best search results.`;
+    }
+
+    if (toolName === 'youtube_transcript' && toolLimit === 1) {
+        return `YOUTUBE TRANSCRIPT LIMIT REACHED: You have already used your one youtube_transcript call. ` +
+            `Continue with your other gathering tools (search, scrape) and proceed to synthesis.`;
+    }
+
+    if (toolName === 'grep') {
+      return `CODE SEARCH LIMIT REACHED: You have used all ${this.limits.grep ?? MAX_GREP_CALLS} local code-search calls. ` +
+        `Proceed with your other tools (search, scrape) and move toward synthesis.`;
+    }
 
     if (category === 'scrape') {
-      return `SCRAPE PROTOCOL COMPLETE: You have completed all ${limit} scrape calls (handshake + 3 batches). ` +
-        `This tool cannot be used again. Proceed immediately to Phase 3: synthesize your findings and submit your report.`;
+      const limit = catLimit ?? getMaxScrapeBatches();
+      const effectiveText = limit > 99 ? 'unlimited' : `${limit} batch${limit === 1 ? '' : 'es'}`;
+      // Generate batch list based on actual limit, but cap display at 6 batches to avoid overly long messages
+      const displayLimit = Math.min(limit, 6);
+      const batchList = displayLimit <= 3 ? 'Batch 1, Batch 2, Batch 3' :
+        Array.from({ length: displayLimit }, (_, i) => `Batch ${i + 1}`).join(', ') + (limit > 6 ? ', ...' : '');
+      return `SCRAPE PROTOCOL COMPLETE: You have completed all ${effectiveText} (${batchList}). ` +
+        `This tool cannot be used again. Proceed immediately to synthesis: compile your findings and submit your full report.`;
     }
+    const limit = catLimit ?? MAX_GATHERING_CALLS;
     return `GATHERING LIMIT REACHED: All ${limit} gathering calls have been used. ` +
       `This tool and all other gathering tools cannot be used again. ` +
-      `Proceed to Phase 2: call scrape with your collected URLs now.`;
+      `Proceed to Step 2: call scrape with your collected URLs now.`;
   }
 
   /**
@@ -103,6 +166,7 @@ export class ToolUsageTracker {
         category,
         callCount: 0,
         limit: this.limits[category as keyof ToolLimits],
+        toolCounts: new Map(),
       });
     }
     return this.usage.get(category)!;
@@ -126,10 +190,13 @@ export class ToolUsageTracker {
 /**
  * Create default tool limits for a researcher
  */
-export function createDefaultToolLimits(): ToolLimits {
+export function createDefaultToolLimits(config?: Config): ToolLimits {
   return {
-    gathering: MAX_GATHERING_CALLS,
-    scrape: MAX_SCRAPE_CALLS,
+    gathering: getMaxGatheringCalls(config),
+    scrape: getMaxScrapeBatches(config),
+    search: 1,
+    youtube_transcript: 1,
+    grep: MAX_GREP_CALLS,
     read: undefined,
   };
 }