@lincoln504/pi-research 1.4.3 → 1.5.1

package/src/tools/scrape.ts

@@ -49,7 +49,14 @@ export function createScrapeTool(options: {
   config?: Config;
 }): ToolDefinition {
   const config = options.config || getConfig(options.ctx.cwd);
-  const maxScrapeBatches = getMaxScrapeBatches(config);
+  // Prefer the tracker's own scrape limit when one is present: it was built via
+  // createDefaultToolLimits(config, cachingActive) and so already reflects any
+  // prompt-caching batch bonus (see getMaxScrapeBatches in constants.ts). Without
+  // this, the tool's own description/promptSnippet/protocol text (below) told the
+  // model a lower, non-caching-aware batch count than the tracker actually
+  // enforced — a real, if harmless, missed-opportunity: the model would stop one
+  // batch short of what it was actually allowed to use.
+  const maxScrapeBatches = options.tracker?.getToolLimit('scrape') ?? getMaxScrapeBatches(config);
   const container = tryGetServiceContainerFromCtx(options.ctx);
 
   // Fallback global state when no orchestration context is provided

package/src/tools/stackexchange.ts

@@ -84,6 +84,20 @@ export function createStackexchangeTool(options: {
           };
       }
 
+      const p = params as StackExchangeParams;
+      const command = p.command;
+
+      // Command-name validation is part of validate-BEFORE-charge too: the schema
+      // types `command` as free-text, so an unknown name used to pass Value.Check,
+      // spend a MAX_GATHERING_CALLS slot, and then die in the dispatch switch with
+      // zero work done — the error message even blamed rate limiting.
+      if (!command || typeof command !== 'string' || !['search', 'get', 'user', 'site'].includes(command)) {
+        return {
+          content: [{ type: 'text', text: `Invalid stackexchange command: ${String(command)}. Valid commands: search, get, user, site.` }],
+          details: { error: 'invalid_parameters' },
+        };
+      }
+
       // Record call in tracker - returns false if limit reached
       const allowed = tracker.recordCall('stackexchange');
       if (!allowed) {
@@ -93,13 +107,6 @@ export function createStackexchangeTool(options: {
           };
       }
 
-      const p = params as StackExchangeParams;
-      const command = p.command;
-
-      if (!command || typeof command !== 'string') {
-        throw new Error('Stack Exchange command is required and must be a string');
-      }
-
       try {
         return await stackexchangeCommand({
           command,

package/src/utils/llm-usage.ts

@@ -63,7 +63,10 @@ function warnIfUnpriced(model: Model<any>, tokens: number, cost: number): void {
  *
  * Zero-valued counters are still emitted (when the provider reported the field at all)
  * so "cache is off" and "provider does not report caching" stay distinguishable: the
- * first shows the counter at 0, the second omits it.
+ * first shows the counter at 0, the second omits it. Caveat: the caller gates on
+ * `tokens > 0 || cost > 0`, so a response whose usage is all-zero across the board
+ * emits nothing — harmless in practice because cache fields are summed into `tokens`,
+ * so a reported cache split implies a nonzero total.
  */
 function recordCacheTokens(parsed: Partial<TokenUsage>, labels: Record<string, string>): void {
   if (typeof parsed.cacheRead === 'number') {

package/src/utils/log-utils.ts

@@ -37,8 +37,11 @@ export function buildDefaultDebugLogPath(): string {
 /**
  * Check whether debug/verbose logging is enabled.
  *
- * Reads PI_RESEARCH_DEBUG env var. This env var is kept in sync with
- * config.DEBUG whenever the config is loaded or saved (see config.ts).
+ * Reads PI_RESEARCH_DEBUG env var. Config SAVES sync config.DEBUG into this
+ * env var (see config.ts buildEnvValues); plain config LOADS do not mutate
+ * process.env, so a DEBUG=true line in config.env enables verbose logging for
+ * a process only after a save has run in it — export PI_RESEARCH_DEBUG=true
+ * in the environment to guarantee it from process start.
  * Set to "true" to enable INFO+DEBUG logs.
  */
 export function isVerboseFromEnv(): boolean {

package/src/utils/metrics-summary.ts

@@ -48,6 +48,12 @@ export interface ResearchStats {
   errors: number;
   /** Total LLM tokens consumed. */
   tokens: number;
+  /** Prompt-cache READ tokens (prompt prefix served from the provider's cache at the
+   *  discounted rate). 0 when the provider reported none or does not report caching. */
+  cacheReadTokens: number;
+  /** Prompt-cache WRITE tokens (prefix stored into the provider's cache, billed at a
+   *  premium). Reads pinned at 0 while writes climb = a caching regression. */
+  cacheWriteTokens: number;
   /** Total cost in USD across all LLM calls. */
   cost: number;
   /** Tool usage counts. */
@@ -193,6 +199,12 @@ export function extractRunStats(snapshot: IMetricsSnapshot): ResearchStats | nul
   // Tokens — sum all label variants
   const tokens = sumCounter(counters, 'llm_tokens_total');
 
+  // Prompt-cache split (recordLlmUsage emits these only when the provider reports
+  // the fields, so 0 here covers both "cache off" and "not reported" — the raw
+  // counters keep those distinguishable for anyone reading the snapshot directly).
+  const cacheReadTokens = sumCounter(counters, 'llm_cache_read_tokens_total');
+  const cacheWriteTokens = sumCounter(counters, 'llm_cache_write_tokens_total');
+
   // Cost — sum all label variants
   const cost = sumCounter(counters, 'llm_cost_total');
 
@@ -251,6 +263,8 @@ export function extractRunStats(snapshot: IMetricsSnapshot): ResearchStats | nul
     urlsFailed,
     errors,
     tokens,
+    cacheReadTokens,
+    cacheWriteTokens,
     cost,
     toolUsage,
   };
@@ -315,7 +329,13 @@ export function buildResearchSummary(stats: ResearchStats): string {
   // --- Resource line: tokens, cost, duration ---
   const resourceParts: string[] = [];
   if (stats.tokens > 0) {
-    resourceParts.push(`**${formatTokens(stats.tokens)}** tokens`);
+    // When prompt caching did real work, say so inline: cache reads are prompt
+    // tokens billed at the provider's discounted cached rate, and this is the
+    // only user-visible confirmation that caching is active for their model.
+    const cached = stats.cacheReadTokens > 0
+      ? ` (**${formatTokens(stats.cacheReadTokens)}** from cache)`
+      : '';
+    resourceParts.push(`**${formatTokens(stats.tokens)}** tokens${cached}`);
   }
   // Render the cost whenever tokens were actually billed-through, INCLUDING when it
   // is exactly zero. Gating on `cost > 0` hid a real defect for weeks: a model whose

package/src/utils/tool-usage-tracker.ts

@@ -195,12 +195,15 @@ export class ToolUsageTracker {
 }
 
 /**
- * Create default tool limits for a researcher
+ * Create default tool limits for a researcher.
+ *
+ * @param cachingActive - Whether prompt caching is active for this run's
+ * resolved model; see getMaxScrapeBatches in constants.ts.
  */
-export function createDefaultToolLimits(config?: Config): ToolLimits {
+export function createDefaultToolLimits(config?: Config, cachingActive = false): ToolLimits {
   return {
     gathering: getMaxGatheringCalls(config),
-    scrape: getMaxScrapeBatches(config),
+    scrape: getMaxScrapeBatches(config, cachingActive),
     search: 1,
     youtube_transcript: 1,
     grep: MAX_GREP_CALLS,

package/src/web-research/scraper-utils.ts

@@ -35,8 +35,10 @@ import { isCloudflareBlockError, isTaskTimeoutError, isPoolShutdownError } from
  * and extractPdfToMarkdown (`Could not extract content from PDF`, `PDF too
  * large`) — a malformed/encrypted/scanned-image-only PDF, or one that simply
  * exceeds the deliberate 100MB policy cap, is routine on the open web and
- * just as non-actionable as a stub page or a 404; the remaining cases
- * delegate to the shared browser-layer predicates. Deliberately does NOT
+ * just as non-actionable as a stub page or a 404; the 25MB HTML policy caps
+ * (`HTML response too large` from the fetch layer, `Browser HTML too large`
+ * from the browser layer) are the same deliberate-cap class; the remaining
+ * cases delegate to the shared browser-layer predicates. Deliberately does NOT
  * match 'PDF extraction unavailable' (a pdf-oxide-wasm native-module load
  * failure) — that is an infrastructure fault affecting every URL, not a
  * routine per-PDF outcome, and must still surface at ERROR.
@@ -48,6 +50,8 @@ export function isBenignScrapeFailure(error: unknown): boolean {
     msg.startsWith('Fetch returned stub:') ||             // nav-only / near-empty page
     msg.startsWith('Could not extract content from PDF') || // malformed/encrypted/scanned-only PDF
     msg.startsWith('PDF too large') ||                    // exceeds the deliberate 100MB policy cap
+    msg.startsWith('HTML response too large') ||          // fetch-layer 25MB policy cap — same class as the PDF cap
+    msg.startsWith('Browser HTML too large') ||           // browser-layer 25MB policy cap
     /^HTTP \d{3}\b/.test(msg) ||                          // remote 4xx/5xx status
     isCloudflareBlockError(error) ||
     isTaskTimeoutError(error) ||

package/src/web-research/web-scraper.ts

@@ -75,6 +75,15 @@ async function convertToMarkdown(html: string): Promise<string> {
   return converter(html);
 }
 
+// TODO(hardening, medium): this synchronous WASM parse runs on the MAIN
+// process's event loop — a large/complex PDF can block it for seconds,
+// starving every concurrent scrape, the TUI render loop, and timers. The
+// correct fix is a worker_threads-based extraction (mirroring the browser
+// pool's CPU isolation pattern), but it is deliberately NOT done in this
+// pass: pdf-oxide-wasm is dynamically imported here and unit tests mock the
+// module via vitest, which cannot reach into a real worker thread — moving
+// the parse requires a companion test-seam redesign first. Bounded today by
+// MAX_PDF_SIZE (100MB) and the scrape task's own deadline.
 async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
   if (bytes.length > MAX_PDF_SIZE) {
     const sizeMB = Math.round(bytes.length / 1024 / 1024);

package/src/youtube/index.ts

@@ -10,6 +10,8 @@ import type { AgentToolResult } from '@earendil-works/pi-coding-agent';
 import { selectVideoIds } from './video-id.ts';
 import { fetchVideoTranscripts, type VideoTranscript } from './transcript-client.ts';
 import { metrics } from '../utils/metrics.ts';
+import { truncateWithMarker } from '../utils/text-utils.ts';
+import { MAX_SCRAPE_CONTENT_CHARS_PER_DOC } from '../constants.ts';
 
 export interface YoutubeTranscriptCommandOptions {
   urls: string[];
@@ -131,7 +133,15 @@ function formatResults(results: VideoTranscript[], rejected: string[]): string {
     // model can copy verbatim; omitting the line entirely for untitled videos
     // left them without any prompt-visible citation string to reuse.
     md += `**Cite as:** ${r.url}${r.title ? ` — '${r.title}'${r.author ? ` by ${r.author}` : ''}` : ''}\n`;
-    md += `\n${r.text}\n\n---\n\n`;
+    // Per-document cap on content entering LLM context — parity with the
+    // scrape tool's own MAX_SCRAPE_CONTENT_CHARS_PER_DOC guard. The raw HTTP
+    // body is capped at 32MB (readTextCapped) and the knowledge-store
+    // ingestion path truncates too, but this tool's direct response to the
+    // calling agent had no cap of its own: a single long/dense-captioned
+    // video (a multi-hour livestream VOD, worst case with maxVideos up to 5)
+    // could inject an unbounded, multi-megabyte payload straight into the
+    // researcher's context.
+    md += `\n${truncateWithMarker(r.text ?? '', MAX_SCRAPE_CONTENT_CHARS_PER_DOC)}\n\n---\n\n`;
   }
 
   if (failed.length > 0) {

package/src/youtube/potoken.ts

@@ -110,7 +110,22 @@ export async function mintPoTokens(
     throw new Error('PoToken minter is wedged by a prior mint that never settled; refusing to start a new mint until it resolves.');
   }
   // Chain on the mutex so only one bridged section runs at a time.
-  const run = mintChain.then(() => doMint(identifiers, opts), () => doMint(identifiers, opts));
+  // Re-check `wedged` at DEQUEUE time, not only at the entry check above: a
+  // caller already queued on the chain when the hard release fires would
+  // otherwise start its own doMint() while the abandoned mint may still be
+  // mid-bridge — the exact double-bridge globalThis corruption the flag
+  // exists to prevent. (No false positive in the normal path: the
+  // wedge-clearing handler below is registered on `settled` BEFORE the race
+  // that dequeues the next caller, so a clean settle clears the flag first.)
+  let enteredMint = false;
+  const startIfSafe = () => {
+    if (wedged) {
+      throw new Error('PoToken minter is wedged by a prior mint that never settled; refusing to start a queued mint until it resolves.');
+    }
+    enteredMint = true;
+    return doMint(identifiers, opts);
+  };
+  const run = mintChain.then(startIfSafe, startIfSafe);
   // Keep the chain alive regardless of this call's success/failure, but bound
   // how long the NEXT caller waits on it: if `run` never settles, force the
   // chain open after the hard timeout rather than wedging every future
@@ -128,7 +143,10 @@ export async function mintPoTokens(
   const settled = run.then(() => undefined, () => undefined);
   // Once doMint() actually settles (success or failure), its own `finally`
   // always runs unbridge() for real — safe for a new mint to bridge again.
-  settled.then(() => { wedged = false; });
+  // Gated on `enteredMint`: a caller REJECTED at dequeue by the wedge check
+  // also settles, and letting it clear the flag would reopen minting while
+  // the abandoned mint's bridge may still be installed.
+  settled.then(() => { if (enteredMint) wedged = false; });
   let hardTimer: NodeJS.Timeout;
   const hardRelease = new Promise<void>(resolve => {
     hardTimer = setTimeout(() => {