@lincoln504/pi-research 1.2.0 → 1.3.1

package/src/tools/scrape.ts

@@ -139,11 +139,38 @@ export function createScrapeTool(options: {
       // almost nothing and must not be blocked as if every URL were a full scrape.
       const { kept: dedupedUrls, duplicates } = deduplicateUrls(rawUrls, getGlobalState().researchId);
 
+      // Partition duplicates by whether their content still survives in the session
+      // cache. The content cache is FIFO-capped, so a pool member's content may have
+      // been evicted; those URLs re-enter the fetch list — otherwise they would be
+      // reported as "retrieved from session memory" while appearing nowhere in the
+      // output, and a re-request would just dedup them away again.
+      const duplicateResults: { url: string; markdown: string; success: true }[] = [];
+      const evictedDuplicates: string[] = [];
+      for (const url of duplicates) {
+        const cachedContent = getCachedScrapedContent(getGlobalState().researchId, url);
+        if (cachedContent) {
+          duplicateResults.push({ url, markdown: cachedContent, success: true });
+        } else {
+          evictedDuplicates.push(url);
+        }
+      }
+
+      // Only the first MAX_SCRAPE_URLS candidates are ever fetched. Slice BEFORE the
+      // context gate so the projection bills what will actually be scraped — billing
+      // the full list blocked an 8-URL first batch on a 128k model at zero tokens
+      // used, while the URLs actually fetchable would have passed. Over-cap URLs are
+      // reported explicitly below instead of silently vanishing (they appear neither
+      // in results nor in Failed-to-Scrape, and the session-pool footer excludes the
+      // current batch).
+      const fetchCandidates = [...dedupedUrls, ...evictedDuplicates];
+      const finalUrls = fetchCandidates.slice(0, MAX_SCRAPE_URLS);
+      const skippedUrls = fetchCandidates.slice(MAX_SCRAPE_URLS);
+
       // Context gate: block if projected token usage would exceed threshold
       if (options.getTokensUsed) {
         const ctxWindow = options.contextWindowSize ?? DEFAULT_MODEL_CONTEXT_WINDOW;
         const tokensUsed = options.getTokensUsed();
-        const projected = (tokensUsed + dedupedUrls.length * config.AVG_TOKENS_PER_SCRAPE) / ctxWindow;
+        const projected = (tokensUsed + finalUrls.length * config.AVG_TOKENS_PER_SCRAPE) / ctxWindow;
         if (projected >= config.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING) {
           const pct = Math.round(config.MAX_SCRAPE_TOKEN_FRACTION_FOR_SCRAPING * 100);
           metrics.increment('tool_scrape_calls_total', 1, { status: 'context_limit' });
@@ -160,36 +187,22 @@ export function createScrapeTool(options: {
       }
 
       const callCount = options.tracker?.getToolCallCount('scrape') ?? 0;
-      const batchLabel = `Batch ${callCount + 1}`;
       options.tracker?.recordCall('scrape');
       const scrapeStartTime = Date.now();
-      const dedupNote = duplicates.length > 0
-        ? `**Global Cache Hit**: ${duplicates.length} URL(s) retrieved from session memory (already scraped).\n\n`
+      // Count only content actually re-served. Evicted pool members are in the fetch
+      // list instead, so they surface as fresh results (or in the not-fetched note),
+      // never as a "retrieved" claim with no content attached.
+      let dedupNote = duplicateResults.length > 0
+        ? `**Global Cache Hit**: ${duplicateResults.length} URL(s) retrieved from session memory (already scraped).\n\n`
         : '';
-      metrics.increment('tool_scrape_duplicates_total', duplicates.length);
-
-      // Even for duplicates, we want to return the content if we have it in the session cache
-      const duplicateResults: { url: string; markdown: string; success: true }[] = [];
-      for (const url of duplicates) {
-        const cachedContent = getCachedScrapedContent(getGlobalState().researchId, url);
-        if (cachedContent) {
-          duplicateResults.push({ url, markdown: cachedContent, success: true });
-          options.onUrlScrapeResult?.(url, true);
-        }
+      if (evictedDuplicates.length > 0) {
+        dedupNote += `**Session Cache Evicted**: ${evictedDuplicates.length} previously-scraped URL(s) no longer in session memory — queued for a fresh fetch:\n${evictedDuplicates.map(u => `- ${u}`).join('\n')}\n\n`;
       }
-
-      if (dedupedUrls.length === 0 && duplicateResults.length === 0) {
-        metrics.increment('tool_scrape_calls_total', 1, { status: 'all_duplicates_no_content' });
-        // Add footer with alternative URLs for discovery
-        const earlyFooter = buildSessionPoolFooter(getGlobalState().researchId, rawUrls, options.researcherId);
-        return {
-          content: [{ type: 'text', text: `# ${batchLabel} Skipped\n\nAll URLs were already in the global pool, but no cached content was available.${earlyFooter}` }],
-          details: { all_duplicates: true },
-        };
+      metrics.increment('tool_scrape_duplicates_total', duplicates.length);
+      for (const r of duplicateResults) {
+        options.onUrlScrapeResult?.(r.url, true);
       }
 
-      const finalUrls = dedupedUrls.slice(0, MAX_SCRAPE_URLS);
-
       // Batch 1 uses the configured scrape concurrency; batch 2+ may go wider
       // once the context budget allows — but never wider than the real browser
       // pool (WORKER_THREADS × WORKER_CONCURRENCY), or the extra requests just
@@ -331,6 +344,15 @@ export function createScrapeTool(options: {
         markdown += '\n';
       }
 
+      if (skippedUrls.length > 0) {
+        markdown += `## Not Fetched — Over Batch Cap (${skippedUrls.length})\n\n`;
+        markdown += `Only the first ${MAX_SCRAPE_URLS} URLs of a batch are fetched. The following URLs were NOT fetched — no content for them appears above. Request them in a later scrape batch if still needed:\n`;
+        for (const url of skippedUrls) {
+          markdown += `- ${url}\n`;
+        }
+        markdown += '\n';
+      }
+
       // Append Session URL Pool footer
       markdown += buildSessionPoolFooter(getGlobalState().researchId, rawUrls, options.researcherId);
 
@@ -342,6 +364,7 @@ export function createScrapeTool(options: {
           failed: failedFresh.length,
           cached: cachedResults.length,
           fresh: successfulFresh.length,
+          skipped: skippedUrls.length,
         },
       };
     },

package/src/tools/search.ts

@@ -9,6 +9,7 @@ import { Type, type Static } from 'typebox';
 import { Value } from 'typebox/value';
 import { search } from '../web-research/search.ts';
 import type { ToolUsageTracker } from '../utils/tool-usage-tracker.ts';
+import type { SystemResearchState } from '../orchestration/deep-research-types.ts';
 import { logger } from '../logger.ts';
 import { type Config, getConfig } from '../config.ts';
 import { metrics } from '../utils/metrics.ts';
@@ -18,6 +19,8 @@ export function createSearchTool(options: {
   ctx: ExtensionContext;
   tracker: ToolUsageTracker;
   onProgress?: (links: number) => void;
+  /** Research state accessor; its researchId keys the per-session circuit breaker (mirrors scrape). */
+  getGlobalState?: () => SystemResearchState;
   config?: Config;
 }): ToolDefinition {
   const youtubeEveryN = (options.config ?? getConfig(options.ctx.cwd)).YOUTUBE_QUERY_EVERY_N;
@@ -84,9 +87,15 @@ export function createSearchTool(options: {
 
       try {
         const container = tryGetServiceContainerFromCtx(ctx);
+        // Per-session circuit-breaker scoping: thread the researchId exactly as
+        // the scrape path does (tools/scrape.ts passes getGlobalState().researchId
+        // into scrape() → runBrowserTask). Search previously hard-coded
+        // sessionId=undefined at the runWorkerSearch hop, so its failures always
+        // hit the GLOBAL breaker while scrape's hit the per-run one.
+        const sessionId = options.getGlobalState?.().researchId;
         const results = await search(queries, options.config, signal, (links) => {
           if (options.onProgress) options.onProgress(links);
-        }, container);
+        }, container, sessionId);
         const elapsed = Date.now() - startTime;
         
         const totalResults = results.reduce((sum, r) => sum + r.results.length, 0);

package/src/tools/security.ts

@@ -97,9 +97,48 @@ export function createSecuritySearchTool(options: {
         throw new Error('At least one search term is required');
       }
 
+      // Normalize database names (schema accepts free text): "NVD" must reach the
+      // dispatcher as "nvd". Unknown names are still passed through — the searcher
+      // reports each one in `errors`, which the note below surfaces — but when EVERY
+      // requested name is unknown the call must fail loudly here: running zero
+      // databases and reporting "0 vulnerabilities found" would read as an
+      // authoritative all-clear.
+      const VALID_DATABASES = ['nvd', 'cisa_kev', 'github', 'osv'] as const;
       const databases: readonly ('nvd' | 'cisa_kev' | 'github' | 'osv')[] = p.databases !== undefined && p.databases.length > 0
-        ? p.databases as readonly ('nvd' | 'cisa_kev' | 'github' | 'osv')[]
-        : ['nvd', 'cisa_kev', 'github', 'osv'];
+        ? p.databases.map(db => db.trim().toLowerCase()) as readonly ('nvd' | 'cisa_kev' | 'github' | 'osv')[]
+        : VALID_DATABASES;
+      if (!databases.some(db => (VALID_DATABASES as readonly string[]).includes(db))) {
+        metrics.increment('tool_security_search_calls_total', 1, { status: 'unknown_databases' });
+        return {
+          content: [{
+            type: 'text',
+            text: `# Security Vulnerability Search Failed\n\n**Error:** No recognized databases in request — nothing was searched.\n\n**Requested:** ${p.databases!.join(', ')}\n\n**Valid databases (case-insensitive):** ${VALID_DATABASES.join(', ')}`,
+          }],
+          details: { error: 'unknown_databases', requested: p.databases, validDatabases: VALID_DATABASES },
+        };
+      }
+      // Normalize the severity filter at the single dispatch chokepoint (schema
+      // accepts free text). Only NVD's path normalized (security/index.ts
+      // getSeverityParam): GitHub/OSV received the raw string, so "med" or
+      // "HIGH,CRITICAL" ran NVD unfiltered while their strict filters silently
+      // returned zero. An unrecognized value must fail loudly BEFORE any database
+      // is queried rather than let the databases silently diverge.
+      const VALID_SEVERITIES = ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'] as const;
+      let severity: string | undefined;
+      if (p.severity !== undefined) {
+        const upper = p.severity.trim().toUpperCase();
+        severity = upper === 'MODERATE' ? 'MEDIUM' : upper;
+        if (!(VALID_SEVERITIES as readonly string[]).includes(severity)) {
+          metrics.increment('tool_security_search_calls_total', 1, { status: 'invalid_severity' });
+          return {
+            content: [{
+              type: 'text',
+              text: `[Error] Unknown severity "${p.severity}" — valid: LOW, MEDIUM, HIGH, CRITICAL (MODERATE accepted as MEDIUM). No databases were queried.`,
+            }],
+            details: { error: 'invalid_severity', requested: p.severity, validSeverities: VALID_SEVERITIES },
+          };
+        }
+      }
       const maxResults = p.maxResults ?? 20;
 
       metrics.increment('tool_security_search_terms_total', terms.length);
@@ -110,7 +149,7 @@ export function createSecuritySearchTool(options: {
         const searchParams: SecuritySearchParams = {
           terms,
           databases,
-          severity: p.severity,
+          severity,
           maxResults,
           includeExploited: p.includeExploited ?? false,
           ecosystem: p.ecosystem,
@@ -172,7 +211,13 @@ export function createSecuritySearchTool(options: {
         markdown += '## NIST NVD\n\n';
         if (results.results.nvd.error !== undefined) {
           markdown += `[Error] ${results.results.nvd.error}\n\n`;
-        } else {
+        }
+        // Partial failures: the clients deliberately return whatever they fetched
+        // PLUS an error annotation (e.g. "NVD lookup failed for 1/2 term(s)"), so
+        // the data must render alongside the [Error] line — the header's totals
+        // already count it. Only a total failure (error with zero rows) stays
+        // error-only, so it can't read as an authoritative "found nothing".
+        if (results.results.nvd.error === undefined || results.results.nvd.vulnerabilities.length > 0) {
           markdown += `Found: ${results.results.nvd.count} vulnerabilities\n\n`;
           for (const vuln of results.results.nvd.vulnerabilities.slice(0, 20)) {
             markdown += `### ${vuln.id}\n`;
@@ -212,7 +257,9 @@ export function createSecuritySearchTool(options: {
         markdown += '## CISA Known Exploited Vulnerabilities\n\n';
         if (results.results.cisa_kev.error !== undefined) {
           markdown += `[Error] ${results.results.cisa_kev.error}\n\n`;
-        } else {
+        }
+        // Partial failure: render the data alongside the error (see NVD section).
+        if (results.results.cisa_kev.error === undefined || results.results.cisa_kev.vulnerabilities.length > 0) {
           markdown += `Found: ${results.results.cisa_kev.count} actively exploited vulnerabilities\n\n`;
           for (const vuln of results.results.cisa_kev.vulnerabilities.slice(0, 20)) {
             markdown += `### ${vuln.id}\n`;
@@ -245,8 +292,16 @@ export function createSecuritySearchTool(options: {
         markdown += '## GitHub Security Advisories\n\n';
         if (results.results.github.error !== undefined) {
           markdown += `[Error] ${results.results.github.error}\n\n`;
-        } else {
+        }
+        // Partial failure: render the data alongside the error (see NVD section).
+        if (results.results.github.error === undefined || results.results.github.advisories.length > 0) {
           markdown += `Found: ${results.results.github.count} advisories\n\n`;
+          if (p.includeExploited === true) {
+            // includeExploited reaches only NVD (hasKev) and CISA KEV (inherently
+            // exploited-only); GitHub carries no exploitation signal, so these
+            // results are NOT filtered by it — say so rather than imply they were.
+            markdown += '*Note: GitHub Advisories are not filtered by known-exploitation — that signal exists only in NVD and CISA KEV.*\n\n';
+          }
           for (const adv of results.results.github.advisories.slice(0, 20)) {
             markdown += `### ${adv.id}\n`;
             markdown += `- **Severity:** ${adv.severity}\n`;
@@ -277,8 +332,14 @@ export function createSecuritySearchTool(options: {
         markdown += '## Open Source Vulnerabilities (OSV)\n\n';
         if (results.results.osv.error !== undefined) {
           markdown += `[Error] ${results.results.osv.error}\n\n`;
-        } else {
+        }
+        // Partial failure: render the data alongside the error (see NVD section).
+        if (results.results.osv.error === undefined || results.results.osv.vulnerabilities.length > 0) {
           markdown += `Found: ${results.results.osv.count} vulnerabilities\n\n`;
+          if (p.includeExploited === true) {
+            // Same caveat as GitHub: OSV carries no exploitation signal.
+            markdown += '*Note: OSV results are not filtered by known-exploitation — that signal exists only in NVD and CISA KEV.*\n\n';
+          }
           for (const vuln of results.results.osv.vulnerabilities.slice(0, 20)) {
             markdown += `### ${vuln.id}\n`;
             markdown += `- **Severity:** ${vuln.severity}\n`;

package/src/tui/research-health.ts

@@ -95,17 +95,33 @@ export interface HealthMonitor {
 
 export function createHealthMonitor(): HealthMonitor {
   let timer: NodeJS.Timeout | null = null;
+  // Critical components already warned this run, so a persistently failing check
+  // logs once rather than on every 30s tick.
+  const warnedCritical = new Set<string>();
 
   const start = () => {
     if (timer) return;
-    
+    warnedCritical.clear();
+
+    // Note: the registry API (runAll) only supports { force } — there is no surface
+    // for selecting/skipping individual checks, so each tick runs the full set
+    // (including the real BrowserRuntime page-load probe when the pool is active).
     timer = setInterval(async () => {
       try {
         const health = await healthRegistry.runAll();
-        const failedNonCritical = health.components.filter(c => !c.healthy && !healthRegistry.isCritical(c.component));
+        const failed = health.components.filter(c => !c.healthy);
+        const failedNonCritical = failed.filter(c => !healthRegistry.isCritical(c.component));
         if (failedNonCritical.length > 0) {
           logger.warn(`[research] Periodic health check: non-critical components degraded: ${failedNonCritical.map(c => c.component).join(', ')}`);
         }
+        // Critical failures are worth surfacing too — the run itself keeps going
+        // (per-task timeouts bound the real work), but silently burning probes while
+        // a critical component is down left the monitor with no output at all.
+        for (const c of failed.filter(f => healthRegistry.isCritical(f.component))) {
+          if (warnedCritical.has(c.component)) continue;
+          warnedCritical.add(c.component);
+          logger.warn(`[research] Periodic health check: critical component unhealthy mid-run: ${c.component}${c.error ? ` (${c.error})` : ''}`);
+        }
       } catch (error) {
         logger.debug('[research] Periodic health check failed (non-blocking):', error);
       }

package/src/utils/circuit-breaker.ts

@@ -71,7 +71,7 @@ export class CircuitBreaker {
       const result = await action();
       const duration = Date.now() - startTime;
       metrics.observe('circuit_breaker_call_duration_ms', duration, { breaker: this.options.name, status: 'success' });
-      this.onSuccess();
+      this.onSuccess(admittedAsProbe);
       return result;
     } catch (error) {
       const duration = Date.now() - startTime;
@@ -105,9 +105,16 @@ export class CircuitBreaker {
     }
   }
 
-  private onSuccess() {
+  private onSuccess(admittedAsProbe: boolean) {
     metrics.increment('circuit_breaker_success_total', 1, { breaker: this.options.name, state: this.state });
     if (this.state === 'HALF_OPEN') {
+      // Only a call admitted through the HALF_OPEN gate is evidence of recovery.
+      // A call admitted while CLOSED that merely settles after the breaker tripped
+      // and re-half-opened carries PRE-outage evidence — counting it would close
+      // the circuit while the real probe is still in flight against a dependency
+      // that may still be down. Its success is simply not counted; the in-flight
+      // probe's own outcome decides the transition.
+      if (!admittedAsProbe) return;
       this.successCount++;
       if (this.successCount >= this.options.halfOpenMaxCalls) {
         this.transitionTo('CLOSED');

package/src/utils/disk-space-checker.ts

@@ -36,12 +36,17 @@ export class DiskSpaceChecker {
       if (availableBytes < this.MIN_DISK_SPACE_BYTES) {
         this.hasDiskSpace = false;
         // Write directly to stderr — this check runs inside the logger itself,
-        // so using the logger here would create a circular dependency.
-        process.stderr.write(
-          `[pi-research] Insufficient disk space for logging: ` +
-          `${Math.round(availableBytes / 1024 / 1024)}MB available, ` +
-          `minimum ${this.MIN_DISK_SPACE_BYTES / 1024 / 1024}MB required\n`
-        );
+        // so using the logger here would create a circular dependency. Guarded:
+        // when the parent has already closed the pipe, a bare write throws
+        // EPIPE/ERR_STREAM_DESTROYED and (with no global uncaughtException
+        // handler in the main process) would crash the run over a diagnostic.
+        try {
+          process.stderr.write(
+            `[pi-research] Insufficient disk space for logging: ` +
+            `${Math.round(availableBytes / 1024 / 1024)}MB available, ` +
+            `minimum ${this.MIN_DISK_SPACE_BYTES / 1024 / 1024}MB required\n`
+          );
+        } catch { /* stderr gone — the flag above already halts log writes */ }
       } else {
         this.hasDiskSpace = true;
       }

package/src/utils/json-utils.ts

@@ -96,6 +96,15 @@ function preRepairJson(jsonStr: string): string {
     .replace(/,\s*([}\]])/g, '$1');
 }
 
+/**
+ * Upper bound on the forward candidate scan in extractJsonObject/extractJsonArray.
+ * Prose braces ("Fill {placeholder} …") or citation brackets ("[see note] …") can
+ * precede the real payload, so a closed-but-unparseable candidate must not end the
+ * search — but brace-dense non-JSON text must not turn extraction into a quadratic
+ * walk either, hence the cap.
+ */
+const MAX_JSON_CANDIDATES = 8;
+
 /**
  * Parse a JSON slice, trying it verbatim first and only running preRepairJson on
  * failure. preRepairJson rewrites smart quotes and trailing commas even inside
@@ -114,9 +123,12 @@ function parseRawThenRepair(jsonStr: string): unknown {
 /**
  * Extract JSON object from raw text
  *
- * Finds the first `{` then walks forward with depth-tracking (respecting string
- * literals) to locate its matching `}`. This correctly handles text that contains
- * multiple JSON objects or has trailing content after the object.
+ * Walks forward with depth-tracking (respecting string literals) from each `{`
+ * candidate to its matching `}`. A candidate that closes but fails to parse is a
+ * prose brace (e.g. "Fill {placeholder} like this: {…}"), so the scan advances to
+ * the next `{` — bounded by MAX_JSON_CANDIDATES. Truncated-repair applies at the
+ * first UNCLOSED candidate: past that point there is no further text to scan, and
+ * an unclosed opener is what genuinely cut-off LLM output looks like.
  *
  * @param text - Text to search for JSON object
  * @returns Extraction result with parsed value or error
@@ -124,7 +136,7 @@ function parseRawThenRepair(jsonStr: string): unknown {
 export function extractJsonObject<T = unknown>(
   text: string
 ): JsonExtractionResult<T> {
-  const objStart = text.indexOf('{');
+  let objStart = text.indexOf('{');
 
   if (objStart === -1) {
     return {
@@ -134,54 +146,67 @@ export function extractJsonObject<T = unknown>(
     };
   }
 
-  const { index: objEnd, inString } = findMatchingBracket(text, objStart);
+  let lastParseError = 'unknown parse error';
+  for (
+    let attempt = 0;
+    attempt < MAX_JSON_CANDIDATES && objStart !== -1;
+    attempt++, objStart = text.indexOf('{', objStart + 1)
+  ) {
+    const { index: objEnd, inString } = findMatchingBracket(text, objStart);
 
-  if (objEnd === -1) {
-    // Attempt local repair for truncated JSON
-    logger.debug('[json-utils] JSON object truncated; attempting local repair');
-    let partialText = text.slice(objStart);
-    
-    // If we were inside a string, we MUST close it first
-    if (inString) {
-        partialText += '"';
-    }
+    if (objEnd === -1) {
+      // Attempt local repair for truncated JSON
+      logger.debug('[json-utils] JSON object truncated; attempting local repair');
+      let partialText = text.slice(objStart);
 
-    for (let i = 1; i <= 15; i++) {
-      try {
-        const candidate = preRepairJson(partialText + '}'.repeat(i));
-        const parsed = JSON.parse(candidate);
-        logger.debug(`[json-utils] JSON object truncated; salvaged by appending ${inString ? 'quote and ' : ''}${i} closing braces`);
-        return { success: true, value: parsed as T, method: 'raw-object' };
-      } catch {
-        continue;
+      // If we were inside a string, we MUST close it first
+      if (inString) {
+          partialText += '"';
+      }
+
+      for (let i = 1; i <= 15; i++) {
+        try {
+          const candidate = preRepairJson(partialText + '}'.repeat(i));
+          const parsed = JSON.parse(candidate);
+          logger.debug(`[json-utils] JSON object truncated; salvaged by appending ${inString ? 'quote and ' : ''}${i} closing braces`);
+          return { success: true, value: parsed as T, method: 'raw-object' };
+        } catch {
+          continue;
+        }
       }
+
+      return {
+        success: false,
+        value: undefined,
+        error: 'No matching closing brace found and local repair failed',
+      };
     }
-    
-    return {
-      success: false,
-      value: undefined,
-      error: 'No matching closing brace found and local repair failed',
-    };
-  }
 
-  try {
-    const parsed = parseRawThenRepair(text.slice(objStart, objEnd + 1));
-    return { success: true, value: parsed as T, method: 'raw-object' };
-  } catch (err) {
-    const errorMsg = err instanceof Error ? err.message : String(err);
-    return {
-      success: false,
-      value: undefined,
-      error: `Failed to parse JSON object: ${errorMsg}`,
-    };
+    try {
+      const parsed = parseRawThenRepair(text.slice(objStart, objEnd + 1));
+      return { success: true, value: parsed as T, method: 'raw-object' };
+    } catch (err) {
+      // Closed but unparseable — a prose brace, not the payload. Try the next `{`.
+      lastParseError = err instanceof Error ? err.message : String(err);
+    }
   }
+
+  return {
+    success: false,
+    value: undefined,
+    error: `Failed to parse JSON object: ${lastParseError}`,
+  };
 }
 
 /**
  * Extract JSON array from raw text
  *
- * Looks for a JSON array spanning from the first `[` to the last `]`.
- * Useful for extracting arrays of strings or objects.
+ * Walks forward with depth-tracking (respecting string literals) from each `[`
+ * candidate to its matching `]`. A candidate that closes but fails to parse is a
+ * citation-style bracket (e.g. "[see note] … [\"a\",\"b\"]"), so the scan advances
+ * to the next `[` — bounded by MAX_JSON_CANDIDATES. Truncated-repair applies at
+ * the first UNCLOSED candidate: past that point there is no further text to scan,
+ * and an unclosed opener is what genuinely cut-off LLM output looks like.
  *
  * @param text - Text to search for JSON array
  * @returns Extraction result with parsed value or error
@@ -189,7 +214,7 @@ export function extractJsonObject<T = unknown>(
 export function extractJsonArray<T = unknown>(
   text: string
 ): JsonExtractionResult<T[]> {
-  const arrStart = text.indexOf('[');
+  let arrStart = text.indexOf('[');
 
   if (arrStart === -1) {
     return {
@@ -199,54 +224,69 @@ export function extractJsonArray<T = unknown>(
     };
   }
 
-  const { index: arrEnd, inString } = findMatchingBracket(text, arrStart);
-  if (arrEnd === -1) {
-    // Attempt local repair for truncated JSON
-    logger.debug('[json-utils] JSON array truncated; attempting local repair');
-    let partialText = text.slice(arrStart);
-    
-    if (inString) {
-        partialText += '"';
-    }
+  let lastError = 'unknown parse error';
+  let lastErrorShape: 'not-array' | 'parse' = 'parse';
+  for (
+    let attempt = 0;
+    attempt < MAX_JSON_CANDIDATES && arrStart !== -1;
+    attempt++, arrStart = text.indexOf('[', arrStart + 1)
+  ) {
+    const { index: arrEnd, inString } = findMatchingBracket(text, arrStart);
+    if (arrEnd === -1) {
+      // Attempt local repair for truncated JSON
+      logger.debug('[json-utils] JSON array truncated; attempting local repair');
+      let partialText = text.slice(arrStart);
 
-    for (let i = 1; i <= 15; i++) {
-      try {
-        const candidate = preRepairJson(partialText + ']'.repeat(i));
-        const parsed = JSON.parse(candidate);
-        if (Array.isArray(parsed)) {
-          logger.debug(`[json-utils] JSON array truncated; salvaged by appending ${inString ? 'quote and ' : ''}${i} closing brackets`);
-          return { success: true, value: parsed as T[], method: 'raw-array' };
-        }
-      } catch {
-        continue;
+      if (inString) {
+          partialText += '"';
       }
-    }
 
-    return {
-      success: false,
-      value: undefined,
-      error: 'No matching closing bracket found and local repair failed',
-    };
-  }
+      for (let i = 1; i <= 15; i++) {
+        try {
+          const candidate = preRepairJson(partialText + ']'.repeat(i));
+          const parsed = JSON.parse(candidate);
+          if (Array.isArray(parsed)) {
+            logger.debug(`[json-utils] JSON array truncated; salvaged by appending ${inString ? 'quote and ' : ''}${i} closing brackets`);
+            return { success: true, value: parsed as T[], method: 'raw-array' };
+          }
+        } catch {
+          continue;
+        }
+      }
 
-  try {
-    const parsed = parseRawThenRepair(text.slice(arrStart, arrEnd + 1));
-    if (!Array.isArray(parsed)) {
       return {
         success: false,
         value: undefined,
-        error: 'Parsed value is not an array',
+        error: 'No matching closing bracket found and local repair failed',
       };
     }
-    return { success: true, value: parsed as T[], method: 'raw-array' };
-  } catch (err) {
-    const errorMsg = err instanceof Error ? err.message : String(err);
+
+    try {
+      const parsed = parseRawThenRepair(text.slice(arrStart, arrEnd + 1));
+      if (!Array.isArray(parsed)) {
+        lastErrorShape = 'not-array';
+        continue;
+      }
+      return { success: true, value: parsed as T[], method: 'raw-array' };
+    } catch (err) {
+      // Closed but unparseable — a citation bracket, not the payload. Try the next `[`.
+      lastErrorShape = 'parse';
+      lastError = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  if (lastErrorShape === 'not-array') {
     return {
       success: false,
       value: undefined,
-      error: `Failed to parse JSON array: ${errorMsg}`,
+      error: 'Parsed value is not an array',
     };
   }
+  return {
+    success: false,
+    value: undefined,
+    error: `Failed to parse JSON array: ${lastError}`,
+  };
 }
 
 /**

package/src/utils/llm-usage.ts

@@ -51,11 +51,16 @@ function warnIfUnpriced(model: Model<any>, tokens: number, cost: number): void {
   );
 }
 
-/** Structural subset of the run observer — just the token sink. Kept structural (rather
+/** Structural subset of the run observer — just the token sinks. Kept structural (rather
  *  than importing ResearchObserver from core/) so this foundation-layer helper does not
  *  depend on an upper layer; the full ResearchObserver satisfies it. */
 export interface TokenSink {
   onTokensConsumed?: (tokens: number, cost: number) => void;
+  /** Phase-scoped sinks, fed from the `component` label ('coordinator'/'evaluator');
+   *  they drive the TUI coord/eval cost rows and the SDK planning_tokens /
+   *  evaluation_tokens events. */
+  onPlanningTokens?: (tokens: number, cost: number) => void;
+  onEvaluationTokens?: (tokens: number, cost: number) => void;
 }
 
 export interface RecordUsageOptions {
@@ -86,6 +91,17 @@ export function recordLlmUsage(
     metrics.increment('llm_tokens_total', tokens, labels);
     metrics.increment('llm_cost_total', cost, labels);
     opts.observer?.onTokensConsumed?.(tokens, cost);
+    // Phase-scoped events: coordinator/evaluator call sites label their usage
+    // 'coordinator'/'evaluator' (see planning-service.ts); route those to the
+    // dedicated observer hooks so the TUI coord/eval cost rows and the SDK
+    // planning_tokens/evaluation_tokens events fire. Before this mapping the hooks
+    // had zero emit sites (regression in c90d7f37) — coordinator/evaluator usage
+    // reached only onTokensConsumed, which the TUI observer does not implement.
+    if (opts.component === 'coordinator') {
+      opts.observer?.onPlanningTokens?.(tokens, cost);
+    } else if (opts.component === 'evaluator') {
+      opts.observer?.onEvaluationTokens?.(tokens, cost);
+    }
   }
   return { tokens, cost };
 }