@lincoln504/pi-research 1.3.11 → 1.3.12

package/src/sdk.ts

@@ -44,7 +44,7 @@ import { runHealthCheck } from './healthcheck/index.ts';
 import { buildModelRegistry as sharedBuildModelRegistry, resolveModel } from './core/llm/model-registry-factory.ts';
 import { scrapeSingle } from './web-research/web-scraper.ts';
 import { validateInitialLinks } from './utils/url-utils.ts';
-import { validateUrlForSSRF } from './web-research/scraper-utils.ts';
+import { validateUrlForSSRF, disposeSsrfSafeFetcher } from './web-research/scraper-utils.ts';
 import type { ScrapeResult } from './core/interfaces/scheduler-interfaces.ts';
 import { randomUUID } from 'node:crypto';
 import type { ResearchDepth } from './types/index.ts';
@@ -954,6 +954,15 @@ async function _doShutdown(): Promise<void> {
     errors.push(err instanceof Error ? err : new Error(String(err)));
   }
 
+  // Release the module-level undici Agent's keep-alive sockets. Nothing else
+  // owns it, so a long-lived host would otherwise hold them past teardown.
+  try {
+    await disposeSsrfSafeFetcher();
+  } catch (err) {
+    logger.error('[SDK] Error disposing SSRF fetcher:', err);
+    errors.push(err instanceof Error ? err : new Error(String(err)));
+  }
+
   if (globalContainer) {
     // Dispose all services in one DAG-ordered teardown. disposeCoreServices first
     // clears the embedding-server state while the StateManager is still alive, then

package/src/security/nvd-client.ts

@@ -22,7 +22,6 @@ import type {
 } from './nvd-types.ts';
 import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-research/retry-utils.ts';
 import { CircuitBreaker } from '../utils/circuit-breaker.ts';
-import { safeUnref } from '../utils/safe-unref.ts';
 import { metrics } from '../utils/metrics.ts';
 import { readJsonCapped, BodyTooLargeError, MAX_BULK_JSON_BODY_BYTES } from '../utils/http-body.ts';
 import {
@@ -77,7 +76,15 @@ class NVDRateLimiter {
           signal?.removeEventListener('abort', onAbort);
           resolve();
         }, waitTime);
-        safeUnref(timeoutId);
+        // NOT unref'd. This timer IS the operation, not a watchdog racing one:
+        // nothing else will ever settle this promise, so unref'ing it let Node
+        // consider the process idle while an NVD search was mid-spacing. With no
+        // other ref'd handle the search simply never resumed — the awaited
+        // promise stayed unsettled and the caller got no result and no error.
+        // Every other unref in this codebase guards a timeout against real I/O
+        // that holds the loop open by itself; this one had nothing behind it.
+        // Bounded (≤6s, or ≤600ms with NVD_API_KEY) and abortable below, so
+        // keeping it ref'd cannot delay shutdown beyond one spacing interval.
         const onAbort = () => {
           clearTimeout(timeoutId as unknown as ReturnType<typeof setTimeout>);
           reject(Object.assign(new Error('NVD search aborted'), { name: 'AbortError' }));
@@ -392,6 +399,8 @@ async function fetchPaginated(
   const signal = options?.signal;
   let startIndex = 0;
   let totalPagesFetched = 0;
+  // Whether the one-time jump to the newest window (below) has been performed.
+  let seekedToNewest = false;
 
   while (totalPagesFetched < maxPages && allVulnerabilities.length < maxResults) {
     if (signal?.aborted) break; // stop paginating promptly on cancel
@@ -433,10 +442,40 @@ async function fetchPaginated(
     }
     metrics.increment('nvd_cache_hits_total', 1);
 
+    const responseData = data as Record<string, unknown> | undefined;
+    const reportedTotal = typeof responseData?.['totalResults'] === 'number'
+      ? responseData['totalResults'] as number
+      : undefined;
+
+    // NVD 2.0 returns keyword matches OLDEST FIRST and offers no sort parameter
+    // (`sortBy` is a 404), so paging from index 0 and stopping at maxResults kept
+    // precisely the least useful window: a `security_search` for "openssl"
+    // returned CVE-1999-0428 through 2004 out of 566 matches and could never
+    // reach the ones published this month. Seek once to the tail instead, so the
+    // rows we keep are the most recent. Only fires when the match set is larger
+    // than what we return — a query whose results all fit is already complete, so
+    // the common (and every mocked) case still costs exactly one request.
+    //
+    // Interaction with the `v2` supplemental pass (searchSingleTerm), deliberate:
+    // that pass admits only entries carrying NO v3/v4 metric, and NVD scores
+    // essentially every modern CVE with v3 — so v2-only rows live at the OLD end.
+    // Seeking to the tail therefore makes the supplement return little or nothing
+    // once a match set is large. That is the intended trade, not an oversight: the
+    // supplement was the channel feeding 1999-era advisories into results that are
+    // supposed to describe the current picture. A genuinely legacy product still
+    // works, because its own v2 match set is small enough that no seek fires.
+
+    if (!seekedToNewest && startIndex === 0 && reportedTotal !== undefined && reportedTotal > maxResults) {
+      seekedToNewest = true;
+      startIndex = reportedTotal - maxResults;
+      allVulnerabilities.length = 0; // the oldest window we just read is not what we want
+      totalPagesFetched++;
+      continue;
+    }
+
     allVulnerabilities.push(...admitted);
 
-    const responseData = data as Record<string, unknown> | undefined;
-    if (responseData?.['totalResults'] !== undefined && typeof responseData['totalResults'] === 'number' && startIndex + pageSize >= responseData['totalResults']) {
+    if (reportedTotal !== undefined && startIndex + pageSize >= reportedTotal) {
       break;
     }
 
@@ -447,6 +486,18 @@ async function fetchPaginated(
   metrics.increment('nvd_pagination_requests_total', totalPagesFetched);
   metrics.observe('nvd_pagination_pages_fetched', totalPagesFetched);
 
+  // Newest first within the window. NVD's own order is ascending, so without
+  // this the caller's slice() would again favour the older end of whatever we
+  // gathered. Undated entries sort last rather than jumping to the front.
+  allVulnerabilities.sort((a, b) => {
+    const ta = a.published ? Date.parse(a.published) : NaN;
+    const tb = b.published ? Date.parse(b.published) : NaN;
+    if (Number.isNaN(ta) && Number.isNaN(tb)) return 0;
+    if (Number.isNaN(ta)) return 1;
+    if (Number.isNaN(tb)) return -1;
+    return tb - ta;
+  });
+
   return allVulnerabilities.slice(0, maxResults);
 }
 

package/src/security/osv-client.ts

@@ -230,7 +230,19 @@ export async function searchOSV(
       }
     }
 
-    vulnerabilities.push(...Array.from(uniqueVulns.values()));
+    // Newest first, so the slice(0, maxResults) below keeps the advisories most
+    // likely to matter. OSV returns a package's vulnerabilities in ID order,
+    // which is effectively alphabetical — truncating that dropped whichever
+    // advisories happened to sort late, recency playing no part. Undated entries
+    // sort last rather than displacing dated ones.
+    vulnerabilities.push(...Array.from(uniqueVulns.values()).sort((a, b) => {
+      const ta = a.published ? Date.parse(a.published) : NaN;
+      const tb = b.published ? Date.parse(b.published) : NaN;
+      if (Number.isNaN(ta) && Number.isNaN(tb)) return 0;
+      if (Number.isNaN(ta)) return 1;
+      if (Number.isNaN(tb)) return -1;
+      return tb - ta;
+    }));
 
     if (failures.length > 0 && failures.length === terms.length) {
       throw new Error(`All OSV lookups failed: ${failures.join('; ')}`);

package/src/tools/research-knowledge-search.ts

@@ -114,6 +114,17 @@ export function formatQueriesBlock(queries: string[]): string {
 /** Widget ID for the knowledge search TUI panel */
 const KNOWLEDGE_WIDGET_ID = 'pi-research-knowledge-search';
 
+/**
+ * How many knowledge searches are currently showing the widget.
+ *
+ * The widget ID is a single global slot and `setWidget` replaces rather than
+ * stacks, but this tool is `executionMode: 'parallel'` and can also be driven
+ * concurrently by the `/knowledge-store` slash command. Without a refcount the
+ * first call to finish tears the widget down while a sibling is still running,
+ * leaving an in-flight search with no indicator. Only the last one out removes it.
+ */
+let knowledgeWidgetRefCount = 0;
+
 /**
  * System string returned when the answer is NOT found in the research
  * knowledge store. The exact phrasing is critical — the main pi agent
@@ -140,6 +151,7 @@ export const RESEARCH_KNOWLEDGE_MAYBE_STRING =
  */
 function showKnowledgeSearchWidget(ctx: ExtensionContext): void {
   if (ctx.mode !== 'tui' || !ctx.hasUI) return;
+  knowledgeWidgetRefCount++;
   try {
     const panelFactory = createKnowledgeSearchPanel();
     ctx.ui.setWidget(KNOWLEDGE_WIDGET_ID, panelFactory as any, { placement: 'aboveEditor' });
@@ -149,10 +161,15 @@ function showKnowledgeSearchWidget(ctx: ExtensionContext): void {
 }
 
 /**
- * Remove the knowledge search TUI widget.
+ * Remove the knowledge search TUI widget once the last concurrent search is done.
  */
 function hideKnowledgeSearchWidget(ctx: ExtensionContext): void {
   if (ctx.mode !== 'tui' || !ctx.hasUI) return;
+  // show() incremented under the identical guard, so the counter stays balanced.
+  // Clamp anyway: a stray hide must never drive it negative and wedge the
+  // widget on-screen for the rest of the session.
+  knowledgeWidgetRefCount = Math.max(0, knowledgeWidgetRefCount - 1);
+  if (knowledgeWidgetRefCount > 0) return;
   try {
     ctx.ui.setWidget(KNOWLEDGE_WIDGET_ID, undefined as any, { placement: 'aboveEditor' });
   } catch (err) {

package/src/utils/log-utils.ts

@@ -133,7 +133,22 @@ const KNOWN_TOKEN_PATTERN = new RegExp(
   'g',
 );
 // JSON Web Tokens — header.payload.signature, each a base64url segment.
-const JWT_PATTERN = new RegExp(`${B_LEFT}eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+`, 'g');
+//
+// The segment lengths are bounded for the same load-bearing reason the URL
+// scheme above is, and this pattern was missed when that one was fixed. `.` is
+// NOT in the segment class, so on a run of base64url characters containing no
+// dot, `[A-Za-z0-9_-]+` consumes to the end of the run and then backtracks one
+// character at a time looking for one — and `_`/`-` count as non-alphanumeric,
+// so B_LEFT admits a fresh start position every few characters inside that same
+// run. `_eyJ` repeated across the 40_000-character scan window measured 393ms of
+// blocked event loop per log message; bounding the segments brings the identical
+// input to 41ms (~10x) with no loss of coverage. The bounds are far above any
+// real token: a 1024-char header covers even an x5c certificate chain, and 8192
+// covers payloads no bearer credential approaches.
+const JWT_PATTERN = new RegExp(
+  `${B_LEFT}eyJ[A-Za-z0-9_-]{1,1024}\\.[A-Za-z0-9_-]{1,8192}\\.[A-Za-z0-9_-]{1,4096}`,
+  'g',
+);
 // HTTP Basic credentials: "Basic <base64>".
 const BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, 'gi');
 // HTTP Bearer credentials: "Bearer <token>". The token is often opaque — many

package/src/web-research/scraper-utils.ts

@@ -277,10 +277,18 @@ export interface SsrfSafeFetcher {
   dispatcher: unknown;
 }
 
-// Cached after first construction. `undefined` = not yet attempted, `null` =
-// undici unavailable (fall back to the global fetch with request-time
-// validation only).
-let cachedSsrfFetcher: SsrfSafeFetcher | null | undefined = undefined;
+// Cached after first construction. `undefined` = not yet attempted; the promise
+// resolves to `null` when undici is unavailable (fall back to the global fetch
+// with request-time validation only).
+//
+// The PROMISE is cached, not the resolved value. Caching the value left a window
+// between the `undefined` check and the assignment — an `await import('undici')`
+// wide — during which every concurrent first caller built its OWN Agent. Only the
+// last assignment survived in the cache, so the others were handed live Agents
+// that nothing tracked: they opened keep-alive sockets and no shutdown path could
+// ever close them, which is precisely the leak disposeSsrfSafeFetcher exists to
+// prevent. Scrapes run concurrently by design, so the very first burst hit this.
+let cachedSsrfFetcher: Promise<SsrfSafeFetcher | null> | undefined = undefined;
 
 /**
  * Build (once) an undici Agent whose connector resolves via ssrfSafeLookup, so
@@ -304,19 +312,52 @@ let cachedSsrfFetcher: SsrfSafeFetcher | null | undefined = undefined;
  * Returns null if undici can't be loaded; callers then use the global fetch with
  * no dispatcher (request-time validation still applies).
  */
-export async function getSsrfSafeFetcher(): Promise<SsrfSafeFetcher | null> {
-  if (cachedSsrfFetcher !== undefined) return cachedSsrfFetcher;
+export function getSsrfSafeFetcher(): Promise<SsrfSafeFetcher | null> {
+  // Assigned before the first await, so concurrent callers all join this one
+  // construction rather than each starting their own.
+  cachedSsrfFetcher ??= (async () => {
+    try {
+      const { Agent, fetch: undiciFetch } = await import('undici');
+      return {
+        fetch: undiciFetch as unknown as SsrfSafeFetcher['fetch'],
+        dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } }),
+      };
+    } catch (e) {
+      logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
+      return null;
+    }
+  })();
+  return cachedSsrfFetcher;
+}
+
+/**
+ * Close the cached SSRF Agent and drop the cache.
+ *
+ * The Agent is a process-lifetime singleton holding keep-alive sockets to every
+ * host scraped so far. The CLI hard-exits so it never notices, but an embedding
+ * host (the pi extension) keeps the process alive across SDK
+ * init/shutdown cycles, where those sockets would linger past a teardown that
+ * claims to have released everything.
+ *
+ * Clearing the cache is not optional: the SDK can be re-initialized in the same
+ * process, and a closed Agent left in the cache would fail every subsequent
+ * scrape. Resetting to `undefined` (not `null`) makes the next call rebuild it.
+ */
+export async function disposeSsrfSafeFetcher(): Promise<void> {
+  const pending = cachedSsrfFetcher;
+  cachedSsrfFetcher = undefined;
+  if (pending === undefined) return;
+  // Await the in-flight construction rather than skipping it: a dispose racing
+  // the first getSsrfSafeFetcher() would otherwise leave the Agent it is still
+  // building unreferenced and unclosed.
+  const cached = await pending.catch(() => null);
+  const closable = cached?.dispatcher as { close?: () => Promise<void> } | undefined;
+  if (!closable?.close) return;
   try {
-    const { Agent, fetch: undiciFetch } = await import('undici');
-    cachedSsrfFetcher = {
-      fetch: undiciFetch as unknown as SsrfSafeFetcher['fetch'],
-      dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } }),
-    };
+    await closable.close();
   } catch (e) {
-    logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
-    cachedSsrfFetcher = null;
+    logger.debug(`[Scrapers] Error closing SSRF dispatcher: ${e instanceof Error ? e.message : String(e)}`);
   }
-  return cachedSsrfFetcher;
 }
 
 /**