@lincoln504/pi-research 1.3.10 → 1.3.12

package/src/knowledge/embedder-init.ts

@@ -42,7 +42,23 @@ export async function loadPipelineWithTimeout(
         interOpNumThreads: 1,
       },
     });
-    return await withTimeout(pipelinePromise, timeoutMs, errorMessage);
+    // withTimeout races but cannot cancel: on timeout the underlying pipeline()
+    // keeps loading in the background and its eventual ONNX session would leak
+    // (the caller's catch disposes this.pipeline, which was never assigned).
+    // Dispose the late arrival when it eventually resolves — one orphaned native
+    // session per retry adds up on a slow disk.
+    let timedOut = true;
+    try {
+      const p = await withTimeout(pipelinePromise, timeoutMs, errorMessage);
+      timedOut = false;
+      return p;
+    } finally {
+      if (timedOut) {
+        pipelinePromise
+          .then((late) => (late as unknown as { dispose?: () => Promise<void> }).dispose?.())
+          .catch(() => { /* load failed after the timeout — nothing to dispose */ });
+      }
+    }
   });
 
   return { pipeline: loadedPipeline, errorMessage };
@@ -279,7 +295,23 @@ export async function loadModelOnCPU(
         interOpNumThreads: 1,
       },
     });
-    return await withTimeout(pipelinePromise, initializationTimeoutMs, 'CPU fallback model load timed out');
+    // Same late-arrival disposal as loadPipelineWithTimeout: withTimeout races
+    // but cannot cancel, so on timeout the CPU load keeps running and its
+    // eventual ONNX session would leak. This path runs *after* a WebGPU
+    // failure — i.e. on a host already under memory pressure — and can be
+    // re-entered per retry, so each orphaned session compounds.
+    let timedOut = true;
+    try {
+      const p = await withTimeout(pipelinePromise, initializationTimeoutMs, 'CPU fallback model load timed out');
+      timedOut = false;
+      return p;
+    } finally {
+      if (timedOut) {
+        pipelinePromise
+          .then((late) => (late as unknown as { dispose?: () => Promise<void> }).dispose?.())
+          .catch(() => { /* load failed after the timeout — nothing to dispose */ });
+      }
+    }
   });
   
   return loadedPipeline;

package/src/knowledge/store.ts

@@ -30,11 +30,15 @@ import { normalizeWorkspacePath } from '../utils/text-utils.ts';
 
 /**
  * Detect a LanceDB concurrent-write conflict — the class of error worth retrying
- * with backoff. LanceDB 0.29 surfaces these as "Commit conflict for version N",
+ * with backoff. LanceDB surfaces these as "Commit conflict for version N",
  * "Retryable commit conflict for version N" and "Too many concurrent writes,
- * please retry later:" (strings confirmed in the 0.29.0 native binary). The three
- * legacy strings ("Version mismatch" / "Lock error" / "Commit error") no longer
- * appear in 0.29 but are kept as a safety net for other/older Lance versions.
+ * please retry later:" — all three re-confirmed present in the 0.37.1 native
+ * binary (`@lancedb/lancedb-<platform>`, NOT the `@lancedb/lancedb` JS package;
+ * grepping the latter finds nothing and looks like a regression), as they were
+ * in 0.29.0. The three legacy strings ("Version mismatch" / "Lock error" /
+ * "Commit error") appear in neither and are kept as a safety net for other/older
+ * Lance versions. Re-check these against the native binary on every upgrade: a
+ * changed message silently turns a retryable conflict into a hard failure.
  */
 export function isLanceCommitConflict(err: unknown): boolean {
   const msg = err instanceof Error ? err.message : String(err);
@@ -290,6 +294,15 @@ export class KnowledgeStore implements IKnowledgeStore {
           logger.warn('[store] Failed to read table schema:', schemaErr);
         }
       } else {
+        // Same warm-up as the migration branch above: createTable() throws
+        // "embedder dimension unknown" on a never-initialized embedder. This
+        // branch (no existing table — fresh dbDir or first-ever run) hit exactly
+        // that when the embedding factory handed back its model-mismatch local
+        // fallback, which is constructed but never initialize()d: every one of
+        // the caller's init retries burned against the same cached instance.
+        if (this.options.embedder.getDimension() === null) {
+          await this.withEmbedderReconnect(embedder => embedder.embedMany([' ']));
+        }
         this.table = await this.createTable();
       }
 
@@ -787,6 +800,49 @@ export class KnowledgeStore implements IKnowledgeStore {
     }
   }
 
+  /**
+   * Refresh the timestamp of one CURRENT generation for url+type — the
+   * revalidation "touch" for the writer-queue's content-unchanged skip. Without
+   * it a stable page kept its original timestamp forever: with
+   * KNOWLEDGE_STORE_MAX_SERVE_AGE_DAYS set, the serve-age gate then treated the
+   * entry as permanently stale (a live re-scrape on every run, for exactly the
+   * content that never changes), and TTL eviction eventually deleted an entry
+   * that had been re-verified byte-identical possibly hours earlier.
+   */
+  async touchByUrlAndType(url: string, ingestionType: string, generationTs: number, newTs: number): Promise<void> {
+    return this.trackOperation('touchByUrlAndType', undefined, async () => {
+      let retryCount = 0;
+      const MAX_RETRIES = 5;
+      const BASE_DELAY = 100;
+      const scopeFilter = this.getScopeFilter();
+      while (retryCount <= MAX_RETRIES) {
+        const table = await this.getFreshTable();
+        try {
+          const escapedUrl = url.replace(/'/g, "''");
+          const escapedType = ingestionType.replace(/'/g, "''");
+          // valuesSql with integer-coerced literals: timestamp is an Int64 column,
+          // and both inputs are integer-coerced so the SQL can never carry
+          // anything but digits.
+          await table.update({
+            where: `url = '${escapedUrl}' AND ingestion_type = '${escapedType}' AND timestamp = ${Math.floor(generationTs)} AND (${scopeFilter})`,
+            valuesSql: { timestamp: String(Math.floor(newTs)) },
+          });
+          logger.log(`[store] Refreshed timestamp for unchanged ${ingestionType} chunks of ${url}`);
+          return;
+        } catch (err) {
+          if (isLanceCommitConflict(err)) {
+            retryCount++;
+            if (retryCount > MAX_RETRIES) throw err;
+            const delay = Math.floor(Math.random() * (BASE_DELAY * Math.pow(2, retryCount - 1)));
+            await new Promise(r => setTimeout(r, delay));
+            continue;
+          }
+          throw err;
+        }
+      }
+    });
+  }
+
   async findDocumentsByUrl(url: string): Promise<StoreDocument[]> {
     // Coordinate with close(): findByUrl runs via the writer-queue in the post-run
     // background window (link-description storage), exactly when a Ctrl+C-driven
@@ -1013,7 +1069,9 @@ export class KnowledgeStore implements IKnowledgeStore {
       // no lock, so rows committed DURING the rebuild were absorbed into the
       // baseline as already-indexed and stayed out of the BM25 leg until an
       // unrelated later commit. Each createIndex commits exactly one version
-      // (measured on lancedb 0.29, fresh and replace alike); anything beyond
+      // (measured on lancedb 0.29 and re-measured on 0.37.1 — vector index, FTS
+      // index and a `replace: true` rebuild each advance version() by exactly
+      // 1); anything beyond
       // version+2 means a concurrent commit rebased beneath our index commits, so
       // fall back to the conservative pre-rebuild baseline — the next cleanup
       // sees version > baseline and rebuilds over the missed rows.
@@ -1179,7 +1237,18 @@ export class KnowledgeStore implements IKnowledgeStore {
           backupDirs.map(async name => ({ name, ts: await this.backupDirTimestamp(name) }))
         );
         ranked.sort((a, b) => a.ts - b.ts);
-        for (const { name } of ranked.slice(0, -1)) {
+        // Retain older backups until they age past the store's own TTL, not just
+        // the single newest one. Two sessions configured with DIFFERENT embedding
+        // models alternate migrations against the shared table (each open sees
+        // "wrong model" → backup → fresh table), and keep-only-newest deleted the
+        // only surviving copy of the first model's data after just two
+        // alternations — silent, permanent loss. A backup older than ttlDays holds
+        // only rows eviction would have deleted anyway, so time-based retention
+        // matches the store's own data-retention contract.
+        const ttlDays = this.options.ttlDays ?? 30;
+        const backupCutoff = Math.trunc(Date.now() - (ttlDays * 24 * 60 * 60 * 1000));
+        for (const { name, ts } of ranked.slice(0, -1)) {
+          if (ts >= backupCutoff) continue;
           const fullPath = path.join(this.options.dbDir, name);
           logger.info(`[store] Removing superseded backup directory: ${name}`);
           await fsPromises.rm(fullPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });

package/src/knowledge/writer-queue.ts

@@ -18,6 +18,9 @@ interface WriterQueueOptions {
     findByUrl: (url: string) => Promise<StoreDocument[]>;
     deleteByUrlAndType: (url: string, type: string, olderThan?: number) => Promise<void>;
     addDocuments: (docs: StoreDocument[]) => Promise<void>;
+    /** Revalidation touch for the content-unchanged skip (see that path). Optional
+     *  so lightweight test doubles need not implement it. */
+    touchByUrlAndType?: (url: string, type: string, generationTs: number, newTs: number) => Promise<void>;
   };
   chunker: Chunker | null;
 }
@@ -199,6 +202,31 @@ export class WriterQueue implements IWriterQueue {
             metrics.increment('knowledge_ingest_stale_prune_failed_total', 1);
           });
         }
+        // The content was just re-fetched and verified byte-identical — record
+        // that as freshness. Without this touch, a stable page's timestamp never
+        // moved: the serve-age gate treated it as permanently stale (live scrape
+        // every run) and TTL eviction eventually deleted a revalidated entry.
+        //
+        // Re-check the newest generation first: a CONCURRENT process may have
+        // ingested changed content for this url+type since our dedup read, and
+        // stamping our (now-old) generation with a later timestamp would make the
+        // stale rows outrank the genuinely newer ones. The touch predicate is
+        // pinned to `timestamp = newestTs`, so the residual window is only the
+        // gap between this re-read and the update — self-healing on the next
+        // real scrape either way.
+        if (this.options.store.touchByUrlAndType) {
+          try {
+            const recheck = (await this.options.store.findByUrl(normalizedUrl))
+              .filter(c => c.metadata['ingestionType'] === incomingType);
+            const stillNewest = recheck.length > 0
+              && Math.max(...recheck.map(d => Number(d.timestamp) || 0)) === newestTs;
+            if (stillNewest) {
+              await this.options.store.touchByUrlAndType(normalizedUrl, incomingType, newestTs, Date.now());
+            }
+          } catch (err) {
+            logger.warn(`[writer-queue] Timestamp refresh for unchanged ${normalizedUrl} (${incomingType}) failed:`, err);
+          }
+        }
         logger.log(`[writer-queue] Skipping ${normalizedUrl} (${incomingType}) — content unchanged.`);
         return;
       }

package/src/orchestration/research-orchestration-service.ts

@@ -14,7 +14,7 @@ import type { Model } from '@earendil-works/pi-ai';
 import { resolveResearchModel } from '../core/llm/research-model-resolver.ts';
 import type { QueryResultWithError } from '../web-research/types.ts';
 import type { RunResearchersOptions } from './orchestration-types.ts';
-import { RESEARCHER_LAUNCH_DELAY_MS, resolveExcludedTools } from '../constants.ts';
+import { RESEARCHER_LAUNCH_DELAY_MS, RESEARCH_TOOL_NAMES, resolveExcludedTools } from '../constants.ts';
 import { search } from '../web-research/search.ts';
 import { parseCitations } from '../utils/text-utils.ts';
 import { logger, resetLogger } from '../logger.ts';
@@ -98,6 +98,27 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
     const configDisabledTools = researchConfig.DISABLED_TOOLS
       ? researchConfig.DISABLED_TOOLS.split(',').map((s) => s.trim()).filter(Boolean)
       : [];
+    // Unknown names are a SILENT no-op downstream (the merge is a blind Set union),
+    // so a typo leaves the tool enabled for a whole billed run while the user
+    // believes it is off. `--exclude-tools` fails fast on this in the CLI parser;
+    // its two siblings — the SDK's `excludeTools` option and
+    // PI_RESEARCH_DISABLED_TOOLS — reach the same union with no check at all, and
+    // RESEARCH_TOOL_NAMES already documents itself as the vocabulary all THREE
+    // accept. Warn rather than throw here: this chokepoint is mid-run and serves a
+    // library API and an ambient env var, where aborting a legitimate request over
+    // one unrecognized name is the wrong trade — the CLI keeps its fail-fast.
+    for (const [surface, names] of [
+      ['excludeTools', excludeTools ?? []],
+      ['PI_RESEARCH_DISABLED_TOOLS', configDisabledTools],
+    ] as const) {
+      const unknown = names.filter((t) => !RESEARCH_TOOL_NAMES.includes(t));
+      if (unknown.length > 0) {
+        logger.warn(
+          `[ResearchOrchestrationService] ${surface}: ignoring unknown tool name${unknown.length === 1 ? '' : 's'} ` +
+          `${unknown.map((t) => `"${t}"`).join(', ')} — these exclude nothing. Valid names: ${RESEARCH_TOOL_NAMES.join(', ')}.`
+        );
+      }
+    }
     const mergedExcludeTools = resolveExcludedTools(excludeTools, configDisabledTools);
 
     // Resolve model using centralized priority logic
@@ -129,6 +150,19 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
       });
       logger.info(`[ResearchOrchestrationService] Acquired research run slot ${runSlot.slotIndex} (cap ${semaphore.getMaxSlots()}).`);
     } catch (err) {
+      const isTerminal =
+        err instanceof ResearchRunCapacityError || (err instanceof Error && err.name === 'AbortError');
+      if (isTerminal) {
+        // The queue notice above may already have announced this run to the
+        // observer (onRunQueued); rethrowing with no terminal event breaks the
+        // "exactly one of onComplete/onError" contract both orchestrators keep —
+        // an observer-driven consumer saw run_queued and then silence. No
+        // orchestrator exists yet, so this cannot double-fire. Observer throws
+        // stay isolated, as everywhere else.
+        try {
+          (observer as { onError?: (e: Error) => void } | undefined)?.onError?.(err as Error);
+        } catch { /* observer isolation */ }
+      }
       if (err instanceof ResearchRunCapacityError) throw err; // capacity exhausted → fail fast
       // A cancel while queueing for a slot must stay cancelled. Fail-open here would
       // start the very run the user just aborted.

package/src/orchestration/research-synthesis-service.ts

@@ -11,7 +11,7 @@
 
 import { parseCitations } from '../utils/text-utils.ts';
 import { normalizeCitations, formatCitedLinks, rewriteCitationMarkers, type GlobalCitation } from '../utils/citation-utils.ts';
-import { lastCitedLinksHeaderIndex } from '../utils/text-utils.ts';
+import { lastCitedLinksHeaderIndex, lastCitedLinksHeader } from '../utils/text-utils.ts';
 import { getScrapedLinks, isTranscribedLink, normalizeUrl } from '../utils/shared-links.ts';
 import { stripTrailingLlmPunctuation } from '../utils/url-utils.ts';
 import { logger } from '../logger.ts';
@@ -321,7 +321,9 @@ export class ResearchSynthesisService implements IService {
           if (globalId !== undefined) localToGlobal.set(localKey, globalId);
           else droppedNumbers.add(localKey);
         });
-        let body = synthesis.slice(0, citedHeaderIdx).trimEnd();
+        // lineStart (not the 'C' offset): slicing at textStart left the tolerated
+        // `##`/`**`/`>` marker dangling as a stray line at the end of the body.
+        let body = synthesis.slice(0, lastCitedLinksHeader(synthesis).lineStart).trimEnd();
         if (localToGlobal.size > 0 || droppedNumbers.size > 0) {
           body = rewriteCitationMarkers(body, localToGlobal, (n) => droppedNumbers.has(n));
         }

package/src/orchestration/researcher.ts

@@ -85,17 +85,6 @@ export async function createResearcherSession(options: CreateResearcherSessionOp
 
   logger.info(`[Researcher] Using model for researcher ${researcherId}: ${modelToUse.provider}/${modelToUse.id}`);
 
-  // The session sends model.maxTokens as the per-request max_tokens. Large-context
-  // catalog entries carry ~230k ceilings, and providers that budget-check max_tokens
-  // up front (OpenRouter 402s it against remaining credits) then reject every
-  // researcher call on a low balance even though the real response costs cents.
-  // Clamp the session's copy of the model to RESEARCHER_MAX_TOKENS; Model is plain
-  // catalog data, so a spread clone is safe and the shared registry entry is untouched.
-  const outputCap = config?.RESEARCHER_MAX_TOKENS ?? 16384;
-  const sessionModel = modelToUse.maxTokens > outputCap
-    ? { ...modelToUse, maxTokens: outputCap }
-    : modelToUse;
-
   try {
     const tracker = new ToolUsageTracker(createDefaultToolLimits(config));
 
@@ -149,7 +138,7 @@ export async function createResearcherSession(options: CreateResearcherSessionOp
       tools, // Explicit allowlist (BUG-1 fix)
       sessionManager: SessionManager.inMemory(), // Each researcher gets its own isolated session
       settingsManager: researcherSettings,
-      model: sessionModel,
+      model: modelToUse,
       // pi 0.80.8 replaced the SDK's `modelRegistry`/`authStorage` session options
       // with the async `modelRuntime`. Pass the runtime backing OUR registry when
       // this run built one (standalone CLI/SDK): it carries any explicit API key
@@ -196,7 +185,7 @@ export async function createResearcherSession(options: CreateResearcherSessionOp
 
       sessionRef.session = result.session;
 
-      return { session: result.session, resolvedModel: sessionModel };
+      return { session: result.session, resolvedModel: modelToUse };
     } catch (tailError) {
       const session = result?.session;
       if (session && typeof session.abort === 'function') {

package/src/prompts/system-coordinator.md

@@ -36,7 +36,7 @@ Before splitting the query into researchers, pin down its scope — exactly what
 
 1. **Plan ONLY Round 1**: Split the query into distinct sub-topics, all inside the scope above, each assigned to a researcher. **Do NOT plan multiple rounds in advance** — additional rounds are only delegated reactively when Round 1 findings reveal gaps.
 2. **Assign Goals**: Each researcher gets a focused goal covering a specific angle or time period.
-3. **Temporal Awareness**: Lead with the current month and year shown above. Generate time-sensitive queries using that exact month and year (e.g., append the current year and, where relevant, the current month; "latest", "current status"). Never anchor queries to an older year (e.g. 2024 or 2025) unless the user explicitly asks about the past. Research MUST be grounded in the present month and year.
+3. **Temporal Awareness**: Lead with the current month and year shown above. Generate time-sensitive queries using that exact month and year (e.g., append the current year and, where relevant, the current month; "latest", "current status"). Never anchor queries to an older year than the one shown above unless the user explicitly asks about the past. Research MUST be grounded in the present month and year.
 4. **Query Planning**: For EACH researcher, generate targeted, specific queries within the budget. Fill the budget when the topic is broad enough to warrant it, but don't pad queries for a narrow topic. Each round is planned independently — make Round 1 count.
 5. **YouTube Discovery**: For roughly **one in {{youtube_query_every_n}}** of each researcher's queries, append the word `youtube` to the query text (e.g. `"<topic> explained youtube"`). DuckDuckGo rarely surfaces YouTube on its own, and YouTube links let researchers read video transcripts. Skip this only when the topic is clearly unsuited to video (e.g. a single numeric lookup).
 6. **Seed Search Burst**: All queries across all researchers are combined into a single pre-search pass that seeds the global link pool.

package/src/prompts/system-lead-evaluator.md

@@ -74,7 +74,7 @@ Use unique, targeted queries for any new researchers.
 - **CRITICAL — Queries are mandatory**: Every researcher MUST have at least one query. Never plan a researcher without queries. Researchers receive ONLY the search results you delegate to them.
 - **Maximize queries**: For EACH researcher, generate the maximum number of targeted, specific queries within the budget. Do not plan fewer than needed — fill the budget. Queries should target primary sources and authoritative evidence.
 - **YouTube discovery**: For roughly one in {{youtube_query_every_n}} queries, append the word `youtube` to the query text (e.g. `"<topic> analysis youtube"`). DuckDuckGo rarely surfaces YouTube on its own, and YouTube links let researchers read video transcripts. Skip only when the gap is clearly unsuited to video.
-- **Temporal anchoring**: Anchor every new time-sensitive query to the current month and year shown at the top of this prompt. Use that exact year (and month where relevant); never default to an older year such as 2024 or 2025 unless the user explicitly asks about the past.
+- **Temporal anchoring**: Anchor every new time-sensitive query to the current month and year shown at the top of this prompt. Use that exact year (and month where relevant); never default to an older year than the one shown there unless the user explicitly asks about the past.
 - **Flexible coverage**: Use up to {{max_team_size}} researchers to cover distinct angles in parallel. Scale based on research gaps — a single well-targeted researcher is often sufficient for focused gaps.
 - **Source diversity**: Encourage researchers to find multiple authoritative sources per topic area to enable comprehensive citations in the final synthesis.
 

package/src/sdk.ts

@@ -44,7 +44,7 @@ import { runHealthCheck } from './healthcheck/index.ts';
 import { buildModelRegistry as sharedBuildModelRegistry, resolveModel } from './core/llm/model-registry-factory.ts';
 import { scrapeSingle } from './web-research/web-scraper.ts';
 import { validateInitialLinks } from './utils/url-utils.ts';
-import { validateUrlForSSRF } from './web-research/scraper-utils.ts';
+import { validateUrlForSSRF, disposeSsrfSafeFetcher } from './web-research/scraper-utils.ts';
 import type { ScrapeResult } from './core/interfaces/scheduler-interfaces.ts';
 import { randomUUID } from 'node:crypto';
 import type { ResearchDepth } from './types/index.ts';
@@ -954,6 +954,15 @@ async function _doShutdown(): Promise<void> {
     errors.push(err instanceof Error ? err : new Error(String(err)));
   }
 
+  // Release the module-level undici Agent's keep-alive sockets. Nothing else
+  // owns it, so a long-lived host would otherwise hold them past teardown.
+  try {
+    await disposeSsrfSafeFetcher();
+  } catch (err) {
+    logger.error('[SDK] Error disposing SSRF fetcher:', err);
+    errors.push(err instanceof Error ? err : new Error(String(err)));
+  }
+
   if (globalContainer) {
     // Dispose all services in one DAG-ordered teardown. disposeCoreServices first
     // clears the embedding-server state while the StateManager is still alive, then

package/src/security/nvd-client.ts

@@ -22,7 +22,6 @@ import type {
 } from './nvd-types.ts';
 import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-research/retry-utils.ts';
 import { CircuitBreaker } from '../utils/circuit-breaker.ts';
-import { safeUnref } from '../utils/safe-unref.ts';
 import { metrics } from '../utils/metrics.ts';
 import { readJsonCapped, BodyTooLargeError, MAX_BULK_JSON_BODY_BYTES } from '../utils/http-body.ts';
 import {
@@ -77,7 +76,15 @@ class NVDRateLimiter {
           signal?.removeEventListener('abort', onAbort);
           resolve();
         }, waitTime);
-        safeUnref(timeoutId);
+        // NOT unref'd. This timer IS the operation, not a watchdog racing one:
+        // nothing else will ever settle this promise, so unref'ing it let Node
+        // consider the process idle while an NVD search was mid-spacing. With no
+        // other ref'd handle the search simply never resumed — the awaited
+        // promise stayed unsettled and the caller got no result and no error.
+        // Every other unref in this codebase guards a timeout against real I/O
+        // that holds the loop open by itself; this one had nothing behind it.
+        // Bounded (≤6s, or ≤600ms with NVD_API_KEY) and abortable below, so
+        // keeping it ref'd cannot delay shutdown beyond one spacing interval.
         const onAbort = () => {
           clearTimeout(timeoutId as unknown as ReturnType<typeof setTimeout>);
           reject(Object.assign(new Error('NVD search aborted'), { name: 'AbortError' }));
@@ -392,6 +399,8 @@ async function fetchPaginated(
   const signal = options?.signal;
   let startIndex = 0;
   let totalPagesFetched = 0;
+  // Whether the one-time jump to the newest window (below) has been performed.
+  let seekedToNewest = false;
 
   while (totalPagesFetched < maxPages && allVulnerabilities.length < maxResults) {
     if (signal?.aborted) break; // stop paginating promptly on cancel
@@ -433,10 +442,40 @@ async function fetchPaginated(
     }
     metrics.increment('nvd_cache_hits_total', 1);
 
+    const responseData = data as Record<string, unknown> | undefined;
+    const reportedTotal = typeof responseData?.['totalResults'] === 'number'
+      ? responseData['totalResults'] as number
+      : undefined;
+
+    // NVD 2.0 returns keyword matches OLDEST FIRST and offers no sort parameter
+    // (`sortBy` is a 404), so paging from index 0 and stopping at maxResults kept
+    // precisely the least useful window: a `security_search` for "openssl"
+    // returned CVE-1999-0428 through 2004 out of 566 matches and could never
+    // reach the ones published this month. Seek once to the tail instead, so the
+    // rows we keep are the most recent. Only fires when the match set is larger
+    // than what we return — a query whose results all fit is already complete, so
+    // the common (and every mocked) case still costs exactly one request.
+    //
+    // Interaction with the `v2` supplemental pass (searchSingleTerm), deliberate:
+    // that pass admits only entries carrying NO v3/v4 metric, and NVD scores
+    // essentially every modern CVE with v3 — so v2-only rows live at the OLD end.
+    // Seeking to the tail therefore makes the supplement return little or nothing
+    // once a match set is large. That is the intended trade, not an oversight: the
+    // supplement was the channel feeding 1999-era advisories into results that are
+    // supposed to describe the current picture. A genuinely legacy product still
+    // works, because its own v2 match set is small enough that no seek fires.
+
+    if (!seekedToNewest && startIndex === 0 && reportedTotal !== undefined && reportedTotal > maxResults) {
+      seekedToNewest = true;
+      startIndex = reportedTotal - maxResults;
+      allVulnerabilities.length = 0; // the oldest window we just read is not what we want
+      totalPagesFetched++;
+      continue;
+    }
+
     allVulnerabilities.push(...admitted);
 
-    const responseData = data as Record<string, unknown> | undefined;
-    if (responseData?.['totalResults'] !== undefined && typeof responseData['totalResults'] === 'number' && startIndex + pageSize >= responseData['totalResults']) {
+    if (reportedTotal !== undefined && startIndex + pageSize >= reportedTotal) {
       break;
     }
 
@@ -447,6 +486,18 @@ async function fetchPaginated(
   metrics.increment('nvd_pagination_requests_total', totalPagesFetched);
   metrics.observe('nvd_pagination_pages_fetched', totalPagesFetched);
 
+  // Newest first within the window. NVD's own order is ascending, so without
+  // this the caller's slice() would again favour the older end of whatever we
+  // gathered. Undated entries sort last rather than jumping to the front.
+  allVulnerabilities.sort((a, b) => {
+    const ta = a.published ? Date.parse(a.published) : NaN;
+    const tb = b.published ? Date.parse(b.published) : NaN;
+    if (Number.isNaN(ta) && Number.isNaN(tb)) return 0;
+    if (Number.isNaN(ta)) return 1;
+    if (Number.isNaN(tb)) return -1;
+    return tb - ta;
+  });
+
   return allVulnerabilities.slice(0, maxResults);
 }
 

package/src/security/osv-client.ts

@@ -230,7 +230,19 @@ export async function searchOSV(
       }
     }
 
-    vulnerabilities.push(...Array.from(uniqueVulns.values()));
+    // Newest first, so the slice(0, maxResults) below keeps the advisories most
+    // likely to matter. OSV returns a package's vulnerabilities in ID order,
+    // which is effectively alphabetical — truncating that dropped whichever
+    // advisories happened to sort late, recency playing no part. Undated entries
+    // sort last rather than displacing dated ones.
+    vulnerabilities.push(...Array.from(uniqueVulns.values()).sort((a, b) => {
+      const ta = a.published ? Date.parse(a.published) : NaN;
+      const tb = b.published ? Date.parse(b.published) : NaN;
+      if (Number.isNaN(ta) && Number.isNaN(tb)) return 0;
+      if (Number.isNaN(ta)) return 1;
+      if (Number.isNaN(tb)) return -1;
+      return tb - ta;
+    }));
 
     if (failures.length > 0 && failures.length === terms.length) {
       throw new Error(`All OSV lookups failed: ${failures.join('; ')}`);

package/src/tools/research-knowledge-search.ts

@@ -114,6 +114,17 @@ export function formatQueriesBlock(queries: string[]): string {
 /** Widget ID for the knowledge search TUI panel */
 const KNOWLEDGE_WIDGET_ID = 'pi-research-knowledge-search';
 
+/**
+ * How many knowledge searches are currently showing the widget.
+ *
+ * The widget ID is a single global slot and `setWidget` replaces rather than
+ * stacks, but this tool is `executionMode: 'parallel'` and can also be driven
+ * concurrently by the `/knowledge-store` slash command. Without a refcount the
+ * first call to finish tears the widget down while a sibling is still running,
+ * leaving an in-flight search with no indicator. Only the last one out removes it.
+ */
+let knowledgeWidgetRefCount = 0;
+
 /**
  * System string returned when the answer is NOT found in the research
  * knowledge store. The exact phrasing is critical — the main pi agent
@@ -140,6 +151,7 @@ export const RESEARCH_KNOWLEDGE_MAYBE_STRING =
  */
 function showKnowledgeSearchWidget(ctx: ExtensionContext): void {
   if (ctx.mode !== 'tui' || !ctx.hasUI) return;
+  knowledgeWidgetRefCount++;
   try {
     const panelFactory = createKnowledgeSearchPanel();
     ctx.ui.setWidget(KNOWLEDGE_WIDGET_ID, panelFactory as any, { placement: 'aboveEditor' });
@@ -149,10 +161,15 @@ function showKnowledgeSearchWidget(ctx: ExtensionContext): void {
 }
 
 /**
- * Remove the knowledge search TUI widget.
+ * Remove the knowledge search TUI widget once the last concurrent search is done.
  */
 function hideKnowledgeSearchWidget(ctx: ExtensionContext): void {
   if (ctx.mode !== 'tui' || !ctx.hasUI) return;
+  // show() incremented under the identical guard, so the counter stays balanced.
+  // Clamp anyway: a stray hide must never drive it negative and wedge the
+  // widget on-screen for the rest of the session.
+  knowledgeWidgetRefCount = Math.max(0, knowledgeWidgetRefCount - 1);
+  if (knowledgeWidgetRefCount > 0) return;
   try {
     ctx.ui.setWidget(KNOWLEDGE_WIDGET_ID, undefined as any, { placement: 'aboveEditor' });
   } catch (err) {

package/src/tools/research-tool-definition.ts

@@ -28,7 +28,7 @@ import { redactSecrets } from '../utils/log-utils.ts';
 import { exportResearchReport, appendExportMessage } from '../utils/research-export.ts';
 import { validateAndSanitizeQuery } from '../utils/input-validation.ts';
 import { validateInitialLinks, MAX_INITIAL_LINKS, MAX_INITIAL_LINK_CHARS } from '../utils/url-utils.ts';
-import { startResearchSession, registerSessionAbort, clearSteeringMessages, getPiActivePanels } from '../orchestration/session-state.ts';
+import { startResearchSession, endResearchSession, registerSessionAbort, clearSteeringMessages, getPiActivePanels } from '../orchestration/session-state.ts';
 import { createResearchTuiManager, hideWorkingIndicator } from '../tui/research-tui-manager.ts';
 import { createCleanupFunction } from '../cleanup/research-cleanup.ts';
 import { createResearchObserver, createObserverState, stopObserverWaveAnimation } from '../observers/research-observer-impl.ts';
@@ -339,6 +339,13 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
             registerSessionAbort(piSessionId, sessionResearchId, internalAbort);
 
             cleanup = async () => {
+              // Free what startResearchSession/registerSessionAbort created — the
+              // failures/failureReasons entries and the AbortController live in the
+              // module-level piSessions map keyed by the STABLE pi session id, so
+              // without this every headless run leaked them for the process
+              // lifetime (the TUI branch frees them via createCleanupFunction, the
+              // SDK in its own finally; this branch freed only steering).
+              endResearchSession(piSessionId, sessionResearchId);
               clearSteeringMessages(piSessionId);
             };
             

package/src/utils/citation-utils.ts

@@ -1,4 +1,4 @@
-import { parseCitations, lastCitedLinksHeaderIndex, type Citation } from './text-utils.ts';
+import { parseCitations, lastCitedLinksHeader, type Citation } from './text-utils.ts';
 import { normalizeUrl } from './shared-links.ts';
 
 export interface GlobalCitation extends Citation {
@@ -66,7 +66,9 @@ export function normalizeCitations(reports: ReadonlyMap<string, string>): {
     // line-leading, all-caps header, never an incidental "cited links" mention in
     // prose (an unanchored split there would drop real findings before the
     // evaluator ever sees them).
-    const headerIdx = lastCitedLinksHeaderIndex(report);
+    // lineStart (not textStart): slicing at the 'C' left the tolerated `##`/`**`/`>`
+    // marker dangling at the end of the kept body.
+    const headerIdx = lastCitedLinksHeader(report).lineStart;
     let content = headerIdx >= 0 ? report.slice(0, headerIdx) : report;
 
     // Renumber local [N] markers to their global ids, and DELETE the ones that could

package/src/utils/log-utils.ts

@@ -133,7 +133,22 @@ const KNOWN_TOKEN_PATTERN = new RegExp(
   'g',
 );
 // JSON Web Tokens — header.payload.signature, each a base64url segment.
-const JWT_PATTERN = new RegExp(`${B_LEFT}eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+`, 'g');
+//
+// The segment lengths are bounded for the same load-bearing reason the URL
+// scheme above is, and this pattern was missed when that one was fixed. `.` is
+// NOT in the segment class, so on a run of base64url characters containing no
+// dot, `[A-Za-z0-9_-]+` consumes to the end of the run and then backtracks one
+// character at a time looking for one — and `_`/`-` count as non-alphanumeric,
+// so B_LEFT admits a fresh start position every few characters inside that same
+// run. `_eyJ` repeated across the 40_000-character scan window measured 393ms of
+// blocked event loop per log message; bounding the segments brings the identical
+// input to 41ms (~10x) with no loss of coverage. The bounds are far above any
+// real token: a 1024-char header covers even an x5c certificate chain, and 8192
+// covers payloads no bearer credential approaches.
+const JWT_PATTERN = new RegExp(
+  `${B_LEFT}eyJ[A-Za-z0-9_-]{1,1024}\\.[A-Za-z0-9_-]{1,8192}\\.[A-Za-z0-9_-]{1,4096}`,
+  'g',
+);
 // HTTP Basic credentials: "Basic <base64>".
 const BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, 'gi');
 // HTTP Bearer credentials: "Bearer <token>". The token is often opaque — many

package/src/utils/text-utils.ts

@@ -253,17 +253,31 @@ function isPlausibleCitationUrl(url: string): boolean {
  * against the `CITED LINKS\n...` block that formatCitedLinks() emits.
  */
 export function lastCitedLinksHeaderIndex(text: string): number {
+  return lastCitedLinksHeader(text).textStart;
+}
+
+/**
+ * Both offsets of the last CITED LINKS header: `textStart` points at the 'C'
+ * (what parseCitations slices FROM), `lineStart` points at the start of the
+ * header line BEFORE any `#`/`>`/`**` markers (what body-replacement slices TO).
+ * Slicing the body at textStart left the tolerated markdown marker dangling at
+ * the end of the body — the rebuilt report then carried a stray `##` or `**`
+ * line whenever the model wrote `## CITED LINKS` instead of the plain form.
+ */
+export function lastCitedLinksHeader(text: string): { lineStart: number; textStart: number } {
   // Case-insensitive but LINE-LEADING: a mid-prose "...the cited links below..." has
   // non-whitespace before it on the line, so it cannot match — only a header at the
   // start of a line (after optional markdown markers) does.
   const re = /(^|\n)([ \t]*(?:#{1,6}[ \t]*|>[ \t]*|\*\*)?)CITED LINKS\b/gi;
-  let last = -1;
+  let lineStart = -1;
+  let textStart = -1;
   let m: RegExpExecArray | null;
   while ((m = re.exec(text)) !== null) {
+    lineStart = m.index + m[1]!.length;
     // Offset of the 'C' in CITED LINKS = match start + leading newline + markers.
-    last = m.index + m[1]!.length + m[2]!.length;
+    textStart = m.index + m[1]!.length + m[2]!.length;
   }
-  return last;
+  return { lineStart, textStart };
 }
 
 /**