@lincoln504/pi-research 0.1.13 → 1.0.1

package/src/security/nvd-types.ts

@@ -0,0 +1,179 @@
+/**
+ * NVD (National Vulnerability Database) API Types
+ */
+
+export interface CVSSData {
+  readonly version?: string;
+  readonly vectorString?: string;
+  readonly baseScore?: number;
+  readonly baseSeverity?: string;
+}
+
+export interface CVSSMetricV31 {
+  readonly cvssData: CVSSData;
+}
+
+export interface CVSSMetricV30 {
+  readonly cvssData: CVSSData;
+}
+
+export interface WeaknessDescription {
+  readonly value?: string;
+}
+
+export interface Weakness {
+  readonly description?: WeaknessDescription[];
+}
+
+export interface Reference {
+  readonly url?: string;
+}
+
+export interface CPEMatch {
+  readonly criteria?: string;
+}
+
+export interface Node {
+  readonly cpeMatch?: CPEMatch[];
+}
+
+export interface Configuration {
+  readonly nodes?: Node[];
+}
+
+export interface Description {
+  readonly value?: string;
+}
+
+// CVSS v4.0 mirrors v3 (score + qualitative severity live in cvssData). NVD is actively adding it.
+export interface CVSSMetricV40 {
+  readonly cvssData: CVSSData;
+}
+
+// CVSS v2 differs: cvssData carries the numeric baseScore/vectorString but NO qualitative rating —
+// NVD exposes the v2 baseSeverity (LOW/MEDIUM/HIGH) at the METRIC level instead.
+export interface CVSSMetricV2 {
+  readonly cvssData: CVSSData;
+  readonly baseSeverity?: string;
+}
+
+export interface Metrics {
+  readonly cvssMetricV40?: CVSSMetricV40[];
+  readonly cvssMetricV31?: CVSSMetricV31[];
+  readonly cvssMetricV30?: CVSSMetricV30[];
+  readonly cvssMetricV2?: CVSSMetricV2[];
+}
+
+export interface CVE {
+  readonly id?: string;
+  readonly descriptions?: Description[];
+  readonly published?: string;
+  readonly lastModified?: string;
+  readonly metrics?: Metrics;
+  readonly weaknesses?: Weakness[];
+  readonly references?: Reference[];
+  readonly configurations?: Configuration[];
+}
+
+export interface NVDEntry {
+  readonly cve: CVE;
+}
+
+export interface NVDApiResponse {
+  readonly vulnerabilities?: NVDEntry[];
+}
+
+export type Severity = 'LOW' | 'MEDIUM' | 'HIGH' | 'CRITICAL';
+
+export interface SearchOptions {
+  readonly severity?: Severity;
+  readonly maxResults?: number;
+  readonly includeExploited?: boolean;
+  readonly cweId?: string;
+  readonly startDate?: string;
+  readonly endDate?: string;
+  readonly maxPages?: number;
+  // Caller cancellation; composed with the per-request timeout and used to
+  // short-circuit the rate-limit spacing, pagination, and retry backoff.
+  readonly signal?: AbortSignal;
+}
+
+export interface RetryOptions {
+  readonly maxRetries: number;
+  readonly initialDelay: number;
+  readonly maxDelay: number;
+  readonly signal?: AbortSignal;
+}
+
+// Type guards
+export function isWeaknessDescription(value: unknown): value is WeaknessDescription {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'value' in value
+  );
+}
+
+export function isWeakness(value: unknown): value is Weakness {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'description' in value
+  );
+}
+
+export function isReference(value: unknown): value is Reference {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'url' in value
+  );
+}
+
+export function isCPEMatch(value: unknown): value is CPEMatch {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'criteria' in value
+  );
+}
+
+export function isNode(value: unknown): value is Node {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'cpeMatch' in value
+  );
+}
+
+export function isConfiguration(value: unknown): value is Configuration {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'nodes' in value
+  );
+}
+
+export function isDescription(value: unknown): value is Description {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'value' in value
+  );
+}
+
+export function isNVDEntry(value: unknown): value is NVDEntry {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'cve' in value
+  );
+}
+
+export function isNVDApiResponse(value: unknown): value is NVDApiResponse {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'vulnerabilities' in value
+  );
+}

package/src/security/osv-client.ts

@@ -0,0 +1,246 @@
+/**
+ * OSV (Open Source Vulnerabilities) API Client
+ *
+ * Open Source Vulnerabilities
+ * API: https://api.osv.dev/v1/
+ * Free public API, no authentication required
+ */
+
+import type { Vulnerability, OSVResult } from './types.ts';
+import type { OsvVulnerability, OsvQueryRequest } from './osv-types.ts';
+import { normalizeEcosystem } from './ecosystem.ts';
+import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-research/retry-utils.ts';
+import { logger } from '../logger.ts';
+import { OSV_TIMEOUT_MS, DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY_MS, DEFAULT_MAX_DELAY_MS } from '../constants.ts';
+import { CircuitBreaker } from '../utils/circuit-breaker.ts';
+import { metrics } from '../utils/metrics.ts';
+import {
+  isOsvVulnerability,
+  isOsvQueryResponse,
+  mapOsvItemToVulnerability,
+} from './osv-types.ts';
+
+const osvCircuitBreaker = new CircuitBreaker({
+  failureThreshold: 5,
+  resetTimeoutMs: 10000,
+  name: 'OSV API',
+  isTransientError: isTransientError
+});
+
+const OSV_BASE_URL = 'https://api.osv.dev/v1';
+const DEFAULT_MAX_RESULTS = 20;
+
+/**
+ * Fetch with retry for resilience
+ */
+async function fetchWithRetry(
+  url: string,
+  options: RequestInit,
+  signal?: AbortSignal,
+): Promise<Response> {
+  const endpoint = new URL(url).pathname;
+  return osvCircuitBreaker.execute(async () => {
+    return retryWithBackoff(
+      async () => {
+        const response = await fetch(url, options);
+        if (!response.ok) {
+          const error = new Error(`HTTP ${response.status}: ${response.statusText}`) as Error & { status?: number };
+          error.status = response.status;
+          metrics.increment('osv_errors_total', 1, { endpoint, status_code: response.status.toString() });
+          if (response.status === 429) {
+            metrics.increment('osv_ratelimit_hits_total', 1, { endpoint });
+          }
+          throw error;
+        }
+        metrics.increment('osv_requests_total', 1, { endpoint, status: 'success' });
+        return response;
+      },
+      {
+        maxRetries: DEFAULT_MAX_RETRIES,
+        initialDelay: DEFAULT_INITIAL_DELAY_MS,
+        maxDelay: DEFAULT_MAX_DELAY_MS,
+        label: `OSV API: ${url}`,
+        signal,
+        isTransientError: (error) => {
+          const status = (error as Error & { status?: number })?.status;
+          if (typeof status === 'number') {
+            return status === 429 || status === 403 || status >= 500;
+          }
+          return isTransientError(error);
+        },
+      },
+    );
+  });
+}
+
+/**
+ * Search OSV for vulnerabilities
+ *
+ * @param terms - Package names, CVE IDs, or keywords
+ * @param options - Optional filters
+ * @returns OSV search result with vulnerabilities and optional error
+ */
+export async function searchOSV(
+  terms: string[],
+  options?: {
+    ecosystem?: string;
+    severity?: string;
+    maxResults?: number;
+    includeAffected?: boolean;
+    signal?: AbortSignal;
+  },
+): Promise<OSVResult> {
+  const startTime = Date.now();
+  const maxResults: number = options?.maxResults ?? DEFAULT_MAX_RESULTS;
+  const vulnerabilities: Vulnerability[] = [];
+  let error: string | undefined = undefined;
+
+  try {
+    // Each term is fetched and settled independently so one failed term (a
+    // transient 5xx, a malformed body, an exhausted retry) never discards the
+    // vulnerabilities already gathered for the others. `skipped` flags a package
+    // term that needs the ecosystem parameter.
+    const fetchTerm = async (term: string): Promise<{ vulns: Vulnerability[]; skipped: boolean }> => {
+      const termUpper: string = term.toUpperCase();
+      const isIdLookup =
+        termUpper.startsWith('CVE-') || termUpper.startsWith('GHSA-') || termUpper.startsWith('OSV-');
+      let response: Response;
+
+      if (isIdLookup) {
+        const normalizedId: string = termUpper.startsWith('GHSA-')
+          ? `GHSA-${term.slice(term.indexOf('-') + 1).toLowerCase()}`
+          : termUpper;
+        const url: string = `${OSV_BASE_URL}/vulns/${encodeURIComponent(normalizedId)}`;
+        response = await fetchWithRetry(url, {
+          headers: { 'User-Agent': 'pi-research/2.0', 'Accept': 'application/json' },
+          signal: createTimeoutSignal(OSV_TIMEOUT_MS, options?.signal),
+        }, options?.signal);
+      } else {
+        if (options?.ecosystem === undefined || options.ecosystem === '') {
+          return { vulns: [], skipped: true };
+        }
+        const body: OsvQueryRequest = { package: { name: term, ecosystem: normalizeEcosystem(options.ecosystem, 'osv') } };
+        response = await fetchWithRetry(`${OSV_BASE_URL}/query`, {
+          method: 'POST',
+          headers: {
+            'User-Agent': 'pi-research/2.0',
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(body),
+          signal: createTimeoutSignal(OSV_TIMEOUT_MS, options?.signal),
+        }, options?.signal);
+      }
+
+      let data: unknown;
+      try {
+        data = await response.json();
+      } catch {
+        metrics.increment('osv_search_errors_total', 1, { error_type: 'malformed_json' });
+        throw new Error(`OSV returned a malformed JSON response for "${term}".`);
+      }
+
+      let items: OsvVulnerability[];
+      if (isOsvVulnerability(data)) {
+        items = [data];
+      } else if (isOsvQueryResponse(data)) {
+        items = data.vulns ?? [];
+      } else {
+        logger.warn(`OSV returned unexpected format for "${term}"`);
+        return { vulns: [], skipped: false };
+      }
+
+      const endpoint = isIdLookup ? 'vulns_by_id' : 'query';
+      metrics.increment(items.length > 0 ? 'osv_cache_hits_total' : 'osv_cache_misses_total', 1, { term, endpoint });
+
+      const vulns: Vulnerability[] = [];
+      for (const item of items) {
+        // Withdrawn/retracted advisories are no longer valid vulnerabilities — skip them so they
+        // are not reported as active findings nor counted toward the researcher grounding gate.
+        if (item.withdrawn) continue;
+        const vuln: Vulnerability = mapOsvItemToVulnerability(item);
+        if (options?.severity !== undefined && options.severity !== '') {
+          // OSV/GHSA severity vocabulary is LOW/MODERATE/HIGH/CRITICAL. Accept the
+          // common "MEDIUM" synonym for MODERATE (GitHub maps it the same way at
+          // github-advisories.ts) — otherwise a severity:"medium" filter silently
+          // drops every OSV result.
+          // mapOsvItemToVulnerability canonicalizes severity to LOW/MEDIUM/HIGH/CRITICAL (both the
+          // OSV "MODERATE" and "MEDIUM" spellings map to 'MEDIUM'). Normalize the caller's filter
+          // into the SAME vocabulary — MODERATE→MEDIUM — before comparing. The previous direction
+          // (MEDIUM→MODERATE) compared against a value vuln.severity never holds, silently dropping
+          // every medium-severity OSV result (the most common filter value).
+          const want = options.severity.toUpperCase();
+          const normalized = want === 'MODERATE' ? 'MEDIUM' : want;
+          if (vuln.severity === 'UNKNOWN') {
+            // We could not rank this record. If it nonetheless carries a CVSS vector we
+            // don't score — mapOsvItemToVulnerability only computes CVSS v3, so a v4-only
+            // (increasingly common) or v2-only record stays UNKNOWN — it IS a scored
+            // vulnerability whose number we simply can't compute. Silently dropping it
+            // under a severity filter would hide a possible HIGH/CRITICAL, so surface it
+            // instead (the safe bias for security triage: a false-include beats a hidden
+            // finding). A record with NO severity signal at all stays filtered out so
+            // results are not flooded with genuinely unranked noise.
+            const hasUnscoredVector =
+              (Array.isArray(item.severity) && item.severity.length > 0) ||
+              (Array.isArray(item.affected) &&
+                item.affected.some((a) => Array.isArray(a.severity) && a.severity.length > 0));
+            if (!hasUnscoredVector) continue;
+            metrics.increment('osv_unscored_included_total', 1, { filter: normalized });
+          } else if (vuln.severity !== normalized) {
+            continue;
+          }
+        }
+        vulns.push(vuln);
+      }
+      return { vulns, skipped: false };
+    };
+
+    const settled = await Promise.allSettled(terms.map(fetchTerm));
+    const allVulns: Vulnerability[] = [];
+    let skippedNoEcosystem = 0;
+    const failures: string[] = [];
+    settled.forEach((result, i) => {
+      if (result.status === 'fulfilled') {
+        allVulns.push(...result.value.vulns);
+        if (result.value.skipped) skippedNoEcosystem++;
+      } else {
+        const reason = result.reason instanceof Error ? result.reason.message : String(result.reason);
+        // A 404 on an ID lookup means "no such vuln", not a real failure.
+        if (!reason.includes('HTTP 404')) failures.push(`${terms[i]} (${reason})`);
+      }
+    });
+
+    const uniqueVulns = new Map<string, Vulnerability>();
+    for (const vuln of allVulns) {
+      if (!uniqueVulns.has(vuln.id)) {
+        uniqueVulns.set(vuln.id, vuln);
+      }
+    }
+
+    vulnerabilities.push(...Array.from(uniqueVulns.values()));
+
+    if (failures.length > 0 && failures.length === terms.length) {
+      throw new Error(`All OSV lookups failed: ${failures.join('; ')}`);
+    }
+    const notes: string[] = [];
+    if (failures.length > 0) notes.push(`Some OSV lookups failed: ${failures.join('; ')}`);
+    if (skippedNoEcosystem > 0) {
+      notes.push(`${skippedNoEcosystem} term(s) require the ecosystem parameter for OSV package search (e.g., ecosystem: "npm"). CVE/GHSA/OSV IDs work without it.`);
+    }
+    if (notes.length > 0) error = notes.join(' ');
+
+  } catch (err: unknown) {
+    error = err instanceof Error ? err.message : String(err);
+    metrics.increment('osv_search_errors_total', 1, { error_type: err instanceof Error ? err.name : 'unknown' });
+  } finally {
+    const duration = Date.now() - startTime;
+    metrics.observe('osv_search_duration_ms', duration, { has_error: error ? 'true' : 'false' });
+  }
+
+  return {
+    count: vulnerabilities.length,
+    vulnerabilities: vulnerabilities.slice(0, maxResults),
+    error,
+  };
+}
+

package/src/security/osv-types.ts

@@ -0,0 +1,285 @@
+/**
+ * OSV (Open Source Vulnerabilities) API Types
+ */
+
+import type { Vulnerability } from './types.ts';
+import { scoreCvss3 } from './cvss.ts';
+
+/**
+ * OSV API vulnerability response (from GET /vulns/{id})
+ */
+export interface OsvVulnerability {
+  id: string;
+  summary?: string;
+  details?: string;
+  published?: string;
+  modified?: string;
+  withdrawn?: string;
+  aliases?: string[];
+  related?: string[];
+  affected?: OsvAffected[];
+  severity?: OsvSeverity[];
+  references?: OsvReference[];
+  database_specific?: OsvDatabaseSpecific;
+}
+
+/**
+ * OSV severity entry
+ */
+export interface OsvSeverity {
+  type: string;
+  score: string;
+}
+
+/**
+ * OSV database-specific metadata
+ */
+export interface OsvDatabaseSpecific {
+  severity?: string;
+  cwe?: Array<{ id: string } | string>;
+  [key: string]: unknown;
+}
+
+/**
+ * OSV affected package entry
+ */
+export interface OsvAffected {
+  package?: OsvPackage;
+  ranges?: OsvRange[];
+  versions?: string[];
+  database_specific?: Record<string, unknown>;
+  severity?: OsvSeverity[];
+}
+
+/**
+ * OSV package information
+ */
+export interface OsvPackage {
+  name?: string;
+  ecosystem?: string;
+  purl?: string;
+  ecosystem_specific?: {
+    name?: string;
+    [key: string]: unknown;
+  };
+  [key: string]: unknown;
+}
+
+/**
+ * OSV version range information
+ */
+export interface OsvRange {
+  type: string;
+  repo?: string;
+  events: OsvRangeEvent[];
+  database_specific?: Record<string, unknown>;
+}
+
+/**
+ * OSV range event (introduced, fixed, last_affected, limits)
+ */
+export interface OsvRangeEvent {
+  introduced?: string;
+  fixed?: string;
+  last_affected?: string;
+  limit?: string;
+}
+
+/**
+ * OSV reference entry
+ */
+export interface OsvReference {
+  type?: string;
+  url?: string;
+}
+
+/**
+ * OSV query request body
+ */
+export interface OsvQueryRequest {
+  package: {
+    name: string;
+    ecosystem: string;
+    purl?: string;
+  };
+  version?: string;
+  commit?: string;
+}
+
+/**
+ * OSV query response (from POST /query)
+ */
+export interface OsvQueryResponse {
+  vulns?: OsvVulnerability[];
+  [key: string]: unknown;
+}
+
+// Type guards
+export function isOsvVulnerability(value: unknown): value is OsvVulnerability {
+  if (value === null || typeof value !== 'object') {
+    return false;
+  }
+  const obj = value as Record<string, unknown>;
+  return typeof obj['id'] === 'string';
+}
+
+export function isOsvQueryResponse(value: unknown): value is OsvQueryResponse {
+  if (value === null || typeof value !== 'object') {
+    return false;
+  }
+  const obj = value as Record<string, unknown>;
+  return Array.isArray(obj['vulns']);
+}
+
+// Helper Functions
+export function mapOsvItemToVulnerability(item: OsvVulnerability): Vulnerability {
+  const osvId: string = item.id;
+  const summary: string = item.summary ?? item.details ?? '';
+  const details: string = item.details ?? '';
+  const published: string | undefined = item.published;
+  const modified: string = item.modified ?? '';
+
+  let severityStr: string | undefined;
+  if (item.database_specific !== undefined &&
+      typeof item.database_specific.severity === 'string') {
+    severityStr = item.database_specific.severity;
+  }
+  let severity: string = mapOsvSeverity(severityStr);
+
+  // When OSV has no database_specific.severity (non-GHSA records: CVE/PyPA/Go/RustSec), the only
+  // severity signal is a CVSS vector in item.severity[]. Compute the v3 base score/severity from it
+  // rather than reporting UNKNOWN with no score.
+  let cvssScore: number | undefined;
+  let cvssVector: string | undefined;
+  if (severity === 'UNKNOWN' && Array.isArray(item.severity)) {
+    for (const sev of item.severity) {
+      if (sev && typeof sev.score === 'string' && /^CVSS_V3/i.test(sev.type ?? '')) {
+        const scored = scoreCvss3(sev.score);
+        if (scored) {
+          severity = scored.severity === 'NONE' ? 'UNKNOWN' : scored.severity;
+          cvssScore = scored.score;
+          cvssVector = sev.score;
+          break;
+        }
+      }
+    }
+  }
+
+  const aliases: string[] = item.aliases ?? [];
+  const references: string[] = [...aliases];
+
+  if (item.references !== undefined) {
+    for (const ref of item.references) {
+      if (ref.url !== undefined && ref.url !== '') {
+        references.push(ref.url);
+      }
+    }
+  }
+
+  const affectedProducts: string[] = [];
+  const fixes: string[] = [];
+
+  if (item.affected !== undefined) {
+    for (const affectedEntry of item.affected) {
+      const pkg: OsvPackage | undefined = affectedEntry.package;
+
+      if (pkg !== undefined) {
+        const pkgName: string = pkg.name ??
+                                (pkg.ecosystem_specific !== undefined && typeof pkg.ecosystem_specific.name === 'string'
+                                  ? pkg.ecosystem_specific.name
+                                  : '');
+
+        if (pkgName !== '') {
+          affectedProducts.push(pkgName);
+        }
+
+        if (affectedEntry.ranges !== undefined) {
+          for (const range of affectedEntry.ranges) {
+            // Guard against a malformed range missing `events` (upstream validation only
+            // checks the vuln `id`): an unguarded events.find() would throw a TypeError,
+            // fail the whole term in Promise.allSettled, and discard all its vulns.
+            const events: OsvRangeEvent[] = Array.isArray(range.events) ? range.events : [];
+
+            const introducedEvent: OsvRangeEvent | undefined = events.find(
+              (e: OsvRangeEvent) => e.introduced !== undefined,
+            );
+            const fixedEvent: OsvRangeEvent | undefined = events.find(
+              (e: OsvRangeEvent) => e.fixed !== undefined,
+            );
+            const lastAffectedEvent: OsvRangeEvent | undefined = events.find(
+              (e: OsvRangeEvent) => e.last_affected !== undefined,
+            );
+
+            const versionInfo: string[] = [];
+
+            if (introducedEvent?.introduced !== undefined) {
+              versionInfo.push(`introduced: ${introducedEvent.introduced}`);
+            }
+            if (fixedEvent?.fixed !== undefined) {
+              versionInfo.push(`fixed: ${fixedEvent.fixed}`);
+            }
+            if (lastAffectedEvent?.last_affected !== undefined) {
+              versionInfo.push(`last affected: ${lastAffectedEvent.last_affected}`);
+            }
+
+            if (versionInfo.length > 0 && pkgName !== '') {
+              fixes.push(`${pkgName}: ${versionInfo.join(', ')}`);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  const cwes: string[] = [];
+  if (item.database_specific !== undefined) {
+    if (item.database_specific.cwe !== undefined && Array.isArray(item.database_specific.cwe)) {
+      for (const cwe of item.database_specific.cwe) {
+        if (typeof cwe === 'string') {
+          cwes.push(cwe);
+        } else if (typeof cwe === 'object' && typeof (cwe as { id?: string }).id === 'string') {
+          cwes.push((cwe as { id: string }).id);
+        }
+      }
+    }
+  }
+
+  return {
+    id: osvId,
+    source: 'osv',
+    severity,
+    description: summary !== '' ? summary : details,
+    published,
+    modified,
+    cvssScore,
+    cvssVector,
+    cwes,
+    references,
+    affectedProducts,
+    fixes,
+    knownExploited: false,
+  };
+}
+
+function mapOsvSeverity(severity?: string): string {
+  if (severity === undefined || severity === '') {
+    return 'UNKNOWN';
+  }
+
+  const upper: string = severity.toUpperCase();
+
+  if (upper === 'CRITICAL') {
+    return 'CRITICAL';
+  }
+  if (upper === 'HIGH') {
+    return 'HIGH';
+  }
+  if (upper === 'MEDIUM' || upper === 'MODERATE') {
+    return 'MEDIUM';
+  }
+  if (upper === 'LOW') {
+    return 'LOW';
+  }
+
+  return 'UNKNOWN';
+}