@secureport/core 0.2.1 → 0.4.0

package/src/reconcile.ts

@@ -1,32 +1,436 @@
-import type { Issue } from './issue.js';
+import type { Finding } from './finding.js';
+import type { Issue, IssueEvent } from './issue.js';
+import type { Run, RunKind, RunSummary, SeverityCounts } from './run.js';
+import {
+  DEFAULT_SLA_POLICY,
+  SEVERITY_ORDER,
+  SEVERITY_WEIGHTS,
+  severityRank,
+  slaDueAt,
+  type SlaPolicy,
+} from './severity.js';
+import { coversLocation } from './coverage.js';
 
-/** A raw scanner result, not yet folded into the tracked issue set. */
-export interface Finding {
-  title: string;
-  severity: Issue['severity'];
+/**
+ * Everything reconciliation needs, and nothing it could get for itself.
+ *
+ * The clock and the id generator are arguments because the function is pure:
+ * no `Date.now()`, no `randomUUID()`, no I/O. The same inputs give the same
+ * issues and the same events, on any machine, which is what makes golden-file
+ * testing possible and what lets a run be replayed.
+ */
+export interface ReconcileInput {
+  /**
+   * Every known issue for this run's target.
+   *
+   * Issues for other targets must not be passed in: the fingerprint already
+   * includes the target, so they could never match, and including them would
+   * return them as untouched output and invite a caller to write them back.
+   */
+  readonly issues: readonly Issue[];
+
+  /** The findings this run produced. */
+  readonly findings: readonly Finding[];
+
+  /** The run itself, which supplies the target, the kind and the coverage. */
+  readonly run: Run;
+
+  /** The moment the run is being reconciled at. */
+  readonly now: Date;
+
+  /**
+   * Supplies ids for issues and events.
+   *
+   * Injected rather than generated so the function stays deterministic. A
+   * caller in production passes `randomUUID`; a test passes a counter.
+   */
+  readonly newId: () => string;
+
+  /**
+   * Who to record as having made these changes.
+   *
+   * Defaults to `reconciler`. An audit trail that cannot say who is not one.
+   */
+  readonly actor?: string;
+
+  /** Remediation windows, for recomputing `slaDueAt`. */
+  readonly slaPolicy?: SlaPolicy;
+
+  /**
+   * How many consecutive covering runs may miss an issue before it resolves.
+   *
+   * **The flapping guard.** Defaults to **1 for uploads** — the person
+   * uploading is asserting a complete result — and **2 for scans**, where a
+   * timeout or a rate limit causes a miss that means nothing. Between the first
+   * miss and resolution the issue stays open, and reports can say "not seen in
+   * last run".
+   *
+   * Per organisation in the hosted product; an argument here, because this
+   * function has nowhere to read configuration from.
+   */
+  readonly autoResolveThreshold?: number;
+
+  /** How long the run took, for the summary. */
+  readonly durationMs?: number;
+}
+
+/**
+ * What reconciliation decided.
+ *
+ * Returns **every** issue for the target, changed or not, so a caller can write
+ * the set back without working out which ones moved. Events are returned rather
+ * than applied: persisting them is the caller's transaction, and reconciliation
+ * has no I/O.
+ */
+export interface ReconcileResult {
+  /** Every issue for the target, after reconciliation. */
+  readonly issues: readonly Issue[];
+
+  /** What changed, in the order it was decided. */
+  readonly events: readonly IssueEvent[];
+
+  /** What the run did, in the terms a report opens with. */
+  readonly summary: RunSummary;
 }
 
 /**
- * Merges incoming findings into the tracked issue set. A finding whose title
- * matches an open issue is folded into it; anything new opens an issue.
+ * The key an issue and a finding must share to be the same issue.
  *
- * Pure: no I/O, no clock, no randomness. Callers supply the id for any issue
- * this opens via `nextId`.
+ * **The fingerprint version is part of it, not decoration.** A fingerprint means
+ * nothing without the algorithm that produced it, so an `fp_v1` finding must
+ * never match an `fp_v2` issue — reconciling across versions would silently
+ * merge two things that were never computed the same way. Crossing a version is
+ * what a re-fingerprint migration is for.
+ *
+ * The separator is an escape rather than a literal control character, so the
+ * source stays plain text: an invisible NUL here once passed prettier, eslint,
+ * tsc and the whole test suite, and was only caught because `grep` went quiet
+ * on a file it could no longer read.
  */
-export function reconcile(findings: Finding[], openIssues: Issue[], nextId: () => string): Issue[] {
-  const byTitle = new Map(openIssues.map((issue) => [issue.title, issue]));
+function issueKey(of: {
+  readonly fingerprintVersion: string;
+  readonly fingerprint: string;
+}): string {
+  return `${of.fingerprintVersion}\u0000${of.fingerprint}`;
+}
 
+/** Groups this run's findings by the issue they belong to. */
+function groupByFingerprint(findings: readonly Finding[]): Map<string, Finding[]> {
+  const groups = new Map<string, Finding[]>();
   for (const finding of findings) {
-    const existing = byTitle.get(finding.title);
+    // The version is part of the key. A fingerprint means nothing without the
+    // algorithm that produced it, so a `fp_v1` finding must never match a
+    // `fp_v2` issue — that is what a re-fingerprint migration is for.
+    const key = issueKey(finding);
+    const existing = groups.get(key);
+    if (existing) existing.push(finding);
+    else groups.set(key, [finding]);
+  }
+  return groups;
+}
+
+/** The most urgent severity among a group of findings for one issue. */
+function highestSeverity(findings: readonly Finding[]): Finding {
+  return findings.reduce((worst, candidate) =>
+    severityRank(candidate.detectedSeverity) > severityRank(worst.detectedSeverity)
+      ? candidate
+      : worst,
+  );
+}
+
+/** The default flapping guard for a run kind. */
+function defaultThreshold(kind: RunKind): number {
+  // An upload asserts a complete result, so one miss is enough. A scan can miss
+  // for reasons that say nothing about the issue.
+  return kind === 'upload' ? 1 : 2;
+}
+
+/** An all-zero count, so a report never has to decide what a missing key means. */
+function emptyCounts(): Record<string, number> {
+  return Object.fromEntries(SEVERITY_ORDER.map((s) => [s, 0]));
+}
+
+/** Whole days between two moments, floored. */
+function daysBetween(from: Date, to: Date): number {
+  return Math.max(0, Math.floor((to.getTime() - from.getTime()) / 86_400_000));
+}
+
+/**
+ * Turns a run's findings into issue state.
+ *
+ * **The only writer of issue state, and a pure function.** Given the same
+ * issues, findings, run and clock it produces the same result every time. It
+ * performs no I/O, reads no clock and generates no randomness; the caller
+ * supplies `now` and `newId` and persists what comes back.
+ *
+ * This half handles everything a **present** finding causes:
+ *
+ * - a fingerprint that matches nothing opens an issue (`created`);
+ * - a fingerprint that matches attaches evidence and refreshes `lastSeen`,
+ *   clearing the miss counter (`seen`);
+ * - a `resolved` issue seen again becomes `regressed` (`reopened`) — a returning
+ *   problem is a different and more interesting event than a new one;
+ * - a changed detected severity updates the issue, and updates the *effective*
+ *   severity and the SLA deadline only where nobody has overridden it
+ *   (`severity_detected_changed`);
+ * - an `ignored` issue whose detected severity has risen above what was
+ *   accepted comes back (`unignored`) — accepting the risk of a medium is not
+ *   accepting the risk of the critical it turned out to be.
+ *
+ * And everything an **absent** finding causes:
+ *
+ * - an issue the run covered but did not see takes a miss, and resolves once it
+ *   has missed enough consecutive covering runs (`resolved`);
+ * - an issue **outside** the run's coverage is untouched — not resolved, and not
+ *   even counted as a miss, so a sequence of narrow scans cannot accumulate
+ *   misses against a path none of them looked at (invariant 5);
+ * - a **manual-origin** issue never auto-resolves; a human closes what a human
+ *   opened (invariant 6);
+ * - a suppression whose expiry has passed lapses (`unignored`), regardless of
+ *   coverage — an ignore expiring is a decision timing out, not an observation.
+ *
+ * Finally it computes the {@link RunSummary}: counts by severity for new, still
+ * open, resolved, regressed and ignored, plus the exposure score. Ignored issues
+ * are excluded from every count but their own, and from exposure, and never from
+ * the suppressed appendix (invariant 7).
+ *
+ * `firstSeen` is never reset, by anything here (invariant 4).
+ *
+ * @param input - Issues, findings, the run, the clock and an id source.
+ * @returns Every issue for the target, plus the events that explain the changes.
+ */
+export function reconcile(input: ReconcileInput): ReconcileResult {
+  const { issues, findings, run, now, newId } = input;
+  const actor = input.actor ?? 'reconciler';
+  const slaPolicy = input.slaPolicy ?? DEFAULT_SLA_POLICY;
+
+  const events: IssueEvent[] = [];
+  const emit = (issueId: string, type: IssueEvent['type'], payload?: Record<string, unknown>) => {
+    events.push({
+      id: newId(),
+      orgId: run.orgId,
+      issueId,
+      runId: run.id,
+      type,
+      actor,
+      ...(payload === undefined ? {} : { payload }),
+      createdAt: now,
+    });
+  };
+
+  const byKey = new Map(issues.map((i) => [issueKey(i), i]));
+  const updated = new Map<string, Issue>();
+
+  for (const [key, group] of groupByFingerprint(findings)) {
+    const existing = byKey.get(key);
+    const worst = highestSeverity(group);
+    const findingIds = group.map((f) => f.id);
+
     if (!existing) {
-      byTitle.set(finding.title, {
-        id: nextId(),
-        title: finding.title,
-        severity: finding.severity,
+      const id = newId();
+      const issue: Issue = {
+        id,
+        orgId: run.orgId,
+        targetId: run.targetId,
+        fingerprint: worst.fingerprint,
+        fingerprintVersion: worst.fingerprintVersion,
+        title: worst.title,
+        vulnKey: worst.vulnKey,
+        ...(worst.cwe === undefined ? {} : { cwe: worst.cwe }),
+        location: worst.location,
+        ...(worst.parameter === undefined ? {} : { parameter: worst.parameter }),
         status: 'open',
+        detectedSeverity: worst.detectedSeverity,
+        effectiveSeverity: worst.detectedSeverity,
+        firstSeen: now,
+        lastSeen: now,
+        consecutiveMisses: 0,
+        origin: run.kind,
+        slaDueAt: slaDueAt(worst.detectedSeverity, now, slaPolicy),
+      };
+      updated.set(key, issue);
+      emit(id, 'created', { findingIds, severity: worst.detectedSeverity });
+      continue;
+    }
+
+    // One `seen` per issue per run, not one per finding. `issue_events` is what
+    // every analytic counts, and a weakness two engines both detected would
+    // otherwise register as two sightings of one issue.
+    let issue: Issue = { ...existing, lastSeen: now, consecutiveMisses: 0 };
+    emit(issue.id, 'seen', { findingIds });
+
+    if (issue.status === 'resolved') {
+      issue = { ...issue, status: 'regressed', reopenedAt: now };
+      emit(issue.id, 'reopened', { resolvedAt: existing.resolvedAt?.toISOString() });
+    }
+
+    if (worst.detectedSeverity !== issue.detectedSeverity) {
+      const overridden = issue.severityOverriddenAt !== undefined;
+      const raisedAboveOverride =
+        overridden && severityRank(worst.detectedSeverity) > severityRank(issue.effectiveSeverity);
+
+      emit(issue.id, 'severity_detected_changed', {
+        from: issue.detectedSeverity,
+        to: worst.detectedSeverity,
+        // "Flag for review" in the spec, which names no column to flag it in.
+        // Recorded on the event rather than invented as a field: the detected
+        // severity has risen above a human's override, and somebody should look.
+        needsReview: raisedAboveOverride,
       });
+
+      issue = overridden
+        ? { ...issue, detectedSeverity: worst.detectedSeverity }
+        : {
+            ...issue,
+            detectedSeverity: worst.detectedSeverity,
+            effectiveSeverity: worst.detectedSeverity,
+            slaDueAt: slaDueAt(worst.detectedSeverity, issue.firstSeen, slaPolicy),
+          };
+    }
+
+    if (
+      issue.status === 'ignored' &&
+      issue.severityAtIgnore !== undefined &&
+      severityRank(worst.detectedSeverity) > severityRank(issue.severityAtIgnore)
+    ) {
+      emit(issue.id, 'unignored', {
+        reason: 'severity increased',
+        from: issue.severityAtIgnore,
+        to: worst.detectedSeverity,
+      });
+      issue = {
+        ...issue,
+        status: 'open',
+        ignoreReason: undefined,
+        ignoreComment: undefined,
+        ignoreScope: undefined,
+        ignoreExpiresAt: undefined,
+        ignoredBy: undefined,
+        severityAtIgnore: undefined,
+      };
+    }
+
+    updated.set(key, issue);
+  }
+
+  const created = new Set([...updated.keys()].filter((k) => !byKey.has(k)));
+  const regressedNow = new Set(
+    [...updated].filter(([k, i]) => !created.has(k) && i.status === 'regressed').map(([k]) => k),
+  );
+  const resolvedNow = new Set<string>();
+
+  // ---- what an absent finding causes -------------------------------------
+  //
+  // A run resolves only what it could have found. An issue outside coverage is
+  // left entirely alone: not resolved, and not counted as a miss either, so a
+  // sequence of narrow scans cannot accumulate misses against a path none of
+  // them looked at.
+  const threshold = input.autoResolveThreshold ?? defaultThreshold(run.kind);
+
+  for (const original of issues) {
+    const key = issueKey(original);
+    if (updated.has(key)) continue; // seen this run
+    if (original.status !== 'open' && original.status !== 'regressed') continue;
+    // A human closes what a human opened (invariant 6).
+    if (original.origin === 'manual') continue;
+    if (!coversLocation(original.location, run.coverage)) continue;
+
+    const misses = original.consecutiveMisses + 1;
+    if (misses < threshold) {
+      updated.set(key, { ...original, consecutiveMisses: misses });
+      continue;
     }
+
+    updated.set(key, {
+      ...original,
+      consecutiveMisses: misses,
+      status: 'resolved',
+      resolvedAt: now,
+    });
+    resolvedNow.add(key);
+    emit(original.id, 'resolved', { consecutiveMisses: misses, threshold });
   }
 
-  return [...byTitle.values()];
+  // ---- suppressions that have lapsed --------------------------------------
+  //
+  // Independent of coverage and of whether the issue was seen: an ignore
+  // expiring is a decision timing out, not an observation.
+  for (const original of issues) {
+    const key = issueKey(original);
+    const current = updated.get(key) ?? original;
+    if (current.status !== 'ignored') continue;
+    if (current.ignoreExpiresAt === undefined) continue;
+    if (current.ignoreExpiresAt.getTime() > now.getTime()) continue;
+
+    emit(current.id, 'unignored', {
+      reason: 'expired',
+      expiredAt: current.ignoreExpiresAt.toISOString(),
+    });
+    updated.set(key, {
+      ...current,
+      status: 'open',
+      ignoreReason: undefined,
+      ignoreComment: undefined,
+      ignoreScope: undefined,
+      ignoreExpiresAt: undefined,
+      ignoredBy: undefined,
+      severityAtIgnore: undefined,
+    });
+  }
+
+  // Untouched issues are returned unchanged, in their original order, so a
+  // caller can write the whole set back without diffing.
+  const result = issues.map((i) => updated.get(issueKey(i)) ?? i);
+  for (const [key, issue] of updated) if (!byKey.has(key)) result.push(issue);
+
+  // ---- the summary --------------------------------------------------------
+  const counts = {
+    new: emptyCounts(),
+    stillOpen: emptyCounts(),
+    resolved: emptyCounts(),
+    regressed: emptyCounts(),
+    ignored: emptyCounts(),
+  };
+  let exposureScore = 0;
+
+  for (const issue of result) {
+    const key = issueKey(issue);
+    const severity = issue.effectiveSeverity;
+
+    if (issue.status === 'ignored') {
+      // Excluded from every other count and from exposure, never from the
+      // suppressed appendix (invariant 7).
+      counts.ignored[severity]++;
+      continue;
+    }
+
+    if (created.has(key)) counts.new[severity]++;
+    else if (resolvedNow.has(key)) counts.resolved[severity]++;
+    else if (regressedNow.has(key)) counts.regressed[severity]++;
+    else if (updated.has(key)) counts.stillOpen[severity]++;
+
+    if (issue.status === 'open' || issue.status === 'regressed') {
+      exposureScore += SEVERITY_WEIGHTS[severity] * Math.min(daysBetween(issue.firstSeen, now), 90);
+    }
+  }
+
+  const summary: RunSummary = {
+    runId: run.id,
+    orgId: run.orgId,
+    targetId: run.targetId,
+    kind: run.kind,
+    trigger: run.trigger,
+    engines: run.coverage.engines,
+    coverage: run.coverage,
+    durationMs: input.durationMs ?? 0,
+    new: counts.new as SeverityCounts,
+    stillOpen: counts.stillOpen as SeverityCounts,
+    resolved: counts.resolved as SeverityCounts,
+    regressed: counts.regressed as SeverityCounts,
+    ignored: counts.ignored as SeverityCounts,
+    exposureScore,
+    createdAt: now,
+  };
+
+  return { issues: result, events, summary };
 }