@secureport/core 0.2.1 → 0.4.0

package/src/report/model.ts

@@ -0,0 +1,462 @@
+import type { RunSummary } from '../run.js';
+import type { Severity } from '../severity.js';
+import { SEVERITY_ORDER } from '../severity.js';
+import type { Snapshot, SnapshotIssue } from '../snapshot.js';
+
+/**
+ * Which report to render.
+ *
+ * - `pen` — Penetration Test. Every issue in full, with its evidence.
+ * - `vap` — Vulnerability Assessment. The same issues, tabulated.
+ * - `exec` — Executive Summary. What changed and what it means, no technical
+ *   detail, for a reader who will not open the others.
+ * - `attest` — Attestation Letter. A formal statement that testing was carried
+ *   out, with its limitations stated as prominently as its findings.
+ * - `retest` — Retest. The difference between two runs: what was fixed, what is
+ *   still there, and what came back. The only kind that requires a baseline.
+ *
+ * **They are presentations of one snapshot, not four analyses.** Anything that
+ * differs between them is layout or level of detail; a number that differed
+ * would be a bug.
+ */
+export type ReportKind = 'pen' | 'vap' | 'exec' | 'attest' | 'retest';
+
+/** What a caller can say about a report that the snapshot does not know. */
+export interface ReportOptions {
+  /** Which report to render. */
+  readonly kind: ReportKind;
+
+  /** Overrides the default title. */
+  readonly title?: string;
+
+  /** Who prepared it. White-labelling is a branding option, not a report type. */
+  readonly preparedBy?: string;
+
+  /** Who it is for. */
+  readonly preparedFor?: string;
+
+  /**
+   * When it was rendered.
+   *
+   * An argument, like everywhere else in this package: a report re-rendered
+   * from the same snapshot must come out identical, or it cannot be compared
+   * with the copy somebody was sent.
+   */
+  readonly now: Date;
+}
+
+/**
+ * How the findings in a report were actually produced.
+ *
+ * **This exists because of the honesty rule in `00-DOMAIN.md` §7.** That rule
+ * is written about the Attestation Letter, but a "Penetration Test" report
+ * built entirely from automated scanner output is the same over-claim: it is
+ * the fastest way to lose an auditor's trust, and with it the compliance
+ * business.
+ *
+ * So the basis is derived from the data rather than asserted by the template. A
+ * report over `scan` runs says automated; one that includes findings a person
+ * recorded says so; one over an upload says the results were brought from
+ * elsewhere. Nobody has to remember to be honest.
+ *
+ * It also carries the advice that automated findings **should be verified by a
+ * qualified technician before the report is submitted as audit evidence**. That
+ * is the practical form of the same rule: a scanner's output is a starting
+ * point for an assessment, not the assessment, and saying so protects both the
+ * reader and whoever hands them the document.
+ *
+ * The method is described; **the engines are not named**. Which scanner found
+ * something is an implementation detail, and naming the stack in a
+ * customer-facing report gives away more than it explains.
+ */
+export interface TestingBasis {
+  /** Whether any finding came from a human rather than an engine. */
+  readonly includesManual: boolean;
+
+  /** Whether Secureport originated the traffic. */
+  readonly automated: boolean;
+
+  /** Whether results were brought from another tool. */
+  readonly uploaded: boolean;
+
+  /**
+   * The engines involved, in the order the run recorded them.
+   *
+   * **Available, but deliberately not printed.** Which scanner produced a
+   * finding is an implementation detail of the assessment, not a fact the
+   * reader needs, and naming the stack in a customer-facing document gives
+   * away more than it explains. The prose describes the *method*; this field is
+   * here for a consumer that has a reason to know.
+   */
+  readonly engines: readonly string[];
+
+  /** One sentence, ready to print. */
+  readonly statement: string;
+}
+
+/** A group of issues sharing a severity, most urgent first. */
+export interface ReportSection {
+  /** The severity this section covers. */
+  readonly severity: Severity;
+
+  /** Its issues, most recently seen first. */
+  readonly issues: readonly SnapshotIssue[];
+}
+
+/** Everything a renderer needs, derived once so no two formats disagree. */
+export interface ReportModel {
+  /** Which report this is. */
+  readonly kind: ReportKind;
+
+  /** The title as printed. */
+  readonly title: string;
+
+  /** The snapshot it was built from. */
+  readonly snapshot: Snapshot;
+
+  /** When it was rendered. */
+  readonly generatedAt: Date;
+
+  /** Who prepared it, if stated. */
+  readonly preparedBy?: string;
+
+  /** Who it is for, if stated. */
+  readonly preparedFor?: string;
+
+  /** How the findings were produced. */
+  readonly basis: TestingBasis;
+
+  /**
+   * Outstanding issues grouped by severity, most urgent first, empties dropped.
+   *
+   * **Resolved issues are not here.** A fixed issue is not a finding — it is
+   * evidence of remediation, and listing it under "Findings" makes a report
+   * read as though the work is still outstanding. It has its own section.
+   */
+  readonly sections: readonly ReportSection[];
+
+  /**
+   * Issues this run resolved, most recently seen first.
+   *
+   * Kept and shown rather than dropped: "what you fixed" is the story the
+   * product exists to tell, and a report that silently omits it throws away its
+   * best evidence.
+   */
+  readonly resolved: readonly SnapshotIssue[];
+
+  /** Open and regressed issues, by severity. What the reader owes work on. */
+  readonly outstanding: Readonly<Record<Severity, number>>;
+
+  /** Total outstanding, so a header does not have to sum a table. */
+  readonly outstandingTotal: number;
+
+  /** Exposure at the end of the run. */
+  readonly exposureScore: number;
+
+  /** Issues past their remediation deadline. */
+  readonly breached: number;
+
+  /**
+   * What this report does not establish.
+   *
+   * **Stated as prominently as the findings, and not optional.** An attestation
+   * that lists what was found and stays quiet about what it cannot support is
+   * the over-claim `00-DOMAIN.md` §7 warns about — and it is the fastest way to
+   * lose an auditor's trust, and with it the compliance business.
+   *
+   * Derived, so the limitations match the report rather than being boilerplate
+   * somebody forgot to update: a report over automated results says so, a
+   * partial scope says so, and none of them claims compliance with a standard
+   * nothing here assessed.
+   */
+  readonly limitations: readonly string[];
+
+  /**
+   * What this run established about the issues carried into it.
+   *
+   * Present whenever the snapshot has a baseline, whatever the kind. Absent on
+   * a first run, where there is nothing to compare against.
+   */
+  readonly retest?: RetestOutcome;
+}
+
+/**
+ * What a retest established about one issue.
+ *
+ * - `fixed` — carried in from the baseline and no longer detected by a run that
+ *   covered it. The only verdict that is evidence of remediation.
+ * - `still_present` — detected again by this run.
+ * - `returned` — was resolved and this run found it again.
+ * - `not_retested` — **not detected, and not fixed either.** The run did not
+ *   cover it, or has not missed it often enough to resolve it. A retest that
+ *   quietly counted these as fixed would be reporting a coverage gap as
+ *   remediation, which is the worst thing this document could do.
+ * - `new` — this run's own finding, not part of what was being retested.
+ */
+export type RetestVerdict = 'fixed' | 'still_present' | 'returned' | 'not_retested' | 'new';
+
+/** One issue, with what the retest established about it. */
+export interface RetestEntry {
+  /** The issue as the snapshot sees it. */
+  readonly issue: SnapshotIssue;
+
+  /** What this run established about it. */
+  readonly verdict: RetestVerdict;
+}
+
+/**
+ * The difference between two runs.
+ *
+ * Present whenever the snapshot has a baseline, whatever the report kind — it
+ * is a fact about the data, not a feature of one template. The `retest` report
+ * requires it; the others may use it.
+ */
+export interface RetestOutcome {
+  /** The run being retested against. */
+  readonly baseline: RunSummary;
+
+  /** Every issue, most urgent first, with its verdict. */
+  readonly entries: readonly RetestEntry[];
+
+  /** How many issues fell into each verdict. */
+  readonly counts: Readonly<Record<RetestVerdict, number>>;
+
+  /**
+   * Fixed as a proportion of what could actually be checked, 0–1.
+   *
+   * **`not_retested` is excluded from the denominator, not counted as a
+   * failure.** An issue nobody looked at is neither fixed nor unfixed, and
+   * folding it either way turns a coverage gap into a number somebody quotes.
+   * `null` when there was nothing to check.
+   */
+  readonly fixRate: number | null;
+}
+
+/** Works out what a run established about the issues carried into it. */
+function describeRetest(snapshot: Snapshot): RetestOutcome | undefined {
+  const { baseline } = snapshot;
+  if (baseline === undefined) return undefined;
+
+  const entries: RetestEntry[] = snapshot.issues.map((issue) => ({
+    issue,
+    verdict: verdictFor(issue),
+  }));
+  // `SEVERITY_ORDER` rather than `severityRank`, which counts the other way:
+  // it returns 4 for `critical` so that a larger number is a worse problem.
+  // Sorting on it ascending put the advisories at the top of the table, which
+  // is how this shipped until a golden file showed it.
+  entries.sort((a, b) => {
+    const bySeverity =
+      SEVERITY_ORDER.indexOf(a.issue.issue.effectiveSeverity) -
+      SEVERITY_ORDER.indexOf(b.issue.issue.effectiveSeverity);
+    return bySeverity === 0
+      ? b.issue.issue.lastSeen.getTime() - a.issue.issue.lastSeen.getTime()
+      : bySeverity;
+  });
+
+  const counts: Record<RetestVerdict, number> = {
+    fixed: 0,
+    still_present: 0,
+    returned: 0,
+    not_retested: 0,
+    new: 0,
+  };
+  for (const entry of entries) counts[entry.verdict]++;
+
+  const checkable = counts.fixed + counts.still_present + counts.returned;
+  return {
+    baseline,
+    entries,
+    counts,
+    fixRate: checkable === 0 ? null : counts.fixed / checkable,
+  };
+}
+
+/**
+ * The verdict for one issue.
+ *
+ * Turns on whether this run produced evidence for it, not on its status alone:
+ * an issue left open because the run never covered it looks identical to one
+ * left open because the run found it again, and telling a reader those are the
+ * same thing is the failure this report exists to avoid.
+ */
+function verdictFor(entry: SnapshotIssue): RetestVerdict {
+  if (entry.change === 'resolved') return 'fixed';
+  if (entry.change === 'regressed') return 'returned';
+  if (entry.change === 'new') return 'new';
+  return entry.findings.length > 0 ? 'still_present' : 'not_retested';
+}
+
+/** Works out what a report cannot establish, from the run rather than a template. */
+function describeLimitations(snapshot: Snapshot, basis: TestingBasis): string[] {
+  const limitations: string[] = [];
+
+  if (!basis.includesManual) {
+    limitations.push(
+      'This is evidence of automated security testing. It is not a human-led penetration ' +
+        'test and does not carry the assurance of one.',
+    );
+  }
+  limitations.push(
+    'Findings should be verified by a qualified technician before this document is relied ' +
+      'upon as audit evidence.',
+  );
+  limitations.push(
+    `Testing covered only ${snapshot.run.coverage.paths.join(', ')}. Anything outside that ` +
+      'was not examined, and its absence from this document is not evidence that it is sound.',
+  );
+  limitations.push(
+    `This reflects the state of the target as of ${snapshot.run.createdAt
+      .toISOString()
+      .slice(0, 10)}. It says nothing about the target before or after that date.`,
+  );
+  limitations.push(
+    'Automated testing cannot establish the absence of a vulnerability. A clean result means ' +
+      'nothing was detected, not that nothing is there.',
+  );
+  if (snapshot.suppressed.length > 0) {
+    limitations.push(
+      `${snapshot.suppressed.length} finding${snapshot.suppressed.length === 1 ? ' has' : 's have'} been ` +
+        'suppressed and excluded from the counts above. They are listed in full in the appendix.',
+    );
+  }
+  limitations.push(
+    'This document does not certify compliance with any standard, framework or regulation.',
+  );
+
+  return limitations;
+}
+
+const DEFAULT_TITLES: Readonly<Record<ReportKind, string>> = Object.freeze({
+  pen: 'Penetration Test Report',
+  vap: 'Vulnerability Assessment Report',
+  exec: 'Executive Summary',
+  attest: 'Attestation of Security Testing',
+  retest: 'Retest Report',
+});
+
+/** Works out how the findings were produced, from the run rather than a claim. */
+function describeBasis(snapshot: Snapshot): TestingBasis {
+  const kind = snapshot.run.kind;
+  const engines = snapshot.run.engines;
+  const includesManual = snapshot.issues.some((i) => i.issue.origin === 'manual');
+  const automated = kind === 'scan';
+  const uploaded = kind === 'upload';
+
+  const parts: string[] = [];
+  if (automated) {
+    parts.push(
+      'Findings were produced by automated vulnerability scanning against the scope described below.',
+    );
+  }
+  if (uploaded) {
+    parts.push(
+      'Findings were imported from an external testing tool and reconciled against previous runs.',
+    );
+  }
+  if (kind === 'manual') parts.push('Findings were recorded manually by a tester.');
+  if (includesManual && kind !== 'manual') {
+    parts.push('Some findings were recorded manually by a tester.');
+  }
+  if (!includesManual && kind !== 'manual') {
+    parts.push('No part of this assessment constitutes a manual penetration test.');
+  }
+  if (automated || uploaded) {
+    parts.push(
+      'Automated findings should be verified by a qualified technician before this report is ' +
+        'submitted as audit evidence.',
+    );
+  }
+
+  return { includesManual, automated, uploaded, engines, statement: parts.join(' ') };
+}
+
+/**
+ * Derives everything a report shows, once, from a snapshot.
+ *
+ * Both renderers consume this rather than the snapshot, so a number in the
+ * Markdown and the same number in the HTML cannot drift apart — they are the
+ * same value formatted twice.
+ *
+ * Suppressed issues are already separated by the snapshot and stay separated
+ * here: they never reach {@link ReportModel.sections}, are excluded from
+ * {@link ReportModel.outstanding} and from the exposure score, and appear only
+ * in the suppressed appendix (invariant 7).
+ *
+ * @param snapshot - Issue state as of a run.
+ * @param options - Title, attribution, and the render time.
+ * @returns The model both renderers format.
+ */
+export function buildReportModel(snapshot: Snapshot, options: ReportOptions): ReportModel {
+  const outstanding: Record<Severity, number> = {
+    critical: 0,
+    high: 0,
+    medium: 0,
+    low: 0,
+    advisory: 0,
+  };
+
+  const bySeverity = new Map<Severity, SnapshotIssue[]>();
+  const resolved: SnapshotIssue[] = [];
+  let breached = 0;
+
+  for (const entry of snapshot.issues) {
+    if (entry.issue.status === 'resolved') {
+      resolved.push(entry);
+      continue;
+    }
+
+    const severity = entry.issue.effectiveSeverity;
+    const group = bySeverity.get(severity);
+    if (group) group.push(entry);
+    else bySeverity.set(severity, [entry]);
+
+    if (entry.issue.status === 'open' || entry.issue.status === 'regressed') {
+      outstanding[severity]++;
+      if (entry.slaStatus === 'breached') breached++;
+    }
+  }
+  resolved.sort((a, b) => b.issue.lastSeen.getTime() - a.issue.lastSeen.getTime());
+
+  const sections: ReportSection[] = [];
+  for (const severity of SEVERITY_ORDER) {
+    const issues = bySeverity.get(severity);
+    if (issues === undefined || issues.length === 0) continue;
+    sections.push({
+      severity,
+      // Most recently seen first: what a reader wants at the top of a section
+      // is what the latest run actually found.
+      issues: [...issues].sort((a, b) => b.issue.lastSeen.getTime() - a.issue.lastSeen.getTime()),
+    });
+  }
+
+  const outstandingTotal = SEVERITY_ORDER.reduce((sum, s) => sum + outstanding[s], 0);
+  const basis = describeBasis(snapshot);
+  const retest = describeRetest(snapshot);
+
+  // Rendering a retest of nothing is the over-claim §7 warns about: the
+  // document's whole content is a comparison, and one run does not have one.
+  if (options.kind === 'retest' && retest === undefined) {
+    throw new TypeError(
+      'a retest report needs a baseline: build the snapshot with `previous` set to the run being retested',
+    );
+  }
+
+  return {
+    kind: options.kind,
+    title: options.title ?? DEFAULT_TITLES[options.kind],
+    snapshot,
+    generatedAt: options.now,
+    ...(options.preparedBy === undefined ? {} : { preparedBy: options.preparedBy }),
+    ...(options.preparedFor === undefined ? {} : { preparedFor: options.preparedFor }),
+    basis,
+    limitations: describeLimitations(snapshot, basis),
+    sections,
+    resolved,
+    outstanding,
+    outstandingTotal,
+    exposureScore: snapshot.run.exposureScore,
+    breached,
+    ...(retest === undefined ? {} : { retest }),
+  };
+}

package/src/run.ts

@@ -0,0 +1,163 @@
+import type { Severity } from './severity.js';
+
+/**
+ * What kind of run this was.
+ *
+ * - `scan` — Secureport originated the traffic.
+ * - `upload` — someone brought results from elsewhere.
+ * - `manual` — a human recorded a finding directly.
+ *
+ * **An upload is first-class, not a lesser scan.** Much of the value is in
+ * tracking results a customer already has, and a model that treated uploads as
+ * second-class would make that path feel second-class too.
+ */
+export type RunKind = 'scan' | 'upload' | 'manual';
+
+/**
+ * What set the run going.
+ *
+ * Recorded on every run from the first line of code, because it is what lets
+ * you say "the Action ran 340 times this quarter" — the sentence that shows
+ * automation is actually being used rather than merely installed.
+ */
+export type RunTrigger = 'github_action' | 'cli' | 'api' | 'scheduled' | 'web';
+
+/**
+ * What a run actually exercised.
+ *
+ * The reason auto-resolution is safe. **A run only resolves what it could have
+ * found** (invariant 5): a quick profile that skipped `/admin` must not close
+ * an `/admin` issue, because not looking is not the same as not finding.
+ *
+ * Scan runs record what they genuinely reached. Upload runs take a declared
+ * scope, defaulting to the whole target — the person uploading is asserting
+ * what their results cover.
+ */
+export interface Coverage {
+  /**
+   * Location patterns the run covered, as globs, e.g.
+   * `https://app.example.com/**`.
+   *
+   * An issue whose location matches none of these is untouched by this run,
+   * whatever else happened.
+   */
+  readonly paths: readonly string[];
+
+  /** Ports exercised, where the run was port-aware. */
+  readonly ports?: readonly number[];
+
+  /** Engines that took part. */
+  readonly engines: readonly string[];
+}
+
+/**
+ * One execution against a target.
+ */
+export interface Run {
+  /** Unique id. */
+  readonly id: string;
+
+  /** Organisation this run belongs to. */
+  readonly orgId: string;
+
+  /** Target it ran against. */
+  readonly targetId: string;
+
+  /** What kind of run it was. */
+  readonly kind: RunKind;
+
+  /** What set it going. */
+  readonly trigger: RunTrigger;
+
+  /** What it exercised. */
+  readonly coverage: Coverage;
+
+  /**
+   * The verification this run relied on, for runs that originate traffic.
+   *
+   * Recorded per run rather than per organisation, because verification
+   * belongs to a target: verifying production must never authorise scanning
+   * staging, even on the same wildcard domain (invariant 11).
+   */
+  readonly verificationId?: string;
+
+  /** When it started. */
+  readonly startedAt: Date;
+
+  /** When it finished. */
+  readonly finishedAt?: Date;
+}
+
+/**
+ * A count of issues per {@link Severity}.
+ *
+ * Every severity is present, including zeroes, so a report renders a complete
+ * table without deciding what a missing key means.
+ */
+export type SeverityCounts = Readonly<Record<Severity, number>>;
+
+/**
+ * What a run did, in the terms a reader cares about.
+ *
+ * The change summary every report opens with, and the row analytics aggregate.
+ * Computed once by reconciliation and stored, so no report has to recount
+ * findings at render time.
+ */
+export interface RunSummary {
+  /** The run this summarises. */
+  readonly runId: string;
+
+  /** Organisation it belongs to. */
+  readonly orgId: string;
+
+  /** Target it ran against. */
+  readonly targetId: string;
+
+  /** What kind of run it was. */
+  readonly kind: RunKind;
+
+  /** What set it going. */
+  readonly trigger: RunTrigger;
+
+  /** Engines that took part. */
+  readonly engines: readonly string[];
+
+  /** What the run exercised. */
+  readonly coverage: Coverage;
+
+  /** How long it took. */
+  readonly durationMs: number;
+
+  /** Issues opened for the first time by this run. */
+  readonly new: SeverityCounts;
+
+  /** Issues already open that this run saw again. */
+  readonly stillOpen: SeverityCounts;
+
+  /** Issues this run resolved. */
+  readonly resolved: SeverityCounts;
+
+  /** Issues that were resolved and that this run found again. */
+  readonly regressed: SeverityCounts;
+
+  /**
+   * Issues currently suppressed.
+   *
+   * Reported separately and **never folded into the other counts**: an ignored
+   * issue is excluded from metrics but never from the suppressed appendix
+   * (invariant 7).
+   */
+  readonly ignored: SeverityCounts;
+
+  /**
+   * Exposure at the end of this run.
+   *
+   * `Σ over open issues of severityWeight × min(daysOpen, 90)`. Simple,
+   * explainable and monotone — it can only fall by fixing things or by time not
+   * passing.
+   */
+  readonly exposureScore: number;
+
+  /** When the summary was computed. */
+  readonly createdAt: Date;
+}