@secureport/core 1.0.0 → 2.1.0

package/src/refingerprint.ts

@@ -0,0 +1,282 @@
+import type { Finding } from './finding.js';
+import type { FingerprintInput, VulnKeyInput } from './fingerprint.js';
+import { FINGERPRINT_VERSION, fingerprint, vulnKey } from './fingerprint.js';
+import type { Issue } from './issue.js';
+
+/**
+ * How to compute a fingerprint. Injected so the dry run can be pointed at a
+ * version that does not exist yet.
+ *
+ * **`@secureport/core` only ever contains one algorithm** — the current one,
+ * named by `FINGERPRINT_VERSION`. So on the day a version bump is being
+ * considered, the way to find out what it would do is to hand this planner
+ * the candidate implementation. Against the shipped algorithm the plan is a
+ * no-op by construction, which is exactly what makes it safe to run at any
+ * time and what lets the tests drive real merges and splits without a
+ * released `fp_v2`.
+ */
+export interface FingerprintAlgorithm {
+  readonly version: string;
+  fingerprintOf(input: FingerprintInput): string;
+  vulnKeyOf(input: VulnKeyInput): string;
+}
+
+/** The algorithm this build ships. */
+export const currentAlgorithm: FingerprintAlgorithm = {
+  version: FINGERPRINT_VERSION,
+  fingerprintOf: fingerprint,
+  vulnKeyOf: vulnKey,
+};
+
+/** An issue and the evidence a re-fingerprint would recompute it from. */
+export interface IssueWithEvidence {
+  readonly issue: Issue;
+  /**
+   * The evidence that **defines** this issue, which is not always every
+   * finding linked to it: `gatherForOrg` excludes what a merge attached from
+   * another issue, because that evidence describes a different weakness and
+   * re-deriving a key from it would split the survivor apart.
+   */
+  readonly findings: readonly Finding[];
+}
+
+/**
+ * What would happen to one issue.
+ *
+ * - `unchanged` — recomputes to the fingerprint it already has.
+ * - `moved` — one new fingerprint, different, and nothing else lands on it.
+ *   The issue keeps its identity and its history; only the key changes.
+ * - `merged` — its new fingerprint is shared with at least one other issue.
+ *   They become one, which is the outcome B39 and B41 are chasing.
+ * - `split` — its findings no longer agree on a fingerprint, so the issue
+ *   would become several. The most disruptive outcome and the one that most
+ *   needs reading before an execution.
+ * - `undecidable` — at least one of its findings cannot be re-keyed at all,
+ *   so no honest answer exists for it. See {@link planRefingerprint}.
+ */
+export type RefingerprintOutcome = 'unchanged' | 'moved' | 'merged' | 'split' | 'undecidable';
+
+export interface IssuePlan {
+  readonly issueId: string;
+  readonly targetId: string;
+  readonly title: string;
+  readonly from: string;
+  readonly fromVersion: string;
+  /** Distinct fingerprints this issue's evidence produces at the new version. */
+  readonly to: readonly string[];
+  readonly outcome: RefingerprintOutcome;
+  /** Other issues that land on one of `to`. Empty unless `outcome` involves a merge. */
+  readonly mergesWith: readonly string[];
+  /**
+   * Findings linked to the issue — **not necessarily the number the recompute
+   * read**. A manual issue is keyed from its own columns however much evidence
+   * is attached to it, and a scanner issue with zero findings falls back to
+   * the same path. Both are reported here as they stand so the count stays a
+   * fact about the data rather than about this function.
+   */
+  readonly evidence: number;
+}
+
+export interface RefingerprintPlan {
+  readonly fromVersion: string;
+  readonly toVersion: string;
+  readonly issues: readonly IssuePlan[];
+  readonly counts: Readonly<Record<RefingerprintOutcome, number>>;
+}
+
+/**
+ * Works out what re-fingerprinting every issue would do, and writes nothing.
+ *
+ * **Recomputed from inputs, never by comparing stored strings.** A stored
+ * `fp_v1` fingerprint and a stored `fp_v2` one are not comparable — that is
+ * the whole reason `reconcile()` refuses to run across versions (B45) — so
+ * the only honest question is "what does the new algorithm say about the same
+ * evidence?". That means re-deriving `vulnKey` too: B39's change was to the
+ * `vulnKey` fallback, and reading the stored value would miss precisely the
+ * class of change this migration exists for.
+ *
+ * **A manual issue is keyed from its own columns, evidence or not.** Findings
+ * can be attached to a human-raised issue after the fact; 24 of the 404 manual
+ * issues in the development database are. Re-deriving those from the evidence
+ * would replace the human's `vulnKey` with the scanner's and merge the issue
+ * into whichever upload-origin issue already holds that key — losing the
+ * issue a person raised, in a migration nobody expected to lose anything.
+ *
+ * **An issue an earlier merge retired keeps the key it has**, and this is what
+ * makes the plan reach a fixed point. A merge copies the duplicate's evidence
+ * to the survivor without removing it from the duplicate (B98), so re-deriving
+ * a retired issue from its evidence produces the survivor's key and proposes
+ * the merge that already happened — every run, forever. Checked before the
+ * manual rule, because an issue can be both and retirement is the later
+ * statement about its identity.
+ *
+ * **Findings are never touched, and this never needs them to be.** A finding
+ * is evidence (invariant 1) and `findings` rejects `UPDATE` at the database
+ * (B66); its `fingerprint_version` records which algorithm produced it and
+ * that stays true forever. Only `issues` are re-keyed. B69, 8 September.
+ *
+ * **A finding whose key cannot be re-derived makes its issue `undecidable`,
+ * and does not stop the report.** `vulnKey()` throws when a finding carries
+ * no mapped rule, CWE, category or engine rule id — so a row that was never
+ * produced by an importer at this version (or was produced by a different
+ * one) has no answer. Reporting that is the useful behaviour: a migration
+ * plan that dies on one row tells an operator nothing about the other
+ * thousand, and guessing a key from the stored value would defeat the whole
+ * purpose, since re-deriving `vulnKey` is the point.
+ */
+export function planRefingerprint(
+  input: readonly IssueWithEvidence[],
+  algorithm: FingerprintAlgorithm = currentAlgorithm,
+): RefingerprintPlan {
+  const recomputed = input.map(({ issue, findings }) => {
+    // Retirement is checked first, and the order is load-bearing. An issue an
+    // earlier merge retired is not re-keyed at all: its identity moved to the
+    // survivor, and its evidence — which the merge copied to that survivor —
+    // would re-derive the survivor's key and propose merging the two all over
+    // again, so the plan would never reach a fixed point. A *manual* issue
+    // that was later merged away is both things at once, and retirement is
+    // the later statement about its identity, so it wins. Measured: with the
+    // branches the other way round, 95 issues reported `moved` on every run
+    // while the executor left them alone on every run, forever.
+    if (issue.status === 'ignored' && issue.ignoreReason === 'duplicate') {
+      return { issue, evidence: findings.length, undecidable: false, to: [issue.fingerprint] };
+    }
+
+    // A manual issue is keyed from its own columns: its `vulnKey` and location
+    // were declared by a person, and evidence linked afterwards corroborates
+    // that issue rather than redefining it. Re-deriving would replace the
+    // human's key with the scanner's and dissolve the issue into whichever
+    // upload-origin issue already holds it.
+    //
+    // A *merge survivor* is deliberately not handled here, though it is also
+    // an identity a person asserted. Keying one from its own columns carries a
+    // stale `vulnKey` — written before the mapping table knew the engine's
+    // rule — so the migrated issue stops matching what the importer now
+    // produces, auto-resolves and comes back as new. `gatherForOrg` excludes
+    // the absorbed evidence instead, so a survivor re-derives correctly from
+    // what was always its own.
+    if (issue.origin === 'manual') {
+      return {
+        issue,
+        evidence: findings.length,
+        undecidable: false,
+        to: [fromOwnColumns(issue, algorithm)],
+      };
+    }
+
+    const keys = findings.map((f) => fingerprintOfFinding(issue, f, algorithm));
+    return {
+      issue,
+      evidence: findings.length,
+      undecidable: keys.some((k) => k === undefined),
+      to: [...new Set(keys.filter((k): k is string => k !== undefined))].sort(),
+    };
+  });
+
+  // A scanner-origin issue whose evidence has since been detached has nothing
+  // left to re-derive from, so it falls back to its own columns rather than
+  // vanishing from the report.
+  for (const row of recomputed) {
+    if (row.to.length === 0 && !row.undecidable) {
+      row.to = [fromOwnColumns(row.issue, algorithm)];
+    }
+  }
+
+  const owners = new Map<string, string[]>();
+  for (const row of recomputed) {
+    if (row.undecidable) continue;
+    for (const fp of row.to) {
+      const list = owners.get(fp);
+      if (list) list.push(row.issue.id);
+      else owners.set(fp, [row.issue.id]);
+    }
+  }
+
+  const counts: Record<RefingerprintOutcome, number> = {
+    unchanged: 0,
+    moved: 0,
+    merged: 0,
+    split: 0,
+    undecidable: 0,
+  };
+
+  const issues = recomputed.map(({ issue, evidence, to, undecidable }): IssuePlan => {
+    const mergesWith = [
+      ...new Set(to.flatMap((fp) => owners.get(fp) ?? []).filter((id) => id !== issue.id)),
+    ].sort();
+
+    // Split first: an issue that fragments is the most disruptive outcome
+    // and stays visible even when one of its fragments also collides.
+    const outcome: RefingerprintOutcome = undecidable
+      ? 'undecidable'
+      : to.length > 1
+        ? 'split'
+        : mergesWith.length > 0
+          ? 'merged'
+          : to[0] === issue.fingerprint
+            ? 'unchanged'
+            : 'moved';
+
+    counts[outcome] += 1;
+    return {
+      issueId: issue.id,
+      targetId: issue.targetId,
+      title: issue.title,
+      from: issue.fingerprint,
+      fromVersion: issue.fingerprintVersion,
+      to,
+      outcome,
+      mergesWith,
+      evidence,
+    };
+  });
+
+  const fromVersions = [...new Set(input.map(({ issue }) => issue.fingerprintVersion))].sort();
+  return {
+    fromVersion: fromVersions.join(', ') || algorithm.version,
+    toVersion: algorithm.version,
+    issues,
+    counts,
+  };
+}
+
+/** An issue's fingerprint at the new version, from the issue's own columns. */
+function fromOwnColumns(issue: Issue, algorithm: FingerprintAlgorithm): string {
+  return algorithm.fingerprintOf({
+    targetId: issue.targetId,
+    vulnKey: issue.vulnKey,
+    location: issue.location,
+    ...(issue.parameter === undefined ? {} : { parameter: issue.parameter }),
+  });
+}
+
+/**
+ * One finding's fingerprint at the new version, `vulnKey` re-derived and all,
+ * or `undefined` when the key cannot be derived from what the finding holds.
+ */
+function fingerprintOfFinding(
+  issue: Issue,
+  finding: Finding,
+  algorithm: FingerprintAlgorithm,
+): string | undefined {
+  let key;
+  try {
+    key = algorithm.vulnKeyOf({
+      sourceEngine: finding.sourceEngine,
+      ...(finding.sourceRuleId === undefined ? {} : { sourceRuleId: finding.sourceRuleId }),
+      ...(finding.cwe === undefined ? {} : { cwe: finding.cwe }),
+      ...(finding.category === undefined ? {} : { category: finding.category }),
+    });
+  } catch {
+    return undefined;
+  }
+  return algorithm.fingerprintOf({
+    // The target is the issue's: a finding records the run it came from, and
+    // every run of an issue's evidence is against the same target anyway.
+    targetId: issue.targetId,
+    vulnKey: key,
+    location: finding.location,
+    ...(finding.parameter === undefined ? {} : { parameter: finding.parameter }),
+    ...(finding.port === undefined ? {} : { port: finding.port }),
+  });
+}