@vaultcompass/conductor 0.8.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -48,11 +48,50 @@ export declare function normalizeIntentGuard(raw: unknown, version: string | nul
48
48
  export declare function normalizeGitleaks(raw: unknown, version: string | null, blocked: boolean): NormalizedGateOutput;
49
49
  /** CVSS base score onto the shared four-level ladder, by the CVSS v3 bands. */
50
50
  export declare function cvssToSeverity(score: number | null): Severity;
51
+ /**
52
+ * The vulnerabilities summary line's lockfile fact (issue #72): the count and
53
+ * names of the lockfiles the UMBRELLA handed osv-scanner with --lockfile,
54
+ * never osv-scanner's own results[], which 2.x prints only for a source with
55
+ * findings. A clean scan of one lockfile and a run that scanned none both
56
+ * have results: [], and only this list tells them apart.
57
+ *
58
+ * A file in `emptied` (osv-scanner read no packages from it, audit finding
59
+ * F2) is marked in place, so the count never reads as that many files read.
60
+ */
61
+ export declare function describeLockfiles(lockfiles: readonly string[], emptied?: ReadonlySet<string>): string;
62
+ /**
63
+ * The advisories osv-scanner.toml filtered out of this run, read from
64
+ * osv-scanner's stderr, the only place it says so.
65
+ *
66
+ * Measured on 2.4.0 (tests/fixtures/osv-scanner-2.4.0-filtered.stderr.txt):
67
+ * the JSON carries no trace of a filtered advisory at all, while stderr
68
+ * prints one "<id> and N alias(es) have been filtered out because: <reason>"
69
+ * line per advisory PER LOCKFILE, then "Filtered N vulnerabilities from
70
+ * output", where N counts the same way (two ignored advisories in two
71
+ * lockfiles print 4). The ignoreUntil expiry is printed nowhere, so it is
72
+ * never claimed here.
73
+ *
74
+ * Annotation only, never a verdict: the count and the ids feed the report's
75
+ * suppressed number and one run fact, and nothing reads them for a decision.
76
+ * A stderr without the count line (a policy that lowers osv-scanner's
77
+ * verbosity) gives null, and the caller keeps the old 0.
78
+ */
79
+ export interface OsvSuppressions {
80
+ count: number;
81
+ /** Distinct ids in first-seen order, each with the reason osv printed. */
82
+ ids: Array<{
83
+ id: string;
84
+ reason: string;
85
+ }>;
86
+ }
87
+ export declare function osvSuppressions(stderr: string): OsvSuppressions | null;
51
88
  export declare function normalizeOsvScanner(raw: unknown, version: string | null, blocked: boolean,
52
89
  /** The scan root as a path osv-scanner may have printed it, in any spelling. */
53
90
  roots?: readonly string[],
54
91
  /** The lockfiles the umbrella passed with --lockfile, repository-relative. */
55
- lockfiles?: readonly string[]): NormalizedGateOutput;
92
+ lockfiles?: readonly string[],
93
+ /** osv-scanner's stderr, read only for the osv-scanner.toml suppressions it names. */
94
+ stderr?: string): NormalizedGateOutput;
56
95
  export type GateProblemRule = 'conductor/gate-missing' | 'conductor/gate-output-unparseable' | 'conductor/gate-failed';
57
96
  /**
58
97
  * What to install for a product, spelled so it cannot be mistaken for
@@ -133,4 +172,6 @@ export declare function summariseStderr(stderr: string | null | undefined): stri
133
172
  * the product and never over the message, so a gate that reworded its error
134
173
  * is the same alert rather than a new one every run.
135
174
  */
136
- export declare function normalizeFailedGate(role: GateRole, product: Product, detail: string, stderr?: string | null): Finding;
175
+ export declare function normalizeFailedGate(role: GateRole, product: Product, detail: string, stderr?: string | null,
176
+ /** Structured facts beside the detail, such as a timed-out gate's limit and elapsed time. */
177
+ extra?: Record<string, unknown>): Finding;
package/dist/normalize.js CHANGED
@@ -797,7 +797,36 @@ export function normalizeGitleaks(raw, version, blocked) {
797
797
  seen.alsoIn.push(droppedCommit);
798
798
  }
799
799
  });
800
- const findings = [...byPlace.values()].map(({ finding, alsoIn }) => alsoIn.length === 0 ? finding : { ...finding, details: { ...finding.details, alsoIn } });
800
+ const kept = [...byPlace.values()].map(({ finding, alsoIn }) => alsoIn.length === 0 ? finding : { ...finding, details: { ...finding.details, alsoIn } });
801
+ // THE MERGE COPY ON A MOVED LINE IS NOT COLLAPSED, and that is deliberate
802
+ // (tests/fixtures/gitleaks-8.30.1-merge-doubled.json, measured on 8.30.1).
803
+ // When the base branch moved the lines above a pull request's secret, the
804
+ // merge's first-parent diff reports it at a different StartLine, so the two
805
+ // records differ in StartLine, Commit, Date, Message and Fingerprint and
806
+ // agree on RuleID, File, StartColumn, EndColumn and Entropy. Under --redact
807
+ // nothing in the record identifies the secret itself: Entropy is a
808
+ // character-frequency score, the same for any two secrets that are
809
+ // anagrams, so collapsing on it could fold two different leaks into one
810
+ // SARIF result. Each record is kept, and each names the other here, so a
811
+ // reader sees the likely pair without the report hiding either of them.
812
+ const shape = (finding) => {
813
+ const subject = finding.subject;
814
+ return subject.kind === 'location' && typeof finding.details.entropy === 'number'
815
+ ? JSON.stringify([finding.ruleId, subject.file, subject.column, subject.endColumn ?? null, finding.details.entropy])
816
+ : null;
817
+ };
818
+ const findings = kept.map((finding) => {
819
+ const key = shape(finding);
820
+ const twins = key === null
821
+ ? []
822
+ : kept
823
+ // A merge re-report is by definition another commit: two records
824
+ // of one shape in the same commit are two places, not a pair.
825
+ .filter((other) => other !== finding && shape(other) === key && other.details.commit !== finding.details.commit)
826
+ .map((other) => other.fingerprint?.value)
827
+ .filter((value) => value !== undefined);
828
+ return twins.length === 0 ? finding : { ...finding, details: { ...finding.details, possiblySameSecretAs: twins } };
829
+ });
801
830
  return {
802
831
  findings,
803
832
  run: {
@@ -806,7 +835,10 @@ export function normalizeGitleaks(raw, version, blocked) {
806
835
  suppressed: 0,
807
836
  ignored: 0,
808
837
  diagnostics: [],
809
- details: { entries: entries.length },
838
+ // Both counts, labelled: the raw number of records in gitleaks' report,
839
+ // and the distinct places left after the merge collapse above. One
840
+ // unlabelled `entries` read as the finding count when it was not.
841
+ details: { 'entries-raw': entries.length, 'entries-distinct': findings.length },
810
842
  },
811
843
  diagnostics: [],
812
844
  };
@@ -943,19 +975,68 @@ function relativeToRoots(file, roots) {
943
975
  * never osv-scanner's own results[], which 2.x prints only for a source with
944
976
  * findings. A clean scan of one lockfile and a run that scanned none both
945
977
  * have results: [], and only this list tells them apart.
978
+ *
979
+ * A file in `emptied` (osv-scanner read no packages from it, audit finding
980
+ * F2) is marked in place, so the count never reads as that many files read.
946
981
  */
947
- function describeLockfiles(lockfiles) {
948
- return lockfiles.length === 0 ? '0' : `${lockfiles.length} (${lockfiles.join(', ')})`;
982
+ export function describeLockfiles(lockfiles, emptied = new Set()) {
983
+ const named = lockfiles.map((file) => (emptied.has(file) ? `${file} [no packages read]` : file));
984
+ return lockfiles.length === 0 ? '0' : `${lockfiles.length} (${named.join(', ')})`;
985
+ }
986
+ const OSV_FILTERED_COUNT = /^Filtered (\d+) vulnerabilit(?:y|ies) from output$/;
987
+ // The id is restricted to an advisory id's shape. osv-scanner prints a
988
+ // reason from osv-scanner.toml verbatim, newlines and escape bytes included
989
+ // (measured on 2.4.0), so a reason can forge what looks like a second line;
990
+ // an id that is not id-shaped is not read at all, and every id that is still
991
+ // goes through the same cleaner as the reason.
992
+ const OSV_FILTERED_ID = /^([A-Za-z0-9._:-]+) and \d+ alias(?:es)? ha(?:s|ve) been filtered out because: (.*)$/;
993
+ /** Printable ASCII only, capped: the reason is text from the repository's own config. */
994
+ function cleanReason(reason) {
995
+ const printable = reason.replace(/[^\x20-\x7e]/g, '').trim();
996
+ return printable.length <= 80 ? printable : `${printable.slice(0, 80)}...`;
997
+ }
998
+ export function osvSuppressions(stderr) {
999
+ let count = null;
1000
+ const ids = [];
1001
+ for (const raw of stderr.split('\n')) {
1002
+ const line = raw.trim();
1003
+ const total = OSV_FILTERED_COUNT.exec(line);
1004
+ if (total !== null) {
1005
+ // A count that is not a safe integer is not a count (a 400-digit one
1006
+ // printed as "suppressed Infinity"); it is ignored like a missing one.
1007
+ const value = Number(total[1]);
1008
+ if (Number.isSafeInteger(value)) {
1009
+ count = value;
1010
+ }
1011
+ continue;
1012
+ }
1013
+ const one = OSV_FILTERED_ID.exec(line);
1014
+ const id = one === null ? '' : cleanReason(one[1]);
1015
+ if (one !== null && id !== '' && !ids.some((entry) => entry.id === id)) {
1016
+ ids.push({ id, reason: cleanReason(one[2]) });
1017
+ }
1018
+ }
1019
+ return count === null ? null : { count, ids };
1020
+ }
1021
+ /** osv-scanner's `dependency_groups` on a package (2.4.0 writes ["dev"]), read leniently. */
1022
+ function osvDependencyGroups(entry) {
1023
+ const groups = entry.dependency_groups;
1024
+ return Array.isArray(groups) && groups.every((group) => typeof group === 'string')
1025
+ ? groups
1026
+ : undefined;
949
1027
  }
950
1028
  export function normalizeOsvScanner(raw, version, blocked,
951
1029
  /** The scan root as a path osv-scanner may have printed it, in any spelling. */
952
1030
  roots = [],
953
1031
  /** The lockfiles the umbrella passed with --lockfile, repository-relative. */
954
- lockfiles = []) {
1032
+ lockfiles = [],
1033
+ /** osv-scanner's stderr, read only for the osv-scanner.toml suppressions it names. */
1034
+ stderr = '') {
955
1035
  const product = 'osv-scanner';
956
1036
  const top = needRecord(raw, product, 'the output');
957
1037
  const results = needArray(top.results, product, 'results');
958
1038
  const findings = [];
1039
+ const otherVerdicts = [];
959
1040
  for (const [ri, rawResult] of results.entries()) {
960
1041
  const resultWhere = `results[${ri}]`;
961
1042
  const result = needRecord(rawResult, product, resultWhere);
@@ -969,6 +1050,8 @@ lockfiles = []) {
969
1050
  const name = needString(pkg.name, product, `${packageWhere}.package.name`);
970
1051
  const installed = optionalString(pkg.version, product, `${packageWhere}.package.version`) ?? '';
971
1052
  const ecosystem = optionalString(pkg.ecosystem, product, `${packageWhere}.package.ecosystem`);
1053
+ const dependencyGroups = osvDependencyGroups(entry);
1054
+ otherVerdicts.push(...osvOtherVerdicts(entry, pkg, `${name}@${installed}`));
972
1055
  const groups = (entry.groups === undefined ? [] : needArray(entry.groups, product, `${packageWhere}.groups`)).filter(isRecord);
973
1056
  const vulns = entry.vulnerabilities === undefined
974
1057
  ? []
@@ -1000,31 +1083,104 @@ lockfiles = []) {
1000
1083
  aliases: Array.isArray(vuln.aliases) ? vuln.aliases : [],
1001
1084
  ...(score === null ? {} : { cvss: score }),
1002
1085
  ...(fixedVersion === undefined ? {} : { fixedVersion }),
1086
+ // Carried, never mapped: whether a dev-only dependency carries
1087
+ // the advisory is for the reader, and changes no verdict.
1088
+ ...(dependencyGroups === undefined ? {} : { dependencyGroups }),
1003
1089
  },
1004
1090
  });
1005
1091
  }
1006
1092
  }
1007
1093
  }
1094
+ const suppressions = osvSuppressions(stderr);
1008
1095
  return {
1009
1096
  findings,
1010
1097
  run: {
1011
1098
  // Any known vulnerability fails the gate; a threshold, when an adopter
1012
1099
  // wants one, lives in osv-scanner.toml.
1013
1100
  failOn: 'any',
1014
- suppressed: 0,
1101
+ // osv-scanner's own count of what osv-scanner.toml filtered, from its
1102
+ // stderr (osvSuppressions): one per advisory per lockfile. 0 when it
1103
+ // printed no count, which was the only value this ever had before.
1104
+ suppressed: suppressions?.count ?? 0,
1015
1105
  ignored: 0,
1016
1106
  diagnostics: [],
1017
1107
  details: {
1018
1108
  lockfiles: describeLockfiles(lockfiles),
1019
1109
  // Distinct from the old "sources" name on purpose: this counts only
1020
1110
  // the sources osv-scanner reported findings for, never how many were
1021
- // scanned, and the old name read as the latter.
1022
- 'sources-with-findings': results.length,
1111
+ // scanned. Counted as the distinct lockfiles the FINDINGS name rather
1112
+ // than results.length: osv-scanner 2.4.0 prints one result per
1113
+ // lockfile handed with --lockfile (measured,
1114
+ // tests/fixtures/osv-scanner-2.4.0-two-lockfiles.json), but a result
1115
+ // carrying only a non-advisory verdict (a licence, a deprecation) has
1116
+ // no finding, and counting it said a lockfile had findings that the
1117
+ // report then showed none for.
1118
+ 'sources-with-findings': new Set(findings.map((finding) => (finding.subject.kind === 'package' ? finding.subject.manifest : ''))).size,
1119
+ ...(suppressions === null || suppressions.ids.length === 0
1120
+ ? {}
1121
+ : {
1122
+ 'suppressed-by-osv-scanner.toml': `${suppressions.ids.map((entry) => `${entry.id} (${entry.reason})`).join(', ')}; ` +
1123
+ 'counted by osv-scanner once per advisory per lockfile; expiry not reported by osv-scanner',
1124
+ }),
1023
1125
  },
1024
1126
  },
1025
1127
  diagnostics: [],
1128
+ otherVerdicts,
1026
1129
  };
1027
1130
  }
1131
+ /**
1132
+ * The package-level keys of an osv-scanner result this normalizer knows:
1133
+ * the package itself, its advisories, their groups, and the licences it was
1134
+ * found to carry, and `dependency_groups`, the dev marker osvDependencyGroups
1135
+ * reads (2.4.0 writes ["dev"]; a marker, not a verdict). `license_violations`
1136
+ * is deliberately NOT here: it is a verdict (osv-scanner exits 1 for it,
1137
+ * measured on 2.4.0 with --licenses).
1138
+ */
1139
+ const OSV_KNOWN_PACKAGE_KEYS = new Set([
1140
+ 'package',
1141
+ 'vulnerabilities',
1142
+ 'groups',
1143
+ 'licenses',
1144
+ 'dependency_groups',
1145
+ ]);
1146
+ function isEmptyValue(value) {
1147
+ if (value === undefined || value === null || value === false || value === '') {
1148
+ return true;
1149
+ }
1150
+ if (Array.isArray(value)) {
1151
+ return value.length === 0;
1152
+ }
1153
+ return isRecord(value) && Object.keys(value).length === 0;
1154
+ }
1155
+ /**
1156
+ * Every verdict on one package that is not an advisory, one line each.
1157
+ *
1158
+ * Two shapes, both measured on osv-scanner 2.4.0. `license_violations` beside
1159
+ * `vulnerabilities` (--licenses), and `package.deprecated: true` INSIDE the
1160
+ * package object (--experimental-flag-deprecated-packages; a deprecated
1161
+ * package with no advisory exits 1 too). Beyond those, any other non-empty
1162
+ * PACKAGE-level key this normalizer does not map counts the same, so a
1163
+ * verdict field a later osv-scanner adds fails closed rather than open.
1164
+ * Top-level keys (license_summary, experimental_config) and result-level ones
1165
+ * (source) are summaries and settings, not verdicts on a package, and are
1166
+ * not read.
1167
+ *
1168
+ * KNOWN GAP: inside the package object only `deprecated` is read. A future
1169
+ * verdict field osv-scanner writes INSIDE `package` (a `yanked` flag, say)
1170
+ * is not seen by the catch-all, which looks at package-level keys only.
1171
+ */
1172
+ function osvOtherVerdicts(entry, pkg, label) {
1173
+ const out = [];
1174
+ for (const [key, value] of Object.entries(entry)) {
1175
+ if (!OSV_KNOWN_PACKAGE_KEYS.has(key) && !isEmptyValue(value)) {
1176
+ out.push(`${key} (${label})`);
1177
+ }
1178
+ }
1179
+ if (!isEmptyValue(pkg.deprecated)) {
1180
+ out.push(`deprecated (${label})`);
1181
+ }
1182
+ return out;
1183
+ }
1028
1184
  function gateProblem(ruleId, role, product, message, details) {
1029
1185
  return {
1030
1186
  schemaVersion: 1,
@@ -1167,8 +1323,10 @@ export function summariseStderr(stderr) {
1167
1323
  * the product and never over the message, so a gate that reworded its error
1168
1324
  * is the same alert rather than a new one every run.
1169
1325
  */
1170
- export function normalizeFailedGate(role, product, detail, stderr) {
1326
+ export function normalizeFailedGate(role, product, detail, stderr,
1327
+ /** Structured facts beside the detail, such as a timed-out gate's limit and elapsed time. */
1328
+ extra = {}) {
1171
1329
  const said = summariseStderr(stderr);
1172
1330
  return gateProblem('conductor/gate-failed', role, product, `The "${role}" gate did not complete: ${detail} Nothing was verified by this gate.` +
1173
- (said === null ? '' : ` The gate wrote to stderr: ${said}`), { detail, stderr: said });
1331
+ (said === null ? '' : ` The gate wrote to stderr: ${said}`), { detail, stderr: said, ...extra });
1174
1332
  }
@@ -0,0 +1,84 @@
1
+ import type { Diagnostic, Finding } from './envelope.js';
2
+ import type { PresentOnBase } from './policy.js';
3
+ export type BaselineLabel = 'introduced' | 'present-on-base';
4
+ /**
5
+ * What the comparison did, carried on the gate's outcome so a report (and the
6
+ * exit-code exception) never needs the policy file to explain itself.
7
+ * Present only when a comparison ran (section 3.1).
8
+ */
9
+ export interface BaseComparison {
10
+ /** The trust base the copies were read from. */
11
+ ref: string;
12
+ /** The base policy's present-on-base value, block when it said nothing. */
13
+ policy: PresentOnBase;
14
+ introduced: number;
15
+ presentOnBase: number;
16
+ /** Lockfiles whose findings all count as introduced because no comparison was possible, with why. */
17
+ unavailable: Array<{
18
+ path: string;
19
+ reason: string;
20
+ }>;
21
+ /**
22
+ * Verdicts in the head run's raw output that are not advisories (licence
23
+ * violations, deprecated packages, any unmapped package-level field). Read
24
+ * by condition (f) of gateExitBlocks.
25
+ */
26
+ otherVerdicts: string[];
27
+ }
28
+ /** The comparison key: advisory id, ecosystem, package, installed version, lockfile path. */
29
+ export declare function comparisonKey(finding: Finding): string | null;
30
+ /**
31
+ * Labels head findings against base findings. Pure, so the comparison can be
32
+ * tested without spawning anything.
33
+ *
34
+ * A finding in a lockfile in `unchanged` (byte-equal to the base, its whole
35
+ * include closure too) is present-on-base without a base scan: identical
36
+ * input, config and database. A finding in a lockfile in `unavailable` is
37
+ * introduced whatever the base said. Otherwise it is present-on-base exactly
38
+ * when a base finding has the same key.
39
+ */
40
+ export declare function labelAgainstBase(head: readonly Finding[], base: readonly Finding[], unavailable: ReadonlySet<string>, unchanged: ReadonlySet<string>): Finding[];
41
+ export interface BaseSpawnResult {
42
+ status: number | null;
43
+ stdout: string;
44
+ error: Error | undefined;
45
+ }
46
+ export interface CompareArgs {
47
+ repoRoot: string;
48
+ ref: string;
49
+ workDir: string;
50
+ policy: PresentOnBase;
51
+ version: string | null;
52
+ headFindings: readonly Finding[];
53
+ /** Every file the head run was handed, repository-relative. */
54
+ headLockfiles: readonly string[];
55
+ /** For each requirements file, the head include closure when the head copy passed the pin check. */
56
+ closures: ReadonlyMap<string, readonly string[] | null>;
57
+ otherVerdicts: readonly string[];
58
+ /** The head run's full argv. */
59
+ headArgv: readonly string[];
60
+ /** The base run's full argv for these absolute copy paths, built by the same gateArgs. */
61
+ baseArgv: (copies: readonly string[]) => string[];
62
+ /** Runs the same binary, with the same timeout, from `cwd`. */
63
+ spawn: (argv: readonly string[], cwd: string) => BaseSpawnResult;
64
+ /** Every spelling of a directory a child could print it in. */
65
+ rootSpellings: (dir: string) => string[];
66
+ }
67
+ export interface CompareResult {
68
+ findings: Finding[];
69
+ comparison: BaseComparison;
70
+ diagnostics: Diagnostic[];
71
+ }
72
+ /**
73
+ * The result when the comparison itself failed in a way compareWithBase did
74
+ * not anticipate: every lockfile with findings is unavailable and every
75
+ * finding is introduced. Never could-not-run.
76
+ */
77
+ export declare function comparisonFailed(args: CompareArgs, reason: string): CompareResult;
78
+ /**
79
+ * Runs the comparison. Assumes the caller has established section 3.1: a
80
+ * decided trust base, a head run that exited blocked, at least one finding.
81
+ * Never throws for anything it anticipated; the caller still wraps it, and
82
+ * any throw becomes "every finding introduced", never could-not-run.
83
+ */
84
+ export declare function compareWithBase(args: CompareArgs): CompareResult;