@sagentix-inc/evidence-collector 0.3.5 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +137 -9
  2. package/apps/sos-app/src/data/checks/aws.json +12 -0
  3. package/apps/sos-app/src/data/checks/azure.json +18 -0
  4. package/apps/sos-app/src/data/checks/bamboohr.json +40 -0
  5. package/apps/sos-app/src/data/checks/entra.json +1 -1
  6. package/apps/sos-app/src/data/checks/gcp.json +12 -0
  7. package/apps/sos-app/src/data/checks/intune.json +7 -1
  8. package/apps/sos-app/src/data/checks/jamf.json +8 -2
  9. package/apps/sos-app/src/data/checks/kandji.json +7 -1
  10. package/apps/sos-app/src/data/checks/rippling.json +40 -0
  11. package/apps/sos-app/src/data/checks/vulnscan.json +27 -0
  12. package/apps/sos-app/src/data/checks/web.json +31 -1
  13. package/collector/package.json +1 -1
  14. package/collector/src/cli.js +344 -37
  15. package/collector/src/config.js +12 -0
  16. package/collector/src/doctor.js +382 -0
  17. package/collector/src/policies.js +160 -0
  18. package/collector/src/pool.js +19 -0
  19. package/collector/src/report.js +88 -16
  20. package/collector/src/sources/aws/checks.js +64 -0
  21. package/collector/src/sources/aws.js +6 -1
  22. package/collector/src/sources/azure/api.js +4 -0
  23. package/collector/src/sources/azure/checks.js +91 -1
  24. package/collector/src/sources/azure.js +8 -2
  25. package/collector/src/sources/bamboohr.js +133 -0
  26. package/collector/src/sources/cloudflare/api.js +24 -4
  27. package/collector/src/sources/device_common.js +42 -0
  28. package/collector/src/sources/directory.js +198 -0
  29. package/collector/src/sources/engines.js +2 -1
  30. package/collector/src/sources/entra.js +4 -3
  31. package/collector/src/sources/gcp/project.js +47 -3
  32. package/collector/src/sources/gcp.js +5 -2
  33. package/collector/src/sources/github.js +46 -8
  34. package/collector/src/sources/google_oauth.js +3 -3
  35. package/collector/src/sources/google_workspace.js +4 -3
  36. package/collector/src/sources/hris_common.js +189 -0
  37. package/collector/src/sources/index.js +3 -0
  38. package/collector/src/sources/intune.js +40 -3
  39. package/collector/src/sources/jamf.js +65 -8
  40. package/collector/src/sources/kandji.js +46 -7
  41. package/collector/src/sources/oauth.js +22 -0
  42. package/collector/src/sources/prowler.js +178 -52
  43. package/collector/src/sources/readonly_http.js +27 -5
  44. package/collector/src/sources/rippling.js +114 -0
  45. package/collector/src/sources/shared-cloud.js +14 -6
  46. package/collector/src/sources/vulnscan.js +386 -0
  47. package/collector/src/sources/web.js +106 -11
  48. package/collector/src/sources/web_privacy.js +266 -0
  49. package/package.json +1 -1
@@ -0,0 +1,386 @@
1
+ // Vulnerability scanner results, reconciled with the organization's asset inventory: files the customer's scanner
2
+ // already wrote, read locally. The collector never runs a scan and never calls the scanner's API.
3
+ //
4
+ // Formats read, each page read 2026-11-07:
5
+ // nessus .nessus (NessusClientData_v2 XML), Tenable's "Nessus File Format" (docs.tenable.com/quick-reference/
6
+ // nessus-file-format/Nessus-File-Format.pdf, last revised 2023-05-24): <ReportHost name="192.168.0.10">,
7
+ // HostProperties tags HOST_START / HOST_END / netbios-name, ReportItem severity "0 — informational … 4 —
8
+ // critical" and pluginID. Also read when present, though that document does not show them [Unverified]:
9
+ // host-ip, host-fqdn, hostname, HOST_END_TIMESTAMP. Authentication: plugin 19506 "Nessus Scan Information"
10
+ // (tenable.com/plugins/nessus/19506, modified 2026-07-10) reports "Whether credentialed or third-party patch
11
+ // management checks are possible"; its output line "Credentialed checks : yes|no" is read [Unverified: the
12
+ // public page does not show the line]. One scan, so no first-found date: findings_within_sla is skipped.
13
+ // tenable Tenable Vulnerability Management exports (developer.tenable.com/reference/exports-vulns-request-export):
14
+ // vulnerability records with asset.hostname / asset.fqdn / asset.ipv4, severity, state OPEN|REOPENED|FIXED,
15
+ // first_found, last_found, plugin.id; asset records with last_authenticated_scan_date ("The date and time of
16
+ // the most recent credentialed scan performed on the asset"), compared with last_scan_time [Unverified name].
17
+ // Exports need "at least the Basic [16] user permission" and "Can View" on the assets.
18
+ // qualys Qualys VMDR Host List Detection (/api/2.0/fo/asset/host/vm/detection/?action=list, HOST_LIST_VM_DETECTION_
19
+ // OUTPUT XML; Qualys API (VM and PA) User Guide v10.41, 2026-10-12, cdn2.qualys.com/docs/qualys-api-vmpc-user-
20
+ // guide.pdf): HOST → IP, DNS, LAST_VM_SCANNED_DATE, LAST_VM_AUTH_SCANNED_DATE; DETECTION → QID, TYPE,
21
+ // SEVERITY 1-5, STATUS "New, Active, Re-Opened, Fixed", FIRST_FOUND_DATETIME. "Scanners and Readers view VM
22
+ // scanned hosts": the Reader role can export it.
23
+ // Wiz is not read: its documentation and API reference sit behind a sign-in (docs.wiz.io, 2026-11-07).
24
+ //
25
+ // Remediation windows: FedRAMP's POA&M guidance, "Critical and High risks to be remediated within 30 days of discovery,
26
+ // Moderate … 90 days … Low … 180 days" (fedramp.gov/legacy/playbook/csp/authorization/poam/). FedRAMP's 2026
27
+ // Vulnerability Detection and Response rules (fedramp.gov/2026/reference/vulnerability-detection-and-response/, changelog
28
+ // 2026-10-13) replace them with timeframes by potential agency impact; set --vulnscan-sla to the windows that apply.
29
+ // Scan age: "CSPs must scan operating systems, Web applications, and databases monthly. [RA-5]"; authenticated scans
30
+ // "wherever possible. [RA-5(5)]" (FedRAMP Continuous Monitoring Playbook v1.0, 2025-11-17).
31
+ //
32
+ // The inventory (--vulnscan-inventory) is the list the scans are held to: a CSV with a hostname, fqdn, name, ip,
33
+ // ipv4 or address column, a JSON array of names or of objects with such a field, or one name or address per line.
34
+ //
35
+ // Checks (one fact each):
36
+ // authenticated_coverage every inventory asset appears in a scan whose credentials worked (100% expected)
37
+ // scan_recent every asset (the inventory's, else every scanned host) was scanned within N days
38
+ // findings_within_sla no open finding older than its severity's remediation window
39
+ // Results carry counts, severities, ages and plugin or QID numbers; never a finding's output text. Host names appear
40
+ // only as counts and, for the assets that fail, up to 50 of the inventory's own names.
41
+ import { basename } from "node:path";
42
+ import { closeSync, fstatSync, openSync, readFileSync } from "node:fs";
43
+ import { arr, isObj, str, Results } from "./shared-cloud.js";
44
+ import { MAX_REPORT_BYTES, bySeverity, idList, paths, posNum, readJsonFile } from "./report_files.js";
45
+ export const VULNSCAN_CHECKS = ["authenticated_coverage", "scan_recent", "findings_within_sla"];
46
+ export const SEVERITIES = ["critical", "high", "medium", "low"];
47
+ /** FedRAMP's remediation windows (High 30, Moderate 90, Low 180 days); critical is held to high's. */
48
+ export const DEFAULT_SLA = { critical: 30, high: 30, medium: 90, low: 180 };
49
+ export const DEFAULTS = { maxScanAgeDays: 30 };
50
+ const DAY = 86_400_000;
51
+ /** Tenable's "Nessus Scan Information" plugin: its output says whether credentialed checks ran. */
52
+ const SCAN_INFO_PLUGIN = "19506";
53
+ // ── A small XML reader: linear scans for the flat tags these formats use (no nesting of a tag in itself). ─────
54
+ const ENT = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
55
+ export const xmlText = (s) => s.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1").replace(/&(amp|lt|gt|quot|apos);/g, (_, e) => ENT[e]).trim();
56
+ /** Each `<tag …>…</tag>` in s, as its attribute text and inner text. */
57
+ export function elements(s, tag) {
58
+ const out = [];
59
+ const open = `<${tag}`, close = `</${tag}>`;
60
+ let i = 0;
61
+ while ((i = s.indexOf(open, i)) >= 0) {
62
+ const next = s[i + open.length];
63
+ if (next !== ">" && next !== " " && next !== "\n" && next !== "\t" && next !== "\r" && next !== "/") {
64
+ i += open.length;
65
+ continue;
66
+ }
67
+ const gt = s.indexOf(">", i);
68
+ if (gt < 0)
69
+ break;
70
+ if (s[gt - 1] === "/") {
71
+ out.push({ attrs: s.slice(i + open.length, gt - 1), inner: "" });
72
+ i = gt + 1;
73
+ continue;
74
+ }
75
+ const end = s.indexOf(close, gt);
76
+ if (end < 0)
77
+ break;
78
+ out.push({ attrs: s.slice(i + open.length, gt), inner: s.slice(gt + 1, end) });
79
+ i = end + close.length;
80
+ }
81
+ return out;
82
+ }
83
+ export const attr = (attrs, name) => {
84
+ const k = attrs.indexOf(`${name}="`);
85
+ if (k < 0 || (k > 0 && !/\s/.test(attrs[k - 1])))
86
+ return undefined;
87
+ const v = attrs.slice(k + name.length + 2), e = v.indexOf('"');
88
+ return e < 0 ? undefined : xmlText(v.slice(0, e));
89
+ };
90
+ const child = (s, tag) => { const e = elements(s, tag)[0]; return e ? xmlText(e.inner) : undefined; };
91
+ function readText(path) {
92
+ let fd;
93
+ try {
94
+ fd = openSync(path, "r");
95
+ }
96
+ catch (e) {
97
+ return { error: `${basename(path)} not readable (${e.code ?? "error"})` };
98
+ }
99
+ try {
100
+ const st = fstatSync(fd);
101
+ if (!st.isFile())
102
+ return { error: `${basename(path)} is not a file` };
103
+ if (st.size > MAX_REPORT_BYTES)
104
+ return { error: `${basename(path)} is larger than ${MAX_REPORT_BYTES / 1024 / 1024} MB` };
105
+ let t = readFileSync(fd, "utf8");
106
+ if (t.charCodeAt(0) === 0xfeff)
107
+ t = t.slice(1);
108
+ return { text: t };
109
+ }
110
+ catch (e) {
111
+ return { error: `${basename(path)} not readable (${e.code ?? "error"})` };
112
+ }
113
+ finally {
114
+ closeSync(fd);
115
+ }
116
+ }
117
+ const when = (v) => { if (!v)
118
+ return undefined; const t = Date.parse(v); return Number.isFinite(t) ? t : undefined; };
119
+ const credentialed = (output) => {
120
+ const m = output?.match(/Credentialed checks\s*:\s*(yes|no)/i);
121
+ return m ? m[1].toLowerCase() === "yes" : null;
122
+ };
123
+ const names = (...xs) => [...new Set(xs.filter((x) => !!x).map((x) => x.toLowerCase()))];
124
+ // ── The three formats ───────────────────────────────────────────────────────────────────────────
125
+ const NESSUS_SEV = { "4": "critical", "3": "high", "2": "medium", "1": "low" };
126
+ export function parseNessus(text, file) {
127
+ const hosts = [];
128
+ for (const rh of elements(text, "ReportHost")) {
129
+ const tags = new Map(elements(rh.inner, "tag").map((t) => [attr(t.attrs, "name") ?? "", xmlText(t.inner)]));
130
+ const end = Number(tags.get("HOST_END_TIMESTAMP"));
131
+ const info = elements(rh.inner, "ReportItem").find((x) => attr(x.attrs, "pluginID") === SCAN_INFO_PLUGIN);
132
+ const flag = tags.get("Credentialed_Scan");
133
+ hosts.push({
134
+ names: names(attr(rh.attrs, "name"), tags.get("host-ip"), tags.get("host-fqdn"), tags.get("hostname"), tags.get("netbios-name")),
135
+ scannedAt: Number.isFinite(end) && end > 0 ? end * 1000 : when(tags.get("HOST_END")),
136
+ authenticated: info ? credentialed(child(info.inner, "plugin_output")) : flag ? flag.toLowerCase() === "true" : null,
137
+ });
138
+ }
139
+ return { format: "nessus", file, hosts, findings: null };
140
+ }
141
+ export function parseTenable(records, file) {
142
+ const by = new Map();
143
+ const findings = [];
144
+ for (const r of records.filter(isObj)) {
145
+ // An asset export record (POST /assets/export): "last_authenticated_scan_date: The date and time of the most recent
146
+ // credentialed scan performed on the asset." Authenticated when that is the asset's last scan, as for Qualys.
147
+ if ("last_authenticated_scan_date" in r || "last_scan_time" in r) {
148
+ const many = (k) => [...arr(r[`${k}s`]), r[k]].map((x) => str(x));
149
+ const ns = names(...many("fqdn"), ...many("hostname"), ...many("ipv4"));
150
+ const key = str(r.id) ?? ns[0];
151
+ if (!key)
152
+ continue;
153
+ const last = when(str(r.last_scan_time)), auth = when(str(r.last_authenticated_scan_date));
154
+ const h = by.get(key) ?? { names: [], authenticated: null };
155
+ h.names = names(...h.names, ...ns);
156
+ if (last && (!h.scannedAt || last > h.scannedAt))
157
+ h.scannedAt = last;
158
+ h.authenticated = last === undefined ? (auth !== undefined ? true : h.authenticated) : auth !== undefined && auth >= last - DAY;
159
+ by.set(key, h);
160
+ continue;
161
+ }
162
+ const a = isObj(r.asset) ? r.asset : {};
163
+ const ns = names(str(a.fqdn), str(a.hostname), str(a.ipv4), ...arr(a.ipv4s).map((x) => str(x)));
164
+ const key = str(a.uuid) ?? ns[0];
165
+ if (!key)
166
+ continue;
167
+ const h = by.get(key) ?? { names: [], authenticated: null };
168
+ h.names = names(...h.names, ...ns);
169
+ const plugin = isObj(r.plugin) ? String(r.plugin.id ?? "") : "";
170
+ const last = when(str(r.last_found));
171
+ if (plugin === SCAN_INFO_PLUGIN) {
172
+ const cred = credentialed(str(r.output));
173
+ if (cred !== null && (h.authenticated === null || !h.authenticated))
174
+ h.authenticated = cred;
175
+ if (last && (!h.scannedAt || last > h.scannedAt))
176
+ h.scannedAt = last;
177
+ }
178
+ by.set(key, h);
179
+ const sev = String(r.severity ?? "").toLowerCase();
180
+ const state = String(r.state ?? "").toUpperCase();
181
+ const first = when(str(r.first_found));
182
+ if (SEVERITIES.includes(sev) && (state === "OPEN" || state === "REOPENED") && first)
183
+ findings.push({ host: key, severity: sev, firstFound: first, id: `plugin ${plugin || "?"}` });
184
+ }
185
+ return { format: "tenable", file, hosts: [...by.values()], findings };
186
+ }
187
+ const QUALYS_SEV = { "5": "critical", "4": "high", "3": "medium", "2": "low", "1": "low" };
188
+ export function parseQualys(text, file) {
189
+ const hosts = [], findings = [];
190
+ for (const h of elements(text, "HOST")) {
191
+ const ip = child(h.inner, "IP"), dns = child(h.inner, "DNS"), netbios = child(h.inner, "NETBIOS");
192
+ const vm = when(child(h.inner, "LAST_VM_SCANNED_DATE")), auth = when(child(h.inner, "LAST_VM_AUTH_SCANNED_DATE"));
193
+ const host = { names: names(ip, dns, dns?.split(".")[0], netbios), scannedAt: vm ?? when(child(h.inner, "LAST_SCAN_DATETIME")),
194
+ // Authenticated when the last authenticated VM scan is the last VM scan (Qualys records both dates).
195
+ authenticated: vm === undefined ? null : auth !== undefined && auth >= vm - DAY };
196
+ hosts.push(host);
197
+ for (const d of elements(h.inner, "DETECTION")) {
198
+ if ((child(d.inner, "TYPE") ?? "").toLowerCase() !== "confirmed")
199
+ continue;
200
+ const status = (child(d.inner, "STATUS") ?? "").toLowerCase();
201
+ const sev = QUALYS_SEV[child(d.inner, "SEVERITY") ?? ""], first = when(child(d.inner, "FIRST_FOUND_DATETIME"));
202
+ if (sev && first && status !== "fixed")
203
+ findings.push({ host: host.names[0] ?? "?", severity: sev, firstFound: first, id: `QID ${child(d.inner, "QID") ?? "?"}` });
204
+ }
205
+ }
206
+ return { format: "qualys", file, hosts, findings };
207
+ }
208
+ /** The scan in a file, by its content. */
209
+ export function readScan(path) {
210
+ const file = basename(path);
211
+ if (/\.json$/i.test(path)) {
212
+ const r = readJsonFile(path);
213
+ if ("error" in r)
214
+ return { error: r.error };
215
+ const recs = Array.isArray(r.data) ? r.data : isObj(r.data) && Array.isArray(r.data.vulnerabilities) ? r.data.vulnerabilities : null;
216
+ if (!recs || !recs.some((x) => isObj(x) && ((isObj(x.asset) && ("plugin" in x || "severity" in x)) || "last_authenticated_scan_date" in x || "last_scan_time" in x)))
217
+ return { error: `${file} is not a Tenable vulnerability or asset export (a JSON array of records with asset and plugin, or with last_scan_time)` };
218
+ return parseTenable(recs, file);
219
+ }
220
+ const t = readText(path);
221
+ if ("error" in t)
222
+ return t;
223
+ const head = t.text.slice(0, 4096);
224
+ if (head.includes("<NessusClientData_v2"))
225
+ return parseNessus(t.text, file);
226
+ if (head.includes("HOST_LIST_VM_DETECTION_OUTPUT"))
227
+ return parseQualys(t.text, file);
228
+ return { error: `${file} is neither a .nessus file (NessusClientData_v2), a Qualys host list detection (HOST_LIST_VM_DETECTION_OUTPUT) nor a Tenable JSON export` };
229
+ }
230
+ /** The inventory's asset names, lowercased, from CSV, JSON or one name per line. */
231
+ export function readInventory(path) {
232
+ const t = readText(path);
233
+ if ("error" in t)
234
+ return t;
235
+ const s = t.text.trim();
236
+ const COLS = ["fqdn", "hostname", "host", "name", "dns", "ip", "ipv4", "ip address", "address", "asset"];
237
+ if (s.startsWith("[")) {
238
+ let j;
239
+ try {
240
+ j = JSON.parse(s);
241
+ }
242
+ catch {
243
+ return { error: `${basename(path)} is not valid JSON` };
244
+ }
245
+ return [...new Set(arr(j).map((x) => (typeof x === "string" ? x : isObj(x) ? COLS.map((c) => str(x[c])).find(Boolean) : undefined)).filter((x) => !!x).map((x) => x.trim().toLowerCase()))];
246
+ }
247
+ const lines = s.split(/\r?\n/).filter((l) => l.trim() && !l.startsWith("#"));
248
+ const head = lines[0]?.split(",").map((h) => h.trim().replace(/^"|"$/g, "").toLowerCase()) ?? [];
249
+ const col = COLS.map((c) => head.indexOf(c)).find((i) => i >= 0);
250
+ const rows = col === undefined ? lines.map((l) => l.trim()) : lines.slice(1).map((l) => (l.split(",")[col] ?? "").trim().replace(/^"|"$/g, ""));
251
+ return [...new Set(rows.filter(Boolean).map((x) => x.toLowerCase()))];
252
+ }
253
+ /** The scanned host an inventory name stands for: the same name, IP or short host name. */
254
+ const matcher = (hosts) => {
255
+ // A host seen in several files (an asset export beside a vulnerability export) is one host: the latest scan wins, and
256
+ // a credential record from any file counts (yes over no over unknown).
257
+ const merge = (a, b) => !a ? b : {
258
+ names: names(...a.names, ...b.names), scannedAt: Math.max(a.scannedAt ?? 0, b.scannedAt ?? 0) || undefined,
259
+ authenticated: a.authenticated === true || b.authenticated === true ? true : a.authenticated === false || b.authenticated === false ? false : null
260
+ };
261
+ const idx = new Map();
262
+ for (const h of hosts)
263
+ for (const n of h.names) {
264
+ idx.set(n, merge(idx.get(n), h));
265
+ const s = n.split(".")[0];
266
+ if (s !== n)
267
+ idx.set(s, merge(idx.get(s), h));
268
+ }
269
+ return (name) => idx.get(name) ?? idx.get(name.split(".")[0]);
270
+ };
271
+ export function parseSla(v) {
272
+ const out = { ...DEFAULT_SLA };
273
+ if (typeof v !== "string" || !v.trim())
274
+ return out;
275
+ for (const part of v.split(",")) {
276
+ const [k, n] = part.split("=").map((x) => x.trim().toLowerCase());
277
+ if (!SEVERITIES.includes(k) || !/^\d+$/.test(n ?? "") || Number(n) < 1)
278
+ return `--vulnscan-sla takes severity=days pairs, e.g. critical=15,high=30,medium=90,low=180 (got ${JSON.stringify(part.slice(0, 40))})`;
279
+ out[k] = Number(n);
280
+ }
281
+ return out;
282
+ }
283
+ export function judgeScans(scans, inventory, o, out, resource) {
284
+ const hosts = scans.flatMap((s) => s.hosts);
285
+ const find = matcher(hosts);
286
+ const from = `${scans.length} report(s) (${[...new Set(scans.map((s) => s.format))].join(", ")}), ${hosts.length} scanned host(s)`;
287
+ // authenticated_coverage
288
+ if (!inventory)
289
+ out.skip("authenticated_coverage", resource, `${from}; give --vulnscan-inventory <file> so coverage is measured against the asset inventory, not against what the scanner happened to reach`);
290
+ else if (!inventory.length)
291
+ out.skip("authenticated_coverage", resource, "the inventory file lists no asset");
292
+ else {
293
+ const missing = [], unauth = [], unknown = [];
294
+ for (const a of inventory) {
295
+ const h = find(a);
296
+ if (!h)
297
+ missing.push(a);
298
+ else if (h.authenticated === false)
299
+ unauth.push(a);
300
+ else if (h.authenticated === null)
301
+ unknown.push(a);
302
+ }
303
+ const ok = inventory.length - missing.length - unauth.length - unknown.length;
304
+ const pct = Math.floor((ok / inventory.length) * 1000) / 10;
305
+ const bad = [...missing, ...unauth, ...unknown];
306
+ out.add("authenticated_coverage", resource, bad.length ? "fail" : "pass", `${from}; ${ok} of ${inventory.length} inventory asset(s) (${pct}%) scanned with working credentials; ${missing.length} not in any scan, ${unauth.length} scanned without credentials, ${unknown.length} with no credential record (plugin 19506 or LAST_VM_AUTH_SCANNED_DATE)${idList(bad.sort())}; 100% required`);
307
+ }
308
+ // scan_recent
309
+ const subjects = inventory?.length ? inventory.map((a) => ({ name: a, h: find(a) })) : hosts.map((h) => ({ name: h.names[0] ?? "?", h }));
310
+ if (!subjects.length)
311
+ out.skip("scan_recent", resource, `${from}; nothing to judge`);
312
+ else {
313
+ const limit = o.now.getTime() - o.maxScanAgeDays * DAY;
314
+ const stale = subjects.filter((x) => !x.h || !x.h.scannedAt || x.h.scannedAt < limit);
315
+ const oldest = subjects.map((x) => x.h?.scannedAt).filter((t) => !!t).sort((a, b) => a - b)[0];
316
+ out.add("scan_recent", resource, stale.length ? "fail" : "pass", `${subjects.length} ${inventory?.length ? "inventory asset(s)" : "scanned host(s) (no inventory given)"}; ${stale.length} not scanned in the last ${o.maxScanAgeDays} day(s)${idList(stale.map((x) => x.name).sort())}${oldest ? `; oldest scan ${new Date(oldest).toISOString().slice(0, 10)}` : ""}`);
317
+ }
318
+ // findings_within_sla
319
+ const dated = scans.filter((s) => s.findings !== null);
320
+ if (!dated.length)
321
+ out.skip("findings_within_sla", resource, `${from}; a .nessus file is one scan and carries no first-found date: import a Tenable Vulnerability Management export or a Qualys host list detection to judge remediation time`);
322
+ else {
323
+ const open = dated.flatMap((s) => s.findings);
324
+ const counts = new Map(), late = new Map();
325
+ const lateIds = new Set();
326
+ for (const f of open) {
327
+ counts.set(f.severity, (counts.get(f.severity) ?? 0) + 1);
328
+ if ((o.now.getTime() - f.firstFound) / DAY > o.sla[f.severity]) {
329
+ late.set(f.severity, (late.get(f.severity) ?? 0) + 1);
330
+ lateIds.add(f.id);
331
+ }
332
+ }
333
+ const nLate = [...late.values()].reduce((a, b) => a + b, 0);
334
+ out.add("findings_within_sla", resource, nLate ? "fail" : "pass", `${open.length} open finding(s) by severity: ${bySeverity(counts, SEVERITIES)}; past the window (${SEVERITIES.map((s) => `${s} ${o.sla[s]} d`).join(", ")}): ${bySeverity(late, SEVERITIES)}${idList([...lateIds].sort(), "")}; none past the window required`);
335
+ }
336
+ }
337
+ export const vulnscan = {
338
+ name: "vulnscan",
339
+ options: {
340
+ "vulnscan-report": { type: "string", multiple: true, help: "a scanner result file: .nessus, a Tenable Vulnerability Management export chunk (.json) or a Qualys host list detection (.xml); repeatable" },
341
+ "vulnscan-inventory": { type: "string", help: "the asset inventory the scans are held to: CSV (hostname/fqdn/ip column), JSON array, or one name or address per line" },
342
+ "vulnscan-max-age-days": { type: "string", help: `days since an asset's last scan before it fails (default ${DEFAULTS.maxScanAgeDays}: monthly)` },
343
+ "vulnscan-sla": { type: "string", help: `remediation windows in days, e.g. critical=30,high=30,medium=90,low=180 (the default)` },
344
+ },
345
+ emits: VULNSCAN_CHECKS.map((c) => `vulnscan.${c}`),
346
+ describe(opts) {
347
+ const ps = paths(opts["vulnscan-report"]);
348
+ return `scanner results ${ps.length ? ps.map((p) => basename(p)).join(", ") : "(none given)"}${typeof opts["vulnscan-inventory"] === "string" ? ` against ${basename(opts["vulnscan-inventory"])}` : ""}`;
349
+ },
350
+ validate(opts) {
351
+ const s = parseSla(opts["vulnscan-sla"]);
352
+ return typeof s === "string" ? s : undefined;
353
+ },
354
+ async run(ctx) {
355
+ const out = new Results("vulnscan");
356
+ const files = paths(ctx.opts["vulnscan-report"]);
357
+ if (!files.length) {
358
+ out.skipAll(VULNSCAN_CHECKS, "vulnscan", "give --vulnscan-report <file>: a .nessus export, a Tenable vulnerability export chunk or a Qualys host list detection");
359
+ return out.results;
360
+ }
361
+ const scans = [];
362
+ for (const f of files) {
363
+ const s = readScan(f);
364
+ if ("error" in s)
365
+ out.skipAll(VULNSCAN_CHECKS, `vulnscan ${basename(f)}`, s.error);
366
+ else
367
+ scans.push(s);
368
+ }
369
+ if (!scans.length)
370
+ return out.results;
371
+ const invPath = typeof ctx.opts["vulnscan-inventory"] === "string" ? ctx.opts["vulnscan-inventory"] : undefined;
372
+ let inventory = null;
373
+ if (invPath) {
374
+ const inv = readInventory(invPath);
375
+ if ("error" in inv) {
376
+ out.skipAll(VULNSCAN_CHECKS, `vulnscan inventory ${basename(invPath)}`, inv.error);
377
+ return out.results;
378
+ }
379
+ inventory = inv;
380
+ }
381
+ const sla = parseSla(ctx.opts["vulnscan-sla"]);
382
+ const resource = `vulnscan ${invPath ? `inventory ${basename(invPath)}` : scans.map((s) => s.file).join(", ").slice(0, 200)}`;
383
+ judgeScans(scans, inventory, { now: ctx.now, maxScanAgeDays: posNum(ctx.opts["vulnscan-max-age-days"], DEFAULTS.maxScanAgeDays), sla: typeof sla === "string" ? DEFAULT_SLA : sla }, out, resource);
384
+ return out.results;
385
+ },
386
+ };
@@ -52,6 +52,23 @@
52
52
  // caa a CAA record set at the domain or the nearest parent that has one (RFC 8659 §3 climbs the
53
53
  // tree toward the root). Recommended.
54
54
  //
55
+ // consent_banner the declared page loads a known consent-management platform (web_privacy.ts CMPS: each vendor's
56
+ // script host from its own installation guide, read 2026-11-07). A page loading no script from
57
+ // another site is skipped (nothing for a banner to govern); one GET cannot see a banner a tag
58
+ // manager loads later, so a miss says so. NIST SP 800-53 Rev. 5 PT-4 (consent).
59
+ // privacy_notice_* the privacy notice at --web-privacy-url (written 2026-11-07 against NIST SP 800-53 Rev. 5 PT-5,
60
+ // PIPEDA Schedule 1 Principles 4.5 and 4.8, and the Office of the Privacy Commissioner of Canada's
61
+ // Guidelines for obtaining meaningful consent; web_privacy.ts lists the topics):
62
+ // published HTTP 200, HTML, and its title or first heading names privacy or personal information
63
+ // dated a "last updated" / "effective" / "dernière mise à jour" date, not in the future and
64
+ // at most --web-privacy-max-age-days old (default 365: a yearly review)
65
+ // topics what is collected, purposes, disclosure, retention, access and correction, contact,
66
+ // and consent or withdrawal, each found in English or French
67
+ // privacy_request_channel a privacy mailbox (privacy@, dpo@, …), a request form or a request portal linked from the
68
+ // notice (NIST SP 800-53 Rev. 5 SI-18(4), PIPEDA Principle 4.9 Individual Access).
69
+ // Page bodies (the declared page for consent_banner, the notice for the others) are read only where the host's
70
+ // robots.txt (RFC 9309) allows this collector's User-Agent, at most 2 MB each; robots.txt is read once per host.
71
+ //
55
72
  // The mail domain is --web-mail-domain, or else the last two labels of each URL's host. That default is wrong for
56
73
  // a host under a two-label public suffix (example.gc.ca gives gc.ca), so give --web-mail-domain there.
57
74
  // Results carry host names, header values, protocol versions and dates: never page content, cookies or a person.
@@ -60,15 +77,17 @@ import { resolveCaa, resolveTxt } from "node:dns/promises";
60
77
  import tls from "node:tls";
61
78
  import { COLLECTOR_VERSION } from "../payload.js";
62
79
  import { Emitter } from "./readonly_http.js";
80
+ import { MAX_PAGE_BYTES, judgeConsent, judgeRequestChannel, looksLikeNotice, noticeDate, noticeTopics, pageText, robotsAllows, TOPICS } from "./web_privacy.js";
63
81
  export const WEB_URL_CHECKS = ["tls_min_version", "certificate_valid", "https_redirect", "hsts", "csp", "content_type_options",
64
- "frame_protection", "referrer_policy", "no_version_banner"];
82
+ "frame_protection", "referrer_policy", "no_version_banner", "consent_banner"];
65
83
  export const WEB_DOMAIN_CHECKS = ["spf", "dmarc", "dnssec", "caa"];
66
- export const WEB_CHECKS = [...WEB_URL_CHECKS, ...WEB_DOMAIN_CHECKS];
84
+ export const WEB_PRIVACY_CHECKS = ["privacy_notice_published", "privacy_notice_dated", "privacy_notice_topics", "privacy_request_channel"];
85
+ export const WEB_CHECKS = [...WEB_URL_CHECKS, ...WEB_DOMAIN_CHECKS, ...WEB_PRIVACY_CHECKS];
67
86
  export const USER_AGENT = `sagentix-evidence-collector/${COLLECTOR_VERSION} (+https://github.com/Sagentix-Advisors-Inc/sagentix-collector)`;
68
87
  export const PROOF_PREFIX = "_sagentix-collector";
69
88
  export const PROOF_TXT = "sagentix-collector-verification";
70
89
  export const PROOF_PATH = "/.well-known/sagentix-collector.txt";
71
- export const DEFAULTS = { delayMs: 1000, timeoutMs: 15_000, certMinDays: 14, hstsMinAge: 31_536_000 };
90
+ export const DEFAULTS = { delayMs: 1000, timeoutMs: 15_000, certMinDays: 14, hstsMinAge: 31_536_000, privacyMaxAgeDays: 365 };
72
91
  const TOKEN = /^[A-Za-z0-9_-]{32,128}$/;
73
92
  const DAY = 86_400_000;
74
93
  const dnsMissing = (e) => ["ENODATA", "ENOTFOUND"].includes(String(e.code));
@@ -270,6 +289,17 @@ export function judgeDmarc(record, at, domain = at) {
270
289
  /** Is a TLS error a local limit (this Node cannot offer the old protocol) or a network failure, rather than the server refusing? */
271
290
  const LOCAL_TLS = /NO_PROTOCOLS_AVAILABLE|NO_CIPHERS_AVAILABLE|NO_SUITABLE/i;
272
291
  const NETWORK = /^(timeout|ENOTFOUND|EAI_AGAIN|EHOSTUNREACH|ENETUNREACH|ECONNREFUSED|ETIMEDOUT)$/;
292
+ /** May this collector read the page body at `u`? robots.txt is fetched once per host. */
293
+ async function robotsOk(c, u) {
294
+ if (!c.robots.has(u.hostname)) {
295
+ const r = await get(c, `https://${u.hostname}/robots.txt`, 512 * 1024);
296
+ // RFC 9309 §2.3.1.3: an unavailable file (4xx) means no rule applies; §2.3.1.4: an unreachable one (5xx, network
297
+ // error) means complete disallow. A redirect is not followed, so it counts as unavailable.
298
+ c.robots.set(u.hostname, "error" in r || r.status >= 500 ? null : r.status === 200 ? r.body : "");
299
+ }
300
+ const rules = c.robots.get(u.hostname);
301
+ return typeof rules === "string" && robotsAllows(rules, USER_AGENT, u.pathname + u.search);
302
+ }
273
303
  /** Waits so that requests to one host are at least delayMs apart. */
274
304
  async function pace(c, host) {
275
305
  const prev = c.last.get(host);
@@ -372,9 +402,10 @@ async function urlChecks(c, out, u, prefix, certMinDays, now) {
372
402
  const ok = [301, 302, 307, 308].includes(r80.status) && to?.protocol === "https:" && to.hostname === host;
373
403
  add("https_redirect", ok, ok ? `HTTP ${r80.status} to ${to.origin}` : `http://${host} answered ${r80.status}${loc ? ` to ${loc.slice(0, 120)}` : " without a redirect"}; a redirect to https://${host} is required`);
374
404
  }
375
- const r = await get(c, u.href);
405
+ const bodyOk = await robotsOk(c, u);
406
+ const r = await get(c, u.href, bodyOk ? MAX_PAGE_BYTES : 0);
376
407
  if ("error" in r) {
377
- for (const k of ["hsts", "csp", "content_type_options", "frame_protection", "referrer_policy", "no_version_banner"])
408
+ for (const k of ["hsts", "csp", "content_type_options", "frame_protection", "referrer_policy", "no_version_banner", "consent_banner"])
378
409
  add(k, "skipped", `${u.href} could not be read (${r.error})`);
379
410
  return;
380
411
  }
@@ -385,6 +416,7 @@ async function urlChecks(c, out, u, prefix, certMinDays, now) {
385
416
  if (headerValue(r.headers, "cf-mitigated")?.trim().toLowerCase() === "challenge") {
386
417
  for (const k of ["hsts", "csp", "content_type_options", "frame_protection", "referrer_policy", "no_version_banner"])
387
418
  add(k, "skipped", `${u.href} answered a Cloudflare challenge page (HTTP ${r.status}, cf-mitigated: challenge), not the site; let the collector's address skip bot checks for this host to judge its headers`);
419
+ add("consent_banner", "skipped", `${u.href} answered a Cloudflare challenge page, not the site`);
388
420
  return;
389
421
  }
390
422
  const h = r.headers, note = r.status >= 300 && r.status < 400 ? ` (headers of the ${r.status} answer; the redirect was not followed)` : "";
@@ -401,6 +433,56 @@ async function urlChecks(c, out, u, prefix, certMinDays, now) {
401
433
  add("referrer_policy", rp.pass, rp.detail + note);
402
434
  const b = judgeBanner(h);
403
435
  add("no_version_banner", b.pass, b.detail + note);
436
+ if (!bodyOk)
437
+ add("consent_banner", "skipped", `robots.txt on ${host} does not let this collector read ${u.pathname}, or could not be read`);
438
+ else if (r.status !== 200)
439
+ add("consent_banner", "skipped", `${u.href} answered ${r.status}; the page itself is needed`);
440
+ else {
441
+ const k = judgeConsent(r.body, u);
442
+ add("consent_banner", k.result === "skipped" ? "skipped" : k.result === "pass", k.detail);
443
+ }
444
+ }
445
+ async function privacyChecks(c, out, u, prefix, maxAgeDays, now) {
446
+ const res = `url ${u.href}`.slice(0, 300);
447
+ const add = (check, result, detail) => out.add(check, result, prefix + detail, res);
448
+ const rest = (detail) => { for (const k of WEB_PRIVACY_CHECKS.slice(1))
449
+ add(k, "skipped", detail); };
450
+ if (!(await robotsOk(c, u))) {
451
+ for (const k of WEB_PRIVACY_CHECKS)
452
+ add(k, "skipped", `robots.txt on ${u.hostname} does not let this collector read ${u.pathname}, or could not be read`);
453
+ return;
454
+ }
455
+ const r = await get(c, u.href, MAX_PAGE_BYTES);
456
+ if ("error" in r) {
457
+ add("privacy_notice_published", "fail", `${u.href} could not be read (${r.error}); a reachable notice is required`);
458
+ return rest("the notice could not be read");
459
+ }
460
+ if (headerValue(r.headers, "cf-mitigated")?.trim().toLowerCase() === "challenge") {
461
+ for (const k of WEB_PRIVACY_CHECKS)
462
+ add(k, "skipped", `${u.href} answered a Cloudflare challenge page, not the notice`);
463
+ return;
464
+ }
465
+ const type = headerValue(r.headers, "content-type") ?? "";
466
+ if (r.status !== 200 || !/html/i.test(type)) {
467
+ add("privacy_notice_published", "fail", `${u.href} answered ${r.status}${type ? ` (${type.slice(0, 40)})` : ""}${r.status >= 300 && r.status < 400 ? ", a redirect (not followed): declare the final address" : ""}; HTTP 200 with an HTML page required`);
468
+ return rest("the notice could not be read");
469
+ }
470
+ const text = pageText(r.body);
471
+ const named = looksLikeNotice(r.body);
472
+ add("privacy_notice_published", named ? "pass" : "fail", named ? `HTTP 200, HTML, ${text.length} characters of text; the title or first heading names privacy or personal information`
473
+ : "HTTP 200, but neither the title nor the first heading names privacy or personal information; declare the notice's own address");
474
+ const d = noticeDate(text);
475
+ if (!d)
476
+ add("privacy_notice_dated", "fail", `no "last updated", "effective" or "dernière mise à jour" date found on the notice; a dated notice is required`);
477
+ else {
478
+ const age = Math.floor((now.getTime() - Date.parse(d.date)) / DAY);
479
+ add("privacy_notice_dated", age >= -1 && age <= maxAgeDays ? "pass" : "fail", `"${d.label}" ${d.date}: ${age < -1 ? "in the future" : `${age} day(s) old`}; at most ${maxAgeDays} allowed`);
480
+ }
481
+ const t = noticeTopics(text);
482
+ const names = (ids) => ids.map((id) => TOPICS.find((x) => x.id === id).en).join("; ");
483
+ add("privacy_notice_topics", t.missing.length ? "fail" : "pass", `${t.found.length} of ${TOPICS.length} topics found${t.missing.length ? `; missing: ${names(t.missing)}` : ""} (phrases in English or French; a person confirms the wording)`);
484
+ const ch = judgeRequestChannel(r.body, u);
485
+ add("privacy_request_channel", ch.pass ? "pass" : "fail", ch.detail);
404
486
  }
405
487
  async function dsRecord(c, domain) {
406
488
  const doh = `https://cloudflare-dns.com/dns-query?name=${encodeURIComponent(domain)}&type=DS`;
@@ -476,10 +558,11 @@ async function domainChecks(c, out, domain, prefix) {
476
558
  }
477
559
  export async function runWeb(o, client, now) {
478
560
  const out = new Emitter("web", "web");
479
- const c = { client, delayMs: o.delayMs, timeoutMs: o.timeoutMs, last: new Map() };
561
+ const c = { client, delayMs: o.delayMs, timeoutMs: o.timeoutMs, last: new Map(), robots: new Map() };
480
562
  const proven = new Set();
481
563
  const prefix = o.skipProof ? "control not proven (dry run only): " : "";
482
- for (const host of [...new Set(o.urls.map((u) => u.hostname))]) {
564
+ const privacy = o.privacyUrls ?? [];
565
+ for (const host of [...new Set([...o.urls, ...privacy].map((u) => u.hostname))]) {
483
566
  if (o.skipProof) {
484
567
  proven.add(host);
485
568
  continue;
@@ -487,13 +570,21 @@ export async function runWeb(o, client, now) {
487
570
  const p = await prove(c, host, o.proofs.get(host));
488
571
  if (p.ok)
489
572
  proven.add(host);
490
- else
573
+ else {
491
574
  for (const u of o.urls.filter((x) => x.hostname === host))
492
575
  out.skipAll(WEB_URL_CHECKS, p.why, `url ${u.href}`.slice(0, 300));
576
+ for (const u of privacy.filter((x) => x.hostname === host))
577
+ out.skipAll(WEB_PRIVACY_CHECKS, p.why, `url ${u.href}`.slice(0, 300));
578
+ }
493
579
  }
494
580
  for (const u of o.urls)
495
581
  if (proven.has(u.hostname))
496
582
  await urlChecks(c, out, u, prefix, o.certMinDays, now);
583
+ if (!privacy.length)
584
+ out.skipAll(WEB_PRIVACY_CHECKS, "give --web-privacy-url <https URL of the privacy notice> to read the notice", "web privacy notice");
585
+ for (const u of privacy)
586
+ if (proven.has(u.hostname))
587
+ await privacyChecks(c, out, u, prefix, o.privacyMaxAgeDays ?? DEFAULTS.privacyMaxAgeDays, now);
497
588
  const domains = [...new Set(o.mailDomains.length ? o.mailDomains : o.urls.map((u) => twoLabels(u.hostname)))];
498
589
  for (const d of domains) {
499
590
  // A mail domain is read only beside a proven host: the same domain, a parent, or a sibling under the same two labels.
@@ -514,6 +605,8 @@ export const web = {
514
605
  "web-skip-proof": { type: "boolean", help: "run without proof of control; accepted only with --dry-run" },
515
606
  "web-delay-ms": { type: "string", help: `milliseconds between two requests to one host (default ${DEFAULTS.delayMs}, at least 200)` },
516
607
  "web-cert-min-days": { type: "string", help: `days a certificate must still be valid (default ${DEFAULTS.certMinDays})` },
608
+ "web-privacy-url": { type: "string", multiple: true, help: "https URL of the organization's privacy notice, on a proven host (repeatable: the English and the French notice)" },
609
+ "web-privacy-max-age-days": { type: "string", help: `days since the notice's "last updated" date before it fails (default ${DEFAULTS.privacyMaxAgeDays})` },
517
610
  },
518
611
  emits: WEB_CHECKS.map((c) => `web.${c}`),
519
612
  describe(opts) {
@@ -528,7 +621,7 @@ export const web = {
528
621
  validate(opts) {
529
622
  if (opts["web-skip-proof"] === true && opts["dry-run"] !== true)
530
623
  return "--web-skip-proof is accepted only with --dry-run: a push to sos-app needs proof that the organization controls each host.";
531
- for (const raw of list(opts["web-url"])) {
624
+ for (const raw of [...list(opts["web-url"]), ...list(opts["web-privacy-url"])]) {
532
625
  const u = parseWebUrl(raw);
533
626
  if (typeof u === "string")
534
627
  return u;
@@ -543,7 +636,8 @@ export const web = {
543
636
  },
544
637
  async run(ctx) {
545
638
  const urls = list(ctx.opts["web-url"]).map(parseWebUrl).filter((u) => typeof u !== "string");
546
- if (!urls.length) {
639
+ const privacyUrls = list(ctx.opts["web-privacy-url"]).map(parseWebUrl).filter((u) => typeof u !== "string");
640
+ if (!urls.length && !privacyUrls.length) {
547
641
  const out = new Emitter("web", "web");
548
642
  out.skipAll(WEB_CHECKS, "give --web-url <https URL> (and --web-proof <host>=<token>)");
549
643
  return out.results;
@@ -552,6 +646,7 @@ export const web = {
552
646
  const skipProof = ctx.opts["web-skip-proof"] === true && ctx.opts["dry-run"] === true;
553
647
  const n = (k, d, min = 0) => { const v = Number(ctx.opts[k]); return Number.isFinite(v) && typeof ctx.opts[k] === "string" ? Math.max(min, v) : d; };
554
648
  return runWeb({ urls, proofs: typeof proofs === "string" ? new Map() : proofs, mailDomains: list(ctx.opts["web-mail-domain"]).map((d) => d.toLowerCase()),
555
- skipProof, delayMs: n("web-delay-ms", DEFAULTS.delayMs, 200), timeoutMs: DEFAULTS.timeoutMs, certMinDays: n("web-cert-min-days", DEFAULTS.certMinDays) }, ctx.clients.web ?? realWebClient(), ctx.now);
649
+ skipProof, delayMs: n("web-delay-ms", DEFAULTS.delayMs, 200), timeoutMs: DEFAULTS.timeoutMs, certMinDays: n("web-cert-min-days", DEFAULTS.certMinDays),
650
+ privacyUrls, privacyMaxAgeDays: n("web-privacy-max-age-days", DEFAULTS.privacyMaxAgeDays, 1) }, ctx.clients.web ?? realWebClient(), ctx.now);
556
651
  },
557
652
  };