@sagentix-inc/evidence-collector 0.3.5 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -9
- package/apps/sos-app/src/data/checks/aws.json +12 -0
- package/apps/sos-app/src/data/checks/azure.json +18 -0
- package/apps/sos-app/src/data/checks/bamboohr.json +40 -0
- package/apps/sos-app/src/data/checks/entra.json +1 -1
- package/apps/sos-app/src/data/checks/gcp.json +12 -0
- package/apps/sos-app/src/data/checks/intune.json +7 -1
- package/apps/sos-app/src/data/checks/jamf.json +8 -2
- package/apps/sos-app/src/data/checks/kandji.json +7 -1
- package/apps/sos-app/src/data/checks/rippling.json +40 -0
- package/apps/sos-app/src/data/checks/vulnscan.json +27 -0
- package/apps/sos-app/src/data/checks/web.json +31 -1
- package/collector/package.json +1 -1
- package/collector/src/cli.js +344 -37
- package/collector/src/config.js +12 -0
- package/collector/src/doctor.js +382 -0
- package/collector/src/policies.js +160 -0
- package/collector/src/pool.js +19 -0
- package/collector/src/report.js +88 -16
- package/collector/src/sources/aws/checks.js +64 -0
- package/collector/src/sources/aws.js +6 -1
- package/collector/src/sources/azure/api.js +4 -0
- package/collector/src/sources/azure/checks.js +91 -1
- package/collector/src/sources/azure.js +8 -2
- package/collector/src/sources/bamboohr.js +133 -0
- package/collector/src/sources/cloudflare/api.js +24 -4
- package/collector/src/sources/device_common.js +42 -0
- package/collector/src/sources/directory.js +198 -0
- package/collector/src/sources/engines.js +2 -1
- package/collector/src/sources/entra.js +4 -3
- package/collector/src/sources/gcp/project.js +47 -3
- package/collector/src/sources/gcp.js +5 -2
- package/collector/src/sources/github.js +46 -8
- package/collector/src/sources/google_oauth.js +3 -3
- package/collector/src/sources/google_workspace.js +4 -3
- package/collector/src/sources/hris_common.js +189 -0
- package/collector/src/sources/index.js +3 -0
- package/collector/src/sources/intune.js +40 -3
- package/collector/src/sources/jamf.js +65 -8
- package/collector/src/sources/kandji.js +46 -7
- package/collector/src/sources/oauth.js +22 -0
- package/collector/src/sources/prowler.js +178 -52
- package/collector/src/sources/readonly_http.js +27 -5
- package/collector/src/sources/rippling.js +114 -0
- package/collector/src/sources/shared-cloud.js +14 -6
- package/collector/src/sources/vulnscan.js +386 -0
- package/collector/src/sources/web.js +106 -11
- package/collector/src/sources/web_privacy.js +266 -0
- package/package.json +1 -1
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
// Vulnerability scanner results, reconciled with the organization's asset inventory: files the customer's scanner
|
|
2
|
+
// already wrote, read locally. The collector never runs a scan and never calls the scanner's API.
|
|
3
|
+
//
|
|
4
|
+
// Formats read, each page read 2026-11-07:
|
|
5
|
+
// nessus .nessus (NessusClientData_v2 XML), Tenable's "Nessus File Format" (docs.tenable.com/quick-reference/
|
|
6
|
+
// nessus-file-format/Nessus-File-Format.pdf, last revised 2023-05-24): <ReportHost name="192.168.0.10">,
|
|
7
|
+
// HostProperties tags HOST_START / HOST_END / netbios-name, ReportItem severity "0 — informational … 4 —
|
|
8
|
+
// critical" and pluginID. Also read when present, though that document does not show them [Unverified]:
|
|
9
|
+
// host-ip, host-fqdn, hostname, HOST_END_TIMESTAMP. Authentication: plugin 19506 "Nessus Scan Information"
|
|
10
|
+
// (tenable.com/plugins/nessus/19506, modified 2026-07-10) reports "Whether credentialed or third-party patch
|
|
11
|
+
// management checks are possible"; its output line "Credentialed checks : yes|no" is read [Unverified: the
|
|
12
|
+
// public page does not show the line]. One scan, so no first-found date: findings_within_sla is skipped.
|
|
13
|
+
// tenable Tenable Vulnerability Management exports (developer.tenable.com/reference/exports-vulns-request-export):
|
|
14
|
+
// vulnerability records with asset.hostname / asset.fqdn / asset.ipv4, severity, state OPEN|REOPENED|FIXED,
|
|
15
|
+
// first_found, last_found, plugin.id; asset records with last_authenticated_scan_date ("The date and time of
|
|
16
|
+
// the most recent credentialed scan performed on the asset"), compared with last_scan_time [Unverified name].
|
|
17
|
+
// Exports need "at least the Basic [16] user permission" and "Can View" on the assets.
|
|
18
|
+
// qualys Qualys VMDR Host List Detection (/api/2.0/fo/asset/host/vm/detection/?action=list, HOST_LIST_VM_DETECTION_
|
|
19
|
+
// OUTPUT XML; Qualys API (VM and PA) User Guide v10.41, 2026-10-12, cdn2.qualys.com/docs/qualys-api-vmpc-user-
|
|
20
|
+
// guide.pdf): HOST → IP, DNS, LAST_VM_SCANNED_DATE, LAST_VM_AUTH_SCANNED_DATE; DETECTION → QID, TYPE,
|
|
21
|
+
// SEVERITY 1-5, STATUS "New, Active, Re-Opened, Fixed", FIRST_FOUND_DATETIME. "Scanners and Readers view VM
|
|
22
|
+
// scanned hosts": the Reader role can export it.
|
|
23
|
+
// Wiz is not read: its documentation and API reference sit behind a sign-in (docs.wiz.io, 2026-11-07).
|
|
24
|
+
//
|
|
25
|
+
// Remediation windows: FedRAMP's POA&M guidance, "Critical and High risks to be remediated within 30 days of discovery,
|
|
26
|
+
// Moderate … 90 days … Low … 180 days" (fedramp.gov/legacy/playbook/csp/authorization/poam/). FedRAMP's 2026
|
|
27
|
+
// Vulnerability Detection and Response rules (fedramp.gov/2026/reference/vulnerability-detection-and-response/, changelog
|
|
28
|
+
// 2026-10-13) replace them with timeframes by potential agency impact; set --vulnscan-sla to the windows that apply.
|
|
29
|
+
// Scan age: "CSPs must scan operating systems, Web applications, and databases monthly. [RA-5]"; authenticated scans
|
|
30
|
+
// "wherever possible. [RA-5(5)]" (FedRAMP Continuous Monitoring Playbook v1.0, 2025-11-17).
|
|
31
|
+
//
|
|
32
|
+
// The inventory (--vulnscan-inventory) is the list the scans are held to: a CSV with a hostname, fqdn, name, ip,
|
|
33
|
+
// ipv4 or address column, a JSON array of names or of objects with such a field, or one name or address per line.
|
|
34
|
+
//
|
|
35
|
+
// Checks (one fact each):
|
|
36
|
+
// authenticated_coverage every inventory asset appears in a scan whose credentials worked (100% expected)
|
|
37
|
+
// scan_recent every asset (the inventory's, else every scanned host) was scanned within N days
|
|
38
|
+
// findings_within_sla no open finding older than its severity's remediation window
|
|
39
|
+
// Results carry counts, severities, ages and plugin or QID numbers; never a finding's output text. Host names appear
|
|
40
|
+
// only as counts and, for the assets that fail, up to 50 of the inventory's own names.
|
|
41
|
+
import { basename } from "node:path";
|
|
42
|
+
import { closeSync, fstatSync, openSync, readFileSync } from "node:fs";
|
|
43
|
+
import { arr, isObj, str, Results } from "./shared-cloud.js";
|
|
44
|
+
import { MAX_REPORT_BYTES, bySeverity, idList, paths, posNum, readJsonFile } from "./report_files.js";
|
|
45
|
+
export const VULNSCAN_CHECKS = ["authenticated_coverage", "scan_recent", "findings_within_sla"];
|
|
46
|
+
export const SEVERITIES = ["critical", "high", "medium", "low"];
|
|
47
|
+
/** FedRAMP's remediation windows (High 30, Moderate 90, Low 180 days); critical is held to high's. */
|
|
48
|
+
export const DEFAULT_SLA = { critical: 30, high: 30, medium: 90, low: 180 };
|
|
49
|
+
export const DEFAULTS = { maxScanAgeDays: 30 };
|
|
50
|
+
const DAY = 86_400_000;
|
|
51
|
+
/** Tenable's "Nessus Scan Information" plugin: its output says whether credentialed checks ran. */
|
|
52
|
+
const SCAN_INFO_PLUGIN = "19506";
|
|
53
|
+
// ── A small XML reader: linear scans for the flat tags these formats use (no nesting of a tag in itself). ─────
|
|
54
|
+
const ENT = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
|
|
55
|
+
export const xmlText = (s) => s.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1").replace(/&(amp|lt|gt|quot|apos);/g, (_, e) => ENT[e]).trim();
|
|
56
|
+
/** Each `<tag …>…</tag>` in s, as its attribute text and inner text. */
|
|
57
|
+
export function elements(s, tag) {
|
|
58
|
+
const out = [];
|
|
59
|
+
const open = `<${tag}`, close = `</${tag}>`;
|
|
60
|
+
let i = 0;
|
|
61
|
+
while ((i = s.indexOf(open, i)) >= 0) {
|
|
62
|
+
const next = s[i + open.length];
|
|
63
|
+
if (next !== ">" && next !== " " && next !== "\n" && next !== "\t" && next !== "\r" && next !== "/") {
|
|
64
|
+
i += open.length;
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
const gt = s.indexOf(">", i);
|
|
68
|
+
if (gt < 0)
|
|
69
|
+
break;
|
|
70
|
+
if (s[gt - 1] === "/") {
|
|
71
|
+
out.push({ attrs: s.slice(i + open.length, gt - 1), inner: "" });
|
|
72
|
+
i = gt + 1;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
const end = s.indexOf(close, gt);
|
|
76
|
+
if (end < 0)
|
|
77
|
+
break;
|
|
78
|
+
out.push({ attrs: s.slice(i + open.length, gt), inner: s.slice(gt + 1, end) });
|
|
79
|
+
i = end + close.length;
|
|
80
|
+
}
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
export const attr = (attrs, name) => {
|
|
84
|
+
const k = attrs.indexOf(`${name}="`);
|
|
85
|
+
if (k < 0 || (k > 0 && !/\s/.test(attrs[k - 1])))
|
|
86
|
+
return undefined;
|
|
87
|
+
const v = attrs.slice(k + name.length + 2), e = v.indexOf('"');
|
|
88
|
+
return e < 0 ? undefined : xmlText(v.slice(0, e));
|
|
89
|
+
};
|
|
90
|
+
const child = (s, tag) => { const e = elements(s, tag)[0]; return e ? xmlText(e.inner) : undefined; };
|
|
91
|
+
function readText(path) {
|
|
92
|
+
let fd;
|
|
93
|
+
try {
|
|
94
|
+
fd = openSync(path, "r");
|
|
95
|
+
}
|
|
96
|
+
catch (e) {
|
|
97
|
+
return { error: `${basename(path)} not readable (${e.code ?? "error"})` };
|
|
98
|
+
}
|
|
99
|
+
try {
|
|
100
|
+
const st = fstatSync(fd);
|
|
101
|
+
if (!st.isFile())
|
|
102
|
+
return { error: `${basename(path)} is not a file` };
|
|
103
|
+
if (st.size > MAX_REPORT_BYTES)
|
|
104
|
+
return { error: `${basename(path)} is larger than ${MAX_REPORT_BYTES / 1024 / 1024} MB` };
|
|
105
|
+
let t = readFileSync(fd, "utf8");
|
|
106
|
+
if (t.charCodeAt(0) === 0xfeff)
|
|
107
|
+
t = t.slice(1);
|
|
108
|
+
return { text: t };
|
|
109
|
+
}
|
|
110
|
+
catch (e) {
|
|
111
|
+
return { error: `${basename(path)} not readable (${e.code ?? "error"})` };
|
|
112
|
+
}
|
|
113
|
+
finally {
|
|
114
|
+
closeSync(fd);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const when = (v) => { if (!v)
|
|
118
|
+
return undefined; const t = Date.parse(v); return Number.isFinite(t) ? t : undefined; };
|
|
119
|
+
const credentialed = (output) => {
|
|
120
|
+
const m = output?.match(/Credentialed checks\s*:\s*(yes|no)/i);
|
|
121
|
+
return m ? m[1].toLowerCase() === "yes" : null;
|
|
122
|
+
};
|
|
123
|
+
const names = (...xs) => [...new Set(xs.filter((x) => !!x).map((x) => x.toLowerCase()))];
|
|
124
|
+
// ── The three formats ───────────────────────────────────────────────────────────────────────────
|
|
125
|
+
const NESSUS_SEV = { "4": "critical", "3": "high", "2": "medium", "1": "low" };
|
|
126
|
+
export function parseNessus(text, file) {
|
|
127
|
+
const hosts = [];
|
|
128
|
+
for (const rh of elements(text, "ReportHost")) {
|
|
129
|
+
const tags = new Map(elements(rh.inner, "tag").map((t) => [attr(t.attrs, "name") ?? "", xmlText(t.inner)]));
|
|
130
|
+
const end = Number(tags.get("HOST_END_TIMESTAMP"));
|
|
131
|
+
const info = elements(rh.inner, "ReportItem").find((x) => attr(x.attrs, "pluginID") === SCAN_INFO_PLUGIN);
|
|
132
|
+
const flag = tags.get("Credentialed_Scan");
|
|
133
|
+
hosts.push({
|
|
134
|
+
names: names(attr(rh.attrs, "name"), tags.get("host-ip"), tags.get("host-fqdn"), tags.get("hostname"), tags.get("netbios-name")),
|
|
135
|
+
scannedAt: Number.isFinite(end) && end > 0 ? end * 1000 : when(tags.get("HOST_END")),
|
|
136
|
+
authenticated: info ? credentialed(child(info.inner, "plugin_output")) : flag ? flag.toLowerCase() === "true" : null,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
return { format: "nessus", file, hosts, findings: null };
|
|
140
|
+
}
|
|
141
|
+
export function parseTenable(records, file) {
|
|
142
|
+
const by = new Map();
|
|
143
|
+
const findings = [];
|
|
144
|
+
for (const r of records.filter(isObj)) {
|
|
145
|
+
// An asset export record (POST /assets/export): "last_authenticated_scan_date: The date and time of the most recent
|
|
146
|
+
// credentialed scan performed on the asset." Authenticated when that is the asset's last scan, as for Qualys.
|
|
147
|
+
if ("last_authenticated_scan_date" in r || "last_scan_time" in r) {
|
|
148
|
+
const many = (k) => [...arr(r[`${k}s`]), r[k]].map((x) => str(x));
|
|
149
|
+
const ns = names(...many("fqdn"), ...many("hostname"), ...many("ipv4"));
|
|
150
|
+
const key = str(r.id) ?? ns[0];
|
|
151
|
+
if (!key)
|
|
152
|
+
continue;
|
|
153
|
+
const last = when(str(r.last_scan_time)), auth = when(str(r.last_authenticated_scan_date));
|
|
154
|
+
const h = by.get(key) ?? { names: [], authenticated: null };
|
|
155
|
+
h.names = names(...h.names, ...ns);
|
|
156
|
+
if (last && (!h.scannedAt || last > h.scannedAt))
|
|
157
|
+
h.scannedAt = last;
|
|
158
|
+
h.authenticated = last === undefined ? (auth !== undefined ? true : h.authenticated) : auth !== undefined && auth >= last - DAY;
|
|
159
|
+
by.set(key, h);
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
const a = isObj(r.asset) ? r.asset : {};
|
|
163
|
+
const ns = names(str(a.fqdn), str(a.hostname), str(a.ipv4), ...arr(a.ipv4s).map((x) => str(x)));
|
|
164
|
+
const key = str(a.uuid) ?? ns[0];
|
|
165
|
+
if (!key)
|
|
166
|
+
continue;
|
|
167
|
+
const h = by.get(key) ?? { names: [], authenticated: null };
|
|
168
|
+
h.names = names(...h.names, ...ns);
|
|
169
|
+
const plugin = isObj(r.plugin) ? String(r.plugin.id ?? "") : "";
|
|
170
|
+
const last = when(str(r.last_found));
|
|
171
|
+
if (plugin === SCAN_INFO_PLUGIN) {
|
|
172
|
+
const cred = credentialed(str(r.output));
|
|
173
|
+
if (cred !== null && (h.authenticated === null || !h.authenticated))
|
|
174
|
+
h.authenticated = cred;
|
|
175
|
+
if (last && (!h.scannedAt || last > h.scannedAt))
|
|
176
|
+
h.scannedAt = last;
|
|
177
|
+
}
|
|
178
|
+
by.set(key, h);
|
|
179
|
+
const sev = String(r.severity ?? "").toLowerCase();
|
|
180
|
+
const state = String(r.state ?? "").toUpperCase();
|
|
181
|
+
const first = when(str(r.first_found));
|
|
182
|
+
if (SEVERITIES.includes(sev) && (state === "OPEN" || state === "REOPENED") && first)
|
|
183
|
+
findings.push({ host: key, severity: sev, firstFound: first, id: `plugin ${plugin || "?"}` });
|
|
184
|
+
}
|
|
185
|
+
return { format: "tenable", file, hosts: [...by.values()], findings };
|
|
186
|
+
}
|
|
187
|
+
const QUALYS_SEV = { "5": "critical", "4": "high", "3": "medium", "2": "low", "1": "low" };
|
|
188
|
+
export function parseQualys(text, file) {
|
|
189
|
+
const hosts = [], findings = [];
|
|
190
|
+
for (const h of elements(text, "HOST")) {
|
|
191
|
+
const ip = child(h.inner, "IP"), dns = child(h.inner, "DNS"), netbios = child(h.inner, "NETBIOS");
|
|
192
|
+
const vm = when(child(h.inner, "LAST_VM_SCANNED_DATE")), auth = when(child(h.inner, "LAST_VM_AUTH_SCANNED_DATE"));
|
|
193
|
+
const host = { names: names(ip, dns, dns?.split(".")[0], netbios), scannedAt: vm ?? when(child(h.inner, "LAST_SCAN_DATETIME")),
|
|
194
|
+
// Authenticated when the last authenticated VM scan is the last VM scan (Qualys records both dates).
|
|
195
|
+
authenticated: vm === undefined ? null : auth !== undefined && auth >= vm - DAY };
|
|
196
|
+
hosts.push(host);
|
|
197
|
+
for (const d of elements(h.inner, "DETECTION")) {
|
|
198
|
+
if ((child(d.inner, "TYPE") ?? "").toLowerCase() !== "confirmed")
|
|
199
|
+
continue;
|
|
200
|
+
const status = (child(d.inner, "STATUS") ?? "").toLowerCase();
|
|
201
|
+
const sev = QUALYS_SEV[child(d.inner, "SEVERITY") ?? ""], first = when(child(d.inner, "FIRST_FOUND_DATETIME"));
|
|
202
|
+
if (sev && first && status !== "fixed")
|
|
203
|
+
findings.push({ host: host.names[0] ?? "?", severity: sev, firstFound: first, id: `QID ${child(d.inner, "QID") ?? "?"}` });
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return { format: "qualys", file, hosts, findings };
|
|
207
|
+
}
|
|
208
|
+
/** The scan in a file, by its content. */
|
|
209
|
+
export function readScan(path) {
|
|
210
|
+
const file = basename(path);
|
|
211
|
+
if (/\.json$/i.test(path)) {
|
|
212
|
+
const r = readJsonFile(path);
|
|
213
|
+
if ("error" in r)
|
|
214
|
+
return { error: r.error };
|
|
215
|
+
const recs = Array.isArray(r.data) ? r.data : isObj(r.data) && Array.isArray(r.data.vulnerabilities) ? r.data.vulnerabilities : null;
|
|
216
|
+
if (!recs || !recs.some((x) => isObj(x) && ((isObj(x.asset) && ("plugin" in x || "severity" in x)) || "last_authenticated_scan_date" in x || "last_scan_time" in x)))
|
|
217
|
+
return { error: `${file} is not a Tenable vulnerability or asset export (a JSON array of records with asset and plugin, or with last_scan_time)` };
|
|
218
|
+
return parseTenable(recs, file);
|
|
219
|
+
}
|
|
220
|
+
const t = readText(path);
|
|
221
|
+
if ("error" in t)
|
|
222
|
+
return t;
|
|
223
|
+
const head = t.text.slice(0, 4096);
|
|
224
|
+
if (head.includes("<NessusClientData_v2"))
|
|
225
|
+
return parseNessus(t.text, file);
|
|
226
|
+
if (head.includes("HOST_LIST_VM_DETECTION_OUTPUT"))
|
|
227
|
+
return parseQualys(t.text, file);
|
|
228
|
+
return { error: `${file} is neither a .nessus file (NessusClientData_v2), a Qualys host list detection (HOST_LIST_VM_DETECTION_OUTPUT) nor a Tenable JSON export` };
|
|
229
|
+
}
|
|
230
|
+
/** The inventory's asset names, lowercased, from CSV, JSON or one name per line. */
|
|
231
|
+
export function readInventory(path) {
|
|
232
|
+
const t = readText(path);
|
|
233
|
+
if ("error" in t)
|
|
234
|
+
return t;
|
|
235
|
+
const s = t.text.trim();
|
|
236
|
+
const COLS = ["fqdn", "hostname", "host", "name", "dns", "ip", "ipv4", "ip address", "address", "asset"];
|
|
237
|
+
if (s.startsWith("[")) {
|
|
238
|
+
let j;
|
|
239
|
+
try {
|
|
240
|
+
j = JSON.parse(s);
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
return { error: `${basename(path)} is not valid JSON` };
|
|
244
|
+
}
|
|
245
|
+
return [...new Set(arr(j).map((x) => (typeof x === "string" ? x : isObj(x) ? COLS.map((c) => str(x[c])).find(Boolean) : undefined)).filter((x) => !!x).map((x) => x.trim().toLowerCase()))];
|
|
246
|
+
}
|
|
247
|
+
const lines = s.split(/\r?\n/).filter((l) => l.trim() && !l.startsWith("#"));
|
|
248
|
+
const head = lines[0]?.split(",").map((h) => h.trim().replace(/^"|"$/g, "").toLowerCase()) ?? [];
|
|
249
|
+
const col = COLS.map((c) => head.indexOf(c)).find((i) => i >= 0);
|
|
250
|
+
const rows = col === undefined ? lines.map((l) => l.trim()) : lines.slice(1).map((l) => (l.split(",")[col] ?? "").trim().replace(/^"|"$/g, ""));
|
|
251
|
+
return [...new Set(rows.filter(Boolean).map((x) => x.toLowerCase()))];
|
|
252
|
+
}
|
|
253
|
+
/** The scanned host an inventory name stands for: the same name, IP or short host name. */
|
|
254
|
+
const matcher = (hosts) => {
|
|
255
|
+
// A host seen in several files (an asset export beside a vulnerability export) is one host: the latest scan wins, and
|
|
256
|
+
// a credential record from any file counts (yes over no over unknown).
|
|
257
|
+
const merge = (a, b) => !a ? b : {
|
|
258
|
+
names: names(...a.names, ...b.names), scannedAt: Math.max(a.scannedAt ?? 0, b.scannedAt ?? 0) || undefined,
|
|
259
|
+
authenticated: a.authenticated === true || b.authenticated === true ? true : a.authenticated === false || b.authenticated === false ? false : null
|
|
260
|
+
};
|
|
261
|
+
const idx = new Map();
|
|
262
|
+
for (const h of hosts)
|
|
263
|
+
for (const n of h.names) {
|
|
264
|
+
idx.set(n, merge(idx.get(n), h));
|
|
265
|
+
const s = n.split(".")[0];
|
|
266
|
+
if (s !== n)
|
|
267
|
+
idx.set(s, merge(idx.get(s), h));
|
|
268
|
+
}
|
|
269
|
+
return (name) => idx.get(name) ?? idx.get(name.split(".")[0]);
|
|
270
|
+
};
|
|
271
|
+
export function parseSla(v) {
|
|
272
|
+
const out = { ...DEFAULT_SLA };
|
|
273
|
+
if (typeof v !== "string" || !v.trim())
|
|
274
|
+
return out;
|
|
275
|
+
for (const part of v.split(",")) {
|
|
276
|
+
const [k, n] = part.split("=").map((x) => x.trim().toLowerCase());
|
|
277
|
+
if (!SEVERITIES.includes(k) || !/^\d+$/.test(n ?? "") || Number(n) < 1)
|
|
278
|
+
return `--vulnscan-sla takes severity=days pairs, e.g. critical=15,high=30,medium=90,low=180 (got ${JSON.stringify(part.slice(0, 40))})`;
|
|
279
|
+
out[k] = Number(n);
|
|
280
|
+
}
|
|
281
|
+
return out;
|
|
282
|
+
}
|
|
283
|
+
export function judgeScans(scans, inventory, o, out, resource) {
|
|
284
|
+
const hosts = scans.flatMap((s) => s.hosts);
|
|
285
|
+
const find = matcher(hosts);
|
|
286
|
+
const from = `${scans.length} report(s) (${[...new Set(scans.map((s) => s.format))].join(", ")}), ${hosts.length} scanned host(s)`;
|
|
287
|
+
// authenticated_coverage
|
|
288
|
+
if (!inventory)
|
|
289
|
+
out.skip("authenticated_coverage", resource, `${from}; give --vulnscan-inventory <file> so coverage is measured against the asset inventory, not against what the scanner happened to reach`);
|
|
290
|
+
else if (!inventory.length)
|
|
291
|
+
out.skip("authenticated_coverage", resource, "the inventory file lists no asset");
|
|
292
|
+
else {
|
|
293
|
+
const missing = [], unauth = [], unknown = [];
|
|
294
|
+
for (const a of inventory) {
|
|
295
|
+
const h = find(a);
|
|
296
|
+
if (!h)
|
|
297
|
+
missing.push(a);
|
|
298
|
+
else if (h.authenticated === false)
|
|
299
|
+
unauth.push(a);
|
|
300
|
+
else if (h.authenticated === null)
|
|
301
|
+
unknown.push(a);
|
|
302
|
+
}
|
|
303
|
+
const ok = inventory.length - missing.length - unauth.length - unknown.length;
|
|
304
|
+
const pct = Math.floor((ok / inventory.length) * 1000) / 10;
|
|
305
|
+
const bad = [...missing, ...unauth, ...unknown];
|
|
306
|
+
out.add("authenticated_coverage", resource, bad.length ? "fail" : "pass", `${from}; ${ok} of ${inventory.length} inventory asset(s) (${pct}%) scanned with working credentials; ${missing.length} not in any scan, ${unauth.length} scanned without credentials, ${unknown.length} with no credential record (plugin 19506 or LAST_VM_AUTH_SCANNED_DATE)${idList(bad.sort())}; 100% required`);
|
|
307
|
+
}
|
|
308
|
+
// scan_recent
|
|
309
|
+
const subjects = inventory?.length ? inventory.map((a) => ({ name: a, h: find(a) })) : hosts.map((h) => ({ name: h.names[0] ?? "?", h }));
|
|
310
|
+
if (!subjects.length)
|
|
311
|
+
out.skip("scan_recent", resource, `${from}; nothing to judge`);
|
|
312
|
+
else {
|
|
313
|
+
const limit = o.now.getTime() - o.maxScanAgeDays * DAY;
|
|
314
|
+
const stale = subjects.filter((x) => !x.h || !x.h.scannedAt || x.h.scannedAt < limit);
|
|
315
|
+
const oldest = subjects.map((x) => x.h?.scannedAt).filter((t) => !!t).sort((a, b) => a - b)[0];
|
|
316
|
+
out.add("scan_recent", resource, stale.length ? "fail" : "pass", `${subjects.length} ${inventory?.length ? "inventory asset(s)" : "scanned host(s) (no inventory given)"}; ${stale.length} not scanned in the last ${o.maxScanAgeDays} day(s)${idList(stale.map((x) => x.name).sort())}${oldest ? `; oldest scan ${new Date(oldest).toISOString().slice(0, 10)}` : ""}`);
|
|
317
|
+
}
|
|
318
|
+
// findings_within_sla
|
|
319
|
+
const dated = scans.filter((s) => s.findings !== null);
|
|
320
|
+
if (!dated.length)
|
|
321
|
+
out.skip("findings_within_sla", resource, `${from}; a .nessus file is one scan and carries no first-found date: import a Tenable Vulnerability Management export or a Qualys host list detection to judge remediation time`);
|
|
322
|
+
else {
|
|
323
|
+
const open = dated.flatMap((s) => s.findings);
|
|
324
|
+
const counts = new Map(), late = new Map();
|
|
325
|
+
const lateIds = new Set();
|
|
326
|
+
for (const f of open) {
|
|
327
|
+
counts.set(f.severity, (counts.get(f.severity) ?? 0) + 1);
|
|
328
|
+
if ((o.now.getTime() - f.firstFound) / DAY > o.sla[f.severity]) {
|
|
329
|
+
late.set(f.severity, (late.get(f.severity) ?? 0) + 1);
|
|
330
|
+
lateIds.add(f.id);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const nLate = [...late.values()].reduce((a, b) => a + b, 0);
|
|
334
|
+
out.add("findings_within_sla", resource, nLate ? "fail" : "pass", `${open.length} open finding(s) by severity: ${bySeverity(counts, SEVERITIES)}; past the window (${SEVERITIES.map((s) => `${s} ${o.sla[s]} d`).join(", ")}): ${bySeverity(late, SEVERITIES)}${idList([...lateIds].sort(), "")}; none past the window required`);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
export const vulnscan = {
|
|
338
|
+
name: "vulnscan",
|
|
339
|
+
options: {
|
|
340
|
+
"vulnscan-report": { type: "string", multiple: true, help: "a scanner result file: .nessus, a Tenable Vulnerability Management export chunk (.json) or a Qualys host list detection (.xml); repeatable" },
|
|
341
|
+
"vulnscan-inventory": { type: "string", help: "the asset inventory the scans are held to: CSV (hostname/fqdn/ip column), JSON array, or one name or address per line" },
|
|
342
|
+
"vulnscan-max-age-days": { type: "string", help: `days since an asset's last scan before it fails (default ${DEFAULTS.maxScanAgeDays}: monthly)` },
|
|
343
|
+
"vulnscan-sla": { type: "string", help: `remediation windows in days, e.g. critical=30,high=30,medium=90,low=180 (the default)` },
|
|
344
|
+
},
|
|
345
|
+
emits: VULNSCAN_CHECKS.map((c) => `vulnscan.${c}`),
|
|
346
|
+
describe(opts) {
|
|
347
|
+
const ps = paths(opts["vulnscan-report"]);
|
|
348
|
+
return `scanner results ${ps.length ? ps.map((p) => basename(p)).join(", ") : "(none given)"}${typeof opts["vulnscan-inventory"] === "string" ? ` against ${basename(opts["vulnscan-inventory"])}` : ""}`;
|
|
349
|
+
},
|
|
350
|
+
validate(opts) {
|
|
351
|
+
const s = parseSla(opts["vulnscan-sla"]);
|
|
352
|
+
return typeof s === "string" ? s : undefined;
|
|
353
|
+
},
|
|
354
|
+
async run(ctx) {
|
|
355
|
+
const out = new Results("vulnscan");
|
|
356
|
+
const files = paths(ctx.opts["vulnscan-report"]);
|
|
357
|
+
if (!files.length) {
|
|
358
|
+
out.skipAll(VULNSCAN_CHECKS, "vulnscan", "give --vulnscan-report <file>: a .nessus export, a Tenable vulnerability export chunk or a Qualys host list detection");
|
|
359
|
+
return out.results;
|
|
360
|
+
}
|
|
361
|
+
const scans = [];
|
|
362
|
+
for (const f of files) {
|
|
363
|
+
const s = readScan(f);
|
|
364
|
+
if ("error" in s)
|
|
365
|
+
out.skipAll(VULNSCAN_CHECKS, `vulnscan ${basename(f)}`, s.error);
|
|
366
|
+
else
|
|
367
|
+
scans.push(s);
|
|
368
|
+
}
|
|
369
|
+
if (!scans.length)
|
|
370
|
+
return out.results;
|
|
371
|
+
const invPath = typeof ctx.opts["vulnscan-inventory"] === "string" ? ctx.opts["vulnscan-inventory"] : undefined;
|
|
372
|
+
let inventory = null;
|
|
373
|
+
if (invPath) {
|
|
374
|
+
const inv = readInventory(invPath);
|
|
375
|
+
if ("error" in inv) {
|
|
376
|
+
out.skipAll(VULNSCAN_CHECKS, `vulnscan inventory ${basename(invPath)}`, inv.error);
|
|
377
|
+
return out.results;
|
|
378
|
+
}
|
|
379
|
+
inventory = inv;
|
|
380
|
+
}
|
|
381
|
+
const sla = parseSla(ctx.opts["vulnscan-sla"]);
|
|
382
|
+
const resource = `vulnscan ${invPath ? `inventory ${basename(invPath)}` : scans.map((s) => s.file).join(", ").slice(0, 200)}`;
|
|
383
|
+
judgeScans(scans, inventory, { now: ctx.now, maxScanAgeDays: posNum(ctx.opts["vulnscan-max-age-days"], DEFAULTS.maxScanAgeDays), sla: typeof sla === "string" ? DEFAULT_SLA : sla }, out, resource);
|
|
384
|
+
return out.results;
|
|
385
|
+
},
|
|
386
|
+
};
|
|
@@ -52,6 +52,23 @@
|
|
|
52
52
|
// caa a CAA record set at the domain or the nearest parent that has one (RFC 8659 §3 climbs the
|
|
53
53
|
// tree toward the root). Recommended.
|
|
54
54
|
//
|
|
55
|
+
// consent_banner the declared page loads a known consent-management platform (web_privacy.ts CMPS: each vendor's
|
|
56
|
+
// script host from its own installation guide, read 2026-11-07). A page loading no script from
|
|
57
|
+
// another site is skipped (nothing for a banner to govern); one GET cannot see a banner a tag
|
|
58
|
+
// manager loads later, so a miss says so. NIST SP 800-53 Rev. 5 PT-4 (consent).
|
|
59
|
+
// privacy_notice_* the privacy notice at --web-privacy-url (written 2026-11-07 against NIST SP 800-53 Rev. 5 PT-5,
|
|
60
|
+
// PIPEDA Schedule 1 Principles 4.5 and 4.8, and the Office of the Privacy Commissioner of Canada's
|
|
61
|
+
// Guidelines for obtaining meaningful consent; web_privacy.ts lists the topics):
|
|
62
|
+
// published HTTP 200, HTML, and its title or first heading names privacy or personal information
|
|
63
|
+
// dated a "last updated" / "effective" / "dernière mise à jour" date, not in the future and
|
|
64
|
+
// at most --web-privacy-max-age-days old (default 365: a yearly review)
|
|
65
|
+
// topics what is collected, purposes, disclosure, retention, access and correction, contact,
|
|
66
|
+
// and consent or withdrawal, each found in English or French
|
|
67
|
+
// privacy_request_channel a privacy mailbox (privacy@, dpo@, …), a request form or a request portal linked from the
|
|
68
|
+
// notice (NIST SP 800-53 Rev. 5 SI-18(4), PIPEDA Principle 4.9 Individual Access).
|
|
69
|
+
// Page bodies (the declared page for consent_banner, the notice for the others) are read only where the host's
|
|
70
|
+
// robots.txt (RFC 9309) allows this collector's User-Agent, at most 2 MB each; robots.txt is read once per host.
|
|
71
|
+
//
|
|
55
72
|
// The mail domain is --web-mail-domain, or else the last two labels of each URL's host. That default is wrong for
|
|
56
73
|
// a host under a two-label public suffix (example.gc.ca gives gc.ca), so give --web-mail-domain there.
|
|
57
74
|
// Results carry host names, header values, protocol versions and dates: never page content, cookies or a person.
|
|
@@ -60,15 +77,17 @@ import { resolveCaa, resolveTxt } from "node:dns/promises";
|
|
|
60
77
|
import tls from "node:tls";
|
|
61
78
|
import { COLLECTOR_VERSION } from "../payload.js";
|
|
62
79
|
import { Emitter } from "./readonly_http.js";
|
|
80
|
+
import { MAX_PAGE_BYTES, judgeConsent, judgeRequestChannel, looksLikeNotice, noticeDate, noticeTopics, pageText, robotsAllows, TOPICS } from "./web_privacy.js";
|
|
63
81
|
export const WEB_URL_CHECKS = ["tls_min_version", "certificate_valid", "https_redirect", "hsts", "csp", "content_type_options",
|
|
64
|
-
"frame_protection", "referrer_policy", "no_version_banner"];
|
|
82
|
+
"frame_protection", "referrer_policy", "no_version_banner", "consent_banner"];
|
|
65
83
|
export const WEB_DOMAIN_CHECKS = ["spf", "dmarc", "dnssec", "caa"];
|
|
66
|
-
export const
|
|
84
|
+
export const WEB_PRIVACY_CHECKS = ["privacy_notice_published", "privacy_notice_dated", "privacy_notice_topics", "privacy_request_channel"];
|
|
85
|
+
export const WEB_CHECKS = [...WEB_URL_CHECKS, ...WEB_DOMAIN_CHECKS, ...WEB_PRIVACY_CHECKS];
|
|
67
86
|
export const USER_AGENT = `sagentix-evidence-collector/${COLLECTOR_VERSION} (+https://github.com/Sagentix-Advisors-Inc/sagentix-collector)`;
|
|
68
87
|
export const PROOF_PREFIX = "_sagentix-collector";
|
|
69
88
|
export const PROOF_TXT = "sagentix-collector-verification";
|
|
70
89
|
export const PROOF_PATH = "/.well-known/sagentix-collector.txt";
|
|
71
|
-
export const DEFAULTS = { delayMs: 1000, timeoutMs: 15_000, certMinDays: 14, hstsMinAge: 31_536_000 };
|
|
90
|
+
export const DEFAULTS = { delayMs: 1000, timeoutMs: 15_000, certMinDays: 14, hstsMinAge: 31_536_000, privacyMaxAgeDays: 365 };
|
|
72
91
|
const TOKEN = /^[A-Za-z0-9_-]{32,128}$/;
|
|
73
92
|
const DAY = 86_400_000;
|
|
74
93
|
const dnsMissing = (e) => ["ENODATA", "ENOTFOUND"].includes(String(e.code));
|
|
@@ -270,6 +289,17 @@ export function judgeDmarc(record, at, domain = at) {
|
|
|
270
289
|
/** Is a TLS error a local limit (this Node cannot offer the old protocol) or a network failure, rather than the server refusing? */
|
|
271
290
|
const LOCAL_TLS = /NO_PROTOCOLS_AVAILABLE|NO_CIPHERS_AVAILABLE|NO_SUITABLE/i;
|
|
272
291
|
const NETWORK = /^(timeout|ENOTFOUND|EAI_AGAIN|EHOSTUNREACH|ENETUNREACH|ECONNREFUSED|ETIMEDOUT)$/;
|
|
292
|
+
/** May this collector read the page body at `u`? robots.txt is fetched once per host. */
|
|
293
|
+
async function robotsOk(c, u) {
|
|
294
|
+
if (!c.robots.has(u.hostname)) {
|
|
295
|
+
const r = await get(c, `https://${u.hostname}/robots.txt`, 512 * 1024);
|
|
296
|
+
// RFC 9309 §2.3.1.3: an unavailable file (4xx) means no rule applies; §2.3.1.4: an unreachable one (5xx, network
|
|
297
|
+
// error) means complete disallow. A redirect is not followed, so it counts as unavailable.
|
|
298
|
+
c.robots.set(u.hostname, "error" in r || r.status >= 500 ? null : r.status === 200 ? r.body : "");
|
|
299
|
+
}
|
|
300
|
+
const rules = c.robots.get(u.hostname);
|
|
301
|
+
return typeof rules === "string" && robotsAllows(rules, USER_AGENT, u.pathname + u.search);
|
|
302
|
+
}
|
|
273
303
|
/** Waits so that requests to one host are at least delayMs apart. */
|
|
274
304
|
async function pace(c, host) {
|
|
275
305
|
const prev = c.last.get(host);
|
|
@@ -372,9 +402,10 @@ async function urlChecks(c, out, u, prefix, certMinDays, now) {
|
|
|
372
402
|
const ok = [301, 302, 307, 308].includes(r80.status) && to?.protocol === "https:" && to.hostname === host;
|
|
373
403
|
add("https_redirect", ok, ok ? `HTTP ${r80.status} to ${to.origin}` : `http://${host} answered ${r80.status}${loc ? ` to ${loc.slice(0, 120)}` : " without a redirect"}; a redirect to https://${host} is required`);
|
|
374
404
|
}
|
|
375
|
-
const
|
|
405
|
+
const bodyOk = await robotsOk(c, u);
|
|
406
|
+
const r = await get(c, u.href, bodyOk ? MAX_PAGE_BYTES : 0);
|
|
376
407
|
if ("error" in r) {
|
|
377
|
-
for (const k of ["hsts", "csp", "content_type_options", "frame_protection", "referrer_policy", "no_version_banner"])
|
|
408
|
+
for (const k of ["hsts", "csp", "content_type_options", "frame_protection", "referrer_policy", "no_version_banner", "consent_banner"])
|
|
378
409
|
add(k, "skipped", `${u.href} could not be read (${r.error})`);
|
|
379
410
|
return;
|
|
380
411
|
}
|
|
@@ -385,6 +416,7 @@ async function urlChecks(c, out, u, prefix, certMinDays, now) {
|
|
|
385
416
|
if (headerValue(r.headers, "cf-mitigated")?.trim().toLowerCase() === "challenge") {
|
|
386
417
|
for (const k of ["hsts", "csp", "content_type_options", "frame_protection", "referrer_policy", "no_version_banner"])
|
|
387
418
|
add(k, "skipped", `${u.href} answered a Cloudflare challenge page (HTTP ${r.status}, cf-mitigated: challenge), not the site; let the collector's address skip bot checks for this host to judge its headers`);
|
|
419
|
+
add("consent_banner", "skipped", `${u.href} answered a Cloudflare challenge page, not the site`);
|
|
388
420
|
return;
|
|
389
421
|
}
|
|
390
422
|
const h = r.headers, note = r.status >= 300 && r.status < 400 ? ` (headers of the ${r.status} answer; the redirect was not followed)` : "";
|
|
@@ -401,6 +433,56 @@ async function urlChecks(c, out, u, prefix, certMinDays, now) {
|
|
|
401
433
|
add("referrer_policy", rp.pass, rp.detail + note);
|
|
402
434
|
const b = judgeBanner(h);
|
|
403
435
|
add("no_version_banner", b.pass, b.detail + note);
|
|
436
|
+
if (!bodyOk)
|
|
437
|
+
add("consent_banner", "skipped", `robots.txt on ${host} does not let this collector read ${u.pathname}, or could not be read`);
|
|
438
|
+
else if (r.status !== 200)
|
|
439
|
+
add("consent_banner", "skipped", `${u.href} answered ${r.status}; the page itself is needed`);
|
|
440
|
+
else {
|
|
441
|
+
const k = judgeConsent(r.body, u);
|
|
442
|
+
add("consent_banner", k.result === "skipped" ? "skipped" : k.result === "pass", k.detail);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
async function privacyChecks(c, out, u, prefix, maxAgeDays, now) {
|
|
446
|
+
const res = `url ${u.href}`.slice(0, 300);
|
|
447
|
+
const add = (check, result, detail) => out.add(check, result, prefix + detail, res);
|
|
448
|
+
const rest = (detail) => { for (const k of WEB_PRIVACY_CHECKS.slice(1))
|
|
449
|
+
add(k, "skipped", detail); };
|
|
450
|
+
if (!(await robotsOk(c, u))) {
|
|
451
|
+
for (const k of WEB_PRIVACY_CHECKS)
|
|
452
|
+
add(k, "skipped", `robots.txt on ${u.hostname} does not let this collector read ${u.pathname}, or could not be read`);
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
const r = await get(c, u.href, MAX_PAGE_BYTES);
|
|
456
|
+
if ("error" in r) {
|
|
457
|
+
add("privacy_notice_published", "fail", `${u.href} could not be read (${r.error}); a reachable notice is required`);
|
|
458
|
+
return rest("the notice could not be read");
|
|
459
|
+
}
|
|
460
|
+
if (headerValue(r.headers, "cf-mitigated")?.trim().toLowerCase() === "challenge") {
|
|
461
|
+
for (const k of WEB_PRIVACY_CHECKS)
|
|
462
|
+
add(k, "skipped", `${u.href} answered a Cloudflare challenge page, not the notice`);
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
const type = headerValue(r.headers, "content-type") ?? "";
|
|
466
|
+
if (r.status !== 200 || !/html/i.test(type)) {
|
|
467
|
+
add("privacy_notice_published", "fail", `${u.href} answered ${r.status}${type ? ` (${type.slice(0, 40)})` : ""}${r.status >= 300 && r.status < 400 ? ", a redirect (not followed): declare the final address" : ""}; HTTP 200 with an HTML page required`);
|
|
468
|
+
return rest("the notice could not be read");
|
|
469
|
+
}
|
|
470
|
+
const text = pageText(r.body);
|
|
471
|
+
const named = looksLikeNotice(r.body);
|
|
472
|
+
add("privacy_notice_published", named ? "pass" : "fail", named ? `HTTP 200, HTML, ${text.length} characters of text; the title or first heading names privacy or personal information`
|
|
473
|
+
: "HTTP 200, but neither the title nor the first heading names privacy or personal information; declare the notice's own address");
|
|
474
|
+
const d = noticeDate(text);
|
|
475
|
+
if (!d)
|
|
476
|
+
add("privacy_notice_dated", "fail", `no "last updated", "effective" or "dernière mise à jour" date found on the notice; a dated notice is required`);
|
|
477
|
+
else {
|
|
478
|
+
const age = Math.floor((now.getTime() - Date.parse(d.date)) / DAY);
|
|
479
|
+
add("privacy_notice_dated", age >= -1 && age <= maxAgeDays ? "pass" : "fail", `"${d.label}" ${d.date}: ${age < -1 ? "in the future" : `${age} day(s) old`}; at most ${maxAgeDays} allowed`);
|
|
480
|
+
}
|
|
481
|
+
const t = noticeTopics(text);
|
|
482
|
+
const names = (ids) => ids.map((id) => TOPICS.find((x) => x.id === id).en).join("; ");
|
|
483
|
+
add("privacy_notice_topics", t.missing.length ? "fail" : "pass", `${t.found.length} of ${TOPICS.length} topics found${t.missing.length ? `; missing: ${names(t.missing)}` : ""} (phrases in English or French; a person confirms the wording)`);
|
|
484
|
+
const ch = judgeRequestChannel(r.body, u);
|
|
485
|
+
add("privacy_request_channel", ch.pass ? "pass" : "fail", ch.detail);
|
|
404
486
|
}
|
|
405
487
|
async function dsRecord(c, domain) {
|
|
406
488
|
const doh = `https://cloudflare-dns.com/dns-query?name=${encodeURIComponent(domain)}&type=DS`;
|
|
@@ -476,10 +558,11 @@ async function domainChecks(c, out, domain, prefix) {
|
|
|
476
558
|
}
|
|
477
559
|
export async function runWeb(o, client, now) {
|
|
478
560
|
const out = new Emitter("web", "web");
|
|
479
|
-
const c = { client, delayMs: o.delayMs, timeoutMs: o.timeoutMs, last: new Map() };
|
|
561
|
+
const c = { client, delayMs: o.delayMs, timeoutMs: o.timeoutMs, last: new Map(), robots: new Map() };
|
|
480
562
|
const proven = new Set();
|
|
481
563
|
const prefix = o.skipProof ? "control not proven (dry run only): " : "";
|
|
482
|
-
|
|
564
|
+
const privacy = o.privacyUrls ?? [];
|
|
565
|
+
for (const host of [...new Set([...o.urls, ...privacy].map((u) => u.hostname))]) {
|
|
483
566
|
if (o.skipProof) {
|
|
484
567
|
proven.add(host);
|
|
485
568
|
continue;
|
|
@@ -487,13 +570,21 @@ export async function runWeb(o, client, now) {
|
|
|
487
570
|
const p = await prove(c, host, o.proofs.get(host));
|
|
488
571
|
if (p.ok)
|
|
489
572
|
proven.add(host);
|
|
490
|
-
else
|
|
573
|
+
else {
|
|
491
574
|
for (const u of o.urls.filter((x) => x.hostname === host))
|
|
492
575
|
out.skipAll(WEB_URL_CHECKS, p.why, `url ${u.href}`.slice(0, 300));
|
|
576
|
+
for (const u of privacy.filter((x) => x.hostname === host))
|
|
577
|
+
out.skipAll(WEB_PRIVACY_CHECKS, p.why, `url ${u.href}`.slice(0, 300));
|
|
578
|
+
}
|
|
493
579
|
}
|
|
494
580
|
for (const u of o.urls)
|
|
495
581
|
if (proven.has(u.hostname))
|
|
496
582
|
await urlChecks(c, out, u, prefix, o.certMinDays, now);
|
|
583
|
+
if (!privacy.length)
|
|
584
|
+
out.skipAll(WEB_PRIVACY_CHECKS, "give --web-privacy-url <https URL of the privacy notice> to read the notice", "web privacy notice");
|
|
585
|
+
for (const u of privacy)
|
|
586
|
+
if (proven.has(u.hostname))
|
|
587
|
+
await privacyChecks(c, out, u, prefix, o.privacyMaxAgeDays ?? DEFAULTS.privacyMaxAgeDays, now);
|
|
497
588
|
const domains = [...new Set(o.mailDomains.length ? o.mailDomains : o.urls.map((u) => twoLabels(u.hostname)))];
|
|
498
589
|
for (const d of domains) {
|
|
499
590
|
// A mail domain is read only beside a proven host: the same domain, a parent, or a sibling under the same two labels.
|
|
@@ -514,6 +605,8 @@ export const web = {
|
|
|
514
605
|
"web-skip-proof": { type: "boolean", help: "run without proof of control; accepted only with --dry-run" },
|
|
515
606
|
"web-delay-ms": { type: "string", help: `milliseconds between two requests to one host (default ${DEFAULTS.delayMs}, at least 200)` },
|
|
516
607
|
"web-cert-min-days": { type: "string", help: `days a certificate must still be valid (default ${DEFAULTS.certMinDays})` },
|
|
608
|
+
"web-privacy-url": { type: "string", multiple: true, help: "https URL of the organization's privacy notice, on a proven host (repeatable: the English and the French notice)" },
|
|
609
|
+
"web-privacy-max-age-days": { type: "string", help: `days since the notice's "last updated" date before it fails (default ${DEFAULTS.privacyMaxAgeDays})` },
|
|
517
610
|
},
|
|
518
611
|
emits: WEB_CHECKS.map((c) => `web.${c}`),
|
|
519
612
|
describe(opts) {
|
|
@@ -528,7 +621,7 @@ export const web = {
|
|
|
528
621
|
validate(opts) {
|
|
529
622
|
if (opts["web-skip-proof"] === true && opts["dry-run"] !== true)
|
|
530
623
|
return "--web-skip-proof is accepted only with --dry-run: a push to sos-app needs proof that the organization controls each host.";
|
|
531
|
-
for (const raw of list(opts["web-url"])) {
|
|
624
|
+
for (const raw of [...list(opts["web-url"]), ...list(opts["web-privacy-url"])]) {
|
|
532
625
|
const u = parseWebUrl(raw);
|
|
533
626
|
if (typeof u === "string")
|
|
534
627
|
return u;
|
|
@@ -543,7 +636,8 @@ export const web = {
|
|
|
543
636
|
},
|
|
544
637
|
async run(ctx) {
|
|
545
638
|
const urls = list(ctx.opts["web-url"]).map(parseWebUrl).filter((u) => typeof u !== "string");
|
|
546
|
-
|
|
639
|
+
const privacyUrls = list(ctx.opts["web-privacy-url"]).map(parseWebUrl).filter((u) => typeof u !== "string");
|
|
640
|
+
if (!urls.length && !privacyUrls.length) {
|
|
547
641
|
const out = new Emitter("web", "web");
|
|
548
642
|
out.skipAll(WEB_CHECKS, "give --web-url <https URL> (and --web-proof <host>=<token>)");
|
|
549
643
|
return out.results;
|
|
@@ -552,6 +646,7 @@ export const web = {
|
|
|
552
646
|
const skipProof = ctx.opts["web-skip-proof"] === true && ctx.opts["dry-run"] === true;
|
|
553
647
|
const n = (k, d, min = 0) => { const v = Number(ctx.opts[k]); return Number.isFinite(v) && typeof ctx.opts[k] === "string" ? Math.max(min, v) : d; };
|
|
554
648
|
return runWeb({ urls, proofs: typeof proofs === "string" ? new Map() : proofs, mailDomains: list(ctx.opts["web-mail-domain"]).map((d) => d.toLowerCase()),
|
|
555
|
-
skipProof, delayMs: n("web-delay-ms", DEFAULTS.delayMs, 200), timeoutMs: DEFAULTS.timeoutMs, certMinDays: n("web-cert-min-days", DEFAULTS.certMinDays)
|
|
649
|
+
skipProof, delayMs: n("web-delay-ms", DEFAULTS.delayMs, 200), timeoutMs: DEFAULTS.timeoutMs, certMinDays: n("web-cert-min-days", DEFAULTS.certMinDays),
|
|
650
|
+
privacyUrls, privacyMaxAgeDays: n("web-privacy-max-age-days", DEFAULTS.privacyMaxAgeDays, 1) }, ctx.clients.web ?? realWebClient(), ctx.now);
|
|
556
651
|
},
|
|
557
652
|
};
|