@slicedearth/whoisleuth-cli 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/LICENSES/Retire.js-Apache-2.0.txt +199 -0
- package/NOTICE +76 -0
- package/README.md +84 -0
- package/TRADEMARKS.md +41 -0
- package/bin/whoisleuth.mjs +35 -0
- package/cli/archive-inspect.mjs +198 -0
- package/cli/arguments.mjs +1022 -0
- package/cli/artifact-verify.mjs +210 -0
- package/cli/bulk-checkpoint.mjs +218 -0
- package/cli/bulk-command-runner.mjs +109 -0
- package/cli/bulk-output.mjs +112 -0
- package/cli/bulk.mjs +133 -0
- package/cli/compare.mjs +238 -0
- package/cli/completion.mjs +175 -0
- package/cli/csv.mjs +1 -0
- package/cli/discover.mjs +50 -0
- package/cli/discovery-command-runner.mjs +36 -0
- package/cli/discovery-observation-snapshot.mjs +315 -0
- package/cli/discovery-scan-command-runner.mjs +141 -0
- package/cli/discovery-scan.mjs +325 -0
- package/cli/discovery-snapshot.mjs +133 -0
- package/cli/discovery-workflow.mjs +73 -0
- package/cli/doctor.mjs +137 -0
- package/cli/errors.mjs +21 -0
- package/cli/evidence-command-arguments.mjs +145 -0
- package/cli/evidence-command-runner.mjs +106 -0
- package/cli/evidence-signing.mjs +215 -0
- package/cli/exit-codes.mjs +9 -0
- package/cli/export-evidence.mjs +28 -0
- package/cli/formatters/evidence-report.mjs +3 -0
- package/cli/formatters/html.mjs +69 -0
- package/cli/formatters/json.mjs +135 -0
- package/cli/formatters/markdown.mjs +3 -0
- package/cli/formatters/terminal.mjs +658 -0
- package/cli/http.mjs +27 -0
- package/cli/lookup-command-runner.mjs +87 -0
- package/cli/lookup-diff.mjs +171 -0
- package/cli/mail-review.mjs +204 -0
- package/cli/manual.mjs +34 -0
- package/cli/network-command-runner.mjs +88 -0
- package/cli/output-file.mjs +97 -0
- package/cli/page-compare.mjs +123 -0
- package/cli/posture.mjs +20 -0
- package/cli/progress-events.mjs +53 -0
- package/cli/progress.mjs +56 -0
- package/cli/registry-access.mjs +3 -0
- package/cli/registry-support.mjs +120 -0
- package/cli/risk-calibration.mjs +294 -0
- package/cli/runner-types.mjs +1 -0
- package/cli/runner.mjs +726 -0
- package/cli/saved-lookup.mjs +87 -0
- package/cli/source-reliability.mjs +727 -0
- package/cli/strict-exit.mjs +55 -0
- package/cli/terminal-presentation.mjs +147 -0
- package/docs/cli.md +827 -0
- package/frontend/src/lib/analysis/acquisition-decision-packet.js +93 -0
- package/frontend/src/lib/analysis/acquisition-due-diligence.js +391 -0
- package/frontend/src/lib/analysis/artifact-integrity.js +21 -0
- package/frontend/src/lib/analysis/brand-profile-model.js +421 -0
- package/frontend/src/lib/analysis/bulk-domain-comparison.js +246 -0
- package/frontend/src/lib/analysis/bulk-mail-exposure.js +192 -0
- package/frontend/src/lib/analysis/bulk-retry-plan.js +86 -0
- package/frontend/src/lib/analysis/bulk-review-export.js +62 -0
- package/frontend/src/lib/analysis/bulk-review-model.js +266 -0
- package/frontend/src/lib/analysis/bulk-session-model.js +515 -0
- package/frontend/src/lib/analysis/bulk-sort.js +49 -0
- package/frontend/src/lib/analysis/campaign-model.js +265 -0
- package/frontend/src/lib/analysis/case-evidence-model.js +642 -0
- package/frontend/src/lib/analysis/case-migration-model.js +248 -0
- package/frontend/src/lib/analysis/case-model.js +5 -0
- package/frontend/src/lib/analysis/case-record-contracts.js +44 -0
- package/frontend/src/lib/analysis/case-record-core.js +232 -0
- package/frontend/src/lib/analysis/case-record-model.js +7 -0
- package/frontend/src/lib/analysis/case-record-operations.js +224 -0
- package/frontend/src/lib/analysis/case-response-model.js +640 -0
- package/frontend/src/lib/analysis/case-response-packet.js +573 -0
- package/frontend/src/lib/analysis/case-storage-model.js +93 -0
- package/frontend/src/lib/analysis/detection-rule-model.js +306 -0
- package/frontend/src/lib/analysis/dns-record-normalization.js +11 -0
- package/frontend/src/lib/analysis/http-summary.js +165 -0
- package/frontend/src/lib/analysis/investigation-guide.js +500 -0
- package/frontend/src/lib/analysis/investigation-template-model.js +144 -0
- package/frontend/src/lib/analysis/page-baseline.js +272 -0
- package/frontend/src/lib/analysis/page-similarity.js +179 -0
- package/frontend/src/lib/analysis/relationship-evidence.js +265 -0
- package/frontend/src/lib/analysis/relationship-observation-model.js +423 -0
- package/frontend/src/lib/analysis/scoring.js +135 -0
- package/frontend/src/lib/analysis/shortlist-model.js +209 -0
- package/frontend/src/lib/analysis/utils.js +265 -0
- package/frontend/src/lib/analysis/watchlist-history.js +643 -0
- package/frontend/src/lib/analysis/watchlist-store.js +127 -0
- package/frontend/src/lib/analysis/website-snapshot-model.js +302 -0
- package/frontend/src/lib/analysis/workspace-archive-crypto.js +282 -0
- package/frontend/src/lib/analysis/workspace-archive.js +472 -0
- package/lib/abort.mjs +26 -0
- package/lib/abusech-auth.mjs +19 -0
- package/lib/availability.mjs +716 -0
- package/lib/bounded-contract-normalizers.mjs +61 -0
- package/lib/browser-library-profile.mjs +310 -0
- package/lib/bulk-comparison-evidence.mjs +83 -0
- package/lib/classify.mjs +93 -0
- package/lib/client-behavior-profile.mjs +83 -0
- package/lib/credential-surface-profile.mjs +66 -0
- package/lib/csv.mjs +23 -0
- package/lib/ct-query.mjs +27 -0
- package/lib/ct-search.mjs +393 -0
- package/lib/distributed-operation-budget.mjs +461 -0
- package/lib/dns-delegation-health.mjs +344 -0
- package/lib/dns-intelligence.mjs +739 -0
- package/lib/dns-mx.mjs +24 -0
- package/lib/dns-resolver-selection.mjs +33 -0
- package/lib/domain-posture-analysis.mjs +254 -0
- package/lib/domain-posture-parsers.mjs +358 -0
- package/lib/domain-posture.mjs +745 -0
- package/lib/error-detail.mjs +10 -0
- package/lib/evidence-export.mjs +431 -0
- package/lib/evidence-report-markdown.mjs +71 -0
- package/lib/evidence-report.mjs +391 -0
- package/lib/external-intelligence-risk.mjs +108 -0
- package/lib/favicon.mjs +146 -0
- package/lib/feature-policy.mjs +70 -0
- package/lib/generated/retire-browser-catalog.mjs +8724 -0
- package/lib/generated/unicode-confusables-17.mjs +93 -0
- package/lib/hostname.mjs +38 -0
- package/lib/html-signals.mjs +680 -0
- package/lib/http-intelligence.mjs +232 -0
- package/lib/idn-confusable-policy.mjs +86 -0
- package/lib/idn-confusables.mjs +465 -0
- package/lib/lookup-cache.mjs +130 -0
- package/lib/lookup-progress.mjs +291 -0
- package/lib/lookup-source-progress.mjs +109 -0
- package/lib/lookup-threat-provider-inventory.mjs +13 -0
- package/lib/lookup.mjs +592 -0
- package/lib/observation.mjs +114 -0
- package/lib/observed-network-context.mjs +355 -0
- package/lib/operation-budget.mjs +489 -0
- package/lib/page-fingerprints.mjs +481 -0
- package/lib/page-role-profile.mjs +117 -0
- package/lib/perceptual-hash.mjs +451 -0
- package/lib/rdap-attempts.mjs +34 -0
- package/lib/rdap-bootstrap.mjs +173 -0
- package/lib/rdap-capabilities.mjs +292 -0
- package/lib/rdap-client.mjs +125 -0
- package/lib/rdap-normalization.mjs +691 -0
- package/lib/rdap-registrar.mjs +196 -0
- package/lib/rdap-transport.mjs +39 -0
- package/lib/rdap-types.mjs +1 -0
- package/lib/rdap-validation.mjs +94 -0
- package/lib/rdap.mjs +16 -0
- package/lib/registry-access.mjs +32 -0
- package/lib/registry-capabilities.mjs +117 -0
- package/lib/registry-capability-catalogue.mjs +1206 -0
- package/lib/registry-comparison.mjs +318 -0
- package/lib/registry-dates.mjs +190 -0
- package/lib/registry-insights.mjs +325 -0
- package/lib/response-policy.mjs +256 -0
- package/lib/risk-scoring.mjs +164 -0
- package/lib/safe-fetch.mjs +446 -0
- package/lib/security-txt.mjs +359 -0
- package/lib/service-binding-dns.mjs +608 -0
- package/lib/sslbl-certificates.generated.mjs +143 -0
- package/lib/sslbl-intelligence.mjs +241 -0
- package/lib/static-html-analysis.mjs +340 -0
- package/lib/structured-data-identity.mjs +323 -0
- package/lib/threat-intelligence-contract.mjs +5 -0
- package/lib/threat-intelligence-definition-registry.mjs +34 -0
- package/lib/threat-intelligence-fixtures.mjs +82 -0
- package/lib/threat-intelligence-manifests.mjs +512 -0
- package/lib/threat-intelligence-runtime.mjs +662 -0
- package/lib/threat-intelligence-targets.mjs +168 -0
- package/lib/threat-intelligence-types.mjs +3 -0
- package/lib/threatfox-intelligence.mjs +307 -0
- package/lib/tls-intelligence.mjs +814 -0
- package/lib/typosquat-generator.mjs +884 -0
- package/lib/urlhaus-intelligence.mjs +361 -0
- package/lib/urlscan-intelligence.mjs +349 -0
- package/lib/website-security-posture.mjs +325 -0
- package/lib/website-technology.mjs +437 -0
- package/lib/whois-authority.mjs +98 -0
- package/lib/whois-chain.mjs +261 -0
- package/lib/whois-contacts.mjs +163 -0
- package/lib/whois-contracts.mjs +3 -0
- package/lib/whois-normalization.mjs +27 -0
- package/lib/whois-transport.mjs +100 -0
- package/lib/whois-values.mjs +92 -0
- package/lib/whois.mjs +1187 -0
- package/package.json +55 -0
- package/third-party-notices.txt +5571 -0
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
// Pure Brand Profile normalization and storage model. The browser wrapper owns
|
|
2
|
+
// persistence and downloads; this module owns schema migration, semantic field
|
|
3
|
+
// bounds, import merging, and exact serialized-byte accounting.
|
|
4
|
+
import { normalizeDomain } from "./case-model.js";
|
|
5
|
+
import { normalizePageBaseline } from "./page-baseline.js";
|
|
6
|
+
import { isInformativeFaviconHash } from "./utils.js";
|
|
7
|
+
export const BRAND_PROFILE_SCHEMA = 'whoisleuth.brand-profiles';
|
|
8
|
+
export const BRAND_PROFILE_SCHEMA_VERSION = 5;
|
|
9
|
+
export const MAX_PROFILES = 100;
|
|
10
|
+
export const MAX_PROFILE_VALUES = 200;
|
|
11
|
+
export const MAX_PROFILE_VALUE_INPUTS = MAX_PROFILE_VALUES * 4;
|
|
12
|
+
// Profiles share the origin's browser-storage quota with cases, campaigns,
|
|
13
|
+
// watchlists, and CT history. Fail at a predictable one-megabyte boundary.
|
|
14
|
+
export const MAX_PROFILE_STORE_BYTES = 1024 * 1024;
|
|
15
|
+
export const MAX_PROFILE_NAME_LENGTH = 100;
|
|
16
|
+
export const MAX_PROFILE_TEXT_LENGTH = 200;
|
|
17
|
+
export const MAX_PROFILE_DOMAIN_LENGTH = 253;
|
|
18
|
+
export const MAX_PROFILE_TLD_LENGTH = 63;
|
|
19
|
+
export const MAX_DKIM_SELECTOR_LENGTH = 253;
|
|
20
|
+
export const MAX_DKIM_SELECTORS = 10;
|
|
21
|
+
export const MAX_PROTECTION_ATTESTATIONS = 6;
|
|
22
|
+
export const MAX_DESIRED_POSTURE_BASELINES = 20;
|
|
23
|
+
export const MAX_DESIRED_POSTURE_RECORDS = 32;
|
|
24
|
+
export const MAX_DESIRED_POSTURE_SUPPRESSIONS = 12;
|
|
25
|
+
const SAFE_ID_RE = /^[A-Za-z0-9_-]{1,128}$/;
|
|
26
|
+
const SHA256_RE = /^[a-f0-9]{64}$/i;
|
|
27
|
+
const CONTROL_RE = /[\x00-\x1f\x7f]/;
|
|
28
|
+
const DNS_LABEL_RE = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
|
29
|
+
const MAIL_PROFILES = new Set(['standard', 'defensive_no_mail', 'parked']);
|
|
30
|
+
export const PROTECTION_ATTESTATION_CONTROLS = Object.freeze([
|
|
31
|
+
'registrar_mfa',
|
|
32
|
+
'recovery_email_separation',
|
|
33
|
+
'registry_lock',
|
|
34
|
+
'emergency_contacts',
|
|
35
|
+
'account_audit_logging',
|
|
36
|
+
'zone_backups',
|
|
37
|
+
]);
|
|
38
|
+
const PROTECTION_ATTESTATION_CONTROL_SET = new Set(PROTECTION_ATTESTATION_CONTROLS);
|
|
39
|
+
const PROTECTION_ATTESTATION_STATES = new Set([
|
|
40
|
+
'observed',
|
|
41
|
+
'not_observed',
|
|
42
|
+
'needs_confirmation',
|
|
43
|
+
'unavailable',
|
|
44
|
+
'not_applicable',
|
|
45
|
+
]);
|
|
46
|
+
const DESIRED_POSTURE_FIELDS = new Set([
|
|
47
|
+
'nameservers',
|
|
48
|
+
'ds',
|
|
49
|
+
'mx',
|
|
50
|
+
'caa',
|
|
51
|
+
'tls_issuer',
|
|
52
|
+
'tls_san_patterns',
|
|
53
|
+
'tls_spki',
|
|
54
|
+
'registrar_lock',
|
|
55
|
+
'renewal_review',
|
|
56
|
+
]);
|
|
57
|
+
const POSTURE_CHECK_STATUSES = new Set(['danger', 'info', 'pass', 'warning']);
|
|
58
|
+
function record(value) {
|
|
59
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
60
|
+
? value
|
|
61
|
+
: {};
|
|
62
|
+
}
|
|
63
|
+
function boundedText(value, maximum = MAX_PROFILE_TEXT_LENGTH) {
|
|
64
|
+
if (typeof value !== 'string' || CONTROL_RE.test(value))
|
|
65
|
+
return '';
|
|
66
|
+
return value.slice(0, maximum * 4).replace(/\s+/g, ' ').trim().slice(0, maximum).trim();
|
|
67
|
+
}
|
|
68
|
+
function safeId(value) {
|
|
69
|
+
return typeof value === 'string' && SAFE_ID_RE.test(value) ? value : null;
|
|
70
|
+
}
|
|
71
|
+
function timestamp(value, fallback) {
|
|
72
|
+
if (typeof value === 'string' && value.length <= 64 && !CONTROL_RE.test(value)) {
|
|
73
|
+
const parsed = Date.parse(value);
|
|
74
|
+
if (Number.isFinite(parsed))
|
|
75
|
+
return new Date(parsed).toISOString();
|
|
76
|
+
}
|
|
77
|
+
return fallback;
|
|
78
|
+
}
|
|
79
|
+
function normalizeTld(value) {
|
|
80
|
+
if (typeof value !== 'string' || CONTROL_RE.test(value))
|
|
81
|
+
return '';
|
|
82
|
+
const tld = value.trim().toLowerCase().replace(/^\./, '');
|
|
83
|
+
if (!tld || tld.length > MAX_PROFILE_TLD_LENGTH || !DNS_LABEL_RE.test(tld))
|
|
84
|
+
return '';
|
|
85
|
+
return tld;
|
|
86
|
+
}
|
|
87
|
+
function normalizeSelector(value) {
|
|
88
|
+
if (typeof value !== 'string' || CONTROL_RE.test(value))
|
|
89
|
+
return '';
|
|
90
|
+
const selector = value.trim().toLowerCase().replace(/^\.+|\.+$/g, '');
|
|
91
|
+
if (!selector || selector.length > MAX_DKIM_SELECTOR_LENGTH)
|
|
92
|
+
return '';
|
|
93
|
+
return selector.split('.').every((label) => DNS_LABEL_RE.test(label)) ? selector : '';
|
|
94
|
+
}
|
|
95
|
+
function normalizeList(value, normalize) {
|
|
96
|
+
if (!Array.isArray(value))
|
|
97
|
+
return [];
|
|
98
|
+
const values = [];
|
|
99
|
+
const seen = new Set();
|
|
100
|
+
for (const item of value.slice(0, MAX_PROFILE_VALUE_INPUTS)) {
|
|
101
|
+
const normalized = normalize(item);
|
|
102
|
+
if (!normalized)
|
|
103
|
+
continue;
|
|
104
|
+
const key = normalized.toLowerCase();
|
|
105
|
+
if (seen.has(key))
|
|
106
|
+
continue;
|
|
107
|
+
seen.add(key);
|
|
108
|
+
values.push(normalized);
|
|
109
|
+
if (values.length >= MAX_PROFILE_VALUES)
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
return values;
|
|
113
|
+
}
|
|
114
|
+
export function normalizeProfileDomains(value) {
|
|
115
|
+
return normalizeList(value, (item) => {
|
|
116
|
+
if (typeof item !== 'string' || item.length > MAX_PROFILE_DOMAIN_LENGTH)
|
|
117
|
+
return '';
|
|
118
|
+
return normalizeDomain(item);
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
export function normalizeProfileTextValues(value) {
|
|
122
|
+
return normalizeList(value, (item) => boundedText(item));
|
|
123
|
+
}
|
|
124
|
+
export function normalizeProfileTlds(value) {
|
|
125
|
+
return normalizeList(value, normalizeTld);
|
|
126
|
+
}
|
|
127
|
+
export function normalizeDkimSelectors(value) {
|
|
128
|
+
return normalizeList(value, normalizeSelector).slice(0, MAX_DKIM_SELECTORS);
|
|
129
|
+
}
|
|
130
|
+
function normalizeMailProtectionProfile(value) {
|
|
131
|
+
return typeof value === 'string' && MAIL_PROFILES.has(value)
|
|
132
|
+
? value
|
|
133
|
+
: 'standard';
|
|
134
|
+
}
|
|
135
|
+
export function normalizeProtectionAttestations(value) {
|
|
136
|
+
if (!Array.isArray(value))
|
|
137
|
+
return [];
|
|
138
|
+
const output = [];
|
|
139
|
+
const seen = new Set();
|
|
140
|
+
for (const item of value.slice(0, MAX_PROTECTION_ATTESTATIONS * 4)) {
|
|
141
|
+
const candidate = record(item);
|
|
142
|
+
if (typeof candidate.control !== 'string'
|
|
143
|
+
|| !PROTECTION_ATTESTATION_CONTROL_SET.has(candidate.control)
|
|
144
|
+
|| seen.has(candidate.control)
|
|
145
|
+
|| typeof candidate.state !== 'string'
|
|
146
|
+
|| !PROTECTION_ATTESTATION_STATES.has(candidate.state)) {
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const assertedAt = timestamp(candidate.assertedAt, null);
|
|
150
|
+
if (!assertedAt)
|
|
151
|
+
continue;
|
|
152
|
+
seen.add(candidate.control);
|
|
153
|
+
output.push({
|
|
154
|
+
control: candidate.control,
|
|
155
|
+
state: candidate.state,
|
|
156
|
+
assertedAt,
|
|
157
|
+
expiresAt: timestamp(candidate.expiresAt, null),
|
|
158
|
+
note: boundedText(candidate.note),
|
|
159
|
+
});
|
|
160
|
+
if (output.length >= MAX_PROTECTION_ATTESTATIONS)
|
|
161
|
+
break;
|
|
162
|
+
}
|
|
163
|
+
return output;
|
|
164
|
+
}
|
|
165
|
+
function normalizeDesiredPostureRecords(value, normalizer) {
|
|
166
|
+
if (!Array.isArray(value))
|
|
167
|
+
return [];
|
|
168
|
+
const output = new Set();
|
|
169
|
+
for (const item of value.slice(0, MAX_DESIRED_POSTURE_RECORDS * 4)) {
|
|
170
|
+
const candidate = normalizer
|
|
171
|
+
? normalizer(item)
|
|
172
|
+
: boundedText(item, 500);
|
|
173
|
+
if (candidate)
|
|
174
|
+
output.add(candidate);
|
|
175
|
+
if (output.size >= MAX_DESIRED_POSTURE_RECORDS)
|
|
176
|
+
break;
|
|
177
|
+
}
|
|
178
|
+
return [...output].sort();
|
|
179
|
+
}
|
|
180
|
+
function normalizeDesiredPostureObservation(value) {
|
|
181
|
+
const candidate = record(value);
|
|
182
|
+
const observedAt = timestamp(candidate.observedAt, null);
|
|
183
|
+
if (!observedAt || !Array.isArray(candidate.checks))
|
|
184
|
+
return null;
|
|
185
|
+
const checks = [];
|
|
186
|
+
const seen = new Set();
|
|
187
|
+
for (const item of candidate.checks.slice(0, 64)) {
|
|
188
|
+
const check = record(item);
|
|
189
|
+
const id = boundedText(check.id, 64);
|
|
190
|
+
if (!id
|
|
191
|
+
|| seen.has(id)
|
|
192
|
+
|| typeof check.status !== 'string'
|
|
193
|
+
|| !POSTURE_CHECK_STATUSES.has(check.status))
|
|
194
|
+
continue;
|
|
195
|
+
seen.add(id);
|
|
196
|
+
checks.push({
|
|
197
|
+
id,
|
|
198
|
+
status: check.status,
|
|
199
|
+
records: normalizeDesiredPostureRecords(check.records),
|
|
200
|
+
});
|
|
201
|
+
if (checks.length >= 32)
|
|
202
|
+
break;
|
|
203
|
+
}
|
|
204
|
+
return checks.length ? { observedAt, checks } : null;
|
|
205
|
+
}
|
|
206
|
+
function normalizeTlsSanPatterns(value) {
|
|
207
|
+
if (!Array.isArray(value))
|
|
208
|
+
return [];
|
|
209
|
+
const output = new Set();
|
|
210
|
+
for (const item of value.slice(0, MAX_DESIRED_POSTURE_RECORDS * 4)) {
|
|
211
|
+
const candidate = boundedText(item, MAX_PROFILE_DOMAIN_LENGTH).toLowerCase().replace(/\.$/u, '');
|
|
212
|
+
const wildcard = candidate.startsWith('*.');
|
|
213
|
+
const normalized = normalizeDomain(wildcard ? candidate.slice(2) : candidate);
|
|
214
|
+
if (!normalized)
|
|
215
|
+
continue;
|
|
216
|
+
output.add(wildcard ? `*.${normalized}` : normalized);
|
|
217
|
+
if (output.size >= MAX_DESIRED_POSTURE_RECORDS)
|
|
218
|
+
break;
|
|
219
|
+
}
|
|
220
|
+
return [...output].sort();
|
|
221
|
+
}
|
|
222
|
+
export function normalizeDesiredPostureBaselines(value, officialDomains, fallbackNow = new Date().toISOString()) {
|
|
223
|
+
if (!Array.isArray(value))
|
|
224
|
+
return [];
|
|
225
|
+
const allowedDomains = new Set(officialDomains);
|
|
226
|
+
const output = [];
|
|
227
|
+
const seen = new Set();
|
|
228
|
+
const fallback = timestamp(fallbackNow, new Date(0).toISOString());
|
|
229
|
+
for (const item of value.slice(0, MAX_DESIRED_POSTURE_BASELINES * 4)) {
|
|
230
|
+
const candidate = record(item);
|
|
231
|
+
const domain = normalizeDomain(candidate.domain);
|
|
232
|
+
if (!domain || !allowedDomains.has(domain) || seen.has(domain))
|
|
233
|
+
continue;
|
|
234
|
+
const suppressions = [];
|
|
235
|
+
const suppressionFields = new Set();
|
|
236
|
+
for (const rawSuppression of (Array.isArray(candidate.suppressions) ? candidate.suppressions : [])
|
|
237
|
+
.slice(0, MAX_DESIRED_POSTURE_SUPPRESSIONS * 4)) {
|
|
238
|
+
const suppression = record(rawSuppression);
|
|
239
|
+
const field = boundedText(suppression.field, 40);
|
|
240
|
+
const reason = boundedText(suppression.reason, MAX_PROFILE_TEXT_LENGTH);
|
|
241
|
+
if (!DESIRED_POSTURE_FIELDS.has(field) || !reason || suppressionFields.has(field))
|
|
242
|
+
continue;
|
|
243
|
+
suppressionFields.add(field);
|
|
244
|
+
suppressions.push({
|
|
245
|
+
field,
|
|
246
|
+
reason,
|
|
247
|
+
expiresAt: timestamp(suppression.expiresAt, null),
|
|
248
|
+
});
|
|
249
|
+
if (suppressions.length >= MAX_DESIRED_POSTURE_SUPPRESSIONS)
|
|
250
|
+
break;
|
|
251
|
+
}
|
|
252
|
+
seen.add(domain);
|
|
253
|
+
output.push({
|
|
254
|
+
version: 1,
|
|
255
|
+
domain,
|
|
256
|
+
nameservers: normalizeDesiredPostureRecords(candidate.nameservers, (entry) => normalizeDomain(entry)),
|
|
257
|
+
ds: normalizeDesiredPostureRecords(candidate.ds),
|
|
258
|
+
mx: normalizeDesiredPostureRecords(candidate.mx),
|
|
259
|
+
caa: normalizeDesiredPostureRecords(candidate.caa),
|
|
260
|
+
tlsIssuer: boundedText(candidate.tlsIssuer, MAX_PROFILE_TEXT_LENGTH),
|
|
261
|
+
tlsSanPatterns: normalizeTlsSanPatterns(candidate.tlsSanPatterns),
|
|
262
|
+
tlsSpkiSha256: typeof candidate.tlsSpkiSha256 === 'string' && SHA256_RE.test(candidate.tlsSpkiSha256)
|
|
263
|
+
? candidate.tlsSpkiSha256.toLowerCase()
|
|
264
|
+
: '',
|
|
265
|
+
registrarLock: ['required', 'not_required'].includes(String(candidate.registrarLock))
|
|
266
|
+
? candidate.registrarLock
|
|
267
|
+
: 'unconfigured',
|
|
268
|
+
renewalReviewAt: timestamp(candidate.renewalReviewAt, null),
|
|
269
|
+
suppressions,
|
|
270
|
+
note: boundedText(candidate.note, MAX_PROFILE_TEXT_LENGTH),
|
|
271
|
+
previousObservation: normalizeDesiredPostureObservation(candidate.previousObservation),
|
|
272
|
+
updatedAt: timestamp(candidate.updatedAt, fallback),
|
|
273
|
+
});
|
|
274
|
+
if (output.length >= MAX_DESIRED_POSTURE_BASELINES)
|
|
275
|
+
break;
|
|
276
|
+
}
|
|
277
|
+
return output;
|
|
278
|
+
}
|
|
279
|
+
function normalizeFaviconHash(value) {
|
|
280
|
+
return typeof value === 'string' && SHA256_RE.test(value) ? value.toLowerCase() : '';
|
|
281
|
+
}
|
|
282
|
+
function normalizeFaviconPHash(value) {
|
|
283
|
+
return typeof value === 'string' && isInformativeFaviconHash(value) ? value.toLowerCase() : '';
|
|
284
|
+
}
|
|
285
|
+
/** Normalize one profile while retaining only known, bounded fields. */
|
|
286
|
+
export function normalizeBrandProfile(raw, options = {}) {
|
|
287
|
+
const value = record(raw);
|
|
288
|
+
const existing = options.existing ? record(options.existing) : null;
|
|
289
|
+
const now = timestamp(options.nowIso, new Date().toISOString());
|
|
290
|
+
const officialDomains = normalizeProfileDomains(value.officialDomains);
|
|
291
|
+
const dkimSelectors = normalizeDkimSelectors(value.dkimSelectors);
|
|
292
|
+
const candidateBaseline = Object.prototype.hasOwnProperty.call(value, 'pageBaseline')
|
|
293
|
+
? normalizePageBaseline(value.pageBaseline)
|
|
294
|
+
: normalizePageBaseline(existing?.pageBaseline);
|
|
295
|
+
const pageBaseline = candidateBaseline && officialDomains.includes(candidateBaseline.domain)
|
|
296
|
+
? candidateBaseline
|
|
297
|
+
: null;
|
|
298
|
+
const profileId = safeId(existing?.id) || safeId(value.id) || (typeof options.makeId === 'function' ? safeId(options.makeId()) : null);
|
|
299
|
+
const name = boundedText(value.name, MAX_PROFILE_NAME_LENGTH);
|
|
300
|
+
if (!profileId || !name)
|
|
301
|
+
return null;
|
|
302
|
+
const createdAt = timestamp(existing?.createdAt, null) || timestamp(value.createdAt, now);
|
|
303
|
+
return {
|
|
304
|
+
id: profileId,
|
|
305
|
+
name,
|
|
306
|
+
officialDomains,
|
|
307
|
+
productNames: normalizeProfileTextValues(value.productNames),
|
|
308
|
+
tlds: normalizeProfileTlds(value.tlds),
|
|
309
|
+
approvedPartnerDomains: normalizeProfileDomains(value.approvedPartnerDomains),
|
|
310
|
+
allowlistedDomains: normalizeProfileDomains(value.allowlistedDomains),
|
|
311
|
+
allowlistedRegistrars: normalizeProfileTextValues(value.allowlistedRegistrars),
|
|
312
|
+
dkimSelectors,
|
|
313
|
+
retiredDkimSelectors: normalizeDkimSelectors(value.retiredDkimSelectors)
|
|
314
|
+
.filter((selector) => !dkimSelectors.includes(selector)),
|
|
315
|
+
mailProtectionProfile: normalizeMailProtectionProfile(value.mailProtectionProfile),
|
|
316
|
+
protectionAttestations: normalizeProtectionAttestations(value.protectionAttestations),
|
|
317
|
+
desiredPostureBaselines: normalizeDesiredPostureBaselines(value.desiredPostureBaselines, officialDomains, now),
|
|
318
|
+
trademarkOwner: boundedText(value.trademarkOwner),
|
|
319
|
+
trademarkRegistration: boundedText(value.trademarkRegistration),
|
|
320
|
+
officialFaviconHash: normalizeFaviconHash(value.officialFaviconHash),
|
|
321
|
+
officialFaviconPHash: normalizeFaviconPHash(value.officialFaviconPHash),
|
|
322
|
+
pageBaseline,
|
|
323
|
+
createdAt,
|
|
324
|
+
updatedAt: options.touch === true ? now : timestamp(value.updatedAt, createdAt),
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
function profileList(raw) {
|
|
328
|
+
if (Array.isArray(raw))
|
|
329
|
+
return raw;
|
|
330
|
+
const value = record(raw);
|
|
331
|
+
return Array.isArray(value.profiles) ? value.profiles : [];
|
|
332
|
+
}
|
|
333
|
+
export function brandProfileStoreVersion(raw) {
|
|
334
|
+
if (Array.isArray(raw))
|
|
335
|
+
return 1;
|
|
336
|
+
const value = record(raw);
|
|
337
|
+
return typeof value.version === 'number' && Number.isFinite(value.version) && value.version > 0 ? value.version : null;
|
|
338
|
+
}
|
|
339
|
+
/** Normalize an internal profile collection or current stored envelope. */
|
|
340
|
+
export function normalizeBrandProfileStore(raw) {
|
|
341
|
+
const byId = new Map();
|
|
342
|
+
for (const item of profileList(raw).slice(0, MAX_PROFILES * 4)) {
|
|
343
|
+
const profile = normalizeBrandProfile(item);
|
|
344
|
+
if (!profile)
|
|
345
|
+
continue;
|
|
346
|
+
const previous = byId.get(profile.id);
|
|
347
|
+
if (!previous || profile.updatedAt > previous.updatedAt)
|
|
348
|
+
byId.set(profile.id, profile);
|
|
349
|
+
if (byId.size >= MAX_PROFILES)
|
|
350
|
+
break;
|
|
351
|
+
}
|
|
352
|
+
return { version: BRAND_PROFILE_SCHEMA_VERSION, profiles: [...byId.values()] };
|
|
353
|
+
}
|
|
354
|
+
export function serializeBrandProfileStore(profiles) {
|
|
355
|
+
return JSON.stringify(assertBrandProfileStoreBudget(profiles));
|
|
356
|
+
}
|
|
357
|
+
function byteLength(value) {
|
|
358
|
+
return new TextEncoder().encode(value).byteLength;
|
|
359
|
+
}
|
|
360
|
+
export function assertBrandProfileStoreBudget(profiles) {
|
|
361
|
+
const store = normalizeBrandProfileStore(profiles);
|
|
362
|
+
if (byteLength(JSON.stringify(store)) > MAX_PROFILE_STORE_BYTES) {
|
|
363
|
+
throw new Error('Brand profile storage is full. Export and remove a profile before saving more.');
|
|
364
|
+
}
|
|
365
|
+
return store;
|
|
366
|
+
}
|
|
367
|
+
export function mergeBrandProfiles(localRaw, importedRaw, options = {}) {
|
|
368
|
+
const imported = record(importedRaw);
|
|
369
|
+
if (imported.schema !== BRAND_PROFILE_SCHEMA) {
|
|
370
|
+
throw new Error('This JSON file is not a WHOISleuth Brand Profile export.');
|
|
371
|
+
}
|
|
372
|
+
if (!Array.isArray(imported.profiles)) {
|
|
373
|
+
throw new Error('Expected a current WHOISleuth Brand Profile export.');
|
|
374
|
+
}
|
|
375
|
+
const importedVersion = brandProfileStoreVersion(importedRaw);
|
|
376
|
+
if (importedVersion !== null && importedVersion > BRAND_PROFILE_SCHEMA_VERSION) {
|
|
377
|
+
throw new Error(`This Brand Profile file uses newer schema ${importedVersion}. Update the app before importing it.`);
|
|
378
|
+
}
|
|
379
|
+
if (![2, 3, 4, BRAND_PROFILE_SCHEMA_VERSION].includes(importedVersion ?? 0)) {
|
|
380
|
+
throw new Error(`Expected a WHOISleuth Brand Profile export using schema 2, 3, 4, or ${BRAND_PROFILE_SCHEMA_VERSION}.`);
|
|
381
|
+
}
|
|
382
|
+
const local = normalizeBrandProfileStore(localRaw).profiles;
|
|
383
|
+
const byName = new Map(local.map((profile) => [profile.name.toLowerCase(), profile]));
|
|
384
|
+
const input = profileList(importedRaw);
|
|
385
|
+
let added = 0;
|
|
386
|
+
let updated = 0;
|
|
387
|
+
let skipped = Math.max(0, input.length - MAX_PROFILES * 4);
|
|
388
|
+
for (const item of input.slice(0, MAX_PROFILES * 4)) {
|
|
389
|
+
const rawName = boundedText(record(item).name, MAX_PROFILE_NAME_LENGTH);
|
|
390
|
+
const existing = rawName ? byName.get(rawName.toLowerCase()) : null;
|
|
391
|
+
const profile = normalizeBrandProfile(item, {
|
|
392
|
+
existing,
|
|
393
|
+
touch: true,
|
|
394
|
+
nowIso: options.nowIso,
|
|
395
|
+
makeId: options.makeId,
|
|
396
|
+
});
|
|
397
|
+
if (!profile) {
|
|
398
|
+
skipped++;
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
if (existing) {
|
|
402
|
+
byName.set(profile.name.toLowerCase(), profile);
|
|
403
|
+
updated++;
|
|
404
|
+
}
|
|
405
|
+
else if (byName.size < MAX_PROFILES) {
|
|
406
|
+
byName.set(profile.name.toLowerCase(), profile);
|
|
407
|
+
added++;
|
|
408
|
+
}
|
|
409
|
+
else
|
|
410
|
+
skipped++;
|
|
411
|
+
}
|
|
412
|
+
return { profiles: [...byName.values()], added, updated, skipped };
|
|
413
|
+
}
|
|
414
|
+
export function buildBrandProfileExport(profiles, nowIso = new Date().toISOString()) {
|
|
415
|
+
return {
|
|
416
|
+
schema: BRAND_PROFILE_SCHEMA,
|
|
417
|
+
version: BRAND_PROFILE_SCHEMA_VERSION,
|
|
418
|
+
exportedAt: timestamp(nowIso, new Date().toISOString()),
|
|
419
|
+
profiles: normalizeBrandProfileStore(profiles).profiles,
|
|
420
|
+
};
|
|
421
|
+
}
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { normalizeBulkSessionResult, } from "./bulk-session-model.js";
|
|
2
|
+
import { sha256ArtifactDigest } from "./artifact-integrity.js";
|
|
3
|
+
import { BULK_REVIEW_STALE_AFTER_DAYS } from "./bulk-retry-plan.js";
|
|
4
|
+
export const BULK_DOMAIN_COMPARISON_SCHEMA = 'whoisleuth.domain-comparison';
|
|
5
|
+
export const BULK_DOMAIN_COMPARISON_VERSION = 3;
|
|
6
|
+
function text(value, maximum = 500) {
|
|
7
|
+
return typeof value === 'string'
|
|
8
|
+
? value.replace(/[\u0000-\u001f\u007f]/gu, ' ').replace(/\s+/gu, ' ').trim().slice(0, maximum)
|
|
9
|
+
: '';
|
|
10
|
+
}
|
|
11
|
+
function timestamp(value) {
|
|
12
|
+
const normalized = text(value, 64);
|
|
13
|
+
return normalized && Number.isFinite(Date.parse(normalized))
|
|
14
|
+
? new Date(normalized).toISOString()
|
|
15
|
+
: null;
|
|
16
|
+
}
|
|
17
|
+
function list(value) {
|
|
18
|
+
return [...new Set(value.map((item) => text(item, 253).toLowerCase()).filter(Boolean))]
|
|
19
|
+
.sort()
|
|
20
|
+
.slice(0, 100);
|
|
21
|
+
}
|
|
22
|
+
function normalized(value) {
|
|
23
|
+
if (typeof value === 'string') {
|
|
24
|
+
const normalizedText = text(value).toLowerCase();
|
|
25
|
+
return normalizedText && normalizedText !== '—' ? normalizedText : null;
|
|
26
|
+
}
|
|
27
|
+
if (typeof value === 'boolean' || typeof value === 'number' || value === null)
|
|
28
|
+
return value;
|
|
29
|
+
return list(value);
|
|
30
|
+
}
|
|
31
|
+
function display(value) {
|
|
32
|
+
if (value === null)
|
|
33
|
+
return 'Not observed';
|
|
34
|
+
if (typeof value === 'string') {
|
|
35
|
+
const displayText = text(value);
|
|
36
|
+
return displayText && displayText !== '—' ? displayText : 'Not observed';
|
|
37
|
+
}
|
|
38
|
+
if (typeof value === 'boolean')
|
|
39
|
+
return value ? 'Observed' : 'Not observed';
|
|
40
|
+
if (typeof value === 'number')
|
|
41
|
+
return String(value);
|
|
42
|
+
const values = [...new Set(value.map((item) => text(item, 253)).filter(Boolean))]
|
|
43
|
+
.sort((left, right) => left.localeCompare(right))
|
|
44
|
+
.slice(0, 100);
|
|
45
|
+
return values.join(', ') || 'Not observed';
|
|
46
|
+
}
|
|
47
|
+
function sourceState(value, sources) {
|
|
48
|
+
if (sources.length === 0) {
|
|
49
|
+
return value.sourceCoverage.length > 0 ? 'complete' : 'not_recorded';
|
|
50
|
+
}
|
|
51
|
+
for (const source of sources) {
|
|
52
|
+
const observed = value.sourceCoverage.find((item) => item.source === source);
|
|
53
|
+
if (observed)
|
|
54
|
+
return observed.state;
|
|
55
|
+
}
|
|
56
|
+
return 'not_recorded';
|
|
57
|
+
}
|
|
58
|
+
function unavailableSource(state) {
|
|
59
|
+
return ['error', 'partial', 'skipped', 'unavailable', 'unsupported'].includes(state);
|
|
60
|
+
}
|
|
61
|
+
function state(left, right, leftSourceState, rightSourceState, options = {}) {
|
|
62
|
+
if (options.retained === false)
|
|
63
|
+
return 'not_recorded';
|
|
64
|
+
if (options.conflicting)
|
|
65
|
+
return 'conflicting';
|
|
66
|
+
const leftValue = normalized(left);
|
|
67
|
+
const rightValue = normalized(right);
|
|
68
|
+
const leftMissing = leftValue === null || (Array.isArray(leftValue) && leftValue.length === 0);
|
|
69
|
+
const rightMissing = rightValue === null || (Array.isArray(rightValue) && rightValue.length === 0);
|
|
70
|
+
if (leftMissing || rightMissing) {
|
|
71
|
+
if ((leftMissing && leftSourceState === 'not_recorded')
|
|
72
|
+
|| (rightMissing && rightSourceState === 'not_recorded'))
|
|
73
|
+
return 'not_recorded';
|
|
74
|
+
if ((leftMissing && unavailableSource(leftSourceState))
|
|
75
|
+
|| (rightMissing && unavailableSource(rightSourceState)))
|
|
76
|
+
return 'unavailable';
|
|
77
|
+
if (leftMissing && rightMissing)
|
|
78
|
+
return 'unavailable';
|
|
79
|
+
return 'missing';
|
|
80
|
+
}
|
|
81
|
+
return JSON.stringify(leftValue) === JSON.stringify(rightValue) ? 'equal' : 'different';
|
|
82
|
+
}
|
|
83
|
+
function evidenceHref(value) {
|
|
84
|
+
return typeof value === 'string' && /^#bulk-result-\d{1,4}$/u.test(value) ? value : '';
|
|
85
|
+
}
|
|
86
|
+
function row(id, category, label, left, right, method, context, limitations = [], options = {}) {
|
|
87
|
+
return {
|
|
88
|
+
id,
|
|
89
|
+
category,
|
|
90
|
+
label,
|
|
91
|
+
left: options.retained === false ? 'Not recorded in compact Bulk evidence' : display(left),
|
|
92
|
+
right: options.retained === false ? 'Not recorded in compact Bulk evidence' : display(right),
|
|
93
|
+
state: state(left, right, context.leftSourceState, context.rightSourceState, options),
|
|
94
|
+
method: text(method, 320),
|
|
95
|
+
source: text(context.source, 80),
|
|
96
|
+
leftSourceState: context.leftSourceState,
|
|
97
|
+
rightSourceState: context.rightSourceState,
|
|
98
|
+
observedAt: context.observedAt,
|
|
99
|
+
leftEvidenceHref: context.leftEvidenceHref,
|
|
100
|
+
rightEvidenceHref: context.rightEvidenceHref,
|
|
101
|
+
limitations: limitations.map((item) => text(item, 320)).filter(Boolean).slice(0, 6),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
function sourceSummary(value) {
|
|
105
|
+
return value.sourceCoverage
|
|
106
|
+
.map((item) => `${item.source}:${item.state}`)
|
|
107
|
+
.sort()
|
|
108
|
+
.slice(0, 20);
|
|
109
|
+
}
|
|
110
|
+
function dnsValues(value, type) {
|
|
111
|
+
if (!value.dns)
|
|
112
|
+
return [];
|
|
113
|
+
const records = value.dns.records[type];
|
|
114
|
+
return type === 'caa'
|
|
115
|
+
? records
|
|
116
|
+
.map((item) => `${item.critical} ${item.tag} ${item.value}`)
|
|
117
|
+
: records;
|
|
118
|
+
}
|
|
119
|
+
function context(left, right, source, sources, observedAt, options) {
|
|
120
|
+
return {
|
|
121
|
+
source,
|
|
122
|
+
leftSourceState: sourceState(left, sources),
|
|
123
|
+
rightSourceState: sourceState(right, sources),
|
|
124
|
+
observedAt,
|
|
125
|
+
leftEvidenceHref: evidenceHref(options.leftEvidenceHref),
|
|
126
|
+
rightEvidenceHref: evidenceHref(options.rightEvidenceHref),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function comparisonEvidenceContext(left, right, family, base) {
|
|
130
|
+
const stateFor = (value) => {
|
|
131
|
+
const state = value.comparisonEvidence?.[family].state;
|
|
132
|
+
if (!state)
|
|
133
|
+
return 'not_recorded';
|
|
134
|
+
return state === 'success' ? 'complete' : state;
|
|
135
|
+
};
|
|
136
|
+
return {
|
|
137
|
+
...base,
|
|
138
|
+
leftSourceState: stateFor(left),
|
|
139
|
+
rightSourceState: stateFor(right),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
function freshness(observedAt, now) {
|
|
143
|
+
if (!observedAt)
|
|
144
|
+
return { state: 'unknown', ageDays: null };
|
|
145
|
+
const ageDays = Math.max(0, Math.floor((now - Date.parse(observedAt)) / 86_400_000));
|
|
146
|
+
return {
|
|
147
|
+
state: ageDays >= BULK_REVIEW_STALE_AFTER_DAYS ? 'stale' : 'current',
|
|
148
|
+
ageDays,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
export function buildBulkDomainComparison(leftRaw, rightRaw, observedAtRaw = null, options = {}) {
|
|
152
|
+
const left = normalizeBulkSessionResult(leftRaw);
|
|
153
|
+
const right = normalizeBulkSessionResult(rightRaw);
|
|
154
|
+
if (!left || !right || left.domain === right.domain)
|
|
155
|
+
return null;
|
|
156
|
+
const observedAt = timestamp(observedAtRaw);
|
|
157
|
+
const registry = context(left, right, 'Registry and registrar evidence', ['rdap', 'availability'], observedAt, options);
|
|
158
|
+
const dns = context(left, right, 'DNS evidence', ['dns'], observedAt, options);
|
|
159
|
+
const http = context(left, right, 'HTTP and static page evidence', ['http'], observedAt, options);
|
|
160
|
+
const tls = context(left, right, 'TLS evidence', ['tls'], observedAt, options);
|
|
161
|
+
const compactTechnology = comparisonEvidenceContext(left, right, 'technology', {
|
|
162
|
+
...http,
|
|
163
|
+
source: 'Bounded technology identifiers',
|
|
164
|
+
});
|
|
165
|
+
const compactTls = comparisonEvidenceContext(left, right, 'tls', tls);
|
|
166
|
+
const source = context(left, right, 'Recorded source coverage', [], observedAt, options);
|
|
167
|
+
const registrationConflicting = [left.availability, right.availability]
|
|
168
|
+
.some((value) => ['conflict', 'conflicting'].includes(value.toLowerCase()));
|
|
169
|
+
const rows = [
|
|
170
|
+
row('registration', 'registration', 'Registration', left.availability, right.availability, 'Authority-aware compact availability state', registry, [], { conflicting: registrationConflicting }),
|
|
171
|
+
row('registrar', 'registration', 'Registrar', left.registrar, right.registrar, 'Normalized registrar display identity', registry),
|
|
172
|
+
row('created', 'lifecycle', 'Created', left.createdDate, right.createdDate, 'Source-reported lifecycle date', registry),
|
|
173
|
+
row('expires', 'lifecycle', 'Expires', left.expiryDate, right.expiryDate, 'Source-reported lifecycle date', registry),
|
|
174
|
+
row('nameservers', 'infrastructure', 'Nameservers', left.nameservers, right.nameservers, 'Normalized exact set comparison', registry),
|
|
175
|
+
row('dns-a', 'dns', 'A records', dnsValues(left, 'a'), dnsValues(right, 'a'), 'Normalized exact retained DNS answer set', dns),
|
|
176
|
+
row('dns-aaaa', 'dns', 'AAAA records', dnsValues(left, 'aaaa'), dnsValues(right, 'aaaa'), 'Normalized exact retained DNS answer set', dns),
|
|
177
|
+
row('dns-cname', 'dns', 'CNAME records', dnsValues(left, 'cname'), dnsValues(right, 'cname'), 'Normalized exact retained DNS answer set', dns),
|
|
178
|
+
row('dns-caa', 'dns', 'CAA records', dnsValues(left, 'caa'), dnsValues(right, 'caa'), 'Normalized exact retained DNS answer set', dns),
|
|
179
|
+
row('dnssec', 'dns', 'DNSSEC state', left.dnssec, right.dnssec, 'Separately attributed compact DNSSEC observation', dns),
|
|
180
|
+
row('ip-addresses', 'infrastructure', 'Observed IP addresses', left.relationship.ipAddresses, right.relationship.ipAddresses, 'Normalized exact set comparison', dns, ['Shared hosting and CDNs are common and do not prove common control.']),
|
|
181
|
+
row('tls-source', 'certificate', 'TLS collection', tls.leftSourceState, tls.rightSourceState, 'Retained compact source-state comparison', tls),
|
|
182
|
+
row('certificate', 'certificate', 'Leaf certificate fingerprint', left.relationship.certificateFingerprint, right.relationship.certificateFingerprint, 'Exact SHA-256 comparison', tls, ['Multi-domain certificates and managed hosting are common.']),
|
|
183
|
+
row('tls-issuer', 'certificate', 'TLS issuer label', left.comparisonEvidence?.tls.issuerLabel ?? null, right.comparisonEvidence?.tls.issuerLabel ?? null, 'Normalized retained issuer-label comparison', compactTls, ['Issuer labels are certificate metadata and do not establish common ownership or control.']),
|
|
184
|
+
row('tls-spki', 'certificate', 'TLS public-key fingerprint', left.comparisonEvidence?.tls.spkiSha256 ?? null, right.comparisonEvidence?.tls.spkiSha256 ?? null, 'Exact SPKI SHA-256 comparison', compactTls, ['Public-key reuse can be an investigative lead but does not establish ownership, intent, safety, or maliciousness.']),
|
|
185
|
+
row('mail', 'mail', 'MX observed', left.hasMx, right.hasMx, 'Compact Deep DNS observation', dns),
|
|
186
|
+
row('null-mx', 'mail', 'Null MX observed', left.hasNullMx, right.hasNullMx, 'Compact Deep DNS observation', dns),
|
|
187
|
+
row('spf', 'mail', 'SPF observed', left.hasSpf, right.hasSpf, 'Compact Deep DNS observation', dns),
|
|
188
|
+
row('dmarc', 'mail', 'DMARC observed', left.hasDmarc, right.hasDmarc, 'Compact Deep DNS observation', dns),
|
|
189
|
+
row('website', 'web', 'Website activity', left.activityStatus, right.activityStatus, 'Bounded homepage observation', http),
|
|
190
|
+
row('page-title', 'identity', 'Page title', left.pageTitle, right.pageTitle, 'Normalized bounded title equality', http),
|
|
191
|
+
row('favicon', 'identity', 'Favicon fingerprint', left.faviconHash, right.faviconHash, 'Exact SHA-256 comparison', http),
|
|
192
|
+
row('tracking', 'identity', 'Tracking identifiers', left.relationship.trackingIdentifiers, right.relationship.trackingIdentifiers, 'Normalized exact set comparison', http, ['Shared identifiers are an investigative lead, not proof of ownership.']),
|
|
193
|
+
row('password-field', 'identity', 'Password field', left.hasPasswordField, right.hasPasswordField, 'Bounded static form observation', http),
|
|
194
|
+
row('phishing-language', 'identity', 'Phishing-language indicator', left.phishingLanguageMatch, right.phishingLanguageMatch, 'Bounded explainable page-language signal', http, ['A wording match is a review lead, not proof of malicious intent.']),
|
|
195
|
+
row('official-assets', 'identity', 'Official asset reuse', left.reusesOfficialAssets, right.reusesOfficialAssets, 'Configured Brand Profile comparison', http),
|
|
196
|
+
row('technology', 'technology', 'Technology identifiers', left.comparisonEvidence?.technology.ids ?? [], right.comparisonEvidence?.technology.ids ?? [], 'Normalized exact set comparison of at most 12 curated identifiers', compactTechnology, ['An unmatched identifier is not evidence that a technology is absent. Shared technologies do not establish common ownership or control.']),
|
|
197
|
+
row('source-health', 'source', 'Recorded source coverage', sourceSummary(left), sourceSummary(right), 'Source-by-source compact state comparison', source, ['A source-state difference can reflect collection conditions rather than a domain change.']),
|
|
198
|
+
];
|
|
199
|
+
const counts = {
|
|
200
|
+
conflicting: 0,
|
|
201
|
+
different: 0,
|
|
202
|
+
equal: 0,
|
|
203
|
+
missing: 0,
|
|
204
|
+
not_recorded: 0,
|
|
205
|
+
unavailable: 0,
|
|
206
|
+
};
|
|
207
|
+
for (const comparisonRow of rows)
|
|
208
|
+
counts[comparisonRow.state] += 1;
|
|
209
|
+
return {
|
|
210
|
+
version: BULK_DOMAIN_COMPARISON_VERSION,
|
|
211
|
+
leftDomain: left.domain,
|
|
212
|
+
rightDomain: right.domain,
|
|
213
|
+
observedAt,
|
|
214
|
+
freshness: freshness(observedAt, options.now ?? Date.now()),
|
|
215
|
+
rows,
|
|
216
|
+
counts,
|
|
217
|
+
limitations: [
|
|
218
|
+
'This compares compact settled evidence already present in Bulk and makes no new request.',
|
|
219
|
+
'Missing, unavailable, and differently collected evidence remain distinct from an observed difference.',
|
|
220
|
+
'Equality does not establish common ownership, infrastructure control, intent, safety, or maliciousness.',
|
|
221
|
+
],
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
export async function buildBulkDomainComparisonExport(comparisonRaw, generatedAtRaw = new Date().toISOString()) {
|
|
225
|
+
const comparison = comparisonRaw;
|
|
226
|
+
if (!comparison || comparison.version !== BULK_DOMAIN_COMPARISON_VERSION) {
|
|
227
|
+
throw new Error('Choose two settled domain results before exporting a comparison.');
|
|
228
|
+
}
|
|
229
|
+
const generatedAt = timestamp(generatedAtRaw) || new Date().toISOString();
|
|
230
|
+
const unsigned = {
|
|
231
|
+
schema: BULK_DOMAIN_COMPARISON_SCHEMA,
|
|
232
|
+
version: BULK_DOMAIN_COMPARISON_VERSION,
|
|
233
|
+
generatedAt,
|
|
234
|
+
comparison,
|
|
235
|
+
};
|
|
236
|
+
const digestSha256 = await sha256ArtifactDigest(unsigned);
|
|
237
|
+
const document = {
|
|
238
|
+
...unsigned,
|
|
239
|
+
integrity: { algorithm: 'SHA-256', digestSha256 },
|
|
240
|
+
};
|
|
241
|
+
return {
|
|
242
|
+
document,
|
|
243
|
+
content: `${JSON.stringify(document, null, 2)}\n`,
|
|
244
|
+
filename: `whoisleuth-domain-comparison-${comparison.leftDomain}-and-${comparison.rightDomain}.json`,
|
|
245
|
+
};
|
|
246
|
+
}
|