@slicedearth/whoisleuth-cli 1.34.0 → 1.39.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -2
- package/bin/whoisleuth.mjs +41 -24
- package/cli/arguments.mjs +326 -3
- package/cli/artifact-verify.mjs +60 -14
- package/cli/bulk.mjs +1 -2
- package/cli/compare.mjs +1 -1
- package/cli/completion.mjs +36 -2
- package/cli/doctor.mjs +11 -5
- package/cli/evidence-command-arguments.mjs +1 -1
- package/cli/evidence-command-runner.mjs +2 -2
- package/cli/evidence-signing.mjs +4 -4
- package/cli/export-evidence.mjs +9 -2
- package/cli/formatters/html.mjs +1 -1
- package/cli/formatters/json.mjs +13 -1
- package/cli/formatters/terminal.mjs +14 -4
- package/cli/investigation-plan.mjs +112 -0
- package/cli/local-mmdb-review.mjs +76 -0
- package/cli/lookalike-calibration.mjs +121 -0
- package/cli/lookup-command-runner.mjs +4 -1
- package/cli/lookup-plan.mjs +3 -3
- package/cli/lookup-reconcile.mjs +141 -0
- package/cli/lookup-timeline.mjs +1 -1
- package/cli/mail-review.mjs +48 -3
- package/cli/offline-evidence-review.mjs +159 -0
- package/cli/registry-doctor.mjs +125 -0
- package/cli/registry-support.mjs +1 -4
- package/cli/risk-calibration.mjs +117 -9
- package/cli/runner.mjs +342 -15
- package/cli/saved-lookup.mjs +2 -2
- package/cli/sharing-review.mjs +202 -0
- package/cli/source-reliability.mjs +1 -1
- package/docs/cli-reference.md +1110 -0
- package/docs/cli.md +224 -847
- package/frontend/src/lib/analysis/acquisition-decision-packet.js +1 -1
- package/frontend/src/lib/analysis/acquisition-due-diligence.js +6 -6
- package/frontend/src/lib/analysis/bulk-domain-comparison.js +12 -12
- package/frontend/src/lib/analysis/bulk-mail-exposure.js +1 -1
- package/frontend/src/lib/analysis/bulk-retry-plan.js +3 -10
- package/frontend/src/lib/analysis/bulk-session-model.js +7 -2
- package/frontend/src/lib/analysis/bulk-source-coverage.js +76 -0
- package/frontend/src/lib/analysis/case-evidence-model.js +26 -4
- package/frontend/src/lib/analysis/case-migration-model.js +6 -1
- package/frontend/src/lib/analysis/case-record-contracts.js +4 -2
- package/frontend/src/lib/analysis/case-record-core.js +5 -1
- package/frontend/src/lib/analysis/case-record-operations.js +6 -1
- package/frontend/src/lib/analysis/case-response-packet.js +7 -7
- package/frontend/src/lib/analysis/evidence-coverage-ledger.js +328 -0
- package/frontend/src/lib/analysis/http-summary.js +2 -1
- package/frontend/src/lib/analysis/investigation-capsule.js +105 -0
- package/frontend/src/lib/analysis/lookup-asset-graph.js +806 -0
- package/frontend/src/lib/analysis/lookup-decision-support.js +480 -0
- package/frontend/src/lib/analysis/lookup-investigation-brief.js +153 -0
- package/frontend/src/lib/analysis/lookup-presentation.js +74 -0
- package/frontend/src/lib/analysis/lookup-response.js +2 -0
- package/frontend/src/lib/analysis/lookup-source-refresh.js +304 -0
- package/frontend/src/lib/analysis/lookup-summary-model.js +398 -0
- package/frontend/src/lib/analysis/page-similarity.js +1 -1
- package/frontend/src/lib/analysis/relationship-evidence.js +3 -3
- package/frontend/src/lib/analysis/relationship-observation-model.js +4 -4
- package/frontend/src/lib/analysis/scoring.js +8 -88
- package/frontend/src/lib/analysis/shortlist-model.js +8 -4
- package/frontend/src/lib/analysis/utils.js +7 -40
- package/frontend/src/lib/analysis/visualization-bounds.js +26 -0
- package/frontend/src/lib/analysis/visualization-force-graph.js +352 -0
- package/frontend/src/lib/analysis/visualization-matrix.js +88 -0
- package/frontend/src/lib/analysis/visualization-models.js +370 -0
- package/frontend/src/lib/analysis/visualization-time-series.js +144 -0
- package/frontend/src/lib/analysis/website-snapshot-model.js +57 -3
- package/frontend/src/lib/analysis/workspace-archive-crypto.js +1 -1
- package/frontend/src/lib/analysis/workspace-archive.js +2 -2
- package/lib/analyst-taxonomy.mjs +40 -0
- package/lib/availability.mjs +30 -10
- package/lib/browser-library-profile.mjs +15 -2
- package/lib/bulk-limits.mjs +13 -0
- package/lib/certificate-extension-profile.mjs +71 -0
- package/lib/credential-surface-profile.mjs +1 -1
- package/lib/ct-response-bounds.mjs +9 -0
- package/lib/ct-search.mjs +66 -12
- package/lib/dns-delegation-health.mjs +56 -5
- package/lib/dnssec-evidence-validation.mjs +235 -0
- package/lib/domain-assurance.mjs +368 -0
- package/lib/domain-control-manifest.mjs +382 -0
- package/lib/domain-posture.mjs +11 -10
- package/lib/encrypted-dns-contract.mjs +81 -0
- package/lib/evidence-export.mjs +17 -3
- package/lib/evidence-report-markdown.mjs +1 -1
- package/lib/evidence-report.mjs +4 -1
- package/lib/favicon.mjs +3 -8
- package/lib/generated/cisa-kev-catalog.mjs +1668 -0
- package/lib/html-signals.mjs +4 -0
- package/lib/http-evidence-bounds.mjs +4 -0
- package/lib/http-intelligence.mjs +2 -1
- package/lib/ip-prefix.mjs +87 -0
- package/lib/local-geoip-evidence.mjs +89 -0
- package/lib/lookup-response-contract.mjs +346 -0
- package/lib/observed-network-context.mjs +1 -1
- package/lib/opportunity-scoring.mjs +107 -0
- package/lib/outbound-identity.mjs +9 -0
- package/lib/outbound-request-bounds.mjs +9 -0
- package/lib/page-fingerprints.mjs +1 -1
- package/lib/perceptual-hash-comparison.mjs +34 -0
- package/lib/perceptual-hash.mjs +3 -31
- package/lib/project-metadata.mjs +8 -0
- package/lib/provider-policy-admission.mjs +49 -0
- package/lib/rdap-capabilities.mjs +6 -6
- package/lib/rdap-search-workbench.mjs +247 -0
- package/lib/rdap.mjs +7 -2
- package/lib/registry-access.mjs +1 -1
- package/lib/registry-capabilities.mjs +31 -1
- package/lib/registry-capability-catalogue.mjs +15 -14
- package/lib/registry-insights.mjs +6 -6
- package/lib/response-policy.mjs +81 -6
- package/lib/risk-scoring.mjs +206 -83
- package/lib/rpki-evidence.mjs +105 -0
- package/lib/safe-fetch.mjs +2 -1
- package/lib/scoring-evidence-quality.mjs +107 -0
- package/lib/security-txt.mjs +3 -3
- package/lib/semantic-version.mjs +54 -0
- package/lib/static-html-analysis.mjs +27 -0
- package/lib/threat-intelligence-manifests.mjs +1 -1
- package/lib/threatfox-intelligence.mjs +11 -6
- package/lib/tls-intelligence.mjs +13 -3
- package/lib/tlsa-evidence.mjs +194 -0
- package/lib/urlhaus-intelligence.mjs +11 -6
- package/lib/urlscan-intelligence.mjs +10 -5
- package/lib/website-security-posture.mjs +16 -9
- package/lib/website-technology.mjs +1 -1
- package/lib/whois-chain.mjs +14 -152
- package/lib/whois.mjs +2 -2
- package/package.json +6 -2
- package/third-party-notices.txt +911 -19
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
import { fmtAge, fmtExpiresIn, formatActivityCell, formatPrivacyCell, } from "./scoring.js";
|
|
2
|
+
const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f]/gu;
|
|
3
|
+
const MAX_SIGNALS = 12;
|
|
4
|
+
const MAX_PROVENANCE_ITEMS = 8;
|
|
5
|
+
const MAX_PROVENANCE_TEXT = 320;
|
|
6
|
+
function record(value) {
|
|
7
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
8
|
+
? value
|
|
9
|
+
: {};
|
|
10
|
+
}
|
|
11
|
+
function boundedText(value, maximum = 320) {
|
|
12
|
+
return String(value ?? '')
|
|
13
|
+
.replace(CONTROL_CHARACTERS, ' ')
|
|
14
|
+
.replace(/\s+/gu, ' ')
|
|
15
|
+
.trim()
|
|
16
|
+
.slice(0, maximum);
|
|
17
|
+
}
|
|
18
|
+
function display(value) {
|
|
19
|
+
if (value === null || value === undefined || value === '')
|
|
20
|
+
return '—';
|
|
21
|
+
if (Array.isArray(value)) {
|
|
22
|
+
return value.slice(0, 16).map((item) => boundedText(item, 120)).filter(Boolean).join(', ') || '—';
|
|
23
|
+
}
|
|
24
|
+
if (typeof value === 'object') {
|
|
25
|
+
const item = record(value);
|
|
26
|
+
return display(item.name || item.org || item.handle || item.domain);
|
|
27
|
+
}
|
|
28
|
+
return boundedText(value) || '—';
|
|
29
|
+
}
|
|
30
|
+
function textOrNull(value) {
|
|
31
|
+
const valueText = typeof value === 'string' ? boundedText(value) : '';
|
|
32
|
+
return valueText || null;
|
|
33
|
+
}
|
|
34
|
+
function records(value) {
|
|
35
|
+
return Array.isArray(value)
|
|
36
|
+
? value.slice(0, 24).map(record).filter((item) => Object.keys(item).length > 0)
|
|
37
|
+
: [];
|
|
38
|
+
}
|
|
39
|
+
function textList(value, maximum = MAX_PROVENANCE_ITEMS) {
|
|
40
|
+
if (!Array.isArray(value))
|
|
41
|
+
return [];
|
|
42
|
+
const output = [];
|
|
43
|
+
const seen = new Set();
|
|
44
|
+
for (const item of value) {
|
|
45
|
+
const text = boundedText(item, MAX_PROVENANCE_TEXT);
|
|
46
|
+
if (!text || seen.has(text))
|
|
47
|
+
continue;
|
|
48
|
+
output.push(text);
|
|
49
|
+
seen.add(text);
|
|
50
|
+
if (output.length >= maximum)
|
|
51
|
+
break;
|
|
52
|
+
}
|
|
53
|
+
return output;
|
|
54
|
+
}
|
|
55
|
+
function formatDate(value) {
|
|
56
|
+
if (!value)
|
|
57
|
+
return '—';
|
|
58
|
+
const parsed = new Date(String(value));
|
|
59
|
+
return Number.isNaN(parsed.getTime()) ? boundedText(value) || '—' : parsed.toLocaleString();
|
|
60
|
+
}
|
|
61
|
+
function statusLabel(value) {
|
|
62
|
+
return boundedText(value, 64).replaceAll('_', ' ') || 'unknown';
|
|
63
|
+
}
|
|
64
|
+
function diagnosticDetail(value) {
|
|
65
|
+
const source = record(value);
|
|
66
|
+
const attempts = Array.isArray(source.attempts)
|
|
67
|
+
? source.attempts.slice(0, 8).map((attempt) => (statusLabel(record(attempt).outcome))).join(' → ')
|
|
68
|
+
: '';
|
|
69
|
+
const parts = [
|
|
70
|
+
boundedText(source.endpoint, 2_048),
|
|
71
|
+
source.transportSecurity === 'http' ? 'transport: cleartext HTTP' : '',
|
|
72
|
+
typeof source.httpStatus === 'number' ? `HTTP ${source.httpStatus}` : '',
|
|
73
|
+
attempts ? `attempts: ${attempts}` : '',
|
|
74
|
+
source.resultState ? `result: ${boundedText(source.resultState, 80)}` : '',
|
|
75
|
+
boundedText(source.errorCode, 120),
|
|
76
|
+
source.authoritativeHop ? `authoritative: ${display(source.authoritativeHop)}` : '',
|
|
77
|
+
source.failedHop ? `failed: ${display(source.failedHop)}` : '',
|
|
78
|
+
source.fetchedAt ? `fetched ${formatDate(source.fetchedAt)}` : '',
|
|
79
|
+
source.queriedAt ? `queried ${formatDate(source.queriedAt)}` : '',
|
|
80
|
+
].filter(Boolean);
|
|
81
|
+
return boundedText(parts.join(' · '), 2_400) || 'No additional source detail';
|
|
82
|
+
}
|
|
83
|
+
function sourceLabel(value) {
|
|
84
|
+
return {
|
|
85
|
+
dns: 'Authoritative DNS delegation',
|
|
86
|
+
rdap: 'Registry RDAP',
|
|
87
|
+
whois: 'WHOIS',
|
|
88
|
+
}[boundedText(value, 32)] || 'No conclusive registration source';
|
|
89
|
+
}
|
|
90
|
+
function sourceObservation(source, diagnostics, fallback) {
|
|
91
|
+
const sourceId = boundedText(source, 32);
|
|
92
|
+
const diagnostic = sourceId === 'dns'
|
|
93
|
+
? record(diagnostics.availability)
|
|
94
|
+
: record(diagnostics[sourceId]);
|
|
95
|
+
return formatDate(diagnostic.fetchedAt
|
|
96
|
+
|| diagnostic.queriedAt
|
|
97
|
+
|| diagnostic.observedAt
|
|
98
|
+
|| fallback);
|
|
99
|
+
}
|
|
100
|
+
function sourceCompleteness(source, diagnostics) {
|
|
101
|
+
const sourceId = boundedText(source, 32);
|
|
102
|
+
const diagnostic = sourceId === 'dns'
|
|
103
|
+
? record(diagnostics.availability)
|
|
104
|
+
: record(diagnostics[sourceId]);
|
|
105
|
+
return statusLabel(diagnostic.status);
|
|
106
|
+
}
|
|
107
|
+
function registryLimitations(availability, rdapParsed, whoisParsed) {
|
|
108
|
+
const output = [
|
|
109
|
+
...textList(availability.limitations),
|
|
110
|
+
...textList(whoisParsed.limitations),
|
|
111
|
+
];
|
|
112
|
+
if (rdapParsed.serverTruncated === true) {
|
|
113
|
+
output.push('The registry declared that the RDAP response omitted some data.');
|
|
114
|
+
output.push(...textList(rdapParsed.serverTruncationReasons));
|
|
115
|
+
}
|
|
116
|
+
return textList(output);
|
|
117
|
+
}
|
|
118
|
+
function comparisonConflicts(value, labels) {
|
|
119
|
+
const accepted = new Set(labels);
|
|
120
|
+
return records(record(value).fields)
|
|
121
|
+
.filter((field) => accepted.has(boundedText(field.label, 120)))
|
|
122
|
+
.filter((field) => boundedText(field.status, 64) === 'conflict')
|
|
123
|
+
.map((field) => boundedText(field.assessment
|
|
124
|
+
|| `${display(field.rdapDisplay || field.registryDisplay)} differs from ${display(field.whoisDisplay || field.registrarDisplay)}`, MAX_PROVENANCE_TEXT))
|
|
125
|
+
.filter(Boolean)
|
|
126
|
+
.slice(0, MAX_PROVENANCE_ITEMS);
|
|
127
|
+
}
|
|
128
|
+
function factProvenance(input) {
|
|
129
|
+
return {
|
|
130
|
+
sources: [sourceLabel(input.source)],
|
|
131
|
+
observedAt: sourceObservation(input.source, input.diagnostics, input.observedAt),
|
|
132
|
+
fieldFamilies: input.fields.map((item) => boundedText(item, 160)).filter(Boolean),
|
|
133
|
+
normalization: boundedText(input.normalization, MAX_PROVENANCE_TEXT),
|
|
134
|
+
completeness: sourceCompleteness(input.source, input.diagnostics),
|
|
135
|
+
limitations: textList(input.limitations),
|
|
136
|
+
conflicts: textList(input.conflicts),
|
|
137
|
+
decisionImpact: boundedText(input.decisionImpact, MAX_PROVENANCE_TEXT),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
function pushSignal(signals, signal) {
|
|
141
|
+
if (signal && signals.length < MAX_SIGNALS)
|
|
142
|
+
signals.push(signal);
|
|
143
|
+
}
|
|
144
|
+
function buildSignals(availability, profileSignals, idnAnalysis) {
|
|
145
|
+
const signals = [];
|
|
146
|
+
const trusted = boundedText(profileSignals.trusted, 80);
|
|
147
|
+
if (trusted)
|
|
148
|
+
pushSignal(signals, { label: `Trusted ${trusted}`, tone: 'good' });
|
|
149
|
+
if (profileSignals.faviconMatch === true) {
|
|
150
|
+
pushSignal(signals, { label: 'Favicon match', tone: 'danger' });
|
|
151
|
+
}
|
|
152
|
+
else if (profileSignals.faviconNearMatch === true) {
|
|
153
|
+
pushSignal(signals, { label: 'Favicon near-match', tone: 'warn' });
|
|
154
|
+
}
|
|
155
|
+
if (profileSignals.reusesOfficialAssets === true) {
|
|
156
|
+
pushSignal(signals, { label: 'Reuses official assets', tone: 'danger' });
|
|
157
|
+
}
|
|
158
|
+
if (availability.hasPasswordField === true) {
|
|
159
|
+
pushSignal(signals, { label: 'Password field', tone: 'warn' });
|
|
160
|
+
}
|
|
161
|
+
const phishingLanguage = textOrNull(availability.phishingLanguageMatch);
|
|
162
|
+
if (phishingLanguage) {
|
|
163
|
+
pushSignal(signals, {
|
|
164
|
+
label: 'Phishing language',
|
|
165
|
+
tone: 'danger',
|
|
166
|
+
detail: phishingLanguage,
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
if (idnAnalysis.mixedScript === true) {
|
|
170
|
+
pushSignal(signals, {
|
|
171
|
+
label: 'Mixed-script IDN',
|
|
172
|
+
tone: 'warn',
|
|
173
|
+
detail: 'The Unicode label combines writing scripts.',
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
if (Array.isArray(idnAnalysis.referenceMatches) && idnAnalysis.referenceMatches.length > 0) {
|
|
177
|
+
pushSignal(signals, {
|
|
178
|
+
label: 'Official-domain skeleton match',
|
|
179
|
+
tone: 'warn',
|
|
180
|
+
detail: 'A bounded visual skeleton matches an official domain in the active brand profile.',
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
const domainAgeDays = typeof availability.domainAgeDays === 'number'
|
|
184
|
+
? availability.domainAgeDays
|
|
185
|
+
: null;
|
|
186
|
+
const age = fmtAge(domainAgeDays);
|
|
187
|
+
if (age)
|
|
188
|
+
pushSignal(signals, { label: age, tone: 'neutral' });
|
|
189
|
+
const expiresInDays = typeof availability.expiresInDays === 'number'
|
|
190
|
+
? availability.expiresInDays
|
|
191
|
+
: null;
|
|
192
|
+
const expiry = fmtExpiresIn(expiresInDays);
|
|
193
|
+
if (expiry && expiresInDays !== null) {
|
|
194
|
+
pushSignal(signals, {
|
|
195
|
+
label: expiry,
|
|
196
|
+
tone: expiresInDays <= 60 ? 'warn' : 'neutral',
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
if (typeof availability.privacyProtected === 'boolean') {
|
|
200
|
+
pushSignal(signals, {
|
|
201
|
+
label: formatPrivacyCell(availability.privacyProtected),
|
|
202
|
+
tone: availability.privacyProtected ? 'warn' : 'good',
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
const activityStatus = textOrNull(availability.activityStatus);
|
|
206
|
+
if (activityStatus) {
|
|
207
|
+
const detail = textOrNull(availability.websiteProbeDetail);
|
|
208
|
+
pushSignal(signals, {
|
|
209
|
+
label: formatActivityCell(activityStatus, availability.hasMx === true, availability.hasSpf === true, availability.hasDmarc === true),
|
|
210
|
+
tone: activityStatus === 'active'
|
|
211
|
+
? 'good'
|
|
212
|
+
: activityStatus === 'parked'
|
|
213
|
+
? 'warn'
|
|
214
|
+
: 'neutral',
|
|
215
|
+
...(detail ? { detail } : {}),
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
return signals;
|
|
219
|
+
}
|
|
220
|
+
function diagnosticAttempts(value) {
|
|
221
|
+
return records(value).slice(0, 8).map((attempt) => ({
|
|
222
|
+
endpoint: boundedText(attempt.endpoint, 2_048) || 'Endpoint not recorded',
|
|
223
|
+
outcome: statusLabel(attempt.outcome),
|
|
224
|
+
detail: boundedText(typeof attempt.status === 'number'
|
|
225
|
+
? `HTTP ${attempt.status}${attempt.detail ? ` · ${boundedText(attempt.detail, 240)}` : ''}`
|
|
226
|
+
: attempt.detail, MAX_PROVENANCE_TEXT) || 'No additional attempt detail',
|
|
227
|
+
}));
|
|
228
|
+
}
|
|
229
|
+
function buildDiagnostics(diagnostics, rdapParsed, whoisParsed, registrarRdap) {
|
|
230
|
+
const sources = ['rdap', 'whois', 'availability'].map((source) => {
|
|
231
|
+
const item = record(diagnostics[source]);
|
|
232
|
+
const status = boundedText(item.status, 64);
|
|
233
|
+
const isRdap = source === 'rdap';
|
|
234
|
+
const isWhois = source === 'whois';
|
|
235
|
+
return {
|
|
236
|
+
source,
|
|
237
|
+
status,
|
|
238
|
+
label: statusLabel(status),
|
|
239
|
+
detail: diagnosticDetail(item),
|
|
240
|
+
endpoint: boundedText(item.endpoint, 2_048),
|
|
241
|
+
route: isRdap
|
|
242
|
+
? 'The endpoint was selected through IANA RDAP bootstrap discovery and bounded endpoint failover.'
|
|
243
|
+
: isWhois
|
|
244
|
+
? 'The response follows the bounded WHOIS registry and referral chain shown below.'
|
|
245
|
+
: 'This derived state applies the authority-aware availability rules to the collected registration evidence.',
|
|
246
|
+
observedAt: formatDate(item.fetchedAt || item.queriedAt || item.observedAt),
|
|
247
|
+
conformance: isRdap ? textList(rdapParsed.conformance) : [],
|
|
248
|
+
limitations: isRdap
|
|
249
|
+
? registryLimitations({}, rdapParsed, {})
|
|
250
|
+
: isWhois
|
|
251
|
+
? textList(whoisParsed.limitations)
|
|
252
|
+
: [],
|
|
253
|
+
attempts: diagnosticAttempts(item.attempts),
|
|
254
|
+
};
|
|
255
|
+
});
|
|
256
|
+
if (registrarRdap.status) {
|
|
257
|
+
const registrarDiagnostic = record(record(diagnostics.rdap).registrar);
|
|
258
|
+
const status = boundedText(registrarRdap.status || registrarDiagnostic.status, 64);
|
|
259
|
+
sources.push({
|
|
260
|
+
source: 'registrar RDAP',
|
|
261
|
+
status,
|
|
262
|
+
label: statusLabel(status),
|
|
263
|
+
detail: diagnosticDetail(registrarDiagnostic),
|
|
264
|
+
endpoint: boundedText(registrarRdap.endpoint || registrarDiagnostic.endpoint, 2_048),
|
|
265
|
+
route: 'This is a separately attributed registrar publication discovered from the registry RDAP response.',
|
|
266
|
+
observedAt: formatDate(registrarRdap.fetchedAt || registrarDiagnostic.fetchedAt),
|
|
267
|
+
conformance: textList(record(registrarRdap.parsed).conformance),
|
|
268
|
+
limitations: textList(registrarRdap.limitations),
|
|
269
|
+
attempts: diagnosticAttempts(registrarRdap.attempt ? [registrarRdap.attempt] : registrarDiagnostic.attempts),
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
const reverseDns = record(diagnostics.reverseDns);
|
|
273
|
+
if (reverseDns.status) {
|
|
274
|
+
const status = boundedText(reverseDns.status, 64);
|
|
275
|
+
sources.push({
|
|
276
|
+
source: 'reverse DNS',
|
|
277
|
+
status,
|
|
278
|
+
label: statusLabel(status),
|
|
279
|
+
detail: diagnosticDetail(reverseDns),
|
|
280
|
+
endpoint: '',
|
|
281
|
+
route: 'PTR evidence is non-authoritative context published by the address operator.',
|
|
282
|
+
observedAt: formatDate(reverseDns.observedAt),
|
|
283
|
+
conformance: [],
|
|
284
|
+
limitations: [],
|
|
285
|
+
attempts: [],
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
return sources;
|
|
289
|
+
}
|
|
290
|
+
export function buildLookupSummaryModel(input) {
|
|
291
|
+
const availability = record(input.availability);
|
|
292
|
+
const rdapParsed = record(input.rdapParsed);
|
|
293
|
+
const whoisParsed = record(input.whoisParsed);
|
|
294
|
+
const diagnostics = record(input.diagnostics);
|
|
295
|
+
const source = availability.source;
|
|
296
|
+
const limitations = registryLimitations(availability, rdapParsed, whoisParsed);
|
|
297
|
+
const registryComparison = input.registryComparison;
|
|
298
|
+
const registrarPublicationComparison = input.registrarPublicationComparison;
|
|
299
|
+
return {
|
|
300
|
+
signals: buildSignals(availability, record(input.profileSignals), record(input.idnAnalysis)),
|
|
301
|
+
facts: [
|
|
302
|
+
{
|
|
303
|
+
label: 'Registration',
|
|
304
|
+
value: display(availability.state || whoisParsed.registrationStatus),
|
|
305
|
+
detail: `${display(availability.confidence)} confidence`,
|
|
306
|
+
provenance: factProvenance({
|
|
307
|
+
source,
|
|
308
|
+
diagnostics,
|
|
309
|
+
observedAt: input.resultObservedAt,
|
|
310
|
+
fields: ['RDAP object state', 'WHOIS registration status', 'authoritative DNS delegation'],
|
|
311
|
+
normalization: 'Authority-aware registration evidence is normalised into a stable availability state while inconclusive sources remain unknown.',
|
|
312
|
+
limitations,
|
|
313
|
+
conflicts: comparisonConflicts(registryComparison, ['Registration status']),
|
|
314
|
+
decisionImpact: 'This is the authority-aware input to the availability result. This inspector does not recalculate or override that decision.',
|
|
315
|
+
}),
|
|
316
|
+
},
|
|
317
|
+
{
|
|
318
|
+
label: 'Registrar',
|
|
319
|
+
value: display(availability.registrar || rdapParsed.registrar || whoisParsed.registrar),
|
|
320
|
+
detail: display(whoisParsed.registrarUrl),
|
|
321
|
+
provenance: factProvenance({
|
|
322
|
+
source,
|
|
323
|
+
diagnostics,
|
|
324
|
+
observedAt: input.resultObservedAt,
|
|
325
|
+
fields: ['RDAP registrar entity', 'WHOIS Registrar', 'registrar RDAP registrar entity'],
|
|
326
|
+
normalization: 'Structured registrar identities are reduced to a bounded display name while each registry, registrar, and WHOIS publication remains separate.',
|
|
327
|
+
limitations,
|
|
328
|
+
conflicts: [
|
|
329
|
+
...comparisonConflicts(registryComparison, ['Registrar', 'Registrar IANA ID']),
|
|
330
|
+
...comparisonConflicts(registrarPublicationComparison, ['Registrar', 'Registrar IANA ID']),
|
|
331
|
+
],
|
|
332
|
+
decisionImpact: 'Registrar identity is investigative and contact context. It does not decide whether the domain exists or establish ownership.',
|
|
333
|
+
}),
|
|
334
|
+
},
|
|
335
|
+
{
|
|
336
|
+
label: 'Created',
|
|
337
|
+
value: formatDate(input.createdDate),
|
|
338
|
+
detail: fmtAge(typeof availability.domainAgeDays === 'number' ? availability.domainAgeDays : null) || 'Registry lifecycle date',
|
|
339
|
+
provenance: factProvenance({
|
|
340
|
+
source,
|
|
341
|
+
diagnostics,
|
|
342
|
+
observedAt: input.resultObservedAt,
|
|
343
|
+
fields: ['RDAP registration event or lifecycle', 'WHOIS creation-date family'],
|
|
344
|
+
normalization: 'Recognised lifecycle dates are parsed into an ISO-compatible instant when possible; the original source publication remains authoritative.',
|
|
345
|
+
limitations,
|
|
346
|
+
conflicts: comparisonConflicts(registryComparison, ['Created']),
|
|
347
|
+
decisionImpact: 'Domain age may contribute to an explainable score factor. It does not prove intent, ownership, or continuous operation.',
|
|
348
|
+
}),
|
|
349
|
+
},
|
|
350
|
+
{
|
|
351
|
+
label: 'Expires',
|
|
352
|
+
value: formatDate(input.expiresDate),
|
|
353
|
+
detail: fmtExpiresIn(typeof availability.expiresInDays === 'number' ? availability.expiresInDays : null) || 'Registry lifecycle date',
|
|
354
|
+
provenance: factProvenance({
|
|
355
|
+
source,
|
|
356
|
+
diagnostics,
|
|
357
|
+
observedAt: input.resultObservedAt,
|
|
358
|
+
fields: ['RDAP expiration event or lifecycle', 'WHOIS expiry-date family'],
|
|
359
|
+
normalization: 'Recognised expiry dates are parsed into an ISO-compatible instant when possible without predicting deletion or release.',
|
|
360
|
+
limitations,
|
|
361
|
+
conflicts: comparisonConflicts(registryComparison, ['Expires']),
|
|
362
|
+
decisionImpact: 'Expiry proximity may contribute to an explainable opportunity factor. It does not predict renewal, deletion, release, price, or eligibility.',
|
|
363
|
+
}),
|
|
364
|
+
},
|
|
365
|
+
{
|
|
366
|
+
label: 'Updated',
|
|
367
|
+
value: formatDate(input.updatedDate),
|
|
368
|
+
detail: 'Most recent registry change',
|
|
369
|
+
provenance: factProvenance({
|
|
370
|
+
source,
|
|
371
|
+
diagnostics,
|
|
372
|
+
observedAt: input.resultObservedAt,
|
|
373
|
+
fields: ['RDAP last-changed event or lifecycle', 'WHOIS updated-date family'],
|
|
374
|
+
normalization: 'Recognised update dates are parsed into an ISO-compatible instant when possible and remain a source-reported lifecycle observation.',
|
|
375
|
+
limitations,
|
|
376
|
+
conflicts: comparisonConflicts(registryComparison, ['Updated']),
|
|
377
|
+
decisionImpact: 'The update date is contextual evidence only and does not establish what changed or whether a website is active.',
|
|
378
|
+
}),
|
|
379
|
+
},
|
|
380
|
+
{
|
|
381
|
+
label: 'Website',
|
|
382
|
+
value: display(availability.activityStatus),
|
|
383
|
+
detail: display(availability.websiteProbeDetail),
|
|
384
|
+
provenance: {
|
|
385
|
+
sources: ['Bounded homepage and favicon observation'],
|
|
386
|
+
observedAt: formatDate(input.resultObservedAt),
|
|
387
|
+
fieldFamilies: ['HTTP response', 'redirect chain', 'bounded homepage body', 'favicon response'],
|
|
388
|
+
normalization: 'Observed responses are classified into active, parked, inactive, or unknown context without executing page scripts.',
|
|
389
|
+
completeness: availability.deepScanComplete === false ? 'partial' : 'complete',
|
|
390
|
+
limitations: textList(record(availability.http).limitations),
|
|
391
|
+
conflicts: [],
|
|
392
|
+
decisionImpact: 'Website activity can inform explainable analysis. A failed or missing response never implies inactivity, safety, or domain availability.',
|
|
393
|
+
},
|
|
394
|
+
},
|
|
395
|
+
],
|
|
396
|
+
diagnostics: buildDiagnostics(diagnostics, rdapParsed, whoisParsed, record(input.registrarRdap)),
|
|
397
|
+
};
|
|
398
|
+
}
|
|
@@ -141,7 +141,7 @@ export function comparePageBaselines(rawReference, rawObserved) {
|
|
|
141
141
|
if (!reference || !observed)
|
|
142
142
|
return null;
|
|
143
143
|
const components = [
|
|
144
|
-
digestComponent('normalized_html', '
|
|
144
|
+
digestComponent('normalized_html', 'Normalised HTML', 'Normalized-HTML', reference.normalizedHtml, observed.normalizedHtml),
|
|
145
145
|
visibleTextComponent(reference.visibleText, observed.visibleText),
|
|
146
146
|
domStructureComponent(reference.domStructure, observed.domStructure),
|
|
147
147
|
formComponent(reference.formStructure, observed.formStructure),
|
|
@@ -206,16 +206,16 @@ export function buildScanRelationships(rawRows) {
|
|
|
206
206
|
const output = [];
|
|
207
207
|
for (const [value, domains] of nameserverSets)
|
|
208
208
|
if (domains.size >= 2)
|
|
209
|
-
output.push(group('nameserver_set', 'Shared nameserver set', 'Exact
|
|
209
|
+
output.push(group('nameserver_set', 'Shared nameserver set', 'Exact normalised set', value, [...domains].sort(), 'These domains reported the same normalised nameserver set retained by this scan.'));
|
|
210
210
|
for (const [value, domains] of addresses)
|
|
211
211
|
if (domains.size >= 2)
|
|
212
|
-
output.push(group('ip_address', 'Shared IP address', 'Exact
|
|
212
|
+
output.push(group('ip_address', 'Shared IP address', 'Exact normalised address', value, [...domains].sort(), 'These domains resolved to the same IP address in this scan. Shared hosting and CDNs are common.'));
|
|
213
213
|
for (const [value, domains] of certificates)
|
|
214
214
|
if (domains.size >= 2)
|
|
215
215
|
output.push(group('certificate', 'Shared TLS certificate', 'Exact leaf-certificate SHA-256', value, [...domains].sort(), 'These domains presented the same leaf certificate in this scan. Multi-domain certificates, shared hosting, CDNs, and managed platforms are common.'));
|
|
216
216
|
for (const [value, domains] of identifiers)
|
|
217
217
|
if (domains.size >= 2)
|
|
218
|
-
output.push(group('tracking_identifier', 'Shared tracking identifier', 'Exact public identifier', value, [...domains].sort(), 'These pages exposed the same
|
|
218
|
+
output.push(group('tracking_identifier', 'Shared tracking identifier', 'Exact public identifier', value, [...domains].sort(), 'These pages exposed the same recognised public tracking identifier in bounded static HTML.'));
|
|
219
219
|
const faviconRows = rows.slice(0, MAX_FAVICON_ROWS).map(({ domain, observation }) => ({
|
|
220
220
|
domain,
|
|
221
221
|
faviconHash: typeof observation.faviconHash === 'string' && FAVICON_SHA_RE.test(observation.faviconHash)
|
|
@@ -24,12 +24,12 @@ const TYPES = new Set(RELATIONSHIP_OBSERVATION_TYPES);
|
|
|
24
24
|
const TYPE_METADATA = Object.freeze({
|
|
25
25
|
nameserver_set: Object.freeze({
|
|
26
26
|
label: 'Shared nameserver set',
|
|
27
|
-
method: 'Exact
|
|
28
|
-
description: 'These domains reported the same
|
|
27
|
+
method: 'Exact normalised set',
|
|
28
|
+
description: 'These domains reported the same normalised nameserver set. Shared DNS providers are common.',
|
|
29
29
|
}),
|
|
30
30
|
ip_address: Object.freeze({
|
|
31
31
|
label: 'Shared IP address',
|
|
32
|
-
method: 'Exact
|
|
32
|
+
method: 'Exact normalised address',
|
|
33
33
|
description: 'These domains resolved to the same IP address. Shared hosting, CDNs, and managed platforms are common.',
|
|
34
34
|
}),
|
|
35
35
|
certificate: Object.freeze({
|
|
@@ -40,7 +40,7 @@ const TYPE_METADATA = Object.freeze({
|
|
|
40
40
|
tracking_identifier: Object.freeze({
|
|
41
41
|
label: 'Shared tracking identifier',
|
|
42
42
|
method: 'Exact public identifier',
|
|
43
|
-
description: 'These pages exposed the same
|
|
43
|
+
description: 'These pages exposed the same recognised public tracking identifier in bounded static HTML.',
|
|
44
44
|
}),
|
|
45
45
|
favicon: Object.freeze({
|
|
46
46
|
label: 'Similar favicon',
|
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
export { RISK_MODEL_VERSION, RISK_REVIEW_THRESHOLD, computeRiskScore, explainRiskScore, formatScoreBreakdown, normalizeRiskModelVersion, riskTone, } from "../../../../lib/risk-scoring.mjs";
|
|
2
|
-
|
|
3
|
-
return value && typeof value === 'object' && !Array.isArray(value)
|
|
4
|
-
? value
|
|
5
|
-
: null;
|
|
6
|
-
}
|
|
2
|
+
export { OPPORTUNITY_MODEL_VERSION, OPPORTUNITY_REVIEW_THRESHOLD, computeOpportunityScore, explainOpportunityScore, normalizeOpportunityModelVersion, opportunityTone, } from "../../../../lib/opportunity-scoring.mjs";
|
|
7
3
|
export function fmtAge(value) {
|
|
8
|
-
if (typeof value !== 'number' || !Number.isFinite(value))
|
|
4
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0)
|
|
9
5
|
return null;
|
|
10
|
-
const days = value;
|
|
6
|
+
const days = Math.round(value);
|
|
11
7
|
if (days < 60)
|
|
12
8
|
return `${days}d old`;
|
|
13
9
|
const years = days / 365.25;
|
|
@@ -16,7 +12,7 @@ export function fmtAge(value) {
|
|
|
16
12
|
export function fmtExpiresIn(value) {
|
|
17
13
|
if (typeof value !== 'number' || !Number.isFinite(value))
|
|
18
14
|
return null;
|
|
19
|
-
const days = value;
|
|
15
|
+
const days = Math.round(value);
|
|
20
16
|
return days < 0 ? `expired ${Math.abs(days)}d ago` : `expires in ${days}d`;
|
|
21
17
|
}
|
|
22
18
|
export const ACTIVITY_LABELS = {
|
|
@@ -39,92 +35,16 @@ export function formatActivityCell(v, hasMx, hasSpf, hasDmarc) {
|
|
|
39
35
|
? ACTIVITY_LABELS[v]
|
|
40
36
|
: '—';
|
|
41
37
|
const mailParts = [];
|
|
42
|
-
if (hasMx)
|
|
38
|
+
if (hasMx === true)
|
|
43
39
|
mailParts.push('MX');
|
|
44
|
-
if (hasSpf)
|
|
40
|
+
if (hasSpf === true)
|
|
45
41
|
mailParts.push('SPF');
|
|
46
|
-
if (hasDmarc)
|
|
42
|
+
if (hasDmarc === true)
|
|
47
43
|
mailParts.push('DMARC');
|
|
48
44
|
return mailParts.length ? `${label} · ${mailParts.join('+')}` : label;
|
|
49
45
|
}
|
|
50
|
-
// ---------------------------------------------------------------------------
|
|
51
|
-
// Opportunity score: combines the signals above into one 0-100 number so a
|
|
52
|
-
// large fast-scan result set can be sorted instead of manually scanned.
|
|
53
|
-
// Higher = more actionable (cheap/easy to get), not "more valuable" in an
|
|
54
|
-
// appraisal sense - a parked, aged, publicly-contactable domain scores high
|
|
55
|
-
// because it's an easy approach; an actively-used one scores low because
|
|
56
|
-
// the owner is unlikely to be interested regardless of the domain's worth.
|
|
57
|
-
// ---------------------------------------------------------------------------
|
|
58
|
-
const STATE_BASE_SCORE = {
|
|
59
|
-
for_sale: 95,
|
|
60
|
-
expiring: 85,
|
|
61
|
-
available: 90,
|
|
62
|
-
registered: 40,
|
|
63
|
-
unknown: null,
|
|
64
|
-
error: null,
|
|
65
|
-
};
|
|
66
|
-
const STATE_LABELS = {
|
|
67
|
-
for_sale: 'for sale',
|
|
68
|
-
expiring: 'expiring/pending delete',
|
|
69
|
-
available: 'available',
|
|
70
|
-
registered: 'registered',
|
|
71
|
-
};
|
|
72
|
-
// Builds the opportunity score alongside a factor-by-factor breakdown (base
|
|
73
|
-
// state score, then each signal's contribution) - the single source of
|
|
74
|
-
// truth computeOpportunityScore() below reads its final number from, and
|
|
75
|
-
// what the score chips' tooltips and the CSV export both render from, so
|
|
76
|
-
// the displayed reasoning can never drift from the actual number.
|
|
77
|
-
export function explainOpportunityScore(value) {
|
|
78
|
-
const r = plainRecord(value);
|
|
79
|
-
if (!r)
|
|
80
|
-
return null;
|
|
81
|
-
const rawState = r.availability ?? r.state;
|
|
82
|
-
if (typeof rawState !== 'string')
|
|
83
|
-
return null;
|
|
84
|
-
const state = rawState;
|
|
85
|
-
const base = STATE_BASE_SCORE[state];
|
|
86
|
-
if (base === null || base === undefined)
|
|
87
|
-
return null;
|
|
88
|
-
const factors = [{ label: `Base score for "${STATE_LABELS[state] || state}"`, delta: base }];
|
|
89
|
-
let score = base;
|
|
90
|
-
if (r.activityStatus === 'parked') {
|
|
91
|
-
factors.push({ label: 'Parked/for-sale page', delta: 15 });
|
|
92
|
-
score += 15;
|
|
93
|
-
}
|
|
94
|
-
else if (r.activityStatus === 'no_site') {
|
|
95
|
-
factors.push({ label: 'No site running', delta: 5 });
|
|
96
|
-
score += 5;
|
|
97
|
-
}
|
|
98
|
-
else if (r.activityStatus === 'active') {
|
|
99
|
-
factors.push({ label: 'Active site in use', delta: -20 });
|
|
100
|
-
score -= 20;
|
|
101
|
-
}
|
|
102
|
-
if (r.privacyProtected === false) {
|
|
103
|
-
factors.push({ label: 'Contact info public', delta: 10 });
|
|
104
|
-
score += 10;
|
|
105
|
-
}
|
|
106
|
-
else if (r.privacyProtected === true) {
|
|
107
|
-
factors.push({ label: 'WHOIS privacy protected', delta: -10 });
|
|
108
|
-
score -= 10;
|
|
109
|
-
}
|
|
110
|
-
if (typeof r.domainAgeDays === 'number') {
|
|
111
|
-
const ageBonus = Math.min(20, (r.domainAgeDays / 365) * 2);
|
|
112
|
-
if (ageBonus !== 0) {
|
|
113
|
-
factors.push({ label: `Domain age (${fmtAge(r.domainAgeDays)})`, delta: ageBonus });
|
|
114
|
-
score += ageBonus;
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
if (state === 'registered' && typeof r.expiresInDays === 'number' && r.expiresInDays >= 0 && r.expiresInDays < 30) {
|
|
118
|
-
factors.push({ label: 'Expires within 30 days', delta: 10 }); // might lapse soon even though not yet flagged "expiring"
|
|
119
|
-
score += 10;
|
|
120
|
-
}
|
|
121
|
-
return { score: Math.max(0, Math.min(100, Math.round(score))), factors };
|
|
122
|
-
}
|
|
123
|
-
export function computeOpportunityScore(value) {
|
|
124
|
-
const explained = explainOpportunityScore(value);
|
|
125
|
-
return explained ? explained.score : null;
|
|
126
|
-
}
|
|
127
46
|
export function scoreTone(score) {
|
|
47
|
+
// Kept as a compatibility name for existing UI callers.
|
|
128
48
|
if (score === null)
|
|
129
49
|
return 'neutral';
|
|
130
50
|
if (score >= 70)
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
// and downloads; this module owns schema migration, record normalization,
|
|
3
3
|
// import merging, and exact serialized-byte accounting.
|
|
4
4
|
import { compactWatchlistResults } from "./watchlist-history.js";
|
|
5
|
-
import { normalizeRiskModelVersion } from "./scoring.js";
|
|
5
|
+
import { normalizeOpportunityModelVersion, normalizeRiskModelVersion } from "./scoring.js";
|
|
6
6
|
export const SHORTLIST_SCHEMA = 'whoisleuth.shortlist';
|
|
7
|
-
export const SHORTLIST_SCHEMA_VERSION =
|
|
7
|
+
export const SHORTLIST_SCHEMA_VERSION = 3;
|
|
8
|
+
export const SUPPORTED_SHORTLIST_SCHEMA_VERSIONS = Object.freeze([2, SHORTLIST_SCHEMA_VERSION]);
|
|
8
9
|
export const MAX_SHORTLIST_ENTRIES = 500;
|
|
9
10
|
export const MAX_SHORTLIST_INPUTS = MAX_SHORTLIST_ENTRIES * 4;
|
|
10
11
|
export const MAX_SHORTLIST_STORE_BYTES = 1024 * 1024;
|
|
@@ -84,6 +85,9 @@ export function normalizeShortlistRecord(raw, options = {}) {
|
|
|
84
85
|
riskScore,
|
|
85
86
|
riskFactors: factors(value.riskFactors),
|
|
86
87
|
opportunityScore: score(value.opportunityScore),
|
|
88
|
+
opportunityModelVersion: score(value.opportunityScore) === null
|
|
89
|
+
? null
|
|
90
|
+
: normalizeOpportunityModelVersion(value.opportunityModelVersion),
|
|
87
91
|
savedAt: timestamp(value.savedAt, timestamp(options.fallbackTimestamp, EPOCH)),
|
|
88
92
|
};
|
|
89
93
|
}
|
|
@@ -168,8 +172,8 @@ export function mergeShortlistStores(localRaw, importedRaw) {
|
|
|
168
172
|
if (importedVersion !== null && importedVersion > SHORTLIST_SCHEMA_VERSION) {
|
|
169
173
|
throw new Error(`This shortlist file uses newer schema ${importedVersion}. Update the app before importing it.`);
|
|
170
174
|
}
|
|
171
|
-
if (importedVersion
|
|
172
|
-
throw new Error(`Expected a WHOISleuth shortlist export using schema ${
|
|
175
|
+
if (importedVersion === null || !SUPPORTED_SHORTLIST_SCHEMA_VERSIONS.includes(importedVersion)) {
|
|
176
|
+
throw new Error(`Expected a WHOISleuth shortlist export using schema ${SUPPORTED_SHORTLIST_SCHEMA_VERSIONS.join(' or ')}.`);
|
|
173
177
|
}
|
|
174
178
|
const local = normalizeShortlistStore(localRaw).entries;
|
|
175
179
|
const byDomain = new Map(local.map((record) => [record.domain, record]));
|