@slicedearth/whoisleuth-cli 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/LICENSES/Retire.js-Apache-2.0.txt +199 -0
- package/NOTICE +76 -0
- package/README.md +84 -0
- package/TRADEMARKS.md +41 -0
- package/bin/whoisleuth.mjs +35 -0
- package/cli/archive-inspect.mjs +198 -0
- package/cli/arguments.mjs +1022 -0
- package/cli/artifact-verify.mjs +210 -0
- package/cli/bulk-checkpoint.mjs +218 -0
- package/cli/bulk-command-runner.mjs +109 -0
- package/cli/bulk-output.mjs +112 -0
- package/cli/bulk.mjs +133 -0
- package/cli/compare.mjs +238 -0
- package/cli/completion.mjs +175 -0
- package/cli/csv.mjs +1 -0
- package/cli/discover.mjs +50 -0
- package/cli/discovery-command-runner.mjs +36 -0
- package/cli/discovery-observation-snapshot.mjs +315 -0
- package/cli/discovery-scan-command-runner.mjs +141 -0
- package/cli/discovery-scan.mjs +325 -0
- package/cli/discovery-snapshot.mjs +133 -0
- package/cli/discovery-workflow.mjs +73 -0
- package/cli/doctor.mjs +137 -0
- package/cli/errors.mjs +21 -0
- package/cli/evidence-command-arguments.mjs +145 -0
- package/cli/evidence-command-runner.mjs +106 -0
- package/cli/evidence-signing.mjs +215 -0
- package/cli/exit-codes.mjs +9 -0
- package/cli/export-evidence.mjs +28 -0
- package/cli/formatters/evidence-report.mjs +3 -0
- package/cli/formatters/html.mjs +69 -0
- package/cli/formatters/json.mjs +135 -0
- package/cli/formatters/markdown.mjs +3 -0
- package/cli/formatters/terminal.mjs +658 -0
- package/cli/http.mjs +27 -0
- package/cli/lookup-command-runner.mjs +87 -0
- package/cli/lookup-diff.mjs +171 -0
- package/cli/mail-review.mjs +204 -0
- package/cli/manual.mjs +34 -0
- package/cli/network-command-runner.mjs +88 -0
- package/cli/output-file.mjs +97 -0
- package/cli/page-compare.mjs +123 -0
- package/cli/posture.mjs +20 -0
- package/cli/progress-events.mjs +53 -0
- package/cli/progress.mjs +56 -0
- package/cli/registry-access.mjs +3 -0
- package/cli/registry-support.mjs +120 -0
- package/cli/risk-calibration.mjs +294 -0
- package/cli/runner-types.mjs +1 -0
- package/cli/runner.mjs +726 -0
- package/cli/saved-lookup.mjs +87 -0
- package/cli/source-reliability.mjs +727 -0
- package/cli/strict-exit.mjs +55 -0
- package/cli/terminal-presentation.mjs +147 -0
- package/docs/cli.md +827 -0
- package/frontend/src/lib/analysis/acquisition-decision-packet.js +93 -0
- package/frontend/src/lib/analysis/acquisition-due-diligence.js +391 -0
- package/frontend/src/lib/analysis/artifact-integrity.js +21 -0
- package/frontend/src/lib/analysis/brand-profile-model.js +421 -0
- package/frontend/src/lib/analysis/bulk-domain-comparison.js +246 -0
- package/frontend/src/lib/analysis/bulk-mail-exposure.js +192 -0
- package/frontend/src/lib/analysis/bulk-retry-plan.js +86 -0
- package/frontend/src/lib/analysis/bulk-review-export.js +62 -0
- package/frontend/src/lib/analysis/bulk-review-model.js +266 -0
- package/frontend/src/lib/analysis/bulk-session-model.js +515 -0
- package/frontend/src/lib/analysis/bulk-sort.js +49 -0
- package/frontend/src/lib/analysis/campaign-model.js +265 -0
- package/frontend/src/lib/analysis/case-evidence-model.js +642 -0
- package/frontend/src/lib/analysis/case-migration-model.js +248 -0
- package/frontend/src/lib/analysis/case-model.js +5 -0
- package/frontend/src/lib/analysis/case-record-contracts.js +44 -0
- package/frontend/src/lib/analysis/case-record-core.js +232 -0
- package/frontend/src/lib/analysis/case-record-model.js +7 -0
- package/frontend/src/lib/analysis/case-record-operations.js +224 -0
- package/frontend/src/lib/analysis/case-response-model.js +640 -0
- package/frontend/src/lib/analysis/case-response-packet.js +573 -0
- package/frontend/src/lib/analysis/case-storage-model.js +93 -0
- package/frontend/src/lib/analysis/detection-rule-model.js +306 -0
- package/frontend/src/lib/analysis/dns-record-normalization.js +11 -0
- package/frontend/src/lib/analysis/http-summary.js +165 -0
- package/frontend/src/lib/analysis/investigation-guide.js +500 -0
- package/frontend/src/lib/analysis/investigation-template-model.js +144 -0
- package/frontend/src/lib/analysis/page-baseline.js +272 -0
- package/frontend/src/lib/analysis/page-similarity.js +179 -0
- package/frontend/src/lib/analysis/relationship-evidence.js +265 -0
- package/frontend/src/lib/analysis/relationship-observation-model.js +423 -0
- package/frontend/src/lib/analysis/scoring.js +135 -0
- package/frontend/src/lib/analysis/shortlist-model.js +209 -0
- package/frontend/src/lib/analysis/utils.js +265 -0
- package/frontend/src/lib/analysis/watchlist-history.js +643 -0
- package/frontend/src/lib/analysis/watchlist-store.js +127 -0
- package/frontend/src/lib/analysis/website-snapshot-model.js +302 -0
- package/frontend/src/lib/analysis/workspace-archive-crypto.js +282 -0
- package/frontend/src/lib/analysis/workspace-archive.js +472 -0
- package/lib/abort.mjs +26 -0
- package/lib/abusech-auth.mjs +19 -0
- package/lib/availability.mjs +716 -0
- package/lib/bounded-contract-normalizers.mjs +61 -0
- package/lib/browser-library-profile.mjs +310 -0
- package/lib/bulk-comparison-evidence.mjs +83 -0
- package/lib/classify.mjs +93 -0
- package/lib/client-behavior-profile.mjs +83 -0
- package/lib/credential-surface-profile.mjs +66 -0
- package/lib/csv.mjs +23 -0
- package/lib/ct-query.mjs +27 -0
- package/lib/ct-search.mjs +393 -0
- package/lib/distributed-operation-budget.mjs +461 -0
- package/lib/dns-delegation-health.mjs +344 -0
- package/lib/dns-intelligence.mjs +739 -0
- package/lib/dns-mx.mjs +24 -0
- package/lib/dns-resolver-selection.mjs +33 -0
- package/lib/domain-posture-analysis.mjs +254 -0
- package/lib/domain-posture-parsers.mjs +358 -0
- package/lib/domain-posture.mjs +745 -0
- package/lib/error-detail.mjs +10 -0
- package/lib/evidence-export.mjs +431 -0
- package/lib/evidence-report-markdown.mjs +71 -0
- package/lib/evidence-report.mjs +391 -0
- package/lib/external-intelligence-risk.mjs +108 -0
- package/lib/favicon.mjs +146 -0
- package/lib/feature-policy.mjs +70 -0
- package/lib/generated/retire-browser-catalog.mjs +8724 -0
- package/lib/generated/unicode-confusables-17.mjs +93 -0
- package/lib/hostname.mjs +38 -0
- package/lib/html-signals.mjs +680 -0
- package/lib/http-intelligence.mjs +232 -0
- package/lib/idn-confusable-policy.mjs +86 -0
- package/lib/idn-confusables.mjs +465 -0
- package/lib/lookup-cache.mjs +130 -0
- package/lib/lookup-progress.mjs +291 -0
- package/lib/lookup-source-progress.mjs +109 -0
- package/lib/lookup-threat-provider-inventory.mjs +13 -0
- package/lib/lookup.mjs +592 -0
- package/lib/observation.mjs +114 -0
- package/lib/observed-network-context.mjs +355 -0
- package/lib/operation-budget.mjs +489 -0
- package/lib/page-fingerprints.mjs +481 -0
- package/lib/page-role-profile.mjs +117 -0
- package/lib/perceptual-hash.mjs +451 -0
- package/lib/rdap-attempts.mjs +34 -0
- package/lib/rdap-bootstrap.mjs +173 -0
- package/lib/rdap-capabilities.mjs +292 -0
- package/lib/rdap-client.mjs +125 -0
- package/lib/rdap-normalization.mjs +691 -0
- package/lib/rdap-registrar.mjs +196 -0
- package/lib/rdap-transport.mjs +39 -0
- package/lib/rdap-types.mjs +1 -0
- package/lib/rdap-validation.mjs +94 -0
- package/lib/rdap.mjs +16 -0
- package/lib/registry-access.mjs +32 -0
- package/lib/registry-capabilities.mjs +117 -0
- package/lib/registry-capability-catalogue.mjs +1206 -0
- package/lib/registry-comparison.mjs +318 -0
- package/lib/registry-dates.mjs +190 -0
- package/lib/registry-insights.mjs +325 -0
- package/lib/response-policy.mjs +256 -0
- package/lib/risk-scoring.mjs +164 -0
- package/lib/safe-fetch.mjs +446 -0
- package/lib/security-txt.mjs +359 -0
- package/lib/service-binding-dns.mjs +608 -0
- package/lib/sslbl-certificates.generated.mjs +143 -0
- package/lib/sslbl-intelligence.mjs +241 -0
- package/lib/static-html-analysis.mjs +340 -0
- package/lib/structured-data-identity.mjs +323 -0
- package/lib/threat-intelligence-contract.mjs +5 -0
- package/lib/threat-intelligence-definition-registry.mjs +34 -0
- package/lib/threat-intelligence-fixtures.mjs +82 -0
- package/lib/threat-intelligence-manifests.mjs +512 -0
- package/lib/threat-intelligence-runtime.mjs +662 -0
- package/lib/threat-intelligence-targets.mjs +168 -0
- package/lib/threat-intelligence-types.mjs +3 -0
- package/lib/threatfox-intelligence.mjs +307 -0
- package/lib/tls-intelligence.mjs +814 -0
- package/lib/typosquat-generator.mjs +884 -0
- package/lib/urlhaus-intelligence.mjs +361 -0
- package/lib/urlscan-intelligence.mjs +349 -0
- package/lib/website-security-posture.mjs +325 -0
- package/lib/website-technology.mjs +437 -0
- package/lib/whois-authority.mjs +98 -0
- package/lib/whois-chain.mjs +261 -0
- package/lib/whois-contacts.mjs +163 -0
- package/lib/whois-contracts.mjs +3 -0
- package/lib/whois-normalization.mjs +27 -0
- package/lib/whois-transport.mjs +100 -0
- package/lib/whois-values.mjs +92 -0
- package/lib/whois.mjs +1187 -0
- package/package.json +55 -0
- package/third-party-notices.txt +5571 -0
|
@@ -0,0 +1,716 @@
|
|
|
1
|
+
// Domain availability check.
|
|
2
|
+
//
|
|
3
|
+
// Determines registered vs. available vs. expiring via RDAP/WHOIS, and flags
|
|
4
|
+
// likely for-sale listings by checking for known parking/marketplace
|
|
5
|
+
// nameservers and "for sale" listing language on the domain's own homepage.
|
|
6
|
+
// This does not attempt to price the domain - registries don't publish sale
|
|
7
|
+
// prices, and that's proprietary marketplace data.
|
|
8
|
+
//
|
|
9
|
+
// Shared by the Express server and the Netlify Functions.
|
|
10
|
+
import { promises as dns } from 'node:dns';
|
|
11
|
+
import { fetchRdapRecord } from "./rdap.mjs";
|
|
12
|
+
import { buildWhoisChain, parseWhoisChain } from "./whois.mjs";
|
|
13
|
+
import { safeFetchDetailed, readTextCapped } from "./safe-fetch.mjs";
|
|
14
|
+
import { collectDnsIntelligence, skippedDnsIntelligence } from "./dns-intelligence.mjs";
|
|
15
|
+
import { fetchFaviconHash } from "./favicon.mjs";
|
|
16
|
+
import { extractHtmlSignals } from "./html-signals.mjs";
|
|
17
|
+
import { featureDecision, networkFeaturePolicy } from "./feature-policy.mjs";
|
|
18
|
+
import { buildHttpObservation, failedHttpObservation, skippedHttpObservation } from "./http-intelligence.mjs";
|
|
19
|
+
import { collectTlsIntelligence, skippedTlsObservation } from "./tls-intelligence.mjs";
|
|
20
|
+
import { parseRegistryDate, registryDateIso } from "./registry-dates.mjs";
|
|
21
|
+
import { analyzeWebsiteSecurityPosture } from "./website-security-posture.mjs";
|
|
22
|
+
import { analyzeResponsePolicyHeaders } from "./response-policy.mjs";
|
|
23
|
+
import { nonEmptyErrorMessage } from "./error-detail.mjs";
|
|
24
|
+
const MAX_HOMEPAGE_BYTES = 300000;
|
|
25
|
+
const DNS_DELEGATION_TIMEOUT_MS = 4000;
|
|
26
|
+
const MAX_DELEGATION_NAMESERVERS = 50;
|
|
27
|
+
const MISSING_DNS_CODES = new Set(['ENODATA', 'ENOTFOUND', 'ENONAME', 'NXDOMAIN']);
|
|
28
|
+
function errorRecord(value) {
|
|
29
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
30
|
+
? value
|
|
31
|
+
: {};
|
|
32
|
+
}
|
|
33
|
+
function rdapEventDate(events, action) {
|
|
34
|
+
const date = events.find((event) => event.action === action)?.date;
|
|
35
|
+
return typeof date === 'string' ? date : null;
|
|
36
|
+
}
|
|
37
|
+
// No marketplace (Afternic/Sedo/Dan.com/GoDaddy Auctions/etc.) offers a
|
|
38
|
+
// free, no-auth API to check "is this specific domain listed for sale" -
|
|
39
|
+
// what's checkable without credentials is the parking/landing-page
|
|
40
|
+
// nameservers and homepage copy these services actually use, which is
|
|
41
|
+
// broader coverage of the same signal, not a live cross-marketplace lookup.
|
|
42
|
+
const PARKING_NS_PATTERNS = [
|
|
43
|
+
/sedoparking\.com$/i,
|
|
44
|
+
/sedo\.com$/i,
|
|
45
|
+
/above\.com$/i,
|
|
46
|
+
/bodis\.com$/i,
|
|
47
|
+
/parkingcrew\.net$/i,
|
|
48
|
+
/dan\.com$/i,
|
|
49
|
+
/hugedomains\.com$/i,
|
|
50
|
+
/uniregistry/i,
|
|
51
|
+
/squadhelp/i,
|
|
52
|
+
/afternic/i,
|
|
53
|
+
/voodoo\.com$/i,
|
|
54
|
+
/fabulous\.com$/i,
|
|
55
|
+
/namedrive/i,
|
|
56
|
+
/smartname\.com$/i,
|
|
57
|
+
/domainsponsor\.com$/i,
|
|
58
|
+
/undeveloped\.com$/i,
|
|
59
|
+
/trafficz\.com$/i,
|
|
60
|
+
/dsredirection\.com$/i,
|
|
61
|
+
];
|
|
62
|
+
const FOR_SALE_TEXT_RE = /(this domain (?:name )?(?:may be|is) for sale|buy this domain|domain(?: name)? for sale|make (?:an|your) offer|inquire about (?:this|the) domain|purchase this domain|this domain is available for purchase|backorder this domain|bid on this domain|premium domain for sale|own this domain|click here to buy this domain|would you like to buy this domain|this domain is (?:available for lease|listed for sale))/i;
|
|
63
|
+
const FOR_SALE_PATH_RE = /\/(?:premium-)?domains?-for-sale(?:\/|$)/i;
|
|
64
|
+
function forSaleRedirectSignal(httpObservation) {
|
|
65
|
+
if (!httpObservation || typeof httpObservation !== 'object')
|
|
66
|
+
return null;
|
|
67
|
+
const observation = httpObservation;
|
|
68
|
+
const urls = [
|
|
69
|
+
observation.finalUrl,
|
|
70
|
+
...(Array.isArray(observation.redirects)
|
|
71
|
+
? observation.redirects.map((redirect) => redirect && typeof redirect === 'object' ? redirect.to : null)
|
|
72
|
+
: []),
|
|
73
|
+
];
|
|
74
|
+
for (const value of urls) {
|
|
75
|
+
if (typeof value !== 'string' || value.length > 2048)
|
|
76
|
+
continue;
|
|
77
|
+
try {
|
|
78
|
+
const url = new URL(value);
|
|
79
|
+
if (['http:', 'https:'].includes(url.protocol) && FOR_SALE_PATH_RE.test(url.pathname)) {
|
|
80
|
+
return `for-sale landing-page redirect (${url.origin}${url.pathname})`;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
// Malformed retained provenance cannot establish a sale signal.
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
// ---------------------------------------------------------------------------
|
|
90
|
+
// Acquisition/sourcing signals: domain age, privacy-redaction, expiry
|
|
91
|
+
// proximity, and site-activity status. These read data already fetched by
|
|
92
|
+
// checkDomainAvailability (RDAP/WHOIS dates and registrant info) - none of
|
|
93
|
+
// this triggers extra registry queries.
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
function computeAgeDays(dateStr) {
|
|
96
|
+
const d = parseRegistryDate(dateStr);
|
|
97
|
+
return d ? Math.floor((Date.now() - d.getTime()) / 86400000) : null;
|
|
98
|
+
}
|
|
99
|
+
function computeDaysUntil(dateStr) {
|
|
100
|
+
const d = parseRegistryDate(dateStr);
|
|
101
|
+
return d ? Math.ceil((d.getTime() - Date.now()) / 86400000) : null;
|
|
102
|
+
}
|
|
103
|
+
const PRIVACY_MARKERS = [
|
|
104
|
+
/redacted for privacy/i,
|
|
105
|
+
/data protected/i,
|
|
106
|
+
/privacy\s*protect/i,
|
|
107
|
+
/whoisguard/i,
|
|
108
|
+
/domains by proxy/i,
|
|
109
|
+
/perfect privacy/i,
|
|
110
|
+
/contact privacy/i,
|
|
111
|
+
/private registration/i,
|
|
112
|
+
/identity protect/i,
|
|
113
|
+
/not disclosed/i,
|
|
114
|
+
/withheld for privacy/i,
|
|
115
|
+
];
|
|
116
|
+
// Only ever called after a lookup has already succeeded (this runs inside
|
|
117
|
+
// checkDomainAvailability once RDAP or WHOIS has returned real data), so a
|
|
118
|
+
// missing/empty registrant here isn't "we don't know" - it means the
|
|
119
|
+
// registry gave us a response with no usable contact in it, which for
|
|
120
|
+
// sourcing purposes (can I reach this owner from this data alone?) is the
|
|
121
|
+
// same practical answer as an explicit privacy-proxy service: no.
|
|
122
|
+
function isPrivacyProtected(registrant) {
|
|
123
|
+
if (!registrant)
|
|
124
|
+
return true;
|
|
125
|
+
const blob = [registrant.name, registrant.org, registrant.email].filter(Boolean).join(' ');
|
|
126
|
+
if (!blob)
|
|
127
|
+
return true; // record exists but every contact field is blank - that's redaction
|
|
128
|
+
return PRIVACY_MARKERS.some((re) => re.test(blob));
|
|
129
|
+
}
|
|
130
|
+
// RDAP Lookup retains a bounded multi-value contact inventory, but Bulk and
|
|
131
|
+
// watchlist availability records deliberately keep the historical compact
|
|
132
|
+
// shape. This prevents repeated contact arrays and registry links from
|
|
133
|
+
// expanding browser-local stores or flowing into case evidence implicitly.
|
|
134
|
+
function compactContact(contact) {
|
|
135
|
+
if (!contact || typeof contact !== 'object' || Array.isArray(contact))
|
|
136
|
+
return null;
|
|
137
|
+
const value = contact;
|
|
138
|
+
const compact = {
|
|
139
|
+
handle: value.handle || null,
|
|
140
|
+
name: value.name || null,
|
|
141
|
+
org: value.org || null,
|
|
142
|
+
email: value.email || null,
|
|
143
|
+
phone: value.phone || null,
|
|
144
|
+
address: value.address || null,
|
|
145
|
+
};
|
|
146
|
+
return Object.values(compact).some(Boolean) ? compact : null;
|
|
147
|
+
}
|
|
148
|
+
// A number of ccTLD registries publish neither an RDAP bootstrap entry nor a
|
|
149
|
+
// generally reachable port-43 WHOIS service. A positive NS answer cannot
|
|
150
|
+
// replace registry registration data, but it does prove that the registrable
|
|
151
|
+
// domain has an active DNS delegation. Use that as a bounded, positive-only
|
|
152
|
+
// fallback: no answer is never interpreted as availability because registered
|
|
153
|
+
// domains can legitimately be undelegated.
|
|
154
|
+
async function checkDnsDelegation(domain, { resolver = dns.resolveNs } = {}) {
|
|
155
|
+
let timer;
|
|
156
|
+
try {
|
|
157
|
+
const records = await Promise.race([
|
|
158
|
+
Promise.resolve().then(() => resolver(domain)),
|
|
159
|
+
new Promise((_, reject) => {
|
|
160
|
+
timer = setTimeout(() => reject(new Error('DNS delegation lookup timed out')), DNS_DELEGATION_TIMEOUT_MS);
|
|
161
|
+
}),
|
|
162
|
+
]);
|
|
163
|
+
const validNameservers = [...new Set((Array.isArray(records) ? records : [])
|
|
164
|
+
.filter((value) => typeof value === 'string')
|
|
165
|
+
.map((value) => value.trim().replace(/\.+$/, '').toLowerCase())
|
|
166
|
+
.filter((value) => value.length > 0 && value.length <= 253)
|
|
167
|
+
.filter((value) => value.split('.').every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(label))))]
|
|
168
|
+
.sort();
|
|
169
|
+
const normalized = validNameservers.slice(0, MAX_DELEGATION_NAMESERVERS);
|
|
170
|
+
return {
|
|
171
|
+
delegated: normalized.length > 0,
|
|
172
|
+
nameservers: normalized,
|
|
173
|
+
nameserversTruncated: validNameservers.length > MAX_DELEGATION_NAMESERVERS,
|
|
174
|
+
error: null,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
const error = errorRecord(err);
|
|
179
|
+
if (typeof error.code === 'string' && MISSING_DNS_CODES.has(error.code)) {
|
|
180
|
+
return { delegated: false, nameservers: [], nameserversTruncated: false, error: null };
|
|
181
|
+
}
|
|
182
|
+
return {
|
|
183
|
+
delegated: false,
|
|
184
|
+
nameservers: [],
|
|
185
|
+
nameserversTruncated: false,
|
|
186
|
+
error: nonEmptyErrorMessage(err, String(err)).slice(0, 180),
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
finally {
|
|
190
|
+
clearTimeout(timer);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
// Fetches enough of the homepage for the lightweight HTML signals below and
|
|
194
|
+
// preserves why the probe failed. A failed request is not evidence that a
|
|
195
|
+
// domain has "no site": transient DNS/TLS/network failures, a slow origin, or
|
|
196
|
+
// an HTTP error can all produce the same null body. Keeping that distinction
|
|
197
|
+
// avoids turning an inconclusive probe into a false inactivity claim.
|
|
198
|
+
async function fetchHomepage(domain, { fetcher = safeFetchDetailed } = {}) {
|
|
199
|
+
const headers = { 'User-Agent': 'Mozilla/5.0 (compatible; DomainStatusChecker/1.0)' };
|
|
200
|
+
const failures = [];
|
|
201
|
+
const probeStartedAt = Date.now();
|
|
202
|
+
for (const scheme of ['https', 'http']) {
|
|
203
|
+
const requestUrl = `${scheme}://${domain}`;
|
|
204
|
+
const attemptStartedAt = Date.now();
|
|
205
|
+
const controller = new AbortController();
|
|
206
|
+
const timeout = setTimeout(() => controller.abort(), 6000);
|
|
207
|
+
try {
|
|
208
|
+
const fetched = await fetcher(requestUrl, { signal: controller.signal, headers });
|
|
209
|
+
const fetchedResponse = fetched instanceof Response ? fetched : fetched.response;
|
|
210
|
+
const detail = fetched && typeof fetched === 'object' && 'response' in fetched
|
|
211
|
+
? fetched
|
|
212
|
+
: {
|
|
213
|
+
response: fetchedResponse,
|
|
214
|
+
requestedUrl: requestUrl,
|
|
215
|
+
finalUrl: requestUrl,
|
|
216
|
+
redirectCount: 0,
|
|
217
|
+
redirectLimitReached: false,
|
|
218
|
+
hops: [{ url: requestUrl, status: fetchedResponse.status, location: null, durationMs: Date.now() - attemptStartedAt }],
|
|
219
|
+
durationMs: Date.now() - attemptStartedAt,
|
|
220
|
+
};
|
|
221
|
+
const res = detail.response;
|
|
222
|
+
// A truncated homepage is still fine here - only used to scan for a
|
|
223
|
+
// for-sale text match, not parsed as well-formed content. The
|
|
224
|
+
// timeout stays armed through this read (cleared in `finally` below,
|
|
225
|
+
// not here) - a malicious site could otherwise send headers
|
|
226
|
+
// immediately and then trickle or stall the body forever, hanging
|
|
227
|
+
// this worker with no deadline once the timer above is disarmed.
|
|
228
|
+
if (res.ok) {
|
|
229
|
+
const body = await readTextCapped(res, MAX_HOMEPAGE_BYTES, { includeSha256: true });
|
|
230
|
+
return {
|
|
231
|
+
text: body.text,
|
|
232
|
+
status: 'fetched',
|
|
233
|
+
detail: `Homepage responded over ${scheme.toUpperCase()} (HTTP ${res.status}).`,
|
|
234
|
+
responsePolicy: analyzeResponsePolicyHeaders(res.headers),
|
|
235
|
+
technologyHeaders: {
|
|
236
|
+
'cf-ray': res.headers.get('cf-ray') || '',
|
|
237
|
+
'x-drupal-cache': res.headers.get('x-drupal-cache') || '',
|
|
238
|
+
'x-fastly-request-id': res.headers.get('x-fastly-request-id') || '',
|
|
239
|
+
'x-powered-by': res.headers.get('x-powered-by') || '',
|
|
240
|
+
'x-shopify-stage': res.headers.get('x-shopify-stage') || '',
|
|
241
|
+
'x-sorting-hat-podid': res.headers.get('x-sorting-hat-podid') || '',
|
|
242
|
+
'x-vercel-id': res.headers.get('x-vercel-id') || '',
|
|
243
|
+
},
|
|
244
|
+
http: buildHttpObservation({ ...detail, durationMs: Date.now() - attemptStartedAt }, {
|
|
245
|
+
previousAttempts: failures,
|
|
246
|
+
capturedBodyBytes: body.bytesRead,
|
|
247
|
+
bodyInspected: true,
|
|
248
|
+
bodyTruncated: body.truncated,
|
|
249
|
+
bodySha256: body.sha256,
|
|
250
|
+
}),
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
// A non-2xx HTTP response still conclusively proves that a web service
|
|
254
|
+
// answered on this domain. It may block this probe (403), require auth
|
|
255
|
+
// (401), have a broken homepage route (404), or be unhealthy (5xx), but
|
|
256
|
+
// none of those are equivalent to "no website". We cannot inspect its
|
|
257
|
+
// HTML signals, so preserve a separate responded status and release the
|
|
258
|
+
// unused body.
|
|
259
|
+
await res.body?.cancel().catch(() => { });
|
|
260
|
+
return {
|
|
261
|
+
text: null,
|
|
262
|
+
status: 'responded',
|
|
263
|
+
detail: `Web server responded over ${scheme.toUpperCase()} (HTTP ${res.status}); homepage content was not available for inspection.`,
|
|
264
|
+
responsePolicy: analyzeResponsePolicyHeaders(res.headers),
|
|
265
|
+
http: buildHttpObservation({ ...detail, durationMs: Date.now() - attemptStartedAt }, {
|
|
266
|
+
previousAttempts: failures,
|
|
267
|
+
capturedBodyBytes: 0,
|
|
268
|
+
bodyInspected: false,
|
|
269
|
+
}),
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
catch (err) {
|
|
273
|
+
const error = errorRecord(err);
|
|
274
|
+
const reason = error.name === 'AbortError'
|
|
275
|
+
? 'timed out after 6 seconds'
|
|
276
|
+
: nonEmptyErrorMessage(err, 'request failed')
|
|
277
|
+
.replace(/[\u0000-\u001f\u007f]+/g, ' ')
|
|
278
|
+
.slice(0, 180);
|
|
279
|
+
failures.push({ url: requestUrl, error: `${scheme.toUpperCase()} ${reason}` });
|
|
280
|
+
}
|
|
281
|
+
finally {
|
|
282
|
+
clearTimeout(timeout);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
return {
|
|
286
|
+
text: null,
|
|
287
|
+
status: 'inconclusive',
|
|
288
|
+
detail: failures.length
|
|
289
|
+
? `Could not confirm homepage activity: ${failures.map((attempt) => attempt.error).join('; ')}.`
|
|
290
|
+
: 'Could not confirm homepage activity.',
|
|
291
|
+
http: failedHttpObservation(failures, { durationMs: Date.now() - probeStartedAt }),
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
function deriveWebsiteActivity(homepageStatus, hasFavicon, alreadyParked = false) {
|
|
295
|
+
if (alreadyParked)
|
|
296
|
+
return 'parked';
|
|
297
|
+
if (homepageStatus === 'fetched' || homepageStatus === 'responded' || hasFavicon)
|
|
298
|
+
return 'active';
|
|
299
|
+
return 'unreachable';
|
|
300
|
+
}
|
|
301
|
+
// fast: true skips the WHOIS fallback (no TCP:43 chain) and the homepage
|
|
302
|
+
// fetch (no for-sale/parking detection) - just RDAP plus the signals
|
|
303
|
+
// derivable from it (age, expiry proximity, privacy). Meant for scanning
|
|
304
|
+
// large sourcing candidate lists quickly and gently on registry rate
|
|
305
|
+
// limits; anything it can't resolve (state "unknown") is meant to get a
|
|
306
|
+
// follow-up deep check (fast: false, the default) on the shortlist only.
|
|
307
|
+
async function checkDomainAvailability(domain, options = {}) {
|
|
308
|
+
const fast = options.fast === true;
|
|
309
|
+
const collectDns = options.collectDnsIntelligence || collectDnsIntelligence;
|
|
310
|
+
const collectTls = options.collectTlsIntelligence || collectTlsIntelligence;
|
|
311
|
+
const fetchHomepageForDomain = options.fetchHomepage || fetchHomepage;
|
|
312
|
+
const fetchFaviconForDomain = options.fetchFaviconHash || fetchFaviconHash;
|
|
313
|
+
const featurePolicy = options.featurePolicy || networkFeaturePolicy();
|
|
314
|
+
const rdapEnabled = featureDecision('rdap', featurePolicy).enabled;
|
|
315
|
+
const whoisEnabled = featureDecision('whois', featurePolicy).enabled;
|
|
316
|
+
const dnsIntelligenceEnabled = featureDecision('dns_intelligence', featurePolicy).enabled;
|
|
317
|
+
const websiteProbeEnabled = featureDecision('website_probe', featurePolicy).enabled;
|
|
318
|
+
const tlsIntelligenceEnabled = featureDecision('tls_intelligence', featurePolicy).enabled;
|
|
319
|
+
const deepScanComplete = rdapEnabled
|
|
320
|
+
&& whoisEnabled
|
|
321
|
+
&& dnsIntelligenceEnabled
|
|
322
|
+
&& websiteProbeEnabled
|
|
323
|
+
&& tlsIntelligenceEnabled;
|
|
324
|
+
const hasPreloadedRdap = Object.prototype.hasOwnProperty.call(options, 'rdapRecord');
|
|
325
|
+
const hasPreloadedWhois = Object.prototype.hasOwnProperty.call(options, 'whoisChain');
|
|
326
|
+
const hasPreloadedRdapPromise = Object.prototype.hasOwnProperty.call(options, 'rdapRecordPromise');
|
|
327
|
+
const hasPreloadedWhoisPromise = Object.prototype.hasOwnProperty.call(options, 'whoisChainPromise');
|
|
328
|
+
const hasPreloadedDnsDelegation = Object.prototype.hasOwnProperty.call(options, 'dnsDelegation');
|
|
329
|
+
let nameservers = [];
|
|
330
|
+
let statuses = [];
|
|
331
|
+
let rdapServer = null;
|
|
332
|
+
let rdapFound = false;
|
|
333
|
+
let registrar = null;
|
|
334
|
+
let registrant = null;
|
|
335
|
+
let abuse = null;
|
|
336
|
+
let createdDate = null;
|
|
337
|
+
let expiryDate = null;
|
|
338
|
+
let createdDateIso = null;
|
|
339
|
+
let expiryDateIso = null;
|
|
340
|
+
let registrationSource = null;
|
|
341
|
+
let registrationConfidence = 'high';
|
|
342
|
+
let dnssec = null;
|
|
343
|
+
let registryDnsEvidence = null;
|
|
344
|
+
if (rdapEnabled) {
|
|
345
|
+
try {
|
|
346
|
+
// Shared with /api/rdap (lib/rdap.mts's fetchRdapRecord) rather than a
|
|
347
|
+
// separate fetch+parse here - same registry data either way, and this
|
|
348
|
+
// also picks up that function's short-TTL cache (lib/lookup-cache.mts)
|
|
349
|
+
// and upstream timeout for free.
|
|
350
|
+
const recordValue = hasPreloadedRdapPromise
|
|
351
|
+
? await options.rdapRecordPromise
|
|
352
|
+
: hasPreloadedRdap
|
|
353
|
+
? options.rdapRecord
|
|
354
|
+
: await fetchRdapRecord('domain', domain);
|
|
355
|
+
const record = errorRecord(recordValue);
|
|
356
|
+
const recordRdapServer = typeof record.rdapServer === 'string' ? record.rdapServer : null;
|
|
357
|
+
const upstreamStatus = typeof record.upstreamStatus === 'number' ? record.upstreamStatus : null;
|
|
358
|
+
if (Object.keys(record).length) {
|
|
359
|
+
rdapServer = recordRdapServer;
|
|
360
|
+
if (upstreamStatus === 404) {
|
|
361
|
+
return {
|
|
362
|
+
state: 'available',
|
|
363
|
+
confidence: 'high',
|
|
364
|
+
detail: 'The registry\'s RDAP service has no record for this domain.',
|
|
365
|
+
source: 'rdap',
|
|
366
|
+
rdapServer: recordRdapServer,
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
const parsed = errorRecord(record.parsed);
|
|
370
|
+
if (Object.keys(parsed).length) {
|
|
371
|
+
registryDnsEvidence = parsed;
|
|
372
|
+
statuses = Array.isArray(parsed.statuses)
|
|
373
|
+
? parsed.statuses.filter((status) => typeof status === 'string').map((status) => status.toLowerCase())
|
|
374
|
+
: [];
|
|
375
|
+
nameservers = Array.isArray(parsed.nameservers)
|
|
376
|
+
? parsed.nameservers.filter((nameserver) => typeof nameserver === 'string')
|
|
377
|
+
: [];
|
|
378
|
+
registrar = compactContact(parsed.registrar);
|
|
379
|
+
registrant = compactContact(parsed.registrant);
|
|
380
|
+
abuse = compactContact(parsed.abuse);
|
|
381
|
+
const events = Array.isArray(parsed.events)
|
|
382
|
+
? parsed.events.map(errorRecord)
|
|
383
|
+
: [];
|
|
384
|
+
const lifecycle = errorRecord(parsed.lifecycle);
|
|
385
|
+
createdDate = typeof lifecycle.createdDate === 'string'
|
|
386
|
+
? lifecycle.createdDate
|
|
387
|
+
: rdapEventDate(events, 'registration');
|
|
388
|
+
expiryDate = typeof lifecycle.expiryDate === 'string'
|
|
389
|
+
? lifecycle.expiryDate
|
|
390
|
+
: rdapEventDate(events, 'expiration');
|
|
391
|
+
createdDateIso = typeof lifecycle.createdDateIso === 'string'
|
|
392
|
+
? lifecycle.createdDateIso
|
|
393
|
+
: registryDateIso(createdDate);
|
|
394
|
+
expiryDateIso = typeof lifecycle.expiryDateIso === 'string'
|
|
395
|
+
? lifecycle.expiryDateIso
|
|
396
|
+
: registryDateIso(expiryDate);
|
|
397
|
+
dnssec = typeof parsed.dnssec === 'string' ? parsed.dnssec : null;
|
|
398
|
+
rdapFound = true;
|
|
399
|
+
registrationSource = 'rdap';
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
catch {
|
|
404
|
+
/* fall through to WHOIS-based detection (deep mode only) */
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
const dnsDelegationPromise = !rdapFound && dnsIntelligenceEnabled
|
|
408
|
+
? hasPreloadedDnsDelegation
|
|
409
|
+
? Promise.resolve(options.dnsDelegation)
|
|
410
|
+
: checkDnsDelegation(domain, { resolver: options.resolveNs || dns.resolveNs })
|
|
411
|
+
: null;
|
|
412
|
+
let whoisChain = null;
|
|
413
|
+
let whoisParsed = null;
|
|
414
|
+
if (!rdapFound && !fast && whoisEnabled) {
|
|
415
|
+
try {
|
|
416
|
+
whoisChain = (hasPreloadedWhoisPromise
|
|
417
|
+
? await options.whoisChainPromise
|
|
418
|
+
: hasPreloadedWhois
|
|
419
|
+
? options.whoisChain
|
|
420
|
+
: await buildWhoisChain(domain)) ?? null;
|
|
421
|
+
if (!Array.isArray(whoisChain))
|
|
422
|
+
throw new Error('WHOIS chain unavailable');
|
|
423
|
+
const parsed = parseWhoisChain(whoisChain);
|
|
424
|
+
whoisParsed = parsed;
|
|
425
|
+
if (parsed.notFound) {
|
|
426
|
+
return {
|
|
427
|
+
state: 'available',
|
|
428
|
+
confidence: 'medium',
|
|
429
|
+
detail: `WHOIS reports no matching record for this domain${parsed.notFoundSource ? ` (per ${parsed.notFoundSource})` : ''}.`,
|
|
430
|
+
source: 'whois',
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
if (parsed.nameservers.length)
|
|
434
|
+
nameservers = parsed.nameservers;
|
|
435
|
+
if (parsed.statuses.length)
|
|
436
|
+
statuses = parsed.statuses.map((status) => status.toLowerCase());
|
|
437
|
+
if (parsed.registrar) {
|
|
438
|
+
registrar = {
|
|
439
|
+
handle: null,
|
|
440
|
+
ianaId: parsed.registrarIanaId || null,
|
|
441
|
+
name: parsed.registrar,
|
|
442
|
+
org: null,
|
|
443
|
+
email: parsed.abuseEmail || null,
|
|
444
|
+
phone: parsed.abusePhone || null,
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
if (parsed.registrantName || parsed.registrantOrg || parsed.registrantEmail || parsed.registrantPhone) {
|
|
448
|
+
registrant = {
|
|
449
|
+
handle: null,
|
|
450
|
+
name: parsed.registrantName || null,
|
|
451
|
+
org: parsed.registrantOrg || null,
|
|
452
|
+
email: parsed.registrantEmail || null,
|
|
453
|
+
phone: parsed.registrantPhone || null,
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
if (parsed.abuseEmail || parsed.abusePhone) {
|
|
457
|
+
abuse = { handle: null, name: null, org: parsed.registrar || null, email: parsed.abuseEmail || null, phone: parsed.abusePhone || null };
|
|
458
|
+
}
|
|
459
|
+
createdDate = parsed.createdDate || null;
|
|
460
|
+
expiryDate = parsed.expiryDate || null;
|
|
461
|
+
createdDateIso = parsed.createdDateIso || parsed.lifecycle?.createdDateIso || registryDateIso(createdDate);
|
|
462
|
+
expiryDateIso = parsed.expiryDateIso || parsed.lifecycle?.expiryDateIso || registryDateIso(expiryDate);
|
|
463
|
+
if (!dnssec)
|
|
464
|
+
dnssec = parsed.dnssec || null;
|
|
465
|
+
if (parsed.registrationStatus === 'registered'
|
|
466
|
+
|| registrar || createdDate || expiryDate || nameservers.length > 0 || statuses.length > 0) {
|
|
467
|
+
registrationSource = 'whois';
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
catch {
|
|
471
|
+
/* if both RDAP and WHOIS fail, we simply can't determine availability */
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
// WHOIS ran and was not a confirmed not-found, but produced no positive
|
|
475
|
+
// registration evidence either - e.g. the registry answered inconclusively
|
|
476
|
+
// or every referral hop failed/rate-limited. Report "unknown" rather than
|
|
477
|
+
// fabricating "registered" from an empty record.
|
|
478
|
+
const hasWhoisRegistrationData = whoisParsed?.registrationStatus === 'registered'
|
|
479
|
+
|| Boolean(registrar) || Boolean(createdDate) || Boolean(expiryDate)
|
|
480
|
+
|| nameservers.length > 0 || statuses.length > 0;
|
|
481
|
+
let dnsDelegated = false;
|
|
482
|
+
if (!rdapFound && !hasWhoisRegistrationData && dnsDelegationPromise) {
|
|
483
|
+
const delegation = await dnsDelegationPromise;
|
|
484
|
+
if (delegation && delegation.delegated === true && Array.isArray(delegation.nameservers) && delegation.nameservers.length) {
|
|
485
|
+
dnsDelegated = true;
|
|
486
|
+
nameservers = delegation.nameservers;
|
|
487
|
+
registrationSource = 'dns';
|
|
488
|
+
registrationConfidence = 'medium';
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
if (!rdapFound && !hasWhoisRegistrationData && !dnsDelegated) {
|
|
492
|
+
const disabledSources = [
|
|
493
|
+
!rdapEnabled ? 'RDAP' : null,
|
|
494
|
+
!fast && !whoisEnabled ? 'WHOIS' : null,
|
|
495
|
+
!dnsIntelligenceEnabled ? 'DNS intelligence' : null,
|
|
496
|
+
].filter(Boolean);
|
|
497
|
+
const disabledDetail = disabledSources.length
|
|
498
|
+
? ` ${disabledSources.join(', ')} ${disabledSources.length === 1 ? 'is' : 'are'} disabled by deployment policy.`
|
|
499
|
+
: '';
|
|
500
|
+
return {
|
|
501
|
+
state: 'unknown',
|
|
502
|
+
confidence: 'low',
|
|
503
|
+
detail: fast
|
|
504
|
+
? `No enabled registration source produced a record or authoritative delegation. A fast scan cannot determine registration status.${disabledDetail}`
|
|
505
|
+
: whoisParsed && whoisParsed.failedHop
|
|
506
|
+
? `WHOIS was inconclusive - a referral hop did not answer conclusively (${whoisParsed.failedHop}).`
|
|
507
|
+
: `No enabled registration source returned conclusive data or an authoritative DNS delegation.${disabledDetail}`,
|
|
508
|
+
...(!fast && whoisEnabled ? { source: 'whois' } : {}),
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
const domainAgeDays = computeAgeDays(createdDateIso || createdDate);
|
|
512
|
+
const expiresInDays = computeDaysUntil(expiryDateIso || expiryDate);
|
|
513
|
+
// DNS proves delegation only; it says nothing about whether registry
|
|
514
|
+
// contact data is privacy-protected or merely unavailable.
|
|
515
|
+
const privacyProtected = registrationSource === 'dns' ? null : isPrivacyProtected(registrant);
|
|
516
|
+
const baseInfo = {
|
|
517
|
+
nameservers,
|
|
518
|
+
statuses,
|
|
519
|
+
registrar,
|
|
520
|
+
registrant,
|
|
521
|
+
abuse,
|
|
522
|
+
createdDate,
|
|
523
|
+
expiryDate,
|
|
524
|
+
createdDateIso,
|
|
525
|
+
expiryDateIso,
|
|
526
|
+
rdapServer,
|
|
527
|
+
domainAgeDays,
|
|
528
|
+
expiresInDays,
|
|
529
|
+
privacyProtected,
|
|
530
|
+
dnssec,
|
|
531
|
+
source: registrationSource,
|
|
532
|
+
};
|
|
533
|
+
if (statuses.some((s) => s.includes('pendingdelete') || s.includes('redemptionperiod'))) {
|
|
534
|
+
return {
|
|
535
|
+
state: 'expiring',
|
|
536
|
+
confidence: 'medium',
|
|
537
|
+
detail: 'Domain is in redemption/pending-delete status and may become available soon.',
|
|
538
|
+
...baseInfo,
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
if (fast) {
|
|
542
|
+
return {
|
|
543
|
+
state: 'registered',
|
|
544
|
+
confidence: registrationConfidence,
|
|
545
|
+
detail: registrationSource === 'dns'
|
|
546
|
+
? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable.'
|
|
547
|
+
: 'Domain is registered. Run a deep check for parking/for-sale detection.',
|
|
548
|
+
...baseInfo,
|
|
549
|
+
};
|
|
550
|
+
}
|
|
551
|
+
// Registered - look for for-sale/parked/website signals (homepage fetch,
|
|
552
|
+
// deep mode only), and check for a configured mail exchanger as a
|
|
553
|
+
// phishing-risk signal (a lookalike domain that can receive/send mail is
|
|
554
|
+
// capable of running credential-harvesting or BEC campaigns).
|
|
555
|
+
const nsSignal = nameservers.find((ns) => PARKING_NS_PATTERNS.some((re) => re.test(ns)));
|
|
556
|
+
let forSaleSignal = nsSignal ? `parking nameserver (${nsSignal})` : null;
|
|
557
|
+
let activityStatus = nsSignal ? 'parked' : 'unknown';
|
|
558
|
+
// Homepage + bounded DNS evidence resolve in parallel; the favicon fetch is sequenced after
|
|
559
|
+
// the homepage so it can use any <link rel="icon"> the page declares (many
|
|
560
|
+
// sites serve no /favicon.ico and only point to a CDN PNG this way). One
|
|
561
|
+
// extra round-trip on the already-slow deep path, in exchange for finding
|
|
562
|
+
// favicons the bare /favicon.ico probe would miss.
|
|
563
|
+
const [homepage, dnsIntelligence, tlsIntelligence] = await Promise.all([
|
|
564
|
+
websiteProbeEnabled ? fetchHomepageForDomain(domain).catch((err) => ({
|
|
565
|
+
text: null,
|
|
566
|
+
status: 'inconclusive',
|
|
567
|
+
detail: `Could not confirm homepage activity: ${String(err && err.message ? err.message : 'request failed').slice(0, 180)}.`,
|
|
568
|
+
http: failedHttpObservation([
|
|
569
|
+
{ url: `https://${domain}`, error: String(err && err.message ? err.message : 'request failed') },
|
|
570
|
+
]),
|
|
571
|
+
})) : Promise.resolve({
|
|
572
|
+
text: null,
|
|
573
|
+
status: 'skipped',
|
|
574
|
+
detail: 'Website probing is disabled by deployment policy.',
|
|
575
|
+
http: skippedHttpObservation(),
|
|
576
|
+
}),
|
|
577
|
+
dnsIntelligenceEnabled
|
|
578
|
+
? collectDns(domain, {
|
|
579
|
+
...(options.dnsResolvers ? { resolvers: options.dnsResolvers } : {}),
|
|
580
|
+
includeExtendedContext: options.includeExtendedDnsContext === true,
|
|
581
|
+
includeInheritedCaa: options.includeInheritedCaa === true,
|
|
582
|
+
registryEvidence: registryDnsEvidence,
|
|
583
|
+
})
|
|
584
|
+
: Promise.resolve(skippedDnsIntelligence('DNS intelligence is disabled by deployment policy.', {
|
|
585
|
+
includeExtendedContext: options.includeExtendedDnsContext === true,
|
|
586
|
+
includeInheritedCaa: options.includeInheritedCaa === true,
|
|
587
|
+
})),
|
|
588
|
+
tlsIntelligenceEnabled
|
|
589
|
+
? collectTls(domain)
|
|
590
|
+
: Promise.resolve(skippedTlsObservation()),
|
|
591
|
+
]);
|
|
592
|
+
const page = homepage.text;
|
|
593
|
+
const favicon = websiteProbeEnabled
|
|
594
|
+
? await fetchFaviconForDomain(domain, { html: page || '' }).catch(() => null)
|
|
595
|
+
: null;
|
|
596
|
+
const faviconHash = favicon ? favicon.hash : null;
|
|
597
|
+
const faviconPHash = favicon ? favicon.phash : null;
|
|
598
|
+
let htmlSignals = {
|
|
599
|
+
pageTitle: null,
|
|
600
|
+
hasPasswordField: false,
|
|
601
|
+
phishingLanguageMatch: null,
|
|
602
|
+
hasExternalFormAction: null,
|
|
603
|
+
externalAssetHosts: [],
|
|
604
|
+
pageIdentity: null,
|
|
605
|
+
credentialSurfaceProfile: null,
|
|
606
|
+
structuredDataIdentity: null,
|
|
607
|
+
technologyProfile: null,
|
|
608
|
+
pageRoleProfile: null,
|
|
609
|
+
clientBehaviorProfile: null,
|
|
610
|
+
};
|
|
611
|
+
if (homepage.status === 'fetched') {
|
|
612
|
+
if (page) {
|
|
613
|
+
const saleMatch = page.match(FOR_SALE_TEXT_RE);
|
|
614
|
+
if (saleMatch) {
|
|
615
|
+
forSaleSignal = forSaleSignal || `homepage text ("${saleMatch[0]}")`;
|
|
616
|
+
activityStatus = 'parked';
|
|
617
|
+
}
|
|
618
|
+
const responseContentType = homepage.http?.response?.contentType;
|
|
619
|
+
const pageIdentityEligible = typeof responseContentType !== 'string'
|
|
620
|
+
|| responseContentType.trim() === ''
|
|
621
|
+
|| /^(?:text\/html|application\/xhtml\+xml)(?:\s*;|$)/i.test(responseContentType.trim());
|
|
622
|
+
htmlSignals = extractHtmlSignals(page, domain, {
|
|
623
|
+
...(typeof homepage.http?.finalUrl === 'string' ? { baseUrl: homepage.http.finalUrl } : {}),
|
|
624
|
+
...(typeof homepage.http?.observedAt === 'string' ? { observedAt: homepage.http.observedAt } : {}),
|
|
625
|
+
sourceTruncated: homepage.http?.response?.bodyTruncated === true,
|
|
626
|
+
exactBodyHash: homepage.http?.response?.bodyHash,
|
|
627
|
+
httpServer: homepage.http?.response?.server,
|
|
628
|
+
responseHeaders: homepage.technologyHeaders,
|
|
629
|
+
activityStatus,
|
|
630
|
+
includePageIdentity: pageIdentityEligible,
|
|
631
|
+
...(options.includeCredentialSurfaceProfile !== undefined
|
|
632
|
+
? { includeCredentialSurfaceProfile: options.includeCredentialSurfaceProfile }
|
|
633
|
+
: {}),
|
|
634
|
+
...(options.includeStructuredDataIdentity !== undefined
|
|
635
|
+
? { includeStructuredDataIdentity: options.includeStructuredDataIdentity }
|
|
636
|
+
: {}),
|
|
637
|
+
...(options.includeTechnologyProfile !== undefined
|
|
638
|
+
? { includeTechnologyProfile: options.includeTechnologyProfile }
|
|
639
|
+
: {}),
|
|
640
|
+
});
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
const securityPosture = options.includeSecurityPosture === false ? null : analyzeWebsiteSecurityPosture({
|
|
644
|
+
http: homepage.http,
|
|
645
|
+
responsePolicy: homepage.responsePolicy,
|
|
646
|
+
pageIdentity: htmlSignals.pageIdentity,
|
|
647
|
+
tls: tlsIntelligence,
|
|
648
|
+
dns: dnsIntelligence,
|
|
649
|
+
dnssec,
|
|
650
|
+
observedAt: homepage.http?.observedAt,
|
|
651
|
+
});
|
|
652
|
+
const redirectSaleSignal = forSaleRedirectSignal(homepage.http);
|
|
653
|
+
if (!forSaleSignal && redirectSaleSignal) {
|
|
654
|
+
forSaleSignal = redirectSaleSignal;
|
|
655
|
+
activityStatus = 'parked';
|
|
656
|
+
}
|
|
657
|
+
const faviconProvedActive = homepage.status === 'inconclusive' && Boolean(favicon);
|
|
658
|
+
const websiteProbeStatus = faviconProvedActive ? 'responded' : homepage.status;
|
|
659
|
+
const websiteProbeDetail = faviconProvedActive
|
|
660
|
+
? `${homepage.detail} A favicon responded successfully, confirming an active web service.`
|
|
661
|
+
: homepage.detail;
|
|
662
|
+
if (websiteProbeEnabled) {
|
|
663
|
+
activityStatus = deriveWebsiteActivity(homepage.status, Boolean(favicon), activityStatus === 'parked');
|
|
664
|
+
}
|
|
665
|
+
if (!forSaleSignal) {
|
|
666
|
+
return {
|
|
667
|
+
state: 'registered',
|
|
668
|
+
confidence: registrationConfidence,
|
|
669
|
+
detail: registrationSource === 'dns'
|
|
670
|
+
? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable. No for-sale signals were observed.'
|
|
671
|
+
: 'Domain is registered and shows no for-sale signals.',
|
|
672
|
+
activityStatus,
|
|
673
|
+
websiteProbeStatus,
|
|
674
|
+
websiteProbeDetail,
|
|
675
|
+
http: homepage.http,
|
|
676
|
+
deepScanComplete,
|
|
677
|
+
faviconHash,
|
|
678
|
+
faviconPHash,
|
|
679
|
+
...htmlSignals,
|
|
680
|
+
securityPosture,
|
|
681
|
+
...baseInfo,
|
|
682
|
+
nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
|
|
683
|
+
dns: dnsIntelligence,
|
|
684
|
+
tls: tlsIntelligence,
|
|
685
|
+
hasMx: dnsIntelligence.hasMx,
|
|
686
|
+
hasNullMx: dnsIntelligence.hasNullMx,
|
|
687
|
+
mxHosts: dnsIntelligence.mxHosts,
|
|
688
|
+
hasSpf: dnsIntelligence.hasSpf,
|
|
689
|
+
hasDmarc: dnsIntelligence.hasDmarc,
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
return {
|
|
693
|
+
state: 'for_sale',
|
|
694
|
+
confidence: 'medium',
|
|
695
|
+
detail: `Detected a for-sale listing (${forSaleSignal}).`,
|
|
696
|
+
activityStatus,
|
|
697
|
+
websiteProbeStatus,
|
|
698
|
+
websiteProbeDetail,
|
|
699
|
+
http: homepage.http,
|
|
700
|
+
deepScanComplete,
|
|
701
|
+
faviconHash,
|
|
702
|
+
faviconPHash,
|
|
703
|
+
...htmlSignals,
|
|
704
|
+
securityPosture,
|
|
705
|
+
...baseInfo,
|
|
706
|
+
nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
|
|
707
|
+
dns: dnsIntelligence,
|
|
708
|
+
tls: tlsIntelligence,
|
|
709
|
+
hasMx: dnsIntelligence.hasMx,
|
|
710
|
+
hasNullMx: dnsIntelligence.hasNullMx,
|
|
711
|
+
mxHosts: dnsIntelligence.mxHosts,
|
|
712
|
+
hasSpf: dnsIntelligence.hasSpf,
|
|
713
|
+
hasDmarc: dnsIntelligence.hasDmarc,
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
export { checkDomainAvailability, checkDnsDelegation, fetchHomepage, deriveWebsiteActivity, forSaleRedirectSignal, parseRegistryDate as parseWhoisDate, };
|