@slicedearth/whoisleuth-cli 1.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/LICENSE +661 -0
  2. package/LICENSES/Retire.js-Apache-2.0.txt +199 -0
  3. package/NOTICE +76 -0
  4. package/README.md +84 -0
  5. package/TRADEMARKS.md +41 -0
  6. package/bin/whoisleuth.mjs +35 -0
  7. package/cli/archive-inspect.mjs +198 -0
  8. package/cli/arguments.mjs +1022 -0
  9. package/cli/artifact-verify.mjs +210 -0
  10. package/cli/bulk-checkpoint.mjs +218 -0
  11. package/cli/bulk-command-runner.mjs +109 -0
  12. package/cli/bulk-output.mjs +112 -0
  13. package/cli/bulk.mjs +133 -0
  14. package/cli/compare.mjs +238 -0
  15. package/cli/completion.mjs +175 -0
  16. package/cli/csv.mjs +1 -0
  17. package/cli/discover.mjs +50 -0
  18. package/cli/discovery-command-runner.mjs +36 -0
  19. package/cli/discovery-observation-snapshot.mjs +315 -0
  20. package/cli/discovery-scan-command-runner.mjs +141 -0
  21. package/cli/discovery-scan.mjs +325 -0
  22. package/cli/discovery-snapshot.mjs +133 -0
  23. package/cli/discovery-workflow.mjs +73 -0
  24. package/cli/doctor.mjs +137 -0
  25. package/cli/errors.mjs +21 -0
  26. package/cli/evidence-command-arguments.mjs +145 -0
  27. package/cli/evidence-command-runner.mjs +106 -0
  28. package/cli/evidence-signing.mjs +215 -0
  29. package/cli/exit-codes.mjs +9 -0
  30. package/cli/export-evidence.mjs +28 -0
  31. package/cli/formatters/evidence-report.mjs +3 -0
  32. package/cli/formatters/html.mjs +69 -0
  33. package/cli/formatters/json.mjs +135 -0
  34. package/cli/formatters/markdown.mjs +3 -0
  35. package/cli/formatters/terminal.mjs +658 -0
  36. package/cli/http.mjs +27 -0
  37. package/cli/lookup-command-runner.mjs +87 -0
  38. package/cli/lookup-diff.mjs +171 -0
  39. package/cli/mail-review.mjs +204 -0
  40. package/cli/manual.mjs +34 -0
  41. package/cli/network-command-runner.mjs +88 -0
  42. package/cli/output-file.mjs +97 -0
  43. package/cli/page-compare.mjs +123 -0
  44. package/cli/posture.mjs +20 -0
  45. package/cli/progress-events.mjs +53 -0
  46. package/cli/progress.mjs +56 -0
  47. package/cli/registry-access.mjs +3 -0
  48. package/cli/registry-support.mjs +120 -0
  49. package/cli/risk-calibration.mjs +294 -0
  50. package/cli/runner-types.mjs +1 -0
  51. package/cli/runner.mjs +726 -0
  52. package/cli/saved-lookup.mjs +87 -0
  53. package/cli/source-reliability.mjs +727 -0
  54. package/cli/strict-exit.mjs +55 -0
  55. package/cli/terminal-presentation.mjs +147 -0
  56. package/docs/cli.md +827 -0
  57. package/frontend/src/lib/analysis/acquisition-decision-packet.js +93 -0
  58. package/frontend/src/lib/analysis/acquisition-due-diligence.js +391 -0
  59. package/frontend/src/lib/analysis/artifact-integrity.js +21 -0
  60. package/frontend/src/lib/analysis/brand-profile-model.js +421 -0
  61. package/frontend/src/lib/analysis/bulk-domain-comparison.js +246 -0
  62. package/frontend/src/lib/analysis/bulk-mail-exposure.js +192 -0
  63. package/frontend/src/lib/analysis/bulk-retry-plan.js +86 -0
  64. package/frontend/src/lib/analysis/bulk-review-export.js +62 -0
  65. package/frontend/src/lib/analysis/bulk-review-model.js +266 -0
  66. package/frontend/src/lib/analysis/bulk-session-model.js +515 -0
  67. package/frontend/src/lib/analysis/bulk-sort.js +49 -0
  68. package/frontend/src/lib/analysis/campaign-model.js +265 -0
  69. package/frontend/src/lib/analysis/case-evidence-model.js +642 -0
  70. package/frontend/src/lib/analysis/case-migration-model.js +248 -0
  71. package/frontend/src/lib/analysis/case-model.js +5 -0
  72. package/frontend/src/lib/analysis/case-record-contracts.js +44 -0
  73. package/frontend/src/lib/analysis/case-record-core.js +232 -0
  74. package/frontend/src/lib/analysis/case-record-model.js +7 -0
  75. package/frontend/src/lib/analysis/case-record-operations.js +224 -0
  76. package/frontend/src/lib/analysis/case-response-model.js +640 -0
  77. package/frontend/src/lib/analysis/case-response-packet.js +573 -0
  78. package/frontend/src/lib/analysis/case-storage-model.js +93 -0
  79. package/frontend/src/lib/analysis/detection-rule-model.js +306 -0
  80. package/frontend/src/lib/analysis/dns-record-normalization.js +11 -0
  81. package/frontend/src/lib/analysis/http-summary.js +165 -0
  82. package/frontend/src/lib/analysis/investigation-guide.js +500 -0
  83. package/frontend/src/lib/analysis/investigation-template-model.js +144 -0
  84. package/frontend/src/lib/analysis/page-baseline.js +272 -0
  85. package/frontend/src/lib/analysis/page-similarity.js +179 -0
  86. package/frontend/src/lib/analysis/relationship-evidence.js +265 -0
  87. package/frontend/src/lib/analysis/relationship-observation-model.js +423 -0
  88. package/frontend/src/lib/analysis/scoring.js +135 -0
  89. package/frontend/src/lib/analysis/shortlist-model.js +209 -0
  90. package/frontend/src/lib/analysis/utils.js +265 -0
  91. package/frontend/src/lib/analysis/watchlist-history.js +643 -0
  92. package/frontend/src/lib/analysis/watchlist-store.js +127 -0
  93. package/frontend/src/lib/analysis/website-snapshot-model.js +302 -0
  94. package/frontend/src/lib/analysis/workspace-archive-crypto.js +282 -0
  95. package/frontend/src/lib/analysis/workspace-archive.js +472 -0
  96. package/lib/abort.mjs +26 -0
  97. package/lib/abusech-auth.mjs +19 -0
  98. package/lib/availability.mjs +716 -0
  99. package/lib/bounded-contract-normalizers.mjs +61 -0
  100. package/lib/browser-library-profile.mjs +310 -0
  101. package/lib/bulk-comparison-evidence.mjs +83 -0
  102. package/lib/classify.mjs +93 -0
  103. package/lib/client-behavior-profile.mjs +83 -0
  104. package/lib/credential-surface-profile.mjs +66 -0
  105. package/lib/csv.mjs +23 -0
  106. package/lib/ct-query.mjs +27 -0
  107. package/lib/ct-search.mjs +393 -0
  108. package/lib/distributed-operation-budget.mjs +461 -0
  109. package/lib/dns-delegation-health.mjs +344 -0
  110. package/lib/dns-intelligence.mjs +739 -0
  111. package/lib/dns-mx.mjs +24 -0
  112. package/lib/dns-resolver-selection.mjs +33 -0
  113. package/lib/domain-posture-analysis.mjs +254 -0
  114. package/lib/domain-posture-parsers.mjs +358 -0
  115. package/lib/domain-posture.mjs +745 -0
  116. package/lib/error-detail.mjs +10 -0
  117. package/lib/evidence-export.mjs +431 -0
  118. package/lib/evidence-report-markdown.mjs +71 -0
  119. package/lib/evidence-report.mjs +391 -0
  120. package/lib/external-intelligence-risk.mjs +108 -0
  121. package/lib/favicon.mjs +146 -0
  122. package/lib/feature-policy.mjs +70 -0
  123. package/lib/generated/retire-browser-catalog.mjs +8724 -0
  124. package/lib/generated/unicode-confusables-17.mjs +93 -0
  125. package/lib/hostname.mjs +38 -0
  126. package/lib/html-signals.mjs +680 -0
  127. package/lib/http-intelligence.mjs +232 -0
  128. package/lib/idn-confusable-policy.mjs +86 -0
  129. package/lib/idn-confusables.mjs +465 -0
  130. package/lib/lookup-cache.mjs +130 -0
  131. package/lib/lookup-progress.mjs +291 -0
  132. package/lib/lookup-source-progress.mjs +109 -0
  133. package/lib/lookup-threat-provider-inventory.mjs +13 -0
  134. package/lib/lookup.mjs +592 -0
  135. package/lib/observation.mjs +114 -0
  136. package/lib/observed-network-context.mjs +355 -0
  137. package/lib/operation-budget.mjs +489 -0
  138. package/lib/page-fingerprints.mjs +481 -0
  139. package/lib/page-role-profile.mjs +117 -0
  140. package/lib/perceptual-hash.mjs +451 -0
  141. package/lib/rdap-attempts.mjs +34 -0
  142. package/lib/rdap-bootstrap.mjs +173 -0
  143. package/lib/rdap-capabilities.mjs +292 -0
  144. package/lib/rdap-client.mjs +125 -0
  145. package/lib/rdap-normalization.mjs +691 -0
  146. package/lib/rdap-registrar.mjs +196 -0
  147. package/lib/rdap-transport.mjs +39 -0
  148. package/lib/rdap-types.mjs +1 -0
  149. package/lib/rdap-validation.mjs +94 -0
  150. package/lib/rdap.mjs +16 -0
  151. package/lib/registry-access.mjs +32 -0
  152. package/lib/registry-capabilities.mjs +117 -0
  153. package/lib/registry-capability-catalogue.mjs +1206 -0
  154. package/lib/registry-comparison.mjs +318 -0
  155. package/lib/registry-dates.mjs +190 -0
  156. package/lib/registry-insights.mjs +325 -0
  157. package/lib/response-policy.mjs +256 -0
  158. package/lib/risk-scoring.mjs +164 -0
  159. package/lib/safe-fetch.mjs +446 -0
  160. package/lib/security-txt.mjs +359 -0
  161. package/lib/service-binding-dns.mjs +608 -0
  162. package/lib/sslbl-certificates.generated.mjs +143 -0
  163. package/lib/sslbl-intelligence.mjs +241 -0
  164. package/lib/static-html-analysis.mjs +340 -0
  165. package/lib/structured-data-identity.mjs +323 -0
  166. package/lib/threat-intelligence-contract.mjs +5 -0
  167. package/lib/threat-intelligence-definition-registry.mjs +34 -0
  168. package/lib/threat-intelligence-fixtures.mjs +82 -0
  169. package/lib/threat-intelligence-manifests.mjs +512 -0
  170. package/lib/threat-intelligence-runtime.mjs +662 -0
  171. package/lib/threat-intelligence-targets.mjs +168 -0
  172. package/lib/threat-intelligence-types.mjs +3 -0
  173. package/lib/threatfox-intelligence.mjs +307 -0
  174. package/lib/tls-intelligence.mjs +814 -0
  175. package/lib/typosquat-generator.mjs +884 -0
  176. package/lib/urlhaus-intelligence.mjs +361 -0
  177. package/lib/urlscan-intelligence.mjs +349 -0
  178. package/lib/website-security-posture.mjs +325 -0
  179. package/lib/website-technology.mjs +437 -0
  180. package/lib/whois-authority.mjs +98 -0
  181. package/lib/whois-chain.mjs +261 -0
  182. package/lib/whois-contacts.mjs +163 -0
  183. package/lib/whois-contracts.mjs +3 -0
  184. package/lib/whois-normalization.mjs +27 -0
  185. package/lib/whois-transport.mjs +100 -0
  186. package/lib/whois-values.mjs +92 -0
  187. package/lib/whois.mjs +1187 -0
  188. package/package.json +55 -0
  189. package/third-party-notices.txt +5571 -0
@@ -0,0 +1,716 @@
1
+ // Domain availability check.
2
+ //
3
+ // Determines registered vs. available vs. expiring via RDAP/WHOIS, and flags
4
+ // likely for-sale listings by checking for known parking/marketplace
5
+ // nameservers and "for sale" listing language on the domain's own homepage.
6
+ // This does not attempt to price the domain - registries don't publish sale
7
+ // prices, and that's proprietary marketplace data.
8
+ //
9
+ // Shared by the Express server and the Netlify Functions.
10
+ import { promises as dns } from 'node:dns';
11
+ import { fetchRdapRecord } from "./rdap.mjs";
12
+ import { buildWhoisChain, parseWhoisChain } from "./whois.mjs";
13
+ import { safeFetchDetailed, readTextCapped } from "./safe-fetch.mjs";
14
+ import { collectDnsIntelligence, skippedDnsIntelligence } from "./dns-intelligence.mjs";
15
+ import { fetchFaviconHash } from "./favicon.mjs";
16
+ import { extractHtmlSignals } from "./html-signals.mjs";
17
+ import { featureDecision, networkFeaturePolicy } from "./feature-policy.mjs";
18
+ import { buildHttpObservation, failedHttpObservation, skippedHttpObservation } from "./http-intelligence.mjs";
19
+ import { collectTlsIntelligence, skippedTlsObservation } from "./tls-intelligence.mjs";
20
+ import { parseRegistryDate, registryDateIso } from "./registry-dates.mjs";
21
+ import { analyzeWebsiteSecurityPosture } from "./website-security-posture.mjs";
22
+ import { analyzeResponsePolicyHeaders } from "./response-policy.mjs";
23
+ import { nonEmptyErrorMessage } from "./error-detail.mjs";
24
+ const MAX_HOMEPAGE_BYTES = 300000;
25
+ const DNS_DELEGATION_TIMEOUT_MS = 4000;
26
+ const MAX_DELEGATION_NAMESERVERS = 50;
27
+ const MISSING_DNS_CODES = new Set(['ENODATA', 'ENOTFOUND', 'ENONAME', 'NXDOMAIN']);
28
+ function errorRecord(value) {
29
+ return value && typeof value === 'object' && !Array.isArray(value)
30
+ ? value
31
+ : {};
32
+ }
33
+ function rdapEventDate(events, action) {
34
+ const date = events.find((event) => event.action === action)?.date;
35
+ return typeof date === 'string' ? date : null;
36
+ }
37
+ // No marketplace (Afternic/Sedo/Dan.com/GoDaddy Auctions/etc.) offers a
38
+ // free, no-auth API to check "is this specific domain listed for sale" -
39
+ // what's checkable without credentials is the parking/landing-page
40
+ // nameservers and homepage copy these services actually use, which is
41
+ // broader coverage of the same signal, not a live cross-marketplace lookup.
42
+ const PARKING_NS_PATTERNS = [
43
+ /sedoparking\.com$/i,
44
+ /sedo\.com$/i,
45
+ /above\.com$/i,
46
+ /bodis\.com$/i,
47
+ /parkingcrew\.net$/i,
48
+ /dan\.com$/i,
49
+ /hugedomains\.com$/i,
50
+ /uniregistry/i,
51
+ /squadhelp/i,
52
+ /afternic/i,
53
+ /voodoo\.com$/i,
54
+ /fabulous\.com$/i,
55
+ /namedrive/i,
56
+ /smartname\.com$/i,
57
+ /domainsponsor\.com$/i,
58
+ /undeveloped\.com$/i,
59
+ /trafficz\.com$/i,
60
+ /dsredirection\.com$/i,
61
+ ];
62
+ const FOR_SALE_TEXT_RE = /(this domain (?:name )?(?:may be|is) for sale|buy this domain|domain(?: name)? for sale|make (?:an|your) offer|inquire about (?:this|the) domain|purchase this domain|this domain is available for purchase|backorder this domain|bid on this domain|premium domain for sale|own this domain|click here to buy this domain|would you like to buy this domain|this domain is (?:available for lease|listed for sale))/i;
63
+ const FOR_SALE_PATH_RE = /\/(?:premium-)?domains?-for-sale(?:\/|$)/i;
64
+ function forSaleRedirectSignal(httpObservation) {
65
+ if (!httpObservation || typeof httpObservation !== 'object')
66
+ return null;
67
+ const observation = httpObservation;
68
+ const urls = [
69
+ observation.finalUrl,
70
+ ...(Array.isArray(observation.redirects)
71
+ ? observation.redirects.map((redirect) => redirect && typeof redirect === 'object' ? redirect.to : null)
72
+ : []),
73
+ ];
74
+ for (const value of urls) {
75
+ if (typeof value !== 'string' || value.length > 2048)
76
+ continue;
77
+ try {
78
+ const url = new URL(value);
79
+ if (['http:', 'https:'].includes(url.protocol) && FOR_SALE_PATH_RE.test(url.pathname)) {
80
+ return `for-sale landing-page redirect (${url.origin}${url.pathname})`;
81
+ }
82
+ }
83
+ catch {
84
+ // Malformed retained provenance cannot establish a sale signal.
85
+ }
86
+ }
87
+ return null;
88
+ }
89
+ // ---------------------------------------------------------------------------
90
+ // Acquisition/sourcing signals: domain age, privacy-redaction, expiry
91
+ // proximity, and site-activity status. These read data already fetched by
92
+ // checkDomainAvailability (RDAP/WHOIS dates and registrant info) - none of
93
+ // this triggers extra registry queries.
94
+ // ---------------------------------------------------------------------------
95
+ function computeAgeDays(dateStr) {
96
+ const d = parseRegistryDate(dateStr);
97
+ return d ? Math.floor((Date.now() - d.getTime()) / 86400000) : null;
98
+ }
99
+ function computeDaysUntil(dateStr) {
100
+ const d = parseRegistryDate(dateStr);
101
+ return d ? Math.ceil((d.getTime() - Date.now()) / 86400000) : null;
102
+ }
103
+ const PRIVACY_MARKERS = [
104
+ /redacted for privacy/i,
105
+ /data protected/i,
106
+ /privacy\s*protect/i,
107
+ /whoisguard/i,
108
+ /domains by proxy/i,
109
+ /perfect privacy/i,
110
+ /contact privacy/i,
111
+ /private registration/i,
112
+ /identity protect/i,
113
+ /not disclosed/i,
114
+ /withheld for privacy/i,
115
+ ];
116
+ // Only ever called after a lookup has already succeeded (this runs inside
117
+ // checkDomainAvailability once RDAP or WHOIS has returned real data), so a
118
+ // missing/empty registrant here isn't "we don't know" - it means the
119
+ // registry gave us a response with no usable contact in it, which for
120
+ // sourcing purposes (can I reach this owner from this data alone?) is the
121
+ // same practical answer as an explicit privacy-proxy service: no.
122
+ function isPrivacyProtected(registrant) {
123
+ if (!registrant)
124
+ return true;
125
+ const blob = [registrant.name, registrant.org, registrant.email].filter(Boolean).join(' ');
126
+ if (!blob)
127
+ return true; // record exists but every contact field is blank - that's redaction
128
+ return PRIVACY_MARKERS.some((re) => re.test(blob));
129
+ }
130
+ // RDAP Lookup retains a bounded multi-value contact inventory, but Bulk and
131
+ // watchlist availability records deliberately keep the historical compact
132
+ // shape. This prevents repeated contact arrays and registry links from
133
+ // expanding browser-local stores or flowing into case evidence implicitly.
134
+ function compactContact(contact) {
135
+ if (!contact || typeof contact !== 'object' || Array.isArray(contact))
136
+ return null;
137
+ const value = contact;
138
+ const compact = {
139
+ handle: value.handle || null,
140
+ name: value.name || null,
141
+ org: value.org || null,
142
+ email: value.email || null,
143
+ phone: value.phone || null,
144
+ address: value.address || null,
145
+ };
146
+ return Object.values(compact).some(Boolean) ? compact : null;
147
+ }
148
+ // A number of ccTLD registries publish neither an RDAP bootstrap entry nor a
149
+ // generally reachable port-43 WHOIS service. A positive NS answer cannot
150
+ // replace registry registration data, but it does prove that the registrable
151
+ // domain has an active DNS delegation. Use that as a bounded, positive-only
152
+ // fallback: no answer is never interpreted as availability because registered
153
+ // domains can legitimately be undelegated.
154
+ async function checkDnsDelegation(domain, { resolver = dns.resolveNs } = {}) {
155
+ let timer;
156
+ try {
157
+ const records = await Promise.race([
158
+ Promise.resolve().then(() => resolver(domain)),
159
+ new Promise((_, reject) => {
160
+ timer = setTimeout(() => reject(new Error('DNS delegation lookup timed out')), DNS_DELEGATION_TIMEOUT_MS);
161
+ }),
162
+ ]);
163
+ const validNameservers = [...new Set((Array.isArray(records) ? records : [])
164
+ .filter((value) => typeof value === 'string')
165
+ .map((value) => value.trim().replace(/\.+$/, '').toLowerCase())
166
+ .filter((value) => value.length > 0 && value.length <= 253)
167
+ .filter((value) => value.split('.').every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(label))))]
168
+ .sort();
169
+ const normalized = validNameservers.slice(0, MAX_DELEGATION_NAMESERVERS);
170
+ return {
171
+ delegated: normalized.length > 0,
172
+ nameservers: normalized,
173
+ nameserversTruncated: validNameservers.length > MAX_DELEGATION_NAMESERVERS,
174
+ error: null,
175
+ };
176
+ }
177
+ catch (err) {
178
+ const error = errorRecord(err);
179
+ if (typeof error.code === 'string' && MISSING_DNS_CODES.has(error.code)) {
180
+ return { delegated: false, nameservers: [], nameserversTruncated: false, error: null };
181
+ }
182
+ return {
183
+ delegated: false,
184
+ nameservers: [],
185
+ nameserversTruncated: false,
186
+ error: nonEmptyErrorMessage(err, String(err)).slice(0, 180),
187
+ };
188
+ }
189
+ finally {
190
+ clearTimeout(timer);
191
+ }
192
+ }
193
+ // Fetches enough of the homepage for the lightweight HTML signals below and
194
+ // preserves why the probe failed. A failed request is not evidence that a
195
+ // domain has "no site": transient DNS/TLS/network failures, a slow origin, or
196
+ // an HTTP error can all produce the same null body. Keeping that distinction
197
+ // avoids turning an inconclusive probe into a false inactivity claim.
198
+ async function fetchHomepage(domain, { fetcher = safeFetchDetailed } = {}) {
199
+ const headers = { 'User-Agent': 'Mozilla/5.0 (compatible; DomainStatusChecker/1.0)' };
200
+ const failures = [];
201
+ const probeStartedAt = Date.now();
202
+ for (const scheme of ['https', 'http']) {
203
+ const requestUrl = `${scheme}://${domain}`;
204
+ const attemptStartedAt = Date.now();
205
+ const controller = new AbortController();
206
+ const timeout = setTimeout(() => controller.abort(), 6000);
207
+ try {
208
+ const fetched = await fetcher(requestUrl, { signal: controller.signal, headers });
209
+ const fetchedResponse = fetched instanceof Response ? fetched : fetched.response;
210
+ const detail = fetched && typeof fetched === 'object' && 'response' in fetched
211
+ ? fetched
212
+ : {
213
+ response: fetchedResponse,
214
+ requestedUrl: requestUrl,
215
+ finalUrl: requestUrl,
216
+ redirectCount: 0,
217
+ redirectLimitReached: false,
218
+ hops: [{ url: requestUrl, status: fetchedResponse.status, location: null, durationMs: Date.now() - attemptStartedAt }],
219
+ durationMs: Date.now() - attemptStartedAt,
220
+ };
221
+ const res = detail.response;
222
+ // A truncated homepage is still fine here - only used to scan for a
223
+ // for-sale text match, not parsed as well-formed content. The
224
+ // timeout stays armed through this read (cleared in `finally` below,
225
+ // not here) - a malicious site could otherwise send headers
226
+ // immediately and then trickle or stall the body forever, hanging
227
+ // this worker with no deadline once the timer above is disarmed.
228
+ if (res.ok) {
229
+ const body = await readTextCapped(res, MAX_HOMEPAGE_BYTES, { includeSha256: true });
230
+ return {
231
+ text: body.text,
232
+ status: 'fetched',
233
+ detail: `Homepage responded over ${scheme.toUpperCase()} (HTTP ${res.status}).`,
234
+ responsePolicy: analyzeResponsePolicyHeaders(res.headers),
235
+ technologyHeaders: {
236
+ 'cf-ray': res.headers.get('cf-ray') || '',
237
+ 'x-drupal-cache': res.headers.get('x-drupal-cache') || '',
238
+ 'x-fastly-request-id': res.headers.get('x-fastly-request-id') || '',
239
+ 'x-powered-by': res.headers.get('x-powered-by') || '',
240
+ 'x-shopify-stage': res.headers.get('x-shopify-stage') || '',
241
+ 'x-sorting-hat-podid': res.headers.get('x-sorting-hat-podid') || '',
242
+ 'x-vercel-id': res.headers.get('x-vercel-id') || '',
243
+ },
244
+ http: buildHttpObservation({ ...detail, durationMs: Date.now() - attemptStartedAt }, {
245
+ previousAttempts: failures,
246
+ capturedBodyBytes: body.bytesRead,
247
+ bodyInspected: true,
248
+ bodyTruncated: body.truncated,
249
+ bodySha256: body.sha256,
250
+ }),
251
+ };
252
+ }
253
+ // A non-2xx HTTP response still conclusively proves that a web service
254
+ // answered on this domain. It may block this probe (403), require auth
255
+ // (401), have a broken homepage route (404), or be unhealthy (5xx), but
256
+ // none of those are equivalent to "no website". We cannot inspect its
257
+ // HTML signals, so preserve a separate responded status and release the
258
+ // unused body.
259
+ await res.body?.cancel().catch(() => { });
260
+ return {
261
+ text: null,
262
+ status: 'responded',
263
+ detail: `Web server responded over ${scheme.toUpperCase()} (HTTP ${res.status}); homepage content was not available for inspection.`,
264
+ responsePolicy: analyzeResponsePolicyHeaders(res.headers),
265
+ http: buildHttpObservation({ ...detail, durationMs: Date.now() - attemptStartedAt }, {
266
+ previousAttempts: failures,
267
+ capturedBodyBytes: 0,
268
+ bodyInspected: false,
269
+ }),
270
+ };
271
+ }
272
+ catch (err) {
273
+ const error = errorRecord(err);
274
+ const reason = error.name === 'AbortError'
275
+ ? 'timed out after 6 seconds'
276
+ : nonEmptyErrorMessage(err, 'request failed')
277
+ .replace(/[\u0000-\u001f\u007f]+/g, ' ')
278
+ .slice(0, 180);
279
+ failures.push({ url: requestUrl, error: `${scheme.toUpperCase()} ${reason}` });
280
+ }
281
+ finally {
282
+ clearTimeout(timeout);
283
+ }
284
+ }
285
+ return {
286
+ text: null,
287
+ status: 'inconclusive',
288
+ detail: failures.length
289
+ ? `Could not confirm homepage activity: ${failures.map((attempt) => attempt.error).join('; ')}.`
290
+ : 'Could not confirm homepage activity.',
291
+ http: failedHttpObservation(failures, { durationMs: Date.now() - probeStartedAt }),
292
+ };
293
+ }
294
+ function deriveWebsiteActivity(homepageStatus, hasFavicon, alreadyParked = false) {
295
+ if (alreadyParked)
296
+ return 'parked';
297
+ if (homepageStatus === 'fetched' || homepageStatus === 'responded' || hasFavicon)
298
+ return 'active';
299
+ return 'unreachable';
300
+ }
301
+ // fast: true skips the WHOIS fallback (no TCP:43 chain) and the homepage
302
+ // fetch (no for-sale/parking detection) - just RDAP plus the signals
303
+ // derivable from it (age, expiry proximity, privacy). Meant for scanning
304
+ // large sourcing candidate lists quickly and gently on registry rate
305
+ // limits; anything it can't resolve (state "unknown") is meant to get a
306
+ // follow-up deep check (fast: false, the default) on the shortlist only.
307
+ async function checkDomainAvailability(domain, options = {}) {
308
+ const fast = options.fast === true;
309
+ const collectDns = options.collectDnsIntelligence || collectDnsIntelligence;
310
+ const collectTls = options.collectTlsIntelligence || collectTlsIntelligence;
311
+ const fetchHomepageForDomain = options.fetchHomepage || fetchHomepage;
312
+ const fetchFaviconForDomain = options.fetchFaviconHash || fetchFaviconHash;
313
+ const featurePolicy = options.featurePolicy || networkFeaturePolicy();
314
+ const rdapEnabled = featureDecision('rdap', featurePolicy).enabled;
315
+ const whoisEnabled = featureDecision('whois', featurePolicy).enabled;
316
+ const dnsIntelligenceEnabled = featureDecision('dns_intelligence', featurePolicy).enabled;
317
+ const websiteProbeEnabled = featureDecision('website_probe', featurePolicy).enabled;
318
+ const tlsIntelligenceEnabled = featureDecision('tls_intelligence', featurePolicy).enabled;
319
+ const deepScanComplete = rdapEnabled
320
+ && whoisEnabled
321
+ && dnsIntelligenceEnabled
322
+ && websiteProbeEnabled
323
+ && tlsIntelligenceEnabled;
324
+ const hasPreloadedRdap = Object.prototype.hasOwnProperty.call(options, 'rdapRecord');
325
+ const hasPreloadedWhois = Object.prototype.hasOwnProperty.call(options, 'whoisChain');
326
+ const hasPreloadedRdapPromise = Object.prototype.hasOwnProperty.call(options, 'rdapRecordPromise');
327
+ const hasPreloadedWhoisPromise = Object.prototype.hasOwnProperty.call(options, 'whoisChainPromise');
328
+ const hasPreloadedDnsDelegation = Object.prototype.hasOwnProperty.call(options, 'dnsDelegation');
329
+ let nameservers = [];
330
+ let statuses = [];
331
+ let rdapServer = null;
332
+ let rdapFound = false;
333
+ let registrar = null;
334
+ let registrant = null;
335
+ let abuse = null;
336
+ let createdDate = null;
337
+ let expiryDate = null;
338
+ let createdDateIso = null;
339
+ let expiryDateIso = null;
340
+ let registrationSource = null;
341
+ let registrationConfidence = 'high';
342
+ let dnssec = null;
343
+ let registryDnsEvidence = null;
344
+ if (rdapEnabled) {
345
+ try {
346
+ // Shared with /api/rdap (lib/rdap.mts's fetchRdapRecord) rather than a
347
+ // separate fetch+parse here - same registry data either way, and this
348
+ // also picks up that function's short-TTL cache (lib/lookup-cache.mts)
349
+ // and upstream timeout for free.
350
+ const recordValue = hasPreloadedRdapPromise
351
+ ? await options.rdapRecordPromise
352
+ : hasPreloadedRdap
353
+ ? options.rdapRecord
354
+ : await fetchRdapRecord('domain', domain);
355
+ const record = errorRecord(recordValue);
356
+ const recordRdapServer = typeof record.rdapServer === 'string' ? record.rdapServer : null;
357
+ const upstreamStatus = typeof record.upstreamStatus === 'number' ? record.upstreamStatus : null;
358
+ if (Object.keys(record).length) {
359
+ rdapServer = recordRdapServer;
360
+ if (upstreamStatus === 404) {
361
+ return {
362
+ state: 'available',
363
+ confidence: 'high',
364
+ detail: 'The registry\'s RDAP service has no record for this domain.',
365
+ source: 'rdap',
366
+ rdapServer: recordRdapServer,
367
+ };
368
+ }
369
+ const parsed = errorRecord(record.parsed);
370
+ if (Object.keys(parsed).length) {
371
+ registryDnsEvidence = parsed;
372
+ statuses = Array.isArray(parsed.statuses)
373
+ ? parsed.statuses.filter((status) => typeof status === 'string').map((status) => status.toLowerCase())
374
+ : [];
375
+ nameservers = Array.isArray(parsed.nameservers)
376
+ ? parsed.nameservers.filter((nameserver) => typeof nameserver === 'string')
377
+ : [];
378
+ registrar = compactContact(parsed.registrar);
379
+ registrant = compactContact(parsed.registrant);
380
+ abuse = compactContact(parsed.abuse);
381
+ const events = Array.isArray(parsed.events)
382
+ ? parsed.events.map(errorRecord)
383
+ : [];
384
+ const lifecycle = errorRecord(parsed.lifecycle);
385
+ createdDate = typeof lifecycle.createdDate === 'string'
386
+ ? lifecycle.createdDate
387
+ : rdapEventDate(events, 'registration');
388
+ expiryDate = typeof lifecycle.expiryDate === 'string'
389
+ ? lifecycle.expiryDate
390
+ : rdapEventDate(events, 'expiration');
391
+ createdDateIso = typeof lifecycle.createdDateIso === 'string'
392
+ ? lifecycle.createdDateIso
393
+ : registryDateIso(createdDate);
394
+ expiryDateIso = typeof lifecycle.expiryDateIso === 'string'
395
+ ? lifecycle.expiryDateIso
396
+ : registryDateIso(expiryDate);
397
+ dnssec = typeof parsed.dnssec === 'string' ? parsed.dnssec : null;
398
+ rdapFound = true;
399
+ registrationSource = 'rdap';
400
+ }
401
+ }
402
+ }
403
+ catch {
404
+ /* fall through to WHOIS-based detection (deep mode only) */
405
+ }
406
+ }
407
+ const dnsDelegationPromise = !rdapFound && dnsIntelligenceEnabled
408
+ ? hasPreloadedDnsDelegation
409
+ ? Promise.resolve(options.dnsDelegation)
410
+ : checkDnsDelegation(domain, { resolver: options.resolveNs || dns.resolveNs })
411
+ : null;
412
+ let whoisChain = null;
413
+ let whoisParsed = null;
414
+ if (!rdapFound && !fast && whoisEnabled) {
415
+ try {
416
+ whoisChain = (hasPreloadedWhoisPromise
417
+ ? await options.whoisChainPromise
418
+ : hasPreloadedWhois
419
+ ? options.whoisChain
420
+ : await buildWhoisChain(domain)) ?? null;
421
+ if (!Array.isArray(whoisChain))
422
+ throw new Error('WHOIS chain unavailable');
423
+ const parsed = parseWhoisChain(whoisChain);
424
+ whoisParsed = parsed;
425
+ if (parsed.notFound) {
426
+ return {
427
+ state: 'available',
428
+ confidence: 'medium',
429
+ detail: `WHOIS reports no matching record for this domain${parsed.notFoundSource ? ` (per ${parsed.notFoundSource})` : ''}.`,
430
+ source: 'whois',
431
+ };
432
+ }
433
+ if (parsed.nameservers.length)
434
+ nameservers = parsed.nameservers;
435
+ if (parsed.statuses.length)
436
+ statuses = parsed.statuses.map((status) => status.toLowerCase());
437
+ if (parsed.registrar) {
438
+ registrar = {
439
+ handle: null,
440
+ ianaId: parsed.registrarIanaId || null,
441
+ name: parsed.registrar,
442
+ org: null,
443
+ email: parsed.abuseEmail || null,
444
+ phone: parsed.abusePhone || null,
445
+ };
446
+ }
447
+ if (parsed.registrantName || parsed.registrantOrg || parsed.registrantEmail || parsed.registrantPhone) {
448
+ registrant = {
449
+ handle: null,
450
+ name: parsed.registrantName || null,
451
+ org: parsed.registrantOrg || null,
452
+ email: parsed.registrantEmail || null,
453
+ phone: parsed.registrantPhone || null,
454
+ };
455
+ }
456
+ if (parsed.abuseEmail || parsed.abusePhone) {
457
+ abuse = { handle: null, name: null, org: parsed.registrar || null, email: parsed.abuseEmail || null, phone: parsed.abusePhone || null };
458
+ }
459
+ createdDate = parsed.createdDate || null;
460
+ expiryDate = parsed.expiryDate || null;
461
+ createdDateIso = parsed.createdDateIso || parsed.lifecycle?.createdDateIso || registryDateIso(createdDate);
462
+ expiryDateIso = parsed.expiryDateIso || parsed.lifecycle?.expiryDateIso || registryDateIso(expiryDate);
463
+ if (!dnssec)
464
+ dnssec = parsed.dnssec || null;
465
+ if (parsed.registrationStatus === 'registered'
466
+ || registrar || createdDate || expiryDate || nameservers.length > 0 || statuses.length > 0) {
467
+ registrationSource = 'whois';
468
+ }
469
+ }
470
+ catch {
471
+ /* if both RDAP and WHOIS fail, we simply can't determine availability */
472
+ }
473
+ }
474
+ // WHOIS ran and was not a confirmed not-found, but produced no positive
475
+ // registration evidence either - e.g. the registry answered inconclusively
476
+ // or every referral hop failed/rate-limited. Report "unknown" rather than
477
+ // fabricating "registered" from an empty record.
478
+ const hasWhoisRegistrationData = whoisParsed?.registrationStatus === 'registered'
479
+ || Boolean(registrar) || Boolean(createdDate) || Boolean(expiryDate)
480
+ || nameservers.length > 0 || statuses.length > 0;
481
+ let dnsDelegated = false;
482
+ if (!rdapFound && !hasWhoisRegistrationData && dnsDelegationPromise) {
483
+ const delegation = await dnsDelegationPromise;
484
+ if (delegation && delegation.delegated === true && Array.isArray(delegation.nameservers) && delegation.nameservers.length) {
485
+ dnsDelegated = true;
486
+ nameservers = delegation.nameservers;
487
+ registrationSource = 'dns';
488
+ registrationConfidence = 'medium';
489
+ }
490
+ }
491
+ if (!rdapFound && !hasWhoisRegistrationData && !dnsDelegated) {
492
+ const disabledSources = [
493
+ !rdapEnabled ? 'RDAP' : null,
494
+ !fast && !whoisEnabled ? 'WHOIS' : null,
495
+ !dnsIntelligenceEnabled ? 'DNS intelligence' : null,
496
+ ].filter(Boolean);
497
+ const disabledDetail = disabledSources.length
498
+ ? ` ${disabledSources.join(', ')} ${disabledSources.length === 1 ? 'is' : 'are'} disabled by deployment policy.`
499
+ : '';
500
+ return {
501
+ state: 'unknown',
502
+ confidence: 'low',
503
+ detail: fast
504
+ ? `No enabled registration source produced a record or authoritative delegation. A fast scan cannot determine registration status.${disabledDetail}`
505
+ : whoisParsed && whoisParsed.failedHop
506
+ ? `WHOIS was inconclusive - a referral hop did not answer conclusively (${whoisParsed.failedHop}).`
507
+ : `No enabled registration source returned conclusive data or an authoritative DNS delegation.${disabledDetail}`,
508
+ ...(!fast && whoisEnabled ? { source: 'whois' } : {}),
509
+ };
510
+ }
511
+ const domainAgeDays = computeAgeDays(createdDateIso || createdDate);
512
+ const expiresInDays = computeDaysUntil(expiryDateIso || expiryDate);
513
+ // DNS proves delegation only; it says nothing about whether registry
514
+ // contact data is privacy-protected or merely unavailable.
515
+ const privacyProtected = registrationSource === 'dns' ? null : isPrivacyProtected(registrant);
516
+ const baseInfo = {
517
+ nameservers,
518
+ statuses,
519
+ registrar,
520
+ registrant,
521
+ abuse,
522
+ createdDate,
523
+ expiryDate,
524
+ createdDateIso,
525
+ expiryDateIso,
526
+ rdapServer,
527
+ domainAgeDays,
528
+ expiresInDays,
529
+ privacyProtected,
530
+ dnssec,
531
+ source: registrationSource,
532
+ };
533
+ if (statuses.some((s) => s.includes('pendingdelete') || s.includes('redemptionperiod'))) {
534
+ return {
535
+ state: 'expiring',
536
+ confidence: 'medium',
537
+ detail: 'Domain is in redemption/pending-delete status and may become available soon.',
538
+ ...baseInfo,
539
+ };
540
+ }
541
+ if (fast) {
542
+ return {
543
+ state: 'registered',
544
+ confidence: registrationConfidence,
545
+ detail: registrationSource === 'dns'
546
+ ? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable.'
547
+ : 'Domain is registered. Run a deep check for parking/for-sale detection.',
548
+ ...baseInfo,
549
+ };
550
+ }
551
+ // Registered - look for for-sale/parked/website signals (homepage fetch,
552
+ // deep mode only), and check for a configured mail exchanger as a
553
+ // phishing-risk signal (a lookalike domain that can receive/send mail is
554
+ // capable of running credential-harvesting or BEC campaigns).
555
+ const nsSignal = nameservers.find((ns) => PARKING_NS_PATTERNS.some((re) => re.test(ns)));
556
+ let forSaleSignal = nsSignal ? `parking nameserver (${nsSignal})` : null;
557
+ let activityStatus = nsSignal ? 'parked' : 'unknown';
558
+ // Homepage + bounded DNS evidence resolve in parallel; the favicon fetch is sequenced after
559
+ // the homepage so it can use any <link rel="icon"> the page declares (many
560
+ // sites serve no /favicon.ico and only point to a CDN PNG this way). One
561
+ // extra round-trip on the already-slow deep path, in exchange for finding
562
+ // favicons the bare /favicon.ico probe would miss.
563
+ const [homepage, dnsIntelligence, tlsIntelligence] = await Promise.all([
564
+ websiteProbeEnabled ? fetchHomepageForDomain(domain).catch((err) => ({
565
+ text: null,
566
+ status: 'inconclusive',
567
+ detail: `Could not confirm homepage activity: ${String(err && err.message ? err.message : 'request failed').slice(0, 180)}.`,
568
+ http: failedHttpObservation([
569
+ { url: `https://${domain}`, error: String(err && err.message ? err.message : 'request failed') },
570
+ ]),
571
+ })) : Promise.resolve({
572
+ text: null,
573
+ status: 'skipped',
574
+ detail: 'Website probing is disabled by deployment policy.',
575
+ http: skippedHttpObservation(),
576
+ }),
577
+ dnsIntelligenceEnabled
578
+ ? collectDns(domain, {
579
+ ...(options.dnsResolvers ? { resolvers: options.dnsResolvers } : {}),
580
+ includeExtendedContext: options.includeExtendedDnsContext === true,
581
+ includeInheritedCaa: options.includeInheritedCaa === true,
582
+ registryEvidence: registryDnsEvidence,
583
+ })
584
+ : Promise.resolve(skippedDnsIntelligence('DNS intelligence is disabled by deployment policy.', {
585
+ includeExtendedContext: options.includeExtendedDnsContext === true,
586
+ includeInheritedCaa: options.includeInheritedCaa === true,
587
+ })),
588
+ tlsIntelligenceEnabled
589
+ ? collectTls(domain)
590
+ : Promise.resolve(skippedTlsObservation()),
591
+ ]);
592
+ const page = homepage.text;
593
+ const favicon = websiteProbeEnabled
594
+ ? await fetchFaviconForDomain(domain, { html: page || '' }).catch(() => null)
595
+ : null;
596
+ const faviconHash = favicon ? favicon.hash : null;
597
+ const faviconPHash = favicon ? favicon.phash : null;
598
+ let htmlSignals = {
599
+ pageTitle: null,
600
+ hasPasswordField: false,
601
+ phishingLanguageMatch: null,
602
+ hasExternalFormAction: null,
603
+ externalAssetHosts: [],
604
+ pageIdentity: null,
605
+ credentialSurfaceProfile: null,
606
+ structuredDataIdentity: null,
607
+ technologyProfile: null,
608
+ pageRoleProfile: null,
609
+ clientBehaviorProfile: null,
610
+ };
611
+ if (homepage.status === 'fetched') {
612
+ if (page) {
613
+ const saleMatch = page.match(FOR_SALE_TEXT_RE);
614
+ if (saleMatch) {
615
+ forSaleSignal = forSaleSignal || `homepage text ("${saleMatch[0]}")`;
616
+ activityStatus = 'parked';
617
+ }
618
+ const responseContentType = homepage.http?.response?.contentType;
619
+ const pageIdentityEligible = typeof responseContentType !== 'string'
620
+ || responseContentType.trim() === ''
621
+ || /^(?:text\/html|application\/xhtml\+xml)(?:\s*;|$)/i.test(responseContentType.trim());
622
+ htmlSignals = extractHtmlSignals(page, domain, {
623
+ ...(typeof homepage.http?.finalUrl === 'string' ? { baseUrl: homepage.http.finalUrl } : {}),
624
+ ...(typeof homepage.http?.observedAt === 'string' ? { observedAt: homepage.http.observedAt } : {}),
625
+ sourceTruncated: homepage.http?.response?.bodyTruncated === true,
626
+ exactBodyHash: homepage.http?.response?.bodyHash,
627
+ httpServer: homepage.http?.response?.server,
628
+ responseHeaders: homepage.technologyHeaders,
629
+ activityStatus,
630
+ includePageIdentity: pageIdentityEligible,
631
+ ...(options.includeCredentialSurfaceProfile !== undefined
632
+ ? { includeCredentialSurfaceProfile: options.includeCredentialSurfaceProfile }
633
+ : {}),
634
+ ...(options.includeStructuredDataIdentity !== undefined
635
+ ? { includeStructuredDataIdentity: options.includeStructuredDataIdentity }
636
+ : {}),
637
+ ...(options.includeTechnologyProfile !== undefined
638
+ ? { includeTechnologyProfile: options.includeTechnologyProfile }
639
+ : {}),
640
+ });
641
+ }
642
+ }
643
+ const securityPosture = options.includeSecurityPosture === false ? null : analyzeWebsiteSecurityPosture({
644
+ http: homepage.http,
645
+ responsePolicy: homepage.responsePolicy,
646
+ pageIdentity: htmlSignals.pageIdentity,
647
+ tls: tlsIntelligence,
648
+ dns: dnsIntelligence,
649
+ dnssec,
650
+ observedAt: homepage.http?.observedAt,
651
+ });
652
+ const redirectSaleSignal = forSaleRedirectSignal(homepage.http);
653
+ if (!forSaleSignal && redirectSaleSignal) {
654
+ forSaleSignal = redirectSaleSignal;
655
+ activityStatus = 'parked';
656
+ }
657
+ const faviconProvedActive = homepage.status === 'inconclusive' && Boolean(favicon);
658
+ const websiteProbeStatus = faviconProvedActive ? 'responded' : homepage.status;
659
+ const websiteProbeDetail = faviconProvedActive
660
+ ? `${homepage.detail} A favicon responded successfully, confirming an active web service.`
661
+ : homepage.detail;
662
+ if (websiteProbeEnabled) {
663
+ activityStatus = deriveWebsiteActivity(homepage.status, Boolean(favicon), activityStatus === 'parked');
664
+ }
665
+ if (!forSaleSignal) {
666
+ return {
667
+ state: 'registered',
668
+ confidence: registrationConfidence,
669
+ detail: registrationSource === 'dns'
670
+ ? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable. No for-sale signals were observed.'
671
+ : 'Domain is registered and shows no for-sale signals.',
672
+ activityStatus,
673
+ websiteProbeStatus,
674
+ websiteProbeDetail,
675
+ http: homepage.http,
676
+ deepScanComplete,
677
+ faviconHash,
678
+ faviconPHash,
679
+ ...htmlSignals,
680
+ securityPosture,
681
+ ...baseInfo,
682
+ nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
683
+ dns: dnsIntelligence,
684
+ tls: tlsIntelligence,
685
+ hasMx: dnsIntelligence.hasMx,
686
+ hasNullMx: dnsIntelligence.hasNullMx,
687
+ mxHosts: dnsIntelligence.mxHosts,
688
+ hasSpf: dnsIntelligence.hasSpf,
689
+ hasDmarc: dnsIntelligence.hasDmarc,
690
+ };
691
+ }
692
+ return {
693
+ state: 'for_sale',
694
+ confidence: 'medium',
695
+ detail: `Detected a for-sale listing (${forSaleSignal}).`,
696
+ activityStatus,
697
+ websiteProbeStatus,
698
+ websiteProbeDetail,
699
+ http: homepage.http,
700
+ deepScanComplete,
701
+ faviconHash,
702
+ faviconPHash,
703
+ ...htmlSignals,
704
+ securityPosture,
705
+ ...baseInfo,
706
+ nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
707
+ dns: dnsIntelligence,
708
+ tls: tlsIntelligence,
709
+ hasMx: dnsIntelligence.hasMx,
710
+ hasNullMx: dnsIntelligence.hasNullMx,
711
+ mxHosts: dnsIntelligence.mxHosts,
712
+ hasSpf: dnsIntelligence.hasSpf,
713
+ hasDmarc: dnsIntelligence.hasDmarc,
714
+ };
715
+ }
716
+ export { checkDomainAvailability, checkDnsDelegation, fetchHomepage, deriveWebsiteActivity, forSaleRedirectSignal, parseRegistryDate as parseWhoisDate, };