@slicedearth/whoisleuth-cli 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +1 -1
- package/NOTICE +2 -2
- package/cli/argument-values.mjs +55 -0
- package/cli/arguments.mjs +24 -470
- package/cli/artifact-validation/case-response.mjs +67 -9
- package/cli/artifact-validation/lookup-evidence.mjs +18 -4
- package/cli/artifact-validation/signed-review.mjs +61 -64
- package/cli/artifact-verify.mjs +11 -2
- package/cli/assurance-arguments.mjs +56 -0
- package/cli/assurance-command-definitions.mjs +179 -0
- package/cli/assurance-command-runner.mjs +1 -1
- package/cli/bounded-json.mjs +1 -1
- package/cli/bulk-checkpoint.mjs +1 -1
- package/cli/bulk-command-runner.mjs +6 -8
- package/cli/bulk.mjs +2 -3
- package/cli/case-command.mjs +99 -6
- package/cli/case-pack.mjs +19 -4
- package/cli/ci-report.mjs +6 -3
- package/cli/collection-arguments.mjs +123 -0
- package/cli/collection-command-definitions.mjs +330 -0
- package/cli/collection-preflight.mjs +8 -1
- package/cli/command-argument-grammar.mjs +8 -210
- package/cli/command-definition.mjs +330 -0
- package/cli/command-reference.mjs +55 -1388
- package/cli/command-seeds.mjs +18 -0
- package/cli/compare.mjs +1 -1
- package/cli/config-profile.mjs +1 -1
- package/cli/ct-event-intake.mjs +4 -4
- package/cli/discover.mjs +2 -2
- package/cli/discovery-observation-snapshot.mjs +2 -2
- package/cli/discovery-scan-command-runner.mjs +7 -9
- package/cli/discovery-scan.mjs +1 -1
- package/cli/discovery-snapshot.mjs +2 -2
- package/cli/discovery-workflow.mjs +4 -4
- package/cli/domain-control-monitor.mjs +6 -6
- package/cli/domain-control-observations.mjs +2 -1
- package/cli/domain-feed.mjs +61 -0
- package/cli/errors.mjs +5 -6
- package/cli/evidence-arguments.mjs +46 -0
- package/cli/evidence-command-definitions.mjs +69 -0
- package/cli/evidence-command-runner.mjs +2 -3
- package/cli/evidence-signing.mjs +3 -2
- package/cli/external-observation-mapping.mjs +2 -2
- package/cli/formatters/json.mjs +1 -13
- package/cli/formatters/terminal-command-formats.mjs +6 -0
- package/cli/formatters/terminal-lookup.mjs +6 -0
- package/cli/history-arguments.mjs +39 -0
- package/cli/history-command-definitions.mjs +88 -0
- package/cli/indicator-set-command.mjs +32 -0
- package/cli/inline-command-families.mjs +6 -47
- package/cli/inline-command-runner.mjs +4 -31
- package/cli/input.mjs +2 -2
- package/cli/intake-command.mjs +147 -0
- package/cli/interchange-report.mjs +4 -2
- package/cli/investigation-artifacts.mjs +3 -4
- package/cli/investigation-package-review.mjs +2 -0
- package/cli/investigation-plan.mjs +32 -154
- package/cli/investigation-recipes.mjs +177 -0
- package/cli/investigation-run.mjs +78 -12
- package/cli/local-mmdb-review.mjs +220 -17
- package/cli/lookalike-calibration.mjs +1 -1
- package/cli/lookup-brief.mjs +2 -2
- package/cli/lookup-command-runner.mjs +19 -35
- package/cli/lookup-diff.mjs +10 -9
- package/cli/lookup-plan.mjs +10 -14
- package/cli/lookup-reconcile.mjs +6 -5
- package/cli/lookup-timeline.mjs +7 -6
- package/cli/mail-header-review.mjs +24 -247
- package/cli/mail-review.mjs +9 -7
- package/cli/network-arguments.mjs +51 -0
- package/cli/network-command-definitions.mjs +130 -0
- package/cli/network-command-runner.mjs +43 -31
- package/cli/offline-evidence-review.mjs +85 -6
- package/cli/posture.mjs +1 -1
- package/cli/registry-fixture-scaffold.mjs +1 -1
- package/cli/retained-artifact-diff.mjs +8 -7
- package/cli/review-arguments.mjs +132 -0
- package/cli/review-command-definitions.mjs +279 -0
- package/cli/review-command-runner.mjs +72 -159
- package/cli/risk-calibration.mjs +14 -12
- package/cli/runner.mjs +20 -32
- package/cli/saved-lookup.mjs +2 -2
- package/cli/selected-input-worker.mjs +73 -0
- package/cli/sharing-review.mjs +5 -64
- package/cli/source-reliability.mjs +2 -2
- package/cli/support-arguments.mjs +49 -0
- package/cli/support-command-definitions.mjs +209 -0
- package/cli/support-command-runner.mjs +4 -4
- package/cli/watchlist-review.mjs +43 -0
- package/cli/workflow-arguments.mjs +100 -0
- package/cli/workflow-command-definitions.mjs +103 -0
- package/cli/workflow-command-runner.mjs +40 -1
- package/docs/cli-reference.md +27 -0
- package/docs/cli.md +322 -4
- package/docs/contextual-reviews.md +178 -0
- package/docs/infrastructure-observations.md +126 -0
- package/docs/offline-intake-context.md +213 -0
- package/lib/analyst-taxonomy.mjs +2 -40
- package/lib/application-version.mjs +1 -1
- package/lib/availability.mjs +13 -789
- package/lib/bounded-contract-normalizers.mjs +4 -0
- package/lib/bounded-json.mjs +2 -330
- package/lib/browser-library-profile.mjs +62 -84
- package/lib/candidate-provenance-bounds.mjs +2 -0
- package/lib/classify.mjs +1 -1
- package/lib/credential-form-attribution.mjs +136 -0
- package/lib/credential-surface-profile.mjs +6 -2
- package/lib/ct-query.mjs +2 -27
- package/lib/ct-response-bounds.mjs +2 -9
- package/lib/ct-search.mjs +83 -47
- package/lib/dmarc-discovery.mjs +62 -0
- package/lib/dns-convergence-review.mjs +21 -8
- package/lib/dns-delegation-health.mjs +35 -15
- package/lib/dns-inheritance-review.mjs +53 -63
- package/lib/dns-intelligence.mjs +27 -11
- package/lib/dns-query-coverage.mjs +40 -0
- package/lib/domain-assurance.mjs +1 -1
- package/lib/domain-change-packet.mjs +12 -13
- package/lib/domain-change-review.mjs +32 -14
- package/lib/domain-control-flight-recorder.mjs +103 -27
- package/lib/domain-control-manifest.mjs +25 -17
- package/lib/domain-posture-analysis.mjs +75 -7
- package/lib/domain-posture-parsers.mjs +9 -0
- package/lib/domain-posture.mjs +45 -25
- package/lib/domain-registration.mjs +392 -0
- package/lib/evidence-export-privacy.mjs +27 -17
- package/lib/evidence-export.mjs +3 -3
- package/lib/evidence-report.mjs +5 -1
- package/lib/external-intelligence-risk.mjs +2 -124
- package/lib/favicon.mjs +13 -4
- package/lib/generated/cisa-kev-catalog.mjs +41 -3
- package/lib/generated/registrar-standing-catalogue.mjs +18 -10
- package/lib/generated/retire-browser-catalog.mjs +332 -19
- package/lib/generated/{unicode-confusables-17.mjs → unicode-confusables.mjs} +33 -33
- package/lib/html-signals.mjs +13 -2
- package/lib/http-evidence-bounds.mjs +2 -8
- package/lib/http-intelligence.mjs +13 -1
- package/lib/idn-confusable-policy.mjs +6 -4
- package/lib/idn-confusables.mjs +1 -1
- package/lib/interchange-fidelity-registry.mjs +10 -0
- package/lib/lookup-child-profile-contract.mjs +28 -11
- package/lib/lookup-contract-primitives.mjs +1 -1
- package/lib/lookup-diagnostics.mjs +1 -23
- package/lib/lookup-network-evidence-bounds.mjs +1 -10
- package/lib/lookup-progress.mjs +50 -1
- package/lib/lookup-response-contract.mjs +70 -40
- package/lib/lookup-response.mjs +5 -15
- package/lib/lookup-source-progress.mjs +8 -22
- package/lib/lookup.mjs +1 -1
- package/lib/operation-budget.mjs +4 -0
- package/lib/opportunity-scoring.mjs +2 -107
- package/lib/outbound-identity.mjs +1 -1
- package/lib/page-fingerprints.mjs +18 -1
- package/lib/perceptual-hash-comparison.mjs +2 -34
- package/lib/perceptual-hash.mjs +36 -10
- package/lib/portable-generator.mjs +2 -29
- package/lib/project-metadata.mjs +2 -8
- package/lib/rdap-client.mjs +11 -7
- package/lib/registrable-domain.mjs +2 -56
- package/lib/registrar-standing-contract.mjs +1 -1
- package/lib/registrar-standing.mjs +9 -13
- package/lib/registry-capabilities.mjs +2 -2
- package/lib/registry-capability-catalogue.mjs +27 -19
- package/lib/risk-calibration-summary.mjs +2 -352
- package/lib/risk-scoring.mjs +2 -345
- package/lib/scoring-evidence-quality.mjs +2 -107
- package/lib/semantic-version.mjs +2 -54
- package/lib/service-binding-dns.mjs +46 -2
- package/lib/smtp-transport-review.mjs +48 -19
- package/lib/sslbl-certificates.generated.mjs +142 -139
- package/lib/static-html-analysis.mjs +32 -4
- package/lib/structured-data-identity.mjs +1 -1
- package/lib/technology-evidence-role.mjs +1 -51
- package/lib/threat-intelligence-manifests.mjs +1 -1
- package/lib/threat-intelligence-runtime.mjs +1 -1
- package/lib/threat-intelligence-targets.mjs +1 -1
- package/lib/threat-intelligence-types.mjs +2 -75
- package/lib/tls-intelligence.mjs +30 -5
- package/lib/typosquat-generator.mjs +11 -2
- package/lib/website-enrichment.mjs +404 -0
- package/lib/whois-authority.mjs +43 -5
- package/lib/whois-chain.mjs +12 -3
- package/lib/whois-common-formats.mjs +3 -1
- package/lib/whois-contacts.mjs +13 -2
- package/lib/whois-normalization.mjs +3 -1
- package/lib/whois-parser.mjs +5 -3
- package/package.json +15 -14
- package/packages/analysis/analyst-taxonomy.mjs +40 -0
- package/packages/analysis/bounded-json.mjs +330 -0
- package/packages/analysis/candidate-provenance-bounds.mjs +4 -0
- package/packages/analysis/cli-argument-grammar.mjs +216 -0
- package/packages/analysis/ct-query.mjs +27 -0
- package/packages/analysis/ct-response-bounds.mjs +9 -0
- package/packages/analysis/external-intelligence-risk.mjs +124 -0
- package/packages/analysis/http-evidence-bounds.mjs +8 -0
- package/packages/analysis/lookup-network-evidence-bounds.mjs +10 -0
- package/packages/analysis/opportunity-scoring.mjs +107 -0
- package/packages/analysis/perceptual-hash-comparison.mjs +34 -0
- package/packages/analysis/portable-generator.mjs +29 -0
- package/packages/analysis/project-metadata.mjs +8 -0
- package/packages/analysis/registrable-domain.mjs +56 -0
- package/packages/analysis/risk-calibration-summary.mjs +352 -0
- package/packages/analysis/risk-scoring.mjs +348 -0
- package/packages/analysis/scoring-evidence-quality.mjs +107 -0
- package/packages/analysis/semantic-version.mjs +54 -0
- package/packages/analysis/threat-intelligence-types.mjs +75 -0
- package/packages/cases/case-containment-handoff.mjs +195 -0
- package/packages/cases/case-evidence-links.mjs +151 -0
- package/packages/cases/case-evidence-model.mjs +89 -12
- package/packages/cases/case-evidence-quality.mjs +5 -0
- package/packages/cases/case-export-input.mjs +7 -5
- package/packages/cases/case-incident-context.mjs +52 -0
- package/packages/cases/case-markdown.mjs +11 -0
- package/packages/cases/case-migration-model.mjs +40 -24
- package/packages/cases/case-object-coverage.mjs +33 -0
- package/packages/cases/case-packet-correction.mjs +106 -0
- package/packages/cases/case-recheck-model.mjs +124 -36
- package/packages/cases/case-record-contracts.mjs +1 -1
- package/packages/cases/case-record-core.mjs +1 -1
- package/packages/cases/case-record-operations.mjs +111 -118
- package/packages/cases/case-record-projection.mjs +23 -4
- package/packages/cases/case-record-version-input.mjs +41 -0
- package/packages/cases/case-report.mjs +93 -27
- package/packages/cases/case-requested-evidence.mjs +146 -0
- package/packages/cases/case-response-actions.mjs +159 -1
- package/packages/cases/case-response-model.mjs +16 -1
- package/packages/cases/case-response-object.mjs +119 -0
- package/packages/cases/case-response-outcomes.mjs +108 -20
- package/packages/cases/case-response-packet-render.mjs +35 -12
- package/packages/cases/case-response-packet.mjs +55 -13
- package/packages/cases/case-response-records.mjs +0 -1
- package/packages/cases/case-response-review-inputs.mjs +60 -8
- package/packages/cases/case-storage-model.mjs +1 -1
- package/packages/cases/case-workflow-metadata.mjs +217 -0
- package/packages/cases/case-workflow-migration.mjs +43 -0
- package/packages/cases/http-summary.mjs +6 -1
- package/packages/comparison/favicon-similarity.mjs +1 -1
- package/packages/comparison/page-similarity.mjs +1 -1
- package/packages/comparison/relationship-evidence.mjs +21 -9
- package/packages/comparison/relationship-provenance.mjs +7 -1
- package/packages/contracts/analyst-interchange.mjs +35 -15
- package/packages/contracts/candidate-watch-review.mjs +54 -0
- package/packages/contracts/case-portability.mjs +360 -49
- package/packages/contracts/cli-command-semantics.mjs +5 -0
- package/packages/contracts/cli-grammar.mjs +6 -0
- package/packages/contracts/cli-lookup-brief.mjs +2 -0
- package/packages/contracts/context-review.mjs +25 -0
- package/packages/contracts/document-intake.mjs +11 -0
- package/packages/contracts/domain-control-flight-recorder.mjs +30 -14
- package/packages/contracts/domain-control-manifest.mjs +1 -1
- package/packages/contracts/domain-control-monitor.mjs +13 -2
- package/packages/contracts/domain-control-review.mjs +36 -15
- package/packages/contracts/external-observation-interchange.mjs +96 -28
- package/packages/contracts/har-review.mjs +2 -0
- package/packages/contracts/identity-events.mjs +4 -0
- package/packages/contracts/investigation-run.mjs +2 -0
- package/packages/contracts/ip-address.mjs +10 -0
- package/packages/contracts/lookup-evidence.mjs +3 -2
- package/packages/contracts/mail-authentication.mjs +5 -0
- package/packages/contracts/message-intake.mjs +37 -0
- package/packages/contracts/offline-comparison.mjs +4 -1
- package/packages/contracts/passive-mail.mjs +10 -0
- package/packages/contracts/risk-calibration.mjs +44 -29
- package/packages/contracts/schema-lifecycle.mjs +3 -1
- package/packages/contracts/web-capture.mjs +2 -2
- package/packages/contracts/workspace-portability.mjs +144 -28
- package/packages/evidence/collection-quality.mjs +67 -0
- package/packages/evidence/credential-form-attribution.mjs +60 -0
- package/packages/evidence/dns-query-outcome.mjs +72 -0
- package/packages/evidence/email-recipient.mjs +18 -0
- package/packages/evidence/image-regions.mjs +15 -0
- package/packages/evidence/lookup-target.mjs +5 -1
- package/packages/evidence/observation.mjs +2 -0
- package/packages/evidence/png-pixels.mjs +117 -0
- package/packages/evidence/public-address-policy.mjs +83 -0
- package/packages/evidence/record-identity.mjs +60 -16
- package/packages/evidence/technology-evidence-role.mjs +51 -0
- package/packages/interchange/defensive-indicator-export.mjs +264 -0
- package/packages/interchange/external-findings-import.mjs +36 -29
- package/packages/interchange/json-output.mjs +14 -0
- package/packages/interchange/managed-indicator-export.mjs +36 -0
- package/packages/interchange/managed-indicator-set.mjs +223 -0
- package/packages/interchange/misp-indicator-export.mjs +93 -0
- package/packages/interchange/sharing-policy.mjs +56 -0
- package/packages/interchange/stix-indicator-export.mjs +129 -0
- package/packages/interchange/web-capture-import.mjs +22 -15
- package/packages/investigation/acquisition-decision-packet.mjs +79 -1
- package/packages/investigation/bulk-mail-exposure.mjs +7 -13
- package/packages/investigation/capture-coverage.mjs +57 -0
- package/packages/investigation/connector-provenance-review.mjs +100 -0
- package/packages/investigation/context-review.mjs +35 -0
- package/packages/investigation/document-intake.mjs +87 -0
- package/packages/investigation/docx-intake.mjs +133 -0
- package/packages/investigation/domain-history-review.mjs +132 -0
- package/packages/investigation/domain-transition-review.mjs +49 -0
- package/packages/investigation/evidence-coverage-ledger.mjs +9 -1
- package/packages/investigation/har-intake.mjs +68 -0
- package/packages/investigation/identity-event-intake.mjs +131 -0
- package/packages/investigation/identity-incident-review.mjs +39 -0
- package/packages/investigation/incident-sequence-review.mjs +58 -0
- package/packages/investigation/infrastructure-observation.mjs +234 -0
- package/packages/investigation/intake-context.mjs +284 -0
- package/packages/investigation/intake-indicators.mjs +116 -0
- package/packages/investigation/intake-phones.mjs +155 -0
- package/packages/investigation/intake-report.mjs +20 -0
- package/packages/investigation/investigation-manifest.mjs +35 -9
- package/packages/investigation/link-intake.mjs +209 -0
- package/packages/investigation/lookup-asset-graph.mjs +1 -1
- package/packages/investigation/mail-authentication-review.mjs +222 -0
- package/packages/investigation/mail-header-identity.mjs +209 -0
- package/packages/investigation/message-intake.mjs +235 -0
- package/packages/investigation/page-behaviour.mjs +113 -0
- package/packages/investigation/pdf-intake.mjs +155 -0
- package/packages/investigation/platform-continuity-review.mjs +60 -0
- package/packages/investigation/qr-intake.mjs +56 -0
- package/packages/investigation/requested-action-hints.mjs +11 -0
- package/packages/investigation/selected-input-review.mjs +17 -0
- package/packages/investigation/storefront-review.mjs +70 -0
- package/packages/monitoring/domain-feed.mjs +321 -0
- package/packages/workspace/brand-candidate-workflow.mjs +351 -0
- package/packages/workspace/brand-keyword-campaign.mjs +231 -0
- package/packages/workspace/brand-profile-model.mjs +48 -4
- package/packages/workspace/bulk-review-model.mjs +28 -26
- package/packages/workspace/bulk-session-model.mjs +27 -13
- package/packages/workspace/campaign-model.mjs +1 -1
- package/packages/workspace/candidate-watch-handoff.mjs +195 -0
- package/packages/workspace/detection-rule-model.mjs +11 -3
- package/packages/workspace/hostile-input.mjs +1 -1
- package/packages/workspace/investigation-guide.mjs +2 -2
- package/packages/workspace/investigation-template-model.mjs +26 -2
- package/packages/workspace/page-baseline.mjs +2 -2
- package/packages/workspace/relationship-observation-model.mjs +17 -13
- package/packages/workspace/shortlist-model.mjs +24 -7
- package/packages/workspace/watchlist-history.mjs +59 -16
- package/packages/workspace/watchlist-store.mjs +152 -11
- package/packages/workspace/website-snapshot-model.mjs +28 -3
- package/packages/workspace/workspace-archive-crypto.mjs +1 -1
- package/packages/workspace/workspace-archive.mjs +5 -4
- package/third-party-notices.txt +549 -16
- package/lib/hostname.mjs +0 -2
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
// Bounded website and network enrichment, independent of registration authority.
|
|
2
|
+
import { capturedWebCollectionQuality } from "../packages/evidence/collection-quality.mjs";
|
|
3
|
+
import { isValidAsciiHostname } from "../packages/contracts/domain-name.mjs";
|
|
4
|
+
import { recordOrEmpty as errorRecord } from "./json-record.mjs";
|
|
5
|
+
import { safeFetchDetailed, readTextCapped } from "./safe-fetch.mjs";
|
|
6
|
+
import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
|
|
7
|
+
import { collectDnsIntelligence, skippedDnsIntelligence } from "./dns-intelligence.mjs";
|
|
8
|
+
import { fetchFaviconHash } from "./favicon.mjs";
|
|
9
|
+
import { extractHtmlSignals } from "./html-signals.mjs";
|
|
10
|
+
import { analyzeStaticHtml } from "./static-html-analysis.mjs";
|
|
11
|
+
import { featureDecision, networkFeaturePolicy } from "./feature-policy.mjs";
|
|
12
|
+
import { withoutHttpDeliveryMetadata, buildHttpObservation, failedHttpObservation, skippedHttpObservation } from "./http-intelligence.mjs";
|
|
13
|
+
import { collectTlsIntelligence, skippedTlsObservation } from "./tls-intelligence.mjs";
|
|
14
|
+
import { analyzeWebsiteSecurityPosture } from "./website-security-posture.mjs";
|
|
15
|
+
import { analyzeWebsiteTechnology, captureTechnologyResponseHeaders, } from "./website-technology.mjs";
|
|
16
|
+
import { analyzeResponsePolicyHeaders, qualifyResponsePolicyWithCspMeta, } from "./response-policy.mjs";
|
|
17
|
+
import { nonEmptyErrorMessage } from "./error-detail.mjs";
|
|
18
|
+
import { prepareSelectedLookupUrl } from "../packages/evidence/lookup-target.mjs";
|
|
19
|
+
import { HOMEPAGE_FETCH_TIMEOUT_MS, MAX_HOMEPAGE_BYTES, } from "./outbound-request-bounds.mjs";
|
|
20
|
+
// No marketplace (Afternic/Sedo/Dan.com/GoDaddy Auctions/etc.) offers a
|
|
21
|
+
// free, no-auth API to check "is this specific domain listed for sale" -
|
|
22
|
+
// what's checkable without credentials is the parking/landing-page
|
|
23
|
+
// nameservers and homepage copy these services actually use, which is
|
|
24
|
+
// broader coverage of the same signal, not a live cross-marketplace lookup.
|
|
25
|
+
const PARKING_NS_DOMAINS = [
|
|
26
|
+
'sedoparking.com',
|
|
27
|
+
'sedo.com',
|
|
28
|
+
'above.com',
|
|
29
|
+
'bodis.com',
|
|
30
|
+
'parkingcrew.net',
|
|
31
|
+
'dan.com',
|
|
32
|
+
'hugedomains.com',
|
|
33
|
+
// https://www.atom.com/help/article/how-to-verify-your-standard-premium-domains
|
|
34
|
+
'squadhelp.com',
|
|
35
|
+
// https://blog.afternic.com/lander-selection-redefining-your-use-of-afternic-landing-pages/
|
|
36
|
+
'afternic.com',
|
|
37
|
+
'voodoo.com',
|
|
38
|
+
'fabulous.com',
|
|
39
|
+
'smartname.com',
|
|
40
|
+
'domainsponsor.com',
|
|
41
|
+
'undeveloped.com',
|
|
42
|
+
'trafficz.com',
|
|
43
|
+
'dsredirection.com',
|
|
44
|
+
];
|
|
45
|
+
function isParkingNameserver(value) {
|
|
46
|
+
const hostname = value.trim().replace(/\.$/u, '');
|
|
47
|
+
// Validate before case folding: non-ASCII letters can lowercase to ASCII.
|
|
48
|
+
if (!isValidAsciiHostname(hostname))
|
|
49
|
+
return false;
|
|
50
|
+
const normalised = hostname.toLowerCase();
|
|
51
|
+
return PARKING_NS_DOMAINS.some(domain => normalised === domain || normalised.endsWith(`.${domain}`));
|
|
52
|
+
}
|
|
53
|
+
const FOR_SALE_PATH_RE = /\/(?:premium-)?domains?-for-sale(?:\/|$)/i;
|
|
54
|
+
function forSaleRedirectSignal(httpObservation) {
|
|
55
|
+
if (!httpObservation || typeof httpObservation !== 'object')
|
|
56
|
+
return null;
|
|
57
|
+
const observation = httpObservation;
|
|
58
|
+
const urls = [
|
|
59
|
+
observation.finalUrl,
|
|
60
|
+
...(Array.isArray(observation.redirects)
|
|
61
|
+
? observation.redirects.map((redirect) => redirect && typeof redirect === 'object' ? redirect.to : null)
|
|
62
|
+
: []),
|
|
63
|
+
];
|
|
64
|
+
for (const value of urls) {
|
|
65
|
+
if (typeof value !== 'string' || value.length > 2048)
|
|
66
|
+
continue;
|
|
67
|
+
try {
|
|
68
|
+
const url = new URL(value);
|
|
69
|
+
if (['http:', 'https:'].includes(url.protocol) && FOR_SALE_PATH_RE.test(url.pathname)) {
|
|
70
|
+
return `for-sale landing-page redirect (${url.origin}${url.pathname})`;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
// Malformed retained provenance cannot establish a sale signal.
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
// Fetches enough of the homepage for the lightweight HTML signals below and
|
|
80
|
+
// preserves why the probe failed. A failed request is not evidence that a
|
|
81
|
+
// domain has "no site": transient DNS/TLS/network failures, a slow origin, or
|
|
82
|
+
// an HTTP error can all produce the same null body. Keeping that distinction
|
|
83
|
+
// avoids turning an inconclusive probe into a false inactivity claim.
|
|
84
|
+
async function fetchHomepage(domain, { fetcher = safeFetchDetailed, timeoutMs = HOMEPAGE_FETCH_TIMEOUT_MS, selectedUrl, signal } = {}) {
|
|
85
|
+
const selected = selectedUrl === undefined ? null : prepareSelectedLookupUrl(selectedUrl, domain);
|
|
86
|
+
const requestTimeoutMs = Number.isInteger(timeoutMs) && timeoutMs >= 10 && timeoutMs <= HOMEPAGE_FETCH_TIMEOUT_MS
|
|
87
|
+
? timeoutMs
|
|
88
|
+
: HOMEPAGE_FETCH_TIMEOUT_MS;
|
|
89
|
+
const headers = whoisleuthRequestHeaders();
|
|
90
|
+
const failures = [];
|
|
91
|
+
const probeStartedAt = Date.now();
|
|
92
|
+
for (const requestUrl of selected ? [selected] : [`https://${domain}`, `http://${domain}`]) {
|
|
93
|
+
signal?.throwIfAborted();
|
|
94
|
+
const scheme = new URL(requestUrl).protocol.slice(0, -1);
|
|
95
|
+
const attemptStartedAt = Date.now();
|
|
96
|
+
const controller = new AbortController();
|
|
97
|
+
const timeout = setTimeout(() => controller.abort(), requestTimeoutMs);
|
|
98
|
+
try {
|
|
99
|
+
const fetched = await fetcher(requestUrl, { signal: signal ? AbortSignal.any([signal, controller.signal]) : controller.signal, headers });
|
|
100
|
+
const fetchedResponse = fetched instanceof Response ? fetched : fetched.response;
|
|
101
|
+
const detail = fetched && typeof fetched === 'object' && 'response' in fetched
|
|
102
|
+
? fetched
|
|
103
|
+
: {
|
|
104
|
+
response: fetchedResponse,
|
|
105
|
+
requestedUrl: requestUrl,
|
|
106
|
+
finalUrl: requestUrl,
|
|
107
|
+
redirectCount: 0,
|
|
108
|
+
redirectLimitReached: false,
|
|
109
|
+
hops: [{ url: requestUrl, status: fetchedResponse.status, location: null, durationMs: Date.now() - attemptStartedAt }],
|
|
110
|
+
durationMs: Date.now() - attemptStartedAt,
|
|
111
|
+
};
|
|
112
|
+
const res = detail.response;
|
|
113
|
+
const analysisBase = new URL(detail.finalUrl);
|
|
114
|
+
analysisBase.search = '';
|
|
115
|
+
analysisBase.hash = '';
|
|
116
|
+
// Retain an explicitly partial prefix when the body exceeds its bound;
|
|
117
|
+
// downstream analyses must not treat it as a complete page. The
|
|
118
|
+
// timeout stays armed through this read (cleared in `finally` below,
|
|
119
|
+
// not here) - a malicious site could otherwise send headers
|
|
120
|
+
// immediately and then trickle or stall the body forever, hanging
|
|
121
|
+
// this worker with no deadline once the timer above is disarmed.
|
|
122
|
+
if (res.ok) {
|
|
123
|
+
const body = await readTextCapped(res, MAX_HOMEPAGE_BYTES, { includeSha256: true });
|
|
124
|
+
return {
|
|
125
|
+
text: body.text,
|
|
126
|
+
status: 'fetched',
|
|
127
|
+
detail: `${selected ? 'Selected URL' : 'Homepage'} responded over ${scheme.toUpperCase()} (HTTP ${res.status}).`,
|
|
128
|
+
responsePolicy: analyzeResponsePolicyHeaders(res.headers),
|
|
129
|
+
technologyHeaders: captureTechnologyResponseHeaders(res.headers),
|
|
130
|
+
...(selected ? { analysisBaseUrl: analysisBase.toString() } : {}),
|
|
131
|
+
http: buildHttpObservation({ ...detail, durationMs: Date.now() - attemptStartedAt }, {
|
|
132
|
+
previousAttempts: failures,
|
|
133
|
+
capturedBodyBytes: body.bytesRead,
|
|
134
|
+
bodyInspected: true,
|
|
135
|
+
bodyTruncated: body.truncated,
|
|
136
|
+
bodySha256: body.sha256,
|
|
137
|
+
}),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
// A non-2xx HTTP response still conclusively proves that a web service
|
|
141
|
+
// answered on this domain. It may block this probe (403), require auth
|
|
142
|
+
// (401), have a broken homepage route (404), or be unhealthy (5xx), but
|
|
143
|
+
// none of those are equivalent to "no website". We cannot inspect its
|
|
144
|
+
// HTML signals, so preserve a separate responded status and release the
|
|
145
|
+
// unused body.
|
|
146
|
+
await res.body?.cancel().catch(() => { });
|
|
147
|
+
return {
|
|
148
|
+
text: null,
|
|
149
|
+
status: 'responded',
|
|
150
|
+
detail: `Web server responded over ${scheme.toUpperCase()} (HTTP ${res.status}); ${selected ? 'selected page' : 'homepage'} content was not available for inspection.`,
|
|
151
|
+
responsePolicy: analyzeResponsePolicyHeaders(res.headers),
|
|
152
|
+
technologyHeaders: captureTechnologyResponseHeaders(res.headers),
|
|
153
|
+
http: buildHttpObservation({ ...detail, durationMs: Date.now() - attemptStartedAt }, {
|
|
154
|
+
previousAttempts: failures,
|
|
155
|
+
capturedBodyBytes: 0,
|
|
156
|
+
bodyInspected: false,
|
|
157
|
+
}),
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
catch (err) {
|
|
161
|
+
signal?.throwIfAborted();
|
|
162
|
+
const error = errorRecord(err);
|
|
163
|
+
const reason = error.name === 'AbortError'
|
|
164
|
+
? `timed out after ${requestTimeoutMs} milliseconds`
|
|
165
|
+
: selected ? 'request failed; the selected URL was not retried at another path or scheme'
|
|
166
|
+
: nonEmptyErrorMessage(err, 'request failed')
|
|
167
|
+
.replace(/[\u0000-\u001f\u007f]+/g, ' ')
|
|
168
|
+
.slice(0, 180);
|
|
169
|
+
failures.push({ url: requestUrl, error: `${scheme.toUpperCase()} ${reason}` });
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
clearTimeout(timeout);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return {
|
|
176
|
+
text: null,
|
|
177
|
+
status: 'inconclusive',
|
|
178
|
+
detail: failures.length
|
|
179
|
+
? `Could not confirm ${selected ? 'selected URL' : 'homepage'} activity: ${failures.map((attempt) => attempt.error).join('; ')}.`
|
|
180
|
+
: 'Could not confirm homepage activity.',
|
|
181
|
+
http: failedHttpObservation(failures, { durationMs: Date.now() - probeStartedAt }),
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
function deriveWebsiteActivity(homepageStatus, hasFavicon, alreadyParked = false) {
|
|
185
|
+
if (alreadyParked)
|
|
186
|
+
return 'parked';
|
|
187
|
+
if (homepageStatus === 'fetched' || homepageStatus === 'responded' || hasFavicon)
|
|
188
|
+
return 'active';
|
|
189
|
+
return 'unreachable';
|
|
190
|
+
}
|
|
191
|
+
async function enrichWebsite(context, options, featurePolicy) {
|
|
192
|
+
options.signal?.throwIfAborted();
|
|
193
|
+
const { domain, observationHostname, selectedUrl, registration, registryDnsEvidence, nameservers, dnssec } = context;
|
|
194
|
+
const websiteProbeEnabled = featureDecision('website_probe', featurePolicy).enabled;
|
|
195
|
+
const collectDns = options.collectDnsIntelligence || collectDnsIntelligence;
|
|
196
|
+
const collectTls = options.collectTlsIntelligence || collectTlsIntelligence;
|
|
197
|
+
const fetchHomepageForDomain = options.fetchHomepage || fetchHomepage;
|
|
198
|
+
const fetchFaviconForDomain = options.fetchFaviconHash || fetchFaviconHash;
|
|
199
|
+
const dnsIntelligenceEnabled = featureDecision('dns_intelligence', featurePolicy).enabled;
|
|
200
|
+
const tlsIntelligenceEnabled = featureDecision('tls_intelligence', featurePolicy).enabled;
|
|
201
|
+
const deepScanComplete = ['rdap', 'whois', 'dns_intelligence', 'website_probe', 'tls_intelligence']
|
|
202
|
+
.every((feature) => featureDecision(feature, featurePolicy).enabled);
|
|
203
|
+
// Registered - look for for-sale/parked/website signals (homepage fetch,
|
|
204
|
+
// deep mode only), and check for a configured mail exchanger as a
|
|
205
|
+
// phishing-risk signal (a lookalike domain that can receive/send mail is
|
|
206
|
+
// capable of running credential-harvesting or BEC campaigns).
|
|
207
|
+
const nsSignal = registration.state === 'registered' ? nameservers.find(isParkingNameserver) : undefined;
|
|
208
|
+
let forSaleSignal = nsSignal ? `parking nameserver (${nsSignal})` : null;
|
|
209
|
+
let activityStatus = nsSignal && observationHostname === domain ? 'parked' : 'unknown';
|
|
210
|
+
// Homepage + bounded DNS evidence resolve in parallel; the favicon fetch is sequenced after
|
|
211
|
+
// the homepage so it can use any <link rel="icon"> the page declares (many
|
|
212
|
+
// sites serve no /favicon.ico and only point to a CDN PNG this way). One
|
|
213
|
+
// extra round-trip on the already-slow deep path, in exchange for finding
|
|
214
|
+
// favicons the bare /favicon.ico probe would miss.
|
|
215
|
+
const pending = [
|
|
216
|
+
websiteProbeEnabled ? fetchHomepageForDomain(observationHostname, {
|
|
217
|
+
...(selectedUrl ? { selectedUrl } : {}), ...(options.signal ? { signal: options.signal } : {}),
|
|
218
|
+
}).catch((err) => ({
|
|
219
|
+
text: null,
|
|
220
|
+
status: 'inconclusive',
|
|
221
|
+
detail: selectedUrl ? 'Could not confirm selected URL activity.'
|
|
222
|
+
: `Could not confirm homepage activity: ${String(err && err.message ? err.message : 'request failed').slice(0, 180)}.`,
|
|
223
|
+
http: failedHttpObservation([
|
|
224
|
+
{ url: selectedUrl ?? `https://${observationHostname}`, error: selectedUrl ? 'request failed' : String(err && err.message ? err.message : 'request failed') },
|
|
225
|
+
]),
|
|
226
|
+
})) : Promise.resolve({
|
|
227
|
+
text: null,
|
|
228
|
+
status: 'skipped',
|
|
229
|
+
detail: 'Website probing is disabled by deployment policy.',
|
|
230
|
+
http: skippedHttpObservation(),
|
|
231
|
+
}),
|
|
232
|
+
dnsIntelligenceEnabled
|
|
233
|
+
? collectDns(observationHostname, {
|
|
234
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
235
|
+
...(options.dnsResolvers ? { resolvers: options.dnsResolvers } : {}),
|
|
236
|
+
includeExtendedContext: options.includeExtendedDnsContext === true,
|
|
237
|
+
includeInheritedCaa: options.includeInheritedCaa === true,
|
|
238
|
+
registryEvidence: registryDnsEvidence,
|
|
239
|
+
...(observationHostname !== domain ? { registrationDomain: domain } : {}),
|
|
240
|
+
})
|
|
241
|
+
: Promise.resolve(skippedDnsIntelligence('DNS intelligence is disabled by deployment policy.', {
|
|
242
|
+
includeExtendedContext: options.includeExtendedDnsContext === true,
|
|
243
|
+
includeInheritedCaa: options.includeInheritedCaa === true,
|
|
244
|
+
})),
|
|
245
|
+
tlsIntelligenceEnabled
|
|
246
|
+
? collectTls(observationHostname, { ...(options.signal ? { signal: options.signal } : {}) })
|
|
247
|
+
: Promise.resolve(skippedTlsObservation()),
|
|
248
|
+
];
|
|
249
|
+
const [homepage, dnsIntelligence, tlsIntelligence] = await Promise.all(pending)
|
|
250
|
+
.finally(() => Promise.allSettled(pending));
|
|
251
|
+
options.signal?.throwIfAborted();
|
|
252
|
+
const page = homepage.text;
|
|
253
|
+
const pageBaseUrl = homepage.analysisBaseUrl ?? (typeof homepage.http?.finalUrl === 'string' ? homepage.http.finalUrl : `https://${observationHostname}/`);
|
|
254
|
+
let pageAnalysis = page ? analyzeStaticHtml(page, { baseUrl: pageBaseUrl, includeVisibleText: true }) : undefined;
|
|
255
|
+
let htmlSignals = {
|
|
256
|
+
domainSaleSignal: null,
|
|
257
|
+
pageTitle: null,
|
|
258
|
+
hasPasswordField: false,
|
|
259
|
+
hasExternalPasswordForm: null,
|
|
260
|
+
phishingLanguageMatch: null,
|
|
261
|
+
hasExternalFormAction: null,
|
|
262
|
+
externalAssetHosts: [],
|
|
263
|
+
cspMetaPolicy: null,
|
|
264
|
+
pageIdentity: null,
|
|
265
|
+
credentialSurfaceProfile: null,
|
|
266
|
+
structuredDataIdentity: null,
|
|
267
|
+
technologyProfile: null,
|
|
268
|
+
pageRoleProfile: null,
|
|
269
|
+
clientBehaviorProfile: null,
|
|
270
|
+
};
|
|
271
|
+
if (homepage.status === 'fetched') {
|
|
272
|
+
if (page) {
|
|
273
|
+
const responseContentType = homepage.http?.response?.contentType;
|
|
274
|
+
const pageIdentityEligible = typeof responseContentType !== 'string'
|
|
275
|
+
|| responseContentType.trim() === ''
|
|
276
|
+
|| /^(?:text\/html|application\/xhtml\+xml)(?:\s*;|$)/i.test(responseContentType.trim());
|
|
277
|
+
htmlSignals = await extractHtmlSignals(page, observationHostname, {
|
|
278
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
279
|
+
...(pageAnalysis ? { htmlAnalysis: pageAnalysis } : {}),
|
|
280
|
+
baseUrl: pageBaseUrl,
|
|
281
|
+
...(typeof homepage.http?.observedAt === 'string' ? { observedAt: homepage.http.observedAt } : {}),
|
|
282
|
+
sourceTruncated: homepage.http?.response?.bodyTruncated === true,
|
|
283
|
+
exactBodyHash: homepage.http?.response?.bodyHash,
|
|
284
|
+
httpServer: homepage.http?.response?.server,
|
|
285
|
+
responseHeaders: homepage.technologyHeaders,
|
|
286
|
+
activityStatus,
|
|
287
|
+
includePageIdentity: pageIdentityEligible,
|
|
288
|
+
includePublicationMetadata: options.includePublicationMetadata !== false,
|
|
289
|
+
...(options.includeCredentialSurfaceProfile !== undefined
|
|
290
|
+
? { includeCredentialSurfaceProfile: options.includeCredentialSurfaceProfile }
|
|
291
|
+
: {}),
|
|
292
|
+
...(options.includeStructuredDataIdentity !== undefined
|
|
293
|
+
? { includeStructuredDataIdentity: options.includeStructuredDataIdentity }
|
|
294
|
+
: {}),
|
|
295
|
+
...(options.includeTechnologyProfile !== undefined
|
|
296
|
+
? { includeTechnologyProfile: options.includeTechnologyProfile }
|
|
297
|
+
: {}),
|
|
298
|
+
});
|
|
299
|
+
if (pageIdentityEligible && htmlSignals.domainSaleSignal) {
|
|
300
|
+
if (!selectedUrl && observationHostname === domain)
|
|
301
|
+
forSaleSignal = forSaleSignal || htmlSignals.domainSaleSignal;
|
|
302
|
+
activityStatus = 'parked';
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
if (options.includeTechnologyProfile !== false && htmlSignals.technologyProfile === null
|
|
307
|
+
&& ['fetched', 'responded'].includes(homepage.status)) {
|
|
308
|
+
htmlSignals.technologyProfile = await analyzeWebsiteTechnology({
|
|
309
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
310
|
+
htmlAvailable: false,
|
|
311
|
+
httpServer: homepage.http?.response?.server,
|
|
312
|
+
responseHeaders: homepage.technologyHeaders,
|
|
313
|
+
observedAt: homepage.http?.observedAt,
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
// Complete document consumers before an optional network wait. Only the
|
|
317
|
+
// small icon projection, not every parsed element/token, crosses that wait.
|
|
318
|
+
const faviconEvidence = pageAnalysis ? {
|
|
319
|
+
iconLinks: pageAnalysis.iconLinks, effectiveBaseUrl: pageAnalysis.effectiveBaseUrl,
|
|
320
|
+
} : undefined;
|
|
321
|
+
const pageCollectionComplete = homepage.status === 'fetched' && homepage.http?.complete === true
|
|
322
|
+
&& homepage.http.response?.bodyTruncated !== true && Boolean(pageAnalysis)
|
|
323
|
+
&& !pageAnalysis?.inputLimitReached && !pageAnalysis?.tagLimitReached
|
|
324
|
+
&& !pageAnalysis?.visibleTextLimitReached && !pageAnalysis?.forms.truncated;
|
|
325
|
+
pageAnalysis = undefined;
|
|
326
|
+
const favicon = websiteProbeEnabled
|
|
327
|
+
? await fetchFaviconForDomain(observationHostname, {
|
|
328
|
+
baseUrl: pageBaseUrl, ...(faviconEvidence ? { htmlAnalysis: faviconEvidence } : {}),
|
|
329
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
330
|
+
}).catch(() => null)
|
|
331
|
+
: null;
|
|
332
|
+
options.signal?.throwIfAborted();
|
|
333
|
+
const faviconHash = favicon ? favicon.hash : null;
|
|
334
|
+
const faviconPHash = favicon ? favicon.phash : null;
|
|
335
|
+
const responsePolicy = qualifyResponsePolicyWithCspMeta(homepage.responsePolicy, htmlSignals.cspMetaPolicy);
|
|
336
|
+
const retainedHttp = options.includeDeliveryMetadata === false
|
|
337
|
+
? withoutHttpDeliveryMetadata(homepage.http)
|
|
338
|
+
: homepage.http;
|
|
339
|
+
const { cspMetaPolicy: _cspMetaPolicy, domainSaleSignal: _domainSaleSignal, ...retainedHtmlSignals } = htmlSignals;
|
|
340
|
+
const securityPosture = options.includeSecurityPosture === false ? null : analyzeWebsiteSecurityPosture({
|
|
341
|
+
http: homepage.http,
|
|
342
|
+
responsePolicy,
|
|
343
|
+
pageIdentity: htmlSignals.pageIdentity,
|
|
344
|
+
tls: tlsIntelligence,
|
|
345
|
+
dns: dnsIntelligence,
|
|
346
|
+
dnssec,
|
|
347
|
+
observedAt: homepage.http?.observedAt,
|
|
348
|
+
});
|
|
349
|
+
const redirectSaleSignal = forSaleRedirectSignal(homepage.http);
|
|
350
|
+
if (!selectedUrl && observationHostname === domain && !forSaleSignal && redirectSaleSignal) {
|
|
351
|
+
forSaleSignal = redirectSaleSignal;
|
|
352
|
+
activityStatus = 'parked';
|
|
353
|
+
}
|
|
354
|
+
const faviconProvedActive = !selectedUrl && homepage.status === 'inconclusive' && Boolean(favicon);
|
|
355
|
+
const websiteProbeStatus = faviconProvedActive ? 'responded' : homepage.status;
|
|
356
|
+
const websiteProbeDetail = faviconProvedActive
|
|
357
|
+
? `${homepage.detail} A favicon responded successfully, confirming an active web service.`
|
|
358
|
+
: homepage.detail;
|
|
359
|
+
if (websiteProbeEnabled) {
|
|
360
|
+
activityStatus = deriveWebsiteActivity(homepage.status, !selectedUrl && Boolean(favicon), activityStatus === 'parked');
|
|
361
|
+
}
|
|
362
|
+
const registrationResult = registration.state !== 'registered' ? registration
|
|
363
|
+
: !forSaleSignal ? {
|
|
364
|
+
...registration,
|
|
365
|
+
detail: 'source' in registration && registration.source === 'dns'
|
|
366
|
+
? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable.'
|
|
367
|
+
: selectedUrl ? 'The domain is registered. Website observations concern the explicitly selected URL.'
|
|
368
|
+
: observationHostname === domain ? 'Domain is registered and shows no for-sale signals.'
|
|
369
|
+
: 'The domain is registered. Website observations concern the separately identified hostname.',
|
|
370
|
+
} : {
|
|
371
|
+
...registration,
|
|
372
|
+
state: 'for_sale',
|
|
373
|
+
confidence: 'medium',
|
|
374
|
+
detail: `Detected a for-sale listing (${forSaleSignal}).`,
|
|
375
|
+
};
|
|
376
|
+
return {
|
|
377
|
+
...registrationResult,
|
|
378
|
+
activityStatus,
|
|
379
|
+
websiteProbeStatus,
|
|
380
|
+
websiteProbeDetail,
|
|
381
|
+
http: retainedHttp,
|
|
382
|
+
...(options.observationHostname !== undefined || selectedUrl ? { observationHostname } : {}),
|
|
383
|
+
...(selectedUrl ? { webObservationMode: 'selected_url' } : {}),
|
|
384
|
+
deepScanComplete,
|
|
385
|
+
webCollectionQuality: capturedWebCollectionQuality(!websiteProbeEnabled ? 'not_collected' : pageCollectionComplete ? 'complete'
|
|
386
|
+
: homepage.status === 'fetched' ? 'partial' : 'unavailable',
|
|
387
|
+
// The bounded favicon collector returns no result for both failed and
|
|
388
|
+
// non-image candidates. A miss therefore cannot prove an icon was removed.
|
|
389
|
+
!websiteProbeEnabled ? 'not_collected' : favicon ? 'complete' : 'unknown'),
|
|
390
|
+
faviconHash,
|
|
391
|
+
faviconPHash,
|
|
392
|
+
...retainedHtmlSignals,
|
|
393
|
+
securityPosture,
|
|
394
|
+
nameservers: nameservers.length || observationHostname !== domain ? nameservers : dnsIntelligence.records.ns,
|
|
395
|
+
dns: dnsIntelligence,
|
|
396
|
+
tls: tlsIntelligence,
|
|
397
|
+
hasMx: dnsIntelligence.hasMx,
|
|
398
|
+
hasNullMx: dnsIntelligence.hasNullMx,
|
|
399
|
+
mxHosts: dnsIntelligence.mxHosts,
|
|
400
|
+
hasSpf: dnsIntelligence.hasSpf,
|
|
401
|
+
hasDmarc: dnsIntelligence.hasDmarc,
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
export { enrichWebsite, fetchHomepage, deriveWebsiteActivity, forSaleRedirectSignal };
|
package/lib/whois-authority.mjs
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
// registry response referred by IANA can decide existence; later registrar
|
|
3
3
|
// output is diagnostic and cannot replace an inconclusive registry result.
|
|
4
4
|
import { normalizeWhoisChain } from "./whois-normalization.mjs";
|
|
5
|
+
import { domainToASCII } from 'node:url';
|
|
6
|
+
import { isValidAsciiHostname } from "../packages/contracts/domain-name.mjs";
|
|
5
7
|
import { parseIndentedWhoisValue, whoisFieldLimit, } from "./whois-values.mjs";
|
|
6
8
|
const LINE_NOT_FOUND_PATTERNS = Object.freeze([
|
|
7
9
|
/^[ \t]*(?:%{1,2}[ \t]*)?(?:error(?::\d+)?[ \t:.-]*)?no match(?:[ \t]+for(?:[ \t]+domain)?(?:[ \t]+["']?[a-z0-9.-]{1,253}["']?)?)?[.!]?[ \t]*$/im,
|
|
@@ -50,7 +52,41 @@ export function hasNicKgRegistrationEvidence(text) {
|
|
|
50
52
|
&& /^[ \t]*Record created[ \t]*:[ \t]*\S/im.test(text)
|
|
51
53
|
&& /^[ \t]*Name servers in the listed order[ \t]*:[ \t]*$/im.test(text);
|
|
52
54
|
}
|
|
53
|
-
function
|
|
55
|
+
function canonicalObjectName(value) {
|
|
56
|
+
const name = domainToASCII(value.trim().replace(/^["']|["'.]+$/gu, '')).toLowerCase();
|
|
57
|
+
return isValidAsciiHostname(name) ? name : null;
|
|
58
|
+
}
|
|
59
|
+
/** An echoed object may qualify this response, never a different query. */
|
|
60
|
+
export function whoisResponseMatchesDomain(text, requestedDomain) {
|
|
61
|
+
const expected = canonicalObjectName(requestedDomain);
|
|
62
|
+
if (!expected)
|
|
63
|
+
return false;
|
|
64
|
+
const identityPatterns = [
|
|
65
|
+
/^[ \t*]*(?:domain(?:[ \t]+name)?|domain_name|domainname)[ \t.]*:[ \t]*(\S[^\r\n]*)$/gimu,
|
|
66
|
+
/^[ \t]*\[Domain Name\][ \t]*(\S[^\r\n]*)$/gimu,
|
|
67
|
+
/^[ \t]*Domain[ \t]+([^\s()]+)[ \t]+\([A-Z][A-Z0-9_-]*\)[ \t]*$/gmu,
|
|
68
|
+
/^[ \t]*Domain(?: name)?[ \t]*:[ \t]*\r?\n(?:[ \t]*\r?\n)*[ \t]+(?:name:[ \t]*)?(\S[^\r\n]*)$/gimu,
|
|
69
|
+
];
|
|
70
|
+
for (const pattern of identityPatterns) {
|
|
71
|
+
for (const match of text.matchAll(pattern)) {
|
|
72
|
+
// A few registries publish Unicode plus its parenthesised ASCII form.
|
|
73
|
+
const names = (match[1] ?? '').split(/[ \t()]+/u).filter(Boolean);
|
|
74
|
+
if (!names.length || names.some(name => canonicalObjectName(name) !== expected))
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
for (const pattern of LINE_NOT_FOUND_PATTERNS) {
|
|
79
|
+
for (const match of text.matchAll(new RegExp(pattern.source, `${pattern.flags}g`))) {
|
|
80
|
+
const names = match[0].match(/[a-z0-9-]+(?:\.[a-z0-9-]+)+\.?/giu) ?? [];
|
|
81
|
+
if (names.some(name => canonicalObjectName(name) !== expected))
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
// Known complete negative line formats without an echoed name retain their
|
|
86
|
+
// existing meaning only within the exact IANA-referred registry response.
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
function classifyHopEvidence(hop, index, requestedDomain) {
|
|
54
90
|
if (hop.error)
|
|
55
91
|
return 'error';
|
|
56
92
|
const text = hop.response || '';
|
|
@@ -60,6 +96,8 @@ function classifyHopEvidence(hop, index) {
|
|
|
60
96
|
return 'rate_limited';
|
|
61
97
|
if (NZ_TEMPORARY_FAILURE_RE.test(text))
|
|
62
98
|
return 'rate_limited';
|
|
99
|
+
if (index > 0 && requestedDomain !== undefined && !whoisResponseMatchesDomain(text, requestedDomain))
|
|
100
|
+
return 'inconclusive';
|
|
63
101
|
if (NZ_NOT_FOUND_RE.test(text))
|
|
64
102
|
return 'negative';
|
|
65
103
|
if (LINE_NOT_FOUND_PATTERNS.some((pattern) => pattern.test(text))) {
|
|
@@ -95,12 +133,12 @@ function ianaRegistryReferral(hop) {
|
|
|
95
133
|
}
|
|
96
134
|
return null;
|
|
97
135
|
}
|
|
98
|
-
export function analyzeWhoisChainAuthority(chain) {
|
|
136
|
+
export function analyzeWhoisChainAuthority(chain, requestedDomain) {
|
|
99
137
|
const source = normalizeWhoisChain(chain);
|
|
100
138
|
const evidence = source.map((hop, index) => ({
|
|
101
139
|
server: hop.server,
|
|
102
140
|
index,
|
|
103
|
-
kind: classifyHopEvidence(hop, index),
|
|
141
|
+
kind: classifyHopEvidence(hop, index, requestedDomain),
|
|
104
142
|
}));
|
|
105
143
|
const failed = evidence.filter((item) => item.kind === 'error' || item.kind === 'rate_limited');
|
|
106
144
|
const registryReferral = ianaRegistryReferral(source[0]);
|
|
@@ -135,11 +173,11 @@ export function analyzeWhoisChainAuthority(chain) {
|
|
|
135
173
|
};
|
|
136
174
|
}
|
|
137
175
|
/** Collection status shared by final and incremental Lookup presentation. */
|
|
138
|
-
export function whoisCollectionStatus(chain) {
|
|
176
|
+
export function whoisCollectionStatus(chain, requestedDomain) {
|
|
139
177
|
const source = normalizeWhoisChain(chain);
|
|
140
178
|
if (!source.length || source[0]?.error)
|
|
141
179
|
return 'error';
|
|
142
180
|
if (source.length === 1)
|
|
143
181
|
return 'unsupported';
|
|
144
|
-
return analyzeWhoisChainAuthority(source).chainStatus;
|
|
182
|
+
return analyzeWhoisChainAuthority(source, requestedDomain).chainStatus;
|
|
145
183
|
}
|
package/lib/whois-chain.mjs
CHANGED
|
@@ -146,7 +146,16 @@ export async function buildWhoisChainUncached(query, options = {}) {
|
|
|
146
146
|
return chain;
|
|
147
147
|
}
|
|
148
148
|
export async function buildWhoisChain(query, options = {}) {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
149
|
+
let ownedCollection;
|
|
150
|
+
try {
|
|
151
|
+
return await cached(`whois:${query.toLowerCase()}`, () => {
|
|
152
|
+
ownedCollection = buildWhoisChainUncached(query, options);
|
|
153
|
+
return ownedCollection;
|
|
154
|
+
}, options.signal);
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
// Cache cancellation rejects promptly, but a started transport still belongs
|
|
158
|
+
// to this lookup lease. Drain it before its caller can release that lease.
|
|
159
|
+
await ownedCollection?.catch(() => { });
|
|
160
|
+
}
|
|
152
161
|
}
|
|
@@ -245,7 +245,9 @@ function applyWhoisCommonFormats(text, isRootHop, dialects, context) {
|
|
|
245
245
|
// sometimes just a hostname, sometimes "hostname ip.addr" - rather
|
|
246
246
|
// than a per-line "Name Server:" label. Only meaningful on non-root hops.
|
|
247
247
|
if (!isRootHop && nameservers.size === 0) {
|
|
248
|
-
|
|
248
|
+
// Whitespace after a colon belongs to that colon: two independent runs
|
|
249
|
+
// around an optional colon allow quadratic backtracking on invalid lines.
|
|
250
|
+
const headerMatch = text.match(/^[ \t*]*(?:Name ?[Ss]ervers|Domain Servers|DNS servers)[ \t.]*(?::[ \t]*)?$/mi);
|
|
249
251
|
if (headerMatch) {
|
|
250
252
|
let found = 0;
|
|
251
253
|
for (const line of text.slice((headerMatch.index ?? 0) + headerMatch[0].length).split('\n')) {
|
package/lib/whois-contacts.mjs
CHANGED
|
@@ -30,7 +30,9 @@ export function resolveIsnicRole(text, handle) {
|
|
|
30
30
|
if (!handle)
|
|
31
31
|
return null;
|
|
32
32
|
const escaped = handle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
33
|
-
|
|
33
|
+
// Capture the whole role value, including leading whitespace. A separate
|
|
34
|
+
// whitespace run overlaps this capture and backtracks when the handle fails.
|
|
35
|
+
const headerMatch = text.match(new RegExp(`^[ \\t]*role:(.+)\\r?\\n[ \\t]*nic-hdl:[ \\t]*${escaped}[ \\t]*$`, 'im'));
|
|
34
36
|
if (!headerMatch)
|
|
35
37
|
return null;
|
|
36
38
|
const lines = text
|
|
@@ -143,7 +145,7 @@ export function parseIndentedContactBlock(text, headerRe) {
|
|
|
143
145
|
if (lines.length === 0)
|
|
144
146
|
return null;
|
|
145
147
|
const remaining = [...lines];
|
|
146
|
-
const emailIndex = remaining.findIndex(
|
|
148
|
+
const emailIndex = remaining.findIndex(isIndentedContactEmail);
|
|
147
149
|
const email = emailIndex !== -1
|
|
148
150
|
? remaining.splice(emailIndex, 1)[0]
|
|
149
151
|
: null;
|
|
@@ -161,3 +163,12 @@ export function parseIndentedContactBlock(text, headerRe) {
|
|
|
161
163
|
truncated: allLines.length > lines.length,
|
|
162
164
|
};
|
|
163
165
|
}
|
|
166
|
+
// Preserve the legacy contact classifier (not a mailbox validator), without
|
|
167
|
+
// repeatedly trying a different separator in a long, dot-heavy failed match.
|
|
168
|
+
function isIndentedContactEmail(line) {
|
|
169
|
+
const at = line.indexOf('@');
|
|
170
|
+
if (at <= 0 || line.indexOf('@', at + 1) !== -1 || /\s/.test(line))
|
|
171
|
+
return false;
|
|
172
|
+
const dot = line.indexOf('.', at + 2);
|
|
173
|
+
return dot !== -1 && dot < line.length - 1;
|
|
174
|
+
}
|
|
@@ -21,7 +21,9 @@ export function normalizeWhoisChain(value) {
|
|
|
21
21
|
...(typeof hop.queriedAt === 'string' ? { queriedAt: hop.queriedAt.slice(0, 64) } : {}),
|
|
22
22
|
...(typeof hop.queryProfile === 'string' ? { queryProfile: hop.queryProfile.slice(0, 80) } : {}),
|
|
23
23
|
...(typeof hop.responseEncoding === 'string' ? { responseEncoding: hop.responseEncoding.slice(0, 40) } : {}),
|
|
24
|
-
|
|
24
|
+
// Parser and authority analysis share protocol line endings, while the
|
|
25
|
+
// original hop retains its wire text. Lone and embedded controls remain.
|
|
26
|
+
...(typeof hop.response === 'string' ? { response: hop.response.slice(0, MAX_WHOIS_BYTES).replace(/\r\n/g, '\n') } : {}),
|
|
25
27
|
...(typeof hop.error === 'string' ? { error: hop.error.slice(0, 1000) } : {}),
|
|
26
28
|
});
|
|
27
29
|
}
|
package/lib/whois-parser.mjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
// WHOIS response parsing. Transport, referral-chain and stable public exports
|
|
2
2
|
// remain owned by the facade in whois.mts.
|
|
3
3
|
import { registryDateIso } from "./registry-dates.mjs";
|
|
4
|
-
import { analyzeWhoisChainAuthority } from "./whois-authority.mjs";
|
|
4
|
+
import { analyzeWhoisChainAuthority, whoisResponseMatchesDomain } from "./whois-authority.mjs";
|
|
5
5
|
import { normalizeWhoisChain } from "./whois-normalization.mjs";
|
|
6
6
|
import { applyWhoisCommonFormats } from "./whois-common-formats.mjs";
|
|
7
7
|
import { applyWhoisDialects } from "./whois-dialects.mjs";
|
|
8
8
|
import { WHOIS_FIELD_PATTERNS, normalizedWhoisContact, } from "./whois-parser-support.mjs";
|
|
9
9
|
import { boundedWhoisValue, whoisFieldLimit, } from "./whois-values.mjs";
|
|
10
|
-
function parseWhoisChain(chain) {
|
|
10
|
+
function parseWhoisChain(chain, requestedDomain) {
|
|
11
11
|
const source = normalizeWhoisChain(chain);
|
|
12
12
|
const fields = {};
|
|
13
13
|
const truncatedFields = new Set();
|
|
@@ -18,6 +18,8 @@ function parseWhoisChain(chain) {
|
|
|
18
18
|
const text = hop.response;
|
|
19
19
|
if (!text)
|
|
20
20
|
return;
|
|
21
|
+
if (hopIndex > 0 && requestedDomain !== undefined && !whoisResponseMatchesDomain(text, requestedDomain))
|
|
22
|
+
return;
|
|
21
23
|
// hopIndex 0 is always whois.iana.org, whose "domain:"/"created:"/
|
|
22
24
|
// "changed:" fields describe the TLD's own root delegation record, not
|
|
23
25
|
// the queried domain - e.g. "created: 1992-08-14" for .gt is Guatemala's
|
|
@@ -118,7 +120,7 @@ function parseWhoisChain(chain) {
|
|
|
118
120
|
// Existence is decided authority-aware, not by a global "any hop said no
|
|
119
121
|
// match" flag: positive registry evidence is never overridden by a later
|
|
120
122
|
// registrar hop that failed, rate-limited, or returned "no match".
|
|
121
|
-
const authority = analyzeWhoisChainAuthority(source);
|
|
123
|
+
const authority = analyzeWhoisChainAuthority(source, requestedDomain);
|
|
122
124
|
return {
|
|
123
125
|
...fields,
|
|
124
126
|
nameservers: [...nameservers],
|
package/package.json
CHANGED
|
@@ -33,38 +33,39 @@
|
|
|
33
33
|
"bugs": {
|
|
34
34
|
"url": "https://github.com/slicedearth/whoisleuth/issues"
|
|
35
35
|
},
|
|
36
|
-
"version": "2.
|
|
36
|
+
"version": "2.6.0",
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@
|
|
38
|
+
"@nuintun/qrcode": "5.0.3",
|
|
39
|
+
"@peculiar/x509": "2.1.0",
|
|
40
|
+
"fast-png": "8.0.0",
|
|
39
41
|
"fflate": "0.8.3",
|
|
40
42
|
"maxmind": "5.0.7",
|
|
41
43
|
"parse5": "8.0.1",
|
|
44
|
+
"pdfjs-dist": "6.4.299",
|
|
45
|
+
"postal-mime": "4.0.4",
|
|
42
46
|
"reflect-metadata": "0.2.2",
|
|
43
|
-
"
|
|
44
|
-
"
|
|
47
|
+
"saxes": "6.0.0",
|
|
48
|
+
"tldts": "7.4.16",
|
|
49
|
+
"undici": "8.11.2"
|
|
45
50
|
},
|
|
46
51
|
"files": [
|
|
47
52
|
"bin/**/*.mjs",
|
|
48
53
|
"cli/**/*.mjs",
|
|
49
54
|
"lib/**/*.mjs",
|
|
50
55
|
"frontend/src/lib/**/*.js",
|
|
51
|
-
"packages
|
|
52
|
-
"
|
|
53
|
-
"packages/contracts/**/*.mjs",
|
|
54
|
-
"packages/evidence/**/*.mjs",
|
|
55
|
-
"packages/interchange/**/*.mjs",
|
|
56
|
-
"packages/investigation/**/*.mjs",
|
|
57
|
-
"packages/monitoring/**/*.mjs",
|
|
58
|
-
"packages/workspace/**/*.mjs",
|
|
56
|
+
"packages/**/*.mjs",
|
|
57
|
+
"README.md",
|
|
59
58
|
"docs/cli.md",
|
|
60
59
|
"docs/cli-reference.md",
|
|
60
|
+
"docs/contextual-reviews.md",
|
|
61
|
+
"docs/offline-intake-context.md",
|
|
62
|
+
"docs/infrastructure-observations.md",
|
|
61
63
|
"DISCLOSURE",
|
|
62
64
|
"LICENSE",
|
|
63
|
-
"LICENSES/*.txt",
|
|
64
65
|
"NOTICE",
|
|
65
|
-
"README.md",
|
|
66
66
|
"SECURITY.md",
|
|
67
67
|
"TRADEMARKS.md",
|
|
68
|
+
"LICENSES/Retire.js-Apache-2.0.txt",
|
|
68
69
|
"third-party-notices.txt"
|
|
69
70
|
],
|
|
70
71
|
"publishConfig": {
|