@slicedearth/whoisleuth-cli 1.32.2 → 1.36.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -5
- package/SECURITY.md +35 -0
- package/cli/arguments.mjs +180 -8
- package/cli/artifact-verify.mjs +32 -0
- package/cli/bulk-command-runner.mjs +3 -1
- package/cli/bulk-output.mjs +6 -1
- package/cli/bulk.mjs +1 -2
- package/cli/command-catalogue.mjs +33 -0
- package/cli/completion.mjs +70 -7
- package/cli/doctor.mjs +78 -43
- package/cli/export-evidence.mjs +9 -2
- package/cli/formatters/html.mjs +1 -1
- package/cli/formatters/terminal.mjs +13 -7
- package/cli/investigation-plan.mjs +112 -0
- package/cli/lookup-command-runner.mjs +8 -0
- package/cli/lookup-diff.mjs +16 -5
- package/cli/lookup-plan.mjs +99 -0
- package/cli/lookup-timeline.mjs +123 -0
- package/cli/mail-review.mjs +47 -2
- package/cli/manual.mjs +2 -1
- package/cli/offline-evidence-review.mjs +141 -0
- package/cli/registry-support.mjs +1 -4
- package/cli/runner.mjs +214 -11
- package/docs/cli.md +200 -45
- package/frontend/src/lib/analysis/bulk-domain-comparison.js +12 -6
- package/frontend/src/lib/analysis/bulk-retry-plan.js +3 -10
- package/frontend/src/lib/analysis/bulk-source-coverage.js +76 -0
- package/frontend/src/lib/analysis/http-summary.js +2 -1
- package/frontend/src/lib/analysis/utils.js +7 -40
- package/frontend/src/lib/analysis/website-snapshot-model.js +57 -3
- package/lib/availability.mjs +30 -10
- package/lib/bulk-limits.mjs +13 -0
- package/lib/csv.mjs +1 -1
- package/lib/ct-response-bounds.mjs +9 -0
- package/lib/ct-search.mjs +66 -12
- package/lib/dns-delegation-health.mjs +56 -5
- package/lib/dnssec-evidence-validation.mjs +235 -0
- package/lib/domain-control-manifest.mjs +339 -0
- package/lib/domain-posture.mjs +2 -1
- package/lib/encrypted-dns-contract.mjs +81 -0
- package/lib/evidence-export.mjs +17 -3
- package/lib/evidence-report-markdown.mjs +3 -3
- package/lib/evidence-report.mjs +3 -0
- package/lib/favicon.mjs +3 -8
- package/lib/html-signals.mjs +4 -0
- package/lib/http-evidence-bounds.mjs +4 -0
- package/lib/http-intelligence.mjs +2 -1
- package/lib/ip-prefix.mjs +54 -0
- package/lib/local-geoip-evidence.mjs +89 -0
- package/lib/outbound-identity.mjs +9 -0
- package/lib/outbound-request-bounds.mjs +9 -0
- package/lib/perceptual-hash-comparison.mjs +34 -0
- package/lib/perceptual-hash.mjs +3 -31
- package/lib/project-metadata.mjs +8 -0
- package/lib/provider-policy-admission.mjs +49 -0
- package/lib/rdap-search-workbench.mjs +157 -0
- package/lib/rdap.mjs +7 -2
- package/lib/registry-capabilities.mjs +31 -1
- package/lib/registry-capability-catalogue.mjs +11 -10
- package/lib/response-policy.mjs +81 -6
- package/lib/rpki-evidence.mjs +105 -0
- package/lib/safe-fetch.mjs +2 -1
- package/lib/semantic-version.mjs +54 -0
- package/lib/static-html-analysis.mjs +27 -0
- package/lib/threatfox-intelligence.mjs +10 -5
- package/lib/tlsa-evidence.mjs +194 -0
- package/lib/urlhaus-intelligence.mjs +10 -5
- package/lib/urlscan-intelligence.mjs +10 -5
- package/lib/website-security-posture.mjs +11 -4
- package/lib/whois-chain.mjs +14 -152
- package/lib/whois.mjs +2 -2
- package/package.json +7 -5
- package/third-party-notices.txt +2 -2
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { normalizeDomain } from "./case-model.js";
|
|
2
2
|
export const WEBSITE_SNAPSHOT_SCHEMA = 'whoisleuth.website-profile-snapshots';
|
|
3
|
-
export const WEBSITE_SNAPSHOT_SCHEMA_VERSION =
|
|
3
|
+
export const WEBSITE_SNAPSHOT_SCHEMA_VERSION = 4;
|
|
4
4
|
export const MAX_WEBSITE_SNAPSHOTS = 60;
|
|
5
5
|
export const MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN = 12;
|
|
6
6
|
export const MAX_WEBSITE_SNAPSHOT_STORE_BYTES = 512 * 1024;
|
|
@@ -65,6 +65,23 @@ function source(value) {
|
|
|
65
65
|
const state = text(item?.state, 40);
|
|
66
66
|
return sourceName && state ? { source: sourceName, state } : null;
|
|
67
67
|
}
|
|
68
|
+
function dependency(value) {
|
|
69
|
+
const item = record(value);
|
|
70
|
+
const recordType = text(item?.recordType, 10);
|
|
71
|
+
const target = normalizeDomain(item?.target);
|
|
72
|
+
const state = text(item?.state, 40);
|
|
73
|
+
if (!target
|
|
74
|
+
|| !['CNAME', 'HTTPS', 'NS', 'MX', 'HTTP'].includes(recordType)
|
|
75
|
+
|| !['candidate', 'unresolved', 'active', 'unsupported', 'false_positive'].includes(state))
|
|
76
|
+
return null;
|
|
77
|
+
return {
|
|
78
|
+
recordType: recordType,
|
|
79
|
+
target,
|
|
80
|
+
state: state,
|
|
81
|
+
qualification: text(item?.qualification, 80) || 'inconclusive',
|
|
82
|
+
serviceFamily: nullableText(item?.serviceFamily, 120),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
68
85
|
function nullableBoolean(value) {
|
|
69
86
|
return typeof value === 'boolean' ? value : null;
|
|
70
87
|
}
|
|
@@ -161,6 +178,7 @@ export function normalizeWebsiteProfileSnapshot(raw) {
|
|
|
161
178
|
identity: identity(value?.identity),
|
|
162
179
|
identityValues: identityValues(value?.identityValues),
|
|
163
180
|
sources: values(value?.sources, 16, source),
|
|
181
|
+
dependencies: values(value?.dependencies, 20, dependency),
|
|
164
182
|
certificate: certificate(value?.certificate),
|
|
165
183
|
};
|
|
166
184
|
}
|
|
@@ -242,13 +260,49 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
|
|
|
242
260
|
const before = normalizeWebsiteProfileSnapshot(beforeRaw);
|
|
243
261
|
const after = normalizeWebsiteProfileSnapshot(afterRaw);
|
|
244
262
|
if (!before || !after || before.domain !== after.domain) {
|
|
245
|
-
return {
|
|
263
|
+
return {
|
|
264
|
+
compatible: false,
|
|
265
|
+
changes: [{ field: 'snapshot', state: 'incomparable', before: before?.domain ?? null, after: after?.domain ?? null }],
|
|
266
|
+
dependencyTransitions: [],
|
|
267
|
+
};
|
|
246
268
|
}
|
|
247
269
|
const changes = [
|
|
248
270
|
...compareMap('technology', new Map(before.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}`])), new Map(after.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}`]))),
|
|
249
271
|
...compareMap('posture', new Map(before.posture.map((item) => [item.id, item.state])), new Map(after.posture.map((item) => [item.id, item.state]))),
|
|
250
272
|
...compareMap('source', new Map(before.sources.map((item) => [item.source, item.state])), new Map(after.sources.map((item) => [item.source, item.state]))),
|
|
273
|
+
...compareMap('dependency', new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`])), new Map(after.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`]))),
|
|
251
274
|
];
|
|
275
|
+
const dependencyTransitions = [];
|
|
276
|
+
const beforeDependencies = new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, item]));
|
|
277
|
+
const afterDependencies = new Map(after.dependencies.map((item) => [`${item.recordType}:${item.target}`, item]));
|
|
278
|
+
for (const key of [...new Set([...beforeDependencies.keys(), ...afterDependencies.keys()])].sort()) {
|
|
279
|
+
const left = beforeDependencies.get(key);
|
|
280
|
+
const right = afterDependencies.get(key);
|
|
281
|
+
if (left?.state === 'active' && right?.qualification === 'known_deprovision_pattern') {
|
|
282
|
+
dependencyTransitions.push({
|
|
283
|
+
target: right.target,
|
|
284
|
+
recordType: right.recordType,
|
|
285
|
+
state: 'active_to_deprovision_cue',
|
|
286
|
+
detail: 'An earlier active navigation target now matches a reviewed passive deprovision-page cue. Verify the service account and DNS manually; this does not establish claimability.',
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
else if (left?.state === 'active' && right?.state === 'unresolved') {
|
|
290
|
+
dependencyTransitions.push({
|
|
291
|
+
target: right.target,
|
|
292
|
+
recordType: right.recordType,
|
|
293
|
+
state: 'active_to_unresolved',
|
|
294
|
+
detail: 'An earlier active dependency is unresolved in the later retained observation. Resolver or collection failure remains possible.',
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
else if (!left && right) {
|
|
298
|
+
dependencyTransitions.push({ target: right.target, recordType: right.recordType, state: 'added', detail: 'This dependency first appears in the later retained snapshot.' });
|
|
299
|
+
}
|
|
300
|
+
else if (left && !right && after.complete) {
|
|
301
|
+
dependencyTransitions.push({ target: left.target, recordType: left.recordType, state: 'removed', detail: 'This dependency is not represented in the later complete snapshot. It may have been intentionally removed or replaced.' });
|
|
302
|
+
}
|
|
303
|
+
if (dependencyTransitions.length >= 20)
|
|
304
|
+
break;
|
|
305
|
+
}
|
|
252
306
|
for (const key of Object.keys(before.identity)) {
|
|
253
307
|
const left = before.identity[key];
|
|
254
308
|
const right = after.identity[key];
|
|
@@ -298,5 +352,5 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
|
|
|
298
352
|
if (before.complete !== after.complete || before.truncated !== after.truncated) {
|
|
299
353
|
changes.push({ field: 'completeness', state: 'incomparable', before: `${before.complete}/${before.truncated}`, after: `${after.complete}/${after.truncated}` });
|
|
300
354
|
}
|
|
301
|
-
return { compatible: true, changes };
|
|
355
|
+
return { compatible: true, changes, dependencyTransitions };
|
|
302
356
|
}
|
package/lib/availability.mjs
CHANGED
|
@@ -11,6 +11,7 @@ import { promises as dns } from 'node:dns';
|
|
|
11
11
|
import { fetchRdapRecord } from "./rdap.mjs";
|
|
12
12
|
import { buildWhoisChain, parseWhoisChain } from "./whois.mjs";
|
|
13
13
|
import { safeFetchDetailed, readTextCapped } from "./safe-fetch.mjs";
|
|
14
|
+
import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
|
|
14
15
|
import { collectDnsIntelligence, skippedDnsIntelligence } from "./dns-intelligence.mjs";
|
|
15
16
|
import { fetchFaviconHash } from "./favicon.mjs";
|
|
16
17
|
import { extractHtmlSignals } from "./html-signals.mjs";
|
|
@@ -19,9 +20,10 @@ import { buildHttpObservation, failedHttpObservation, skippedHttpObservation } f
|
|
|
19
20
|
import { collectTlsIntelligence, skippedTlsObservation } from "./tls-intelligence.mjs";
|
|
20
21
|
import { parseRegistryDate, registryDateIso } from "./registry-dates.mjs";
|
|
21
22
|
import { analyzeWebsiteSecurityPosture } from "./website-security-posture.mjs";
|
|
22
|
-
import { analyzeResponsePolicyHeaders } from "./response-policy.mjs";
|
|
23
|
+
import { analyzeResponsePolicyHeaders, qualifyResponsePolicyWithCspMeta, } from "./response-policy.mjs";
|
|
23
24
|
import { nonEmptyErrorMessage } from "./error-detail.mjs";
|
|
24
|
-
|
|
25
|
+
import { registryServiceAdmissionFor } from "./registry-capabilities.mjs";
|
|
26
|
+
import { HOMEPAGE_FETCH_TIMEOUT_MS, MAX_HOMEPAGE_BYTES, } from "./outbound-request-bounds.mjs";
|
|
25
27
|
const DNS_DELEGATION_TIMEOUT_MS = 4000;
|
|
26
28
|
const MAX_DELEGATION_NAMESERVERS = 50;
|
|
27
29
|
const MISSING_DNS_CODES = new Set(['ENODATA', 'ENOTFOUND', 'ENONAME', 'NXDOMAIN']);
|
|
@@ -196,14 +198,14 @@ async function checkDnsDelegation(domain, { resolver = dns.resolveNs } = {}) {
|
|
|
196
198
|
// an HTTP error can all produce the same null body. Keeping that distinction
|
|
197
199
|
// avoids turning an inconclusive probe into a false inactivity claim.
|
|
198
200
|
async function fetchHomepage(domain, { fetcher = safeFetchDetailed } = {}) {
|
|
199
|
-
const headers =
|
|
201
|
+
const headers = whoisleuthRequestHeaders();
|
|
200
202
|
const failures = [];
|
|
201
203
|
const probeStartedAt = Date.now();
|
|
202
204
|
for (const scheme of ['https', 'http']) {
|
|
203
205
|
const requestUrl = `${scheme}://${domain}`;
|
|
204
206
|
const attemptStartedAt = Date.now();
|
|
205
207
|
const controller = new AbortController();
|
|
206
|
-
const timeout = setTimeout(() => controller.abort(),
|
|
208
|
+
const timeout = setTimeout(() => controller.abort(), HOMEPAGE_FETCH_TIMEOUT_MS);
|
|
207
209
|
try {
|
|
208
210
|
const fetched = await fetcher(requestUrl, { signal: controller.signal, headers });
|
|
209
211
|
const fetchedResponse = fetched instanceof Response ? fetched : fetched.response;
|
|
@@ -298,6 +300,20 @@ function deriveWebsiteActivity(homepageStatus, hasFavicon, alreadyParked = false
|
|
|
298
300
|
return 'active';
|
|
299
301
|
return 'unreachable';
|
|
300
302
|
}
|
|
303
|
+
function registryPolicyDetail(domain, fast) {
|
|
304
|
+
const details = [];
|
|
305
|
+
const rdapAdmission = registryServiceAdmissionFor(domain, 'rdap');
|
|
306
|
+
if (rdapAdmission && !rdapAdmission.allowed) {
|
|
307
|
+
details.push('RDAP was not queried because the registry capability profile records no IANA-published RDAP service for this suffix.');
|
|
308
|
+
}
|
|
309
|
+
const whoisAdmission = fast ? null : registryServiceAdmissionFor(domain, 'whois');
|
|
310
|
+
if (whoisAdmission && !whoisAdmission.allowed) {
|
|
311
|
+
details.push(whoisAdmission.state === 'permission_required'
|
|
312
|
+
? 'WHOIS was not queried because registry permission or source authorization is required.'
|
|
313
|
+
: 'WHOIS was not queried because IANA publishes no domain WHOIS service for this suffix.');
|
|
314
|
+
}
|
|
315
|
+
return details.length ? ` ${details.join(' ')}` : '';
|
|
316
|
+
}
|
|
301
317
|
// fast: true skips the WHOIS fallback (no TCP:43 chain) and the homepage
|
|
302
318
|
// fetch (no for-sale/parking detection) - just RDAP plus the signals
|
|
303
319
|
// derivable from it (age, expiry proximity, privacy). Meant for scanning
|
|
@@ -497,14 +513,15 @@ async function checkDomainAvailability(domain, options = {}) {
|
|
|
497
513
|
const disabledDetail = disabledSources.length
|
|
498
514
|
? ` ${disabledSources.join(', ')} ${disabledSources.length === 1 ? 'is' : 'are'} disabled by deployment policy.`
|
|
499
515
|
: '';
|
|
516
|
+
const policyDetail = registryPolicyDetail(domain, fast);
|
|
500
517
|
return {
|
|
501
518
|
state: 'unknown',
|
|
502
519
|
confidence: 'low',
|
|
503
520
|
detail: fast
|
|
504
|
-
? `No enabled registration source produced a record or authoritative delegation. A fast scan cannot determine registration status.${disabledDetail}`
|
|
521
|
+
? `No enabled registration source produced a record or authoritative delegation. A fast scan cannot determine registration status.${disabledDetail}${policyDetail}`
|
|
505
522
|
: whoisParsed && whoisParsed.failedHop
|
|
506
|
-
? `WHOIS was inconclusive - a referral hop did not answer conclusively (${whoisParsed.failedHop})
|
|
507
|
-
: `No enabled registration source returned conclusive data or an authoritative DNS delegation.${disabledDetail}`,
|
|
523
|
+
? `WHOIS was inconclusive - a referral hop did not answer conclusively (${whoisParsed.failedHop}).${policyDetail}`
|
|
524
|
+
: `No enabled registration source returned conclusive data or an authoritative DNS delegation.${disabledDetail}${policyDetail}`,
|
|
508
525
|
...(!fast && whoisEnabled ? { source: 'whois' } : {}),
|
|
509
526
|
};
|
|
510
527
|
}
|
|
@@ -601,6 +618,7 @@ async function checkDomainAvailability(domain, options = {}) {
|
|
|
601
618
|
phishingLanguageMatch: null,
|
|
602
619
|
hasExternalFormAction: null,
|
|
603
620
|
externalAssetHosts: [],
|
|
621
|
+
cspMetaPolicy: null,
|
|
604
622
|
pageIdentity: null,
|
|
605
623
|
credentialSurfaceProfile: null,
|
|
606
624
|
structuredDataIdentity: null,
|
|
@@ -640,9 +658,11 @@ async function checkDomainAvailability(domain, options = {}) {
|
|
|
640
658
|
});
|
|
641
659
|
}
|
|
642
660
|
}
|
|
661
|
+
const responsePolicy = qualifyResponsePolicyWithCspMeta(homepage.responsePolicy, htmlSignals.cspMetaPolicy);
|
|
662
|
+
const { cspMetaPolicy: _cspMetaPolicy, ...retainedHtmlSignals } = htmlSignals;
|
|
643
663
|
const securityPosture = options.includeSecurityPosture === false ? null : analyzeWebsiteSecurityPosture({
|
|
644
664
|
http: homepage.http,
|
|
645
|
-
responsePolicy
|
|
665
|
+
responsePolicy,
|
|
646
666
|
pageIdentity: htmlSignals.pageIdentity,
|
|
647
667
|
tls: tlsIntelligence,
|
|
648
668
|
dns: dnsIntelligence,
|
|
@@ -676,7 +696,7 @@ async function checkDomainAvailability(domain, options = {}) {
|
|
|
676
696
|
deepScanComplete,
|
|
677
697
|
faviconHash,
|
|
678
698
|
faviconPHash,
|
|
679
|
-
...
|
|
699
|
+
...retainedHtmlSignals,
|
|
680
700
|
securityPosture,
|
|
681
701
|
...baseInfo,
|
|
682
702
|
nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
|
|
@@ -700,7 +720,7 @@ async function checkDomainAvailability(domain, options = {}) {
|
|
|
700
720
|
deepScanComplete,
|
|
701
721
|
faviconHash,
|
|
702
722
|
faviconPHash,
|
|
703
|
-
...
|
|
723
|
+
...retainedHtmlSignals,
|
|
704
724
|
securityPosture,
|
|
705
725
|
...baseInfo,
|
|
706
726
|
nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// Shared collection ceilings for both the browser and local CLI. These limits
|
|
2
|
+
// bound one analyst-triggered job; deployment rate and operation budgets remain
|
|
3
|
+
// separate controls around the individual lookup requests.
|
|
4
|
+
export const MAX_FAST_BULK_QUERIES = 500;
|
|
5
|
+
export const MAX_DEEP_BULK_QUERIES = 50;
|
|
6
|
+
export const MAX_FAST_BULK_CONCURRENCY = 8;
|
|
7
|
+
export const MAX_DEEP_BULK_CONCURRENCY = 3;
|
|
8
|
+
export function bulkQueryLimit(mode) {
|
|
9
|
+
return mode === 'deep' ? MAX_DEEP_BULK_QUERIES : MAX_FAST_BULK_QUERIES;
|
|
10
|
+
}
|
|
11
|
+
export function bulkConcurrencyLimit(mode) {
|
|
12
|
+
return mode === 'deep' ? MAX_DEEP_BULK_CONCURRENCY : MAX_FAST_BULK_CONCURRENCY;
|
|
13
|
+
}
|
package/lib/csv.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Shared bounded CSV serialization for browser and CLI exports. Values can
|
|
2
2
|
// contain upstream evidence, so normalize controls and prevent spreadsheet
|
|
3
3
|
// formula execution before applying ordinary CSV quoting.
|
|
4
|
-
const CSV_FORMULA_TRIGGER_RE = /^
|
|
4
|
+
const CSV_FORMULA_TRIGGER_RE = /^[=+\-@]/u;
|
|
5
5
|
export const MAX_CSV_CELL_LENGTH = 32_768;
|
|
6
6
|
function csvText(value) {
|
|
7
7
|
const raw = Array.isArray(value)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Shared response-contract bounds for certificate-log search. The collector
|
|
2
|
+
// and browser normalizer import these values so a valid backend response is
|
|
3
|
+
// not accidentally clipped by a stale client-side mirror.
|
|
4
|
+
export const MAX_CT_RESPONSE_RESULTS = 500;
|
|
5
|
+
export const MAX_CT_RESPONSE_HOSTNAMES_PER_MATCH = 50;
|
|
6
|
+
export const MAX_CT_RESPONSE_TIMESTAMP_LENGTH = 64;
|
|
7
|
+
export const MAX_CT_RESPONSE_CERTIFICATE_GROUPS = 100;
|
|
8
|
+
export const MAX_CT_RESPONSE_DOMAINS_PER_GROUP = 20;
|
|
9
|
+
export const MAX_CT_RESPONSE_HOSTNAMES_PER_GROUP = 30;
|
package/lib/ct-search.mjs
CHANGED
|
@@ -9,9 +9,11 @@
|
|
|
9
9
|
import { parse } from 'tldts';
|
|
10
10
|
import { normalizeCtQuery } from "./ct-query.mjs";
|
|
11
11
|
import { safeFetch, readTextCapped } from "./safe-fetch.mjs";
|
|
12
|
+
import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
|
|
12
13
|
import { createObservation } from "./observation.mjs";
|
|
14
|
+
import { MAX_CT_RESPONSE_CERTIFICATE_GROUPS, MAX_CT_RESPONSE_DOMAINS_PER_GROUP, MAX_CT_RESPONSE_HOSTNAMES_PER_GROUP, MAX_CT_RESPONSE_HOSTNAMES_PER_MATCH, MAX_CT_RESPONSE_RESULTS, MAX_CT_RESPONSE_TIMESTAMP_LENGTH, } from "./ct-response-bounds.mjs";
|
|
13
15
|
const CRT_SH_TIMEOUT_MS = 20000;
|
|
14
|
-
const MAX_RESULTS =
|
|
16
|
+
const MAX_RESULTS = MAX_CT_RESPONSE_RESULTS;
|
|
15
17
|
// crt.sh's response size scales with how many certificates ever matched the
|
|
16
18
|
// keyword, not with MAX_RESULTS - a broad single-word keyword can have
|
|
17
19
|
// millions of matching certificates, and unlike the domain-homepage fetch in
|
|
@@ -40,7 +42,10 @@ const MAX_TIMEOUT_RETRIES = 1;
|
|
|
40
42
|
const MAX_CT_ROWS = 50_000;
|
|
41
43
|
// Structured-match bounds.
|
|
42
44
|
const MAX_MATCHES = MAX_RESULTS; // 500
|
|
43
|
-
const MAX_HOSTNAMES_PER_MATCH =
|
|
45
|
+
const MAX_HOSTNAMES_PER_MATCH = MAX_CT_RESPONSE_HOSTNAMES_PER_MATCH;
|
|
46
|
+
const MAX_CERTIFICATE_GROUPS = MAX_CT_RESPONSE_CERTIFICATE_GROUPS;
|
|
47
|
+
const MAX_DOMAINS_PER_CERTIFICATE_GROUP = MAX_CT_RESPONSE_DOMAINS_PER_GROUP;
|
|
48
|
+
const MAX_HOSTNAMES_PER_CERTIFICATE_GROUP = MAX_CT_RESPONSE_HOSTNAMES_PER_GROUP;
|
|
44
49
|
const HOSTNAME_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/;
|
|
45
50
|
// Maximum length for a digit-only certificate ID string before it is
|
|
46
51
|
// rejected as overlong. Bounds decimal crt.sh record identifiers before
|
|
@@ -49,7 +54,7 @@ const MAX_ID_DIGITS = 32;
|
|
|
49
54
|
// Maximum length for a serial_number string before it is rejected.
|
|
50
55
|
const MAX_SERIAL_LENGTH = 128;
|
|
51
56
|
// Maximum length for an entry_timestamp string before it is rejected.
|
|
52
|
-
const MAX_TIMESTAMP_LENGTH =
|
|
57
|
+
const MAX_TIMESTAMP_LENGTH = MAX_CT_RESPONSE_TIMESTAMP_LENGTH;
|
|
53
58
|
function normalizeHostname(raw) {
|
|
54
59
|
let h = raw.trim().toLowerCase();
|
|
55
60
|
if (h.startsWith('*.'))
|
|
@@ -68,7 +73,7 @@ async function fetchCrtSh(keyword, attempt = 0, dependencies = {}) {
|
|
|
68
73
|
let res;
|
|
69
74
|
try {
|
|
70
75
|
res = await fetcher(`https://crt.sh/?q=${encodeURIComponent(keyword)}&output=json`, {
|
|
71
|
-
headers: { Accept: 'application/json'
|
|
76
|
+
headers: whoisleuthRequestHeaders({ Accept: 'application/json' }),
|
|
72
77
|
signal: controller.signal,
|
|
73
78
|
});
|
|
74
79
|
}
|
|
@@ -249,6 +254,7 @@ function validateEntryTimestamp(value) {
|
|
|
249
254
|
* lastObservedAt: string | null,
|
|
250
255
|
* certificateCount: number
|
|
251
256
|
* }>,
|
|
257
|
+
* certificateGroups: CtCertificateGroup[],
|
|
252
258
|
* truncated: boolean
|
|
253
259
|
* }}
|
|
254
260
|
*/
|
|
@@ -268,14 +274,31 @@ function summarizeCtResults(rows) {
|
|
|
268
274
|
// Map<registrableDomain, { hostnames: Set, certIds: Set,
|
|
269
275
|
// firstObservedAt: string|null, lastObservedAt: string|null }>
|
|
270
276
|
const groupMap = new Map();
|
|
277
|
+
const certificateMap = new Map();
|
|
271
278
|
for (let i = 0; i < rows.length; i++) {
|
|
272
279
|
const row = rows[i];
|
|
273
280
|
if (!row || typeof row !== 'object')
|
|
274
281
|
continue;
|
|
275
282
|
const certId = resolveCertId(row, i);
|
|
276
283
|
// Collect hostnames from name_value and common_name.
|
|
277
|
-
const
|
|
284
|
+
const rawBlob = `${row.name_value || ''}\n${row.common_name || ''}`;
|
|
285
|
+
const blob = rawBlob;
|
|
278
286
|
const ts = validateEntryTimestamp(row.entry_timestamp);
|
|
287
|
+
let certificate = certificateMap.get(certId);
|
|
288
|
+
if (!certificate) {
|
|
289
|
+
certificate = {
|
|
290
|
+
domains: new Set(),
|
|
291
|
+
hostnames: new Set(),
|
|
292
|
+
observedAt: ts,
|
|
293
|
+
wildcardObserved: false,
|
|
294
|
+
};
|
|
295
|
+
certificateMap.set(certId, certificate);
|
|
296
|
+
}
|
|
297
|
+
else if (ts !== null && (certificate.observedAt === null || ts > certificate.observedAt)) {
|
|
298
|
+
certificate.observedAt = ts;
|
|
299
|
+
}
|
|
300
|
+
if (rawBlob.split('\n').some((line) => line.trim().startsWith('*.')))
|
|
301
|
+
certificate.wildcardObserved = true;
|
|
279
302
|
for (const line of blob.split('\n')) {
|
|
280
303
|
const host = normalizeHostname(line);
|
|
281
304
|
if (!host)
|
|
@@ -293,6 +316,8 @@ function summarizeCtResults(rows) {
|
|
|
293
316
|
}
|
|
294
317
|
if (!regDomain)
|
|
295
318
|
continue;
|
|
319
|
+
certificate.hostnames.add(host);
|
|
320
|
+
certificate.domains.add(regDomain);
|
|
296
321
|
let group = groupMap.get(regDomain);
|
|
297
322
|
if (!group) {
|
|
298
323
|
group = {
|
|
@@ -348,11 +373,37 @@ function summarizeCtResults(rows) {
|
|
|
348
373
|
}
|
|
349
374
|
return a.domain.localeCompare(b.domain);
|
|
350
375
|
});
|
|
376
|
+
const certificateGroups = [...certificateMap.entries()]
|
|
377
|
+
.map(([certificateKey, group]) => ({
|
|
378
|
+
certificateKey: certificateKey.slice(0, 180),
|
|
379
|
+
domains: [...group.domains].sort().slice(0, MAX_DOMAINS_PER_CERTIFICATE_GROUP),
|
|
380
|
+
hostnames: [...group.hostnames].sort().slice(0, MAX_HOSTNAMES_PER_CERTIFICATE_GROUP),
|
|
381
|
+
observedAt: group.observedAt,
|
|
382
|
+
wildcardObserved: group.wildcardObserved,
|
|
383
|
+
domainCount: group.domains.size,
|
|
384
|
+
hostnameCount: group.hostnames.size,
|
|
385
|
+
}))
|
|
386
|
+
.filter((group) => group.domains.length > 0)
|
|
387
|
+
.sort((left, right) => (right.domainCount - left.domainCount
|
|
388
|
+
|| right.hostnameCount - left.hostnameCount
|
|
389
|
+
|| (right.observedAt ?? '').localeCompare(left.observedAt ?? '')
|
|
390
|
+
|| left.certificateKey.localeCompare(right.certificateKey)));
|
|
391
|
+
const certificateGroupsTruncated = certificateGroups.length > MAX_CERTIFICATE_GROUPS
|
|
392
|
+
|| certificateGroups.some((group) => (group.domainCount > MAX_DOMAINS_PER_CERTIFICATE_GROUP
|
|
393
|
+
|| group.hostnameCount > MAX_HOSTNAMES_PER_CERTIFICATE_GROUP));
|
|
351
394
|
const matchTruncated = matches.length > MAX_MATCHES;
|
|
352
395
|
const perMatchTruncated = matches.some((m) => (groupMap.get(m.domain)?.hostnames.size || 0) > MAX_HOSTNAMES_PER_MATCH);
|
|
353
396
|
return {
|
|
354
397
|
domains: sortedLegacy.slice(0, MAX_RESULTS),
|
|
355
398
|
matches: matches.slice(0, MAX_MATCHES),
|
|
399
|
+
certificateGroups: certificateGroups.slice(0, MAX_CERTIFICATE_GROUPS).map((group) => ({
|
|
400
|
+
certificateKey: group.certificateKey,
|
|
401
|
+
domains: group.domains,
|
|
402
|
+
hostnames: group.hostnames,
|
|
403
|
+
observedAt: group.observedAt,
|
|
404
|
+
wildcardObserved: group.wildcardObserved,
|
|
405
|
+
})),
|
|
406
|
+
certificateGroupsTruncated,
|
|
356
407
|
truncated: legacyTruncated || matchTruncated || perMatchTruncated,
|
|
357
408
|
};
|
|
358
409
|
}
|
|
@@ -364,12 +415,12 @@ async function searchCertificateTransparency(keyword, dependencies = {}) {
|
|
|
364
415
|
const trimmed = normalizeCtQuery(keyword);
|
|
365
416
|
if (!trimmed) {
|
|
366
417
|
return {
|
|
367
|
-
domains: [], certCount: 0, truncated: false, matches: [],
|
|
418
|
+
domains: [], certCount: 0, truncated: false, matches: [], certificateGroups: [], certificateGroupsTruncated: false,
|
|
368
419
|
observation: createObservation({
|
|
369
420
|
status: 'success', observedAt: new Date().toISOString(), source: 'certificate_transparency',
|
|
370
421
|
durationMs: Date.now() - startedAt, complete: true, truncated: false,
|
|
371
422
|
limitations: ['Certificate Transparency observations indicate public certificate logging, not current site activity or maliciousness.'],
|
|
372
|
-
diagnostics: { certificateRows: 0, matches: 0 },
|
|
423
|
+
diagnostics: { certificateRows: 0, matches: 0, certificateGroups: 0 },
|
|
373
424
|
}),
|
|
374
425
|
};
|
|
375
426
|
}
|
|
@@ -379,14 +430,17 @@ async function searchCertificateTransparency(keyword, dependencies = {}) {
|
|
|
379
430
|
certCount: data.length,
|
|
380
431
|
...summary,
|
|
381
432
|
observation: createObservation({
|
|
382
|
-
status: summary.truncated ? 'partial' : 'success',
|
|
433
|
+
status: summary.truncated || summary.certificateGroupsTruncated ? 'partial' : 'success',
|
|
383
434
|
observedAt: new Date().toISOString(),
|
|
384
435
|
source: 'certificate_transparency',
|
|
385
436
|
durationMs: Date.now() - startedAt,
|
|
386
|
-
complete: !summary.truncated,
|
|
387
|
-
truncated: summary.truncated,
|
|
388
|
-
limitations: [
|
|
389
|
-
|
|
437
|
+
complete: !summary.truncated && !summary.certificateGroupsTruncated,
|
|
438
|
+
truncated: summary.truncated || summary.certificateGroupsTruncated,
|
|
439
|
+
limitations: [
|
|
440
|
+
'Certificate Transparency observations indicate public certificate logging, not current site activity or maliciousness.',
|
|
441
|
+
...(summary.certificateGroupsTruncated ? ['The optional certificate-group projection was capped independently of the registrable-domain result set.'] : []),
|
|
442
|
+
],
|
|
443
|
+
diagnostics: { certificateRows: data.length, matches: summary.matches.length, certificateGroups: summary.certificateGroups.length, certificateGroupsTruncated: summary.certificateGroupsTruncated },
|
|
390
444
|
}),
|
|
391
445
|
};
|
|
392
446
|
}
|
|
@@ -13,6 +13,7 @@ const MAX_NAMESERVERS = 16;
|
|
|
13
13
|
const MAX_ERROR_LENGTH = 180;
|
|
14
14
|
const DNS_DELEGATION_TIMEOUT_MS = 2200;
|
|
15
15
|
const LAME_CODES = new Set(['ENOTAUTH', 'EREFUSED']);
|
|
16
|
+
const MAX_SOA_VALUE = 0xffff_ffff;
|
|
16
17
|
function record(value) {
|
|
17
18
|
return value && typeof value === 'object' && !Array.isArray(value)
|
|
18
19
|
? value
|
|
@@ -57,6 +58,27 @@ function errorDetail(error) {
|
|
|
57
58
|
.trim()
|
|
58
59
|
.slice(0, MAX_ERROR_LENGTH) || 'DNS query failed';
|
|
59
60
|
}
|
|
61
|
+
function soaProjection(value) {
|
|
62
|
+
const source = record(value);
|
|
63
|
+
const nsname = hostname(source.nsname);
|
|
64
|
+
const hostmaster = hostname(source.hostmaster);
|
|
65
|
+
const number = (field) => (typeof field === 'number'
|
|
66
|
+
&& Number.isSafeInteger(field)
|
|
67
|
+
&& field >= 0
|
|
68
|
+
&& field <= MAX_SOA_VALUE
|
|
69
|
+
? field
|
|
70
|
+
: null);
|
|
71
|
+
const projection = {
|
|
72
|
+
nsname,
|
|
73
|
+
hostmaster,
|
|
74
|
+
serial: number(source.serial),
|
|
75
|
+
refresh: number(source.refresh),
|
|
76
|
+
retry: number(source.retry),
|
|
77
|
+
expire: number(source.expire),
|
|
78
|
+
minttl: number(source.minttl),
|
|
79
|
+
};
|
|
80
|
+
return Object.values(projection).every((field) => field === null) ? null : projection;
|
|
81
|
+
}
|
|
60
82
|
function withTimeout(promise, timeoutMs) {
|
|
61
83
|
return new Promise((resolve, reject) => {
|
|
62
84
|
const timer = setTimeout(() => reject(Object.assign(new Error('DNS query timed out'), { code: 'ETIMEOUT' })), timeoutMs);
|
|
@@ -85,16 +107,20 @@ async function defaultAuthorityQuery(input) {
|
|
|
85
107
|
? soa.reason
|
|
86
108
|
: null;
|
|
87
109
|
if (reason !== null) {
|
|
110
|
+
const soaValue = soa.status === 'fulfilled' ? soaProjection(soa.value) : null;
|
|
88
111
|
return {
|
|
89
112
|
nameservers: nameservers.status === 'fulfilled' ? nameservers.value : [],
|
|
90
|
-
soaPrimary:
|
|
113
|
+
soaPrimary: soaValue?.nsname ?? null,
|
|
114
|
+
soa: soaValue,
|
|
91
115
|
errorCode: errorCode(reason),
|
|
92
116
|
error: errorDetail(reason),
|
|
93
117
|
};
|
|
94
118
|
}
|
|
119
|
+
const soaValue = soa.status === 'fulfilled' ? soaProjection(soa.value) : null;
|
|
95
120
|
return {
|
|
96
121
|
nameservers: nameservers.status === 'fulfilled' ? nameservers.value : [],
|
|
97
|
-
soaPrimary:
|
|
122
|
+
soaPrimary: soaValue?.nsname ?? null,
|
|
123
|
+
soa: soaValue,
|
|
98
124
|
errorCode: null,
|
|
99
125
|
error: null,
|
|
100
126
|
};
|
|
@@ -201,8 +227,10 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
|
|
|
201
227
|
const code = boundedText(query.errorCode, 40);
|
|
202
228
|
const error = boundedText(query.error, MAX_ERROR_LENGTH);
|
|
203
229
|
const nameservers = hostnames(query.nameservers);
|
|
204
|
-
const
|
|
205
|
-
|
|
230
|
+
const soa = soaProjection(query.soa)
|
|
231
|
+
?? soaProjection({ nsname: query.soaPrimary });
|
|
232
|
+
const soaPrimary = soa?.nsname ?? hostname(query.soaPrimary);
|
|
233
|
+
const hasAnswer = nameservers.length > 0 || soaPrimary !== null || soa !== null;
|
|
206
234
|
const state = error && hasAnswer
|
|
207
235
|
? 'partial'
|
|
208
236
|
: code && LAME_CODES.has(code)
|
|
@@ -210,7 +238,7 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
|
|
|
210
238
|
: error || !hasAnswer
|
|
211
239
|
? 'unreachable'
|
|
212
240
|
: 'success';
|
|
213
|
-
return { address, state, nameservers, soaPrimary, errorCode: code, error };
|
|
241
|
+
return { address, state, nameservers, soaPrimary, soa, errorCode: code, error };
|
|
214
242
|
}
|
|
215
243
|
catch (error) {
|
|
216
244
|
const code = errorCode(error);
|
|
@@ -219,6 +247,7 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
|
|
|
219
247
|
state: LAME_CODES.has(code) ? 'lame' : 'unreachable',
|
|
220
248
|
nameservers: [],
|
|
221
249
|
soaPrimary: null,
|
|
250
|
+
soa: null,
|
|
222
251
|
errorCode: code || null,
|
|
223
252
|
error: errorDetail(error),
|
|
224
253
|
};
|
|
@@ -240,6 +269,7 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
|
|
|
240
269
|
: 'unreachable',
|
|
241
270
|
nameservers: retained?.nameservers ?? [],
|
|
242
271
|
soaPrimary: retained?.soaPrimary ?? null,
|
|
272
|
+
soa: retained?.soa ?? null,
|
|
243
273
|
attempts,
|
|
244
274
|
};
|
|
245
275
|
}));
|
|
@@ -250,6 +280,18 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
|
|
|
250
280
|
const authoritySets = successfulAuthorities.map((authority) => authority.nameservers);
|
|
251
281
|
const inconsistentAuthoritySets = authoritySets.some((set) => (!sameSet(set, authoritySets[0] ?? [])
|
|
252
282
|
|| (parentNameservers.length > 0 && !sameSet(set, parentNameservers))));
|
|
283
|
+
const retainedSoa = authorities
|
|
284
|
+
.filter((authority) => authority.state === 'success' || authority.state === 'partial')
|
|
285
|
+
.map((authority) => ({ nameserver: authority.nameserver, soa: authority.soa }))
|
|
286
|
+
.filter((entry) => entry.soa !== null);
|
|
287
|
+
const soaSerials = [...new Set(retainedSoa
|
|
288
|
+
.map((entry) => entry.soa.serial)
|
|
289
|
+
.filter((serial) => serial !== null))];
|
|
290
|
+
const soaPrimaries = [...new Set(retainedSoa
|
|
291
|
+
.map((entry) => entry.soa.nsname)
|
|
292
|
+
.filter((primary) => primary !== null))];
|
|
293
|
+
const soaSerialConflict = soaSerials.length > 1;
|
|
294
|
+
const soaPrimaryConflict = soaPrimaries.length > 1;
|
|
253
295
|
const inBailiwick = candidates.filter((nameserver) => (nameserver === domain || nameserver.endsWith(`.${domain}`)));
|
|
254
296
|
const missingGlue = inBailiwick.filter((nameserver) => (!(registry.nameserverDetails.find((item) => item.name === nameserver)?.addresses.length)));
|
|
255
297
|
const dnssecMismatch = (registry.delegationSigned === true && registry.dsRecordCount === 0) || (registry.delegationSigned === false && registry.dsRecordCount > 0);
|
|
@@ -278,6 +320,15 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
|
|
|
278
320
|
: inconsistentAuthoritySets
|
|
279
321
|
? 'Direct nameserver answers are inconsistent'
|
|
280
322
|
: 'Direct nameserver answers agree with the observed parent view', successfulAuthorities.map((authority) => `${authority.nameserver}: ${authority.nameservers.join(', ') || 'no NS answer'}`).join(' | ') || 'No direct answers.', 'Align the NS set served by every authority with the intended parent delegation before cutover.'),
|
|
323
|
+
finding('authority_soa_consistency', 'Authoritative SOA consistency', retainedSoa.length < 2 || soaSerials.length < 1 || soaPrimaries.length < 1
|
|
324
|
+
? 'unknown'
|
|
325
|
+
: soaSerialConflict || soaPrimaryConflict ? 'warning' : 'healthy', retainedSoa.length < 2
|
|
326
|
+
? 'Fewer than two authority SOA observations were available'
|
|
327
|
+
: soaSerialConflict
|
|
328
|
+
? 'Authoritative servers published different SOA serials'
|
|
329
|
+
: soaPrimaryConflict
|
|
330
|
+
? 'Authoritative servers published different SOA primary nameservers'
|
|
331
|
+
: 'Observed authority SOA serials and primary nameservers agree', retainedSoa.map((entry) => (`${entry.nameserver}: serial ${entry.soa.serial ?? 'unavailable'}, primary ${entry.soa.nsname ?? 'unavailable'}, refresh ${entry.soa.refresh ?? 'unavailable'}, retry ${entry.soa.retry ?? 'unavailable'}, expire ${entry.soa.expire ?? 'unavailable'}`)).join(' | ') || 'No complete SOA observation was available.', 'Confirm zone propagation and the intended SOA primary before relying on a DNS change or secondary-server state.'),
|
|
281
332
|
finding('in_bailiwick_glue', 'In-bailiwick glue', !inBailiwick.length ? 'healthy' : missingGlue.length ? 'warning' : 'healthy', !inBailiwick.length
|
|
282
333
|
? 'No in-bailiwick nameserver requires registry glue'
|
|
283
334
|
: missingGlue.length
|