@slicedearth/whoisleuth-cli 1.32.2 → 1.36.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +43 -5
  2. package/SECURITY.md +35 -0
  3. package/cli/arguments.mjs +180 -8
  4. package/cli/artifact-verify.mjs +32 -0
  5. package/cli/bulk-command-runner.mjs +3 -1
  6. package/cli/bulk-output.mjs +6 -1
  7. package/cli/bulk.mjs +1 -2
  8. package/cli/command-catalogue.mjs +33 -0
  9. package/cli/completion.mjs +70 -7
  10. package/cli/doctor.mjs +78 -43
  11. package/cli/export-evidence.mjs +9 -2
  12. package/cli/formatters/html.mjs +1 -1
  13. package/cli/formatters/terminal.mjs +13 -7
  14. package/cli/investigation-plan.mjs +112 -0
  15. package/cli/lookup-command-runner.mjs +8 -0
  16. package/cli/lookup-diff.mjs +16 -5
  17. package/cli/lookup-plan.mjs +99 -0
  18. package/cli/lookup-timeline.mjs +123 -0
  19. package/cli/mail-review.mjs +47 -2
  20. package/cli/manual.mjs +2 -1
  21. package/cli/offline-evidence-review.mjs +141 -0
  22. package/cli/registry-support.mjs +1 -4
  23. package/cli/runner.mjs +214 -11
  24. package/docs/cli.md +200 -45
  25. package/frontend/src/lib/analysis/bulk-domain-comparison.js +12 -6
  26. package/frontend/src/lib/analysis/bulk-retry-plan.js +3 -10
  27. package/frontend/src/lib/analysis/bulk-source-coverage.js +76 -0
  28. package/frontend/src/lib/analysis/http-summary.js +2 -1
  29. package/frontend/src/lib/analysis/utils.js +7 -40
  30. package/frontend/src/lib/analysis/website-snapshot-model.js +57 -3
  31. package/lib/availability.mjs +30 -10
  32. package/lib/bulk-limits.mjs +13 -0
  33. package/lib/csv.mjs +1 -1
  34. package/lib/ct-response-bounds.mjs +9 -0
  35. package/lib/ct-search.mjs +66 -12
  36. package/lib/dns-delegation-health.mjs +56 -5
  37. package/lib/dnssec-evidence-validation.mjs +235 -0
  38. package/lib/domain-control-manifest.mjs +339 -0
  39. package/lib/domain-posture.mjs +2 -1
  40. package/lib/encrypted-dns-contract.mjs +81 -0
  41. package/lib/evidence-export.mjs +17 -3
  42. package/lib/evidence-report-markdown.mjs +3 -3
  43. package/lib/evidence-report.mjs +3 -0
  44. package/lib/favicon.mjs +3 -8
  45. package/lib/html-signals.mjs +4 -0
  46. package/lib/http-evidence-bounds.mjs +4 -0
  47. package/lib/http-intelligence.mjs +2 -1
  48. package/lib/ip-prefix.mjs +54 -0
  49. package/lib/local-geoip-evidence.mjs +89 -0
  50. package/lib/outbound-identity.mjs +9 -0
  51. package/lib/outbound-request-bounds.mjs +9 -0
  52. package/lib/perceptual-hash-comparison.mjs +34 -0
  53. package/lib/perceptual-hash.mjs +3 -31
  54. package/lib/project-metadata.mjs +8 -0
  55. package/lib/provider-policy-admission.mjs +49 -0
  56. package/lib/rdap-search-workbench.mjs +157 -0
  57. package/lib/rdap.mjs +7 -2
  58. package/lib/registry-capabilities.mjs +31 -1
  59. package/lib/registry-capability-catalogue.mjs +11 -10
  60. package/lib/response-policy.mjs +81 -6
  61. package/lib/rpki-evidence.mjs +105 -0
  62. package/lib/safe-fetch.mjs +2 -1
  63. package/lib/semantic-version.mjs +54 -0
  64. package/lib/static-html-analysis.mjs +27 -0
  65. package/lib/threatfox-intelligence.mjs +10 -5
  66. package/lib/tlsa-evidence.mjs +194 -0
  67. package/lib/urlhaus-intelligence.mjs +10 -5
  68. package/lib/urlscan-intelligence.mjs +10 -5
  69. package/lib/website-security-posture.mjs +11 -4
  70. package/lib/whois-chain.mjs +14 -152
  71. package/lib/whois.mjs +2 -2
  72. package/package.json +7 -5
  73. package/third-party-notices.txt +2 -2
@@ -1,6 +1,6 @@
1
1
  import { normalizeDomain } from "./case-model.js";
2
2
  export const WEBSITE_SNAPSHOT_SCHEMA = 'whoisleuth.website-profile-snapshots';
3
- export const WEBSITE_SNAPSHOT_SCHEMA_VERSION = 3;
3
+ export const WEBSITE_SNAPSHOT_SCHEMA_VERSION = 4;
4
4
  export const MAX_WEBSITE_SNAPSHOTS = 60;
5
5
  export const MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN = 12;
6
6
  export const MAX_WEBSITE_SNAPSHOT_STORE_BYTES = 512 * 1024;
@@ -65,6 +65,23 @@ function source(value) {
65
65
  const state = text(item?.state, 40);
66
66
  return sourceName && state ? { source: sourceName, state } : null;
67
67
  }
68
+ function dependency(value) {
69
+ const item = record(value);
70
+ const recordType = text(item?.recordType, 10);
71
+ const target = normalizeDomain(item?.target);
72
+ const state = text(item?.state, 40);
73
+ if (!target
74
+ || !['CNAME', 'HTTPS', 'NS', 'MX', 'HTTP'].includes(recordType)
75
+ || !['candidate', 'unresolved', 'active', 'unsupported', 'false_positive'].includes(state))
76
+ return null;
77
+ return {
78
+ recordType: recordType,
79
+ target,
80
+ state: state,
81
+ qualification: text(item?.qualification, 80) || 'inconclusive',
82
+ serviceFamily: nullableText(item?.serviceFamily, 120),
83
+ };
84
+ }
68
85
  function nullableBoolean(value) {
69
86
  return typeof value === 'boolean' ? value : null;
70
87
  }
@@ -161,6 +178,7 @@ export function normalizeWebsiteProfileSnapshot(raw) {
161
178
  identity: identity(value?.identity),
162
179
  identityValues: identityValues(value?.identityValues),
163
180
  sources: values(value?.sources, 16, source),
181
+ dependencies: values(value?.dependencies, 20, dependency),
164
182
  certificate: certificate(value?.certificate),
165
183
  };
166
184
  }
@@ -242,13 +260,49 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
242
260
  const before = normalizeWebsiteProfileSnapshot(beforeRaw);
243
261
  const after = normalizeWebsiteProfileSnapshot(afterRaw);
244
262
  if (!before || !after || before.domain !== after.domain) {
245
- return { compatible: false, changes: [{ field: 'snapshot', state: 'incomparable', before: before?.domain ?? null, after: after?.domain ?? null }] };
263
+ return {
264
+ compatible: false,
265
+ changes: [{ field: 'snapshot', state: 'incomparable', before: before?.domain ?? null, after: after?.domain ?? null }],
266
+ dependencyTransitions: [],
267
+ };
246
268
  }
247
269
  const changes = [
248
270
  ...compareMap('technology', new Map(before.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}`])), new Map(after.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}`]))),
249
271
  ...compareMap('posture', new Map(before.posture.map((item) => [item.id, item.state])), new Map(after.posture.map((item) => [item.id, item.state]))),
250
272
  ...compareMap('source', new Map(before.sources.map((item) => [item.source, item.state])), new Map(after.sources.map((item) => [item.source, item.state]))),
273
+ ...compareMap('dependency', new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`])), new Map(after.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`]))),
251
274
  ];
275
+ const dependencyTransitions = [];
276
+ const beforeDependencies = new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, item]));
277
+ const afterDependencies = new Map(after.dependencies.map((item) => [`${item.recordType}:${item.target}`, item]));
278
+ for (const key of [...new Set([...beforeDependencies.keys(), ...afterDependencies.keys()])].sort()) {
279
+ const left = beforeDependencies.get(key);
280
+ const right = afterDependencies.get(key);
281
+ if (left?.state === 'active' && right?.qualification === 'known_deprovision_pattern') {
282
+ dependencyTransitions.push({
283
+ target: right.target,
284
+ recordType: right.recordType,
285
+ state: 'active_to_deprovision_cue',
286
+ detail: 'An earlier active navigation target now matches a reviewed passive deprovision-page cue. Verify the service account and DNS manually; this does not establish claimability.',
287
+ });
288
+ }
289
+ else if (left?.state === 'active' && right?.state === 'unresolved') {
290
+ dependencyTransitions.push({
291
+ target: right.target,
292
+ recordType: right.recordType,
293
+ state: 'active_to_unresolved',
294
+ detail: 'An earlier active dependency is unresolved in the later retained observation. Resolver or collection failure remains possible.',
295
+ });
296
+ }
297
+ else if (!left && right) {
298
+ dependencyTransitions.push({ target: right.target, recordType: right.recordType, state: 'added', detail: 'This dependency first appears in the later retained snapshot.' });
299
+ }
300
+ else if (left && !right && after.complete) {
301
+ dependencyTransitions.push({ target: left.target, recordType: left.recordType, state: 'removed', detail: 'This dependency is not represented in the later complete snapshot. It may have been intentionally removed or replaced.' });
302
+ }
303
+ if (dependencyTransitions.length >= 20)
304
+ break;
305
+ }
252
306
  for (const key of Object.keys(before.identity)) {
253
307
  const left = before.identity[key];
254
308
  const right = after.identity[key];
@@ -298,5 +352,5 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
298
352
  if (before.complete !== after.complete || before.truncated !== after.truncated) {
299
353
  changes.push({ field: 'completeness', state: 'incomparable', before: `${before.complete}/${before.truncated}`, after: `${after.complete}/${after.truncated}` });
300
354
  }
301
- return { compatible: true, changes };
355
+ return { compatible: true, changes, dependencyTransitions };
302
356
  }
@@ -11,6 +11,7 @@ import { promises as dns } from 'node:dns';
11
11
  import { fetchRdapRecord } from "./rdap.mjs";
12
12
  import { buildWhoisChain, parseWhoisChain } from "./whois.mjs";
13
13
  import { safeFetchDetailed, readTextCapped } from "./safe-fetch.mjs";
14
+ import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
14
15
  import { collectDnsIntelligence, skippedDnsIntelligence } from "./dns-intelligence.mjs";
15
16
  import { fetchFaviconHash } from "./favicon.mjs";
16
17
  import { extractHtmlSignals } from "./html-signals.mjs";
@@ -19,9 +20,10 @@ import { buildHttpObservation, failedHttpObservation, skippedHttpObservation } f
19
20
  import { collectTlsIntelligence, skippedTlsObservation } from "./tls-intelligence.mjs";
20
21
  import { parseRegistryDate, registryDateIso } from "./registry-dates.mjs";
21
22
  import { analyzeWebsiteSecurityPosture } from "./website-security-posture.mjs";
22
- import { analyzeResponsePolicyHeaders } from "./response-policy.mjs";
23
+ import { analyzeResponsePolicyHeaders, qualifyResponsePolicyWithCspMeta, } from "./response-policy.mjs";
23
24
  import { nonEmptyErrorMessage } from "./error-detail.mjs";
24
- const MAX_HOMEPAGE_BYTES = 300000;
25
+ import { registryServiceAdmissionFor } from "./registry-capabilities.mjs";
26
+ import { HOMEPAGE_FETCH_TIMEOUT_MS, MAX_HOMEPAGE_BYTES, } from "./outbound-request-bounds.mjs";
25
27
  const DNS_DELEGATION_TIMEOUT_MS = 4000;
26
28
  const MAX_DELEGATION_NAMESERVERS = 50;
27
29
  const MISSING_DNS_CODES = new Set(['ENODATA', 'ENOTFOUND', 'ENONAME', 'NXDOMAIN']);
@@ -196,14 +198,14 @@ async function checkDnsDelegation(domain, { resolver = dns.resolveNs } = {}) {
196
198
  // an HTTP error can all produce the same null body. Keeping that distinction
197
199
  // avoids turning an inconclusive probe into a false inactivity claim.
198
200
  async function fetchHomepage(domain, { fetcher = safeFetchDetailed } = {}) {
199
- const headers = { 'User-Agent': 'Mozilla/5.0 (compatible; DomainStatusChecker/1.0)' };
201
+ const headers = whoisleuthRequestHeaders();
200
202
  const failures = [];
201
203
  const probeStartedAt = Date.now();
202
204
  for (const scheme of ['https', 'http']) {
203
205
  const requestUrl = `${scheme}://${domain}`;
204
206
  const attemptStartedAt = Date.now();
205
207
  const controller = new AbortController();
206
- const timeout = setTimeout(() => controller.abort(), 6000);
208
+ const timeout = setTimeout(() => controller.abort(), HOMEPAGE_FETCH_TIMEOUT_MS);
207
209
  try {
208
210
  const fetched = await fetcher(requestUrl, { signal: controller.signal, headers });
209
211
  const fetchedResponse = fetched instanceof Response ? fetched : fetched.response;
@@ -298,6 +300,20 @@ function deriveWebsiteActivity(homepageStatus, hasFavicon, alreadyParked = false
298
300
  return 'active';
299
301
  return 'unreachable';
300
302
  }
303
+ function registryPolicyDetail(domain, fast) {
304
+ const details = [];
305
+ const rdapAdmission = registryServiceAdmissionFor(domain, 'rdap');
306
+ if (rdapAdmission && !rdapAdmission.allowed) {
307
+ details.push('RDAP was not queried because the registry capability profile records no IANA-published RDAP service for this suffix.');
308
+ }
309
+ const whoisAdmission = fast ? null : registryServiceAdmissionFor(domain, 'whois');
310
+ if (whoisAdmission && !whoisAdmission.allowed) {
311
+ details.push(whoisAdmission.state === 'permission_required'
312
+ ? 'WHOIS was not queried because registry permission or source authorization is required.'
313
+ : 'WHOIS was not queried because IANA publishes no domain WHOIS service for this suffix.');
314
+ }
315
+ return details.length ? ` ${details.join(' ')}` : '';
316
+ }
301
317
  // fast: true skips the WHOIS fallback (no TCP:43 chain) and the homepage
302
318
  // fetch (no for-sale/parking detection) - just RDAP plus the signals
303
319
  // derivable from it (age, expiry proximity, privacy). Meant for scanning
@@ -497,14 +513,15 @@ async function checkDomainAvailability(domain, options = {}) {
497
513
  const disabledDetail = disabledSources.length
498
514
  ? ` ${disabledSources.join(', ')} ${disabledSources.length === 1 ? 'is' : 'are'} disabled by deployment policy.`
499
515
  : '';
516
+ const policyDetail = registryPolicyDetail(domain, fast);
500
517
  return {
501
518
  state: 'unknown',
502
519
  confidence: 'low',
503
520
  detail: fast
504
- ? `No enabled registration source produced a record or authoritative delegation. A fast scan cannot determine registration status.${disabledDetail}`
521
+ ? `No enabled registration source produced a record or authoritative delegation. A fast scan cannot determine registration status.${disabledDetail}${policyDetail}`
505
522
  : whoisParsed && whoisParsed.failedHop
506
- ? `WHOIS was inconclusive - a referral hop did not answer conclusively (${whoisParsed.failedHop}).`
507
- : `No enabled registration source returned conclusive data or an authoritative DNS delegation.${disabledDetail}`,
523
+ ? `WHOIS was inconclusive - a referral hop did not answer conclusively (${whoisParsed.failedHop}).${policyDetail}`
524
+ : `No enabled registration source returned conclusive data or an authoritative DNS delegation.${disabledDetail}${policyDetail}`,
508
525
  ...(!fast && whoisEnabled ? { source: 'whois' } : {}),
509
526
  };
510
527
  }
@@ -601,6 +618,7 @@ async function checkDomainAvailability(domain, options = {}) {
601
618
  phishingLanguageMatch: null,
602
619
  hasExternalFormAction: null,
603
620
  externalAssetHosts: [],
621
+ cspMetaPolicy: null,
604
622
  pageIdentity: null,
605
623
  credentialSurfaceProfile: null,
606
624
  structuredDataIdentity: null,
@@ -640,9 +658,11 @@ async function checkDomainAvailability(domain, options = {}) {
640
658
  });
641
659
  }
642
660
  }
661
+ const responsePolicy = qualifyResponsePolicyWithCspMeta(homepage.responsePolicy, htmlSignals.cspMetaPolicy);
662
+ const { cspMetaPolicy: _cspMetaPolicy, ...retainedHtmlSignals } = htmlSignals;
643
663
  const securityPosture = options.includeSecurityPosture === false ? null : analyzeWebsiteSecurityPosture({
644
664
  http: homepage.http,
645
- responsePolicy: homepage.responsePolicy,
665
+ responsePolicy,
646
666
  pageIdentity: htmlSignals.pageIdentity,
647
667
  tls: tlsIntelligence,
648
668
  dns: dnsIntelligence,
@@ -676,7 +696,7 @@ async function checkDomainAvailability(domain, options = {}) {
676
696
  deepScanComplete,
677
697
  faviconHash,
678
698
  faviconPHash,
679
- ...htmlSignals,
699
+ ...retainedHtmlSignals,
680
700
  securityPosture,
681
701
  ...baseInfo,
682
702
  nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
@@ -700,7 +720,7 @@ async function checkDomainAvailability(domain, options = {}) {
700
720
  deepScanComplete,
701
721
  faviconHash,
702
722
  faviconPHash,
703
- ...htmlSignals,
723
+ ...retainedHtmlSignals,
704
724
  securityPosture,
705
725
  ...baseInfo,
706
726
  nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
@@ -0,0 +1,13 @@
1
+ // Shared collection ceilings for both the browser and local CLI. These limits
2
+ // bound one analyst-triggered job; deployment rate and operation budgets remain
3
+ // separate controls around the individual lookup requests.
4
+ export const MAX_FAST_BULK_QUERIES = 500;
5
+ export const MAX_DEEP_BULK_QUERIES = 50;
6
+ export const MAX_FAST_BULK_CONCURRENCY = 8;
7
+ export const MAX_DEEP_BULK_CONCURRENCY = 3;
8
+ export function bulkQueryLimit(mode) {
9
+ return mode === 'deep' ? MAX_DEEP_BULK_QUERIES : MAX_FAST_BULK_QUERIES;
10
+ }
11
+ export function bulkConcurrencyLimit(mode) {
12
+ return mode === 'deep' ? MAX_DEEP_BULK_CONCURRENCY : MAX_FAST_BULK_CONCURRENCY;
13
+ }
package/lib/csv.mjs CHANGED
@@ -1,7 +1,7 @@
1
1
  // Shared bounded CSV serialization for browser and CLI exports. Values can
2
2
  // contain upstream evidence, so normalize controls and prevent spreadsheet
3
3
  // formula execution before applying ordinary CSV quoting.
4
- const CSV_FORMULA_TRIGGER_RE = /^(?:[\t\r\n ]*[=+\-@]|[\t\r\n])/u;
4
+ const CSV_FORMULA_TRIGGER_RE = /^[=+\-@]/u;
5
5
  export const MAX_CSV_CELL_LENGTH = 32_768;
6
6
  function csvText(value) {
7
7
  const raw = Array.isArray(value)
@@ -0,0 +1,9 @@
1
+ // Shared response-contract bounds for certificate-log search. The collector
2
+ // and browser normalizer import these values so a valid backend response is
3
+ // not accidentally clipped by a stale client-side mirror.
4
+ export const MAX_CT_RESPONSE_RESULTS = 500;
5
+ export const MAX_CT_RESPONSE_HOSTNAMES_PER_MATCH = 50;
6
+ export const MAX_CT_RESPONSE_TIMESTAMP_LENGTH = 64;
7
+ export const MAX_CT_RESPONSE_CERTIFICATE_GROUPS = 100;
8
+ export const MAX_CT_RESPONSE_DOMAINS_PER_GROUP = 20;
9
+ export const MAX_CT_RESPONSE_HOSTNAMES_PER_GROUP = 30;
package/lib/ct-search.mjs CHANGED
@@ -9,9 +9,11 @@
9
9
  import { parse } from 'tldts';
10
10
  import { normalizeCtQuery } from "./ct-query.mjs";
11
11
  import { safeFetch, readTextCapped } from "./safe-fetch.mjs";
12
+ import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
12
13
  import { createObservation } from "./observation.mjs";
14
+ import { MAX_CT_RESPONSE_CERTIFICATE_GROUPS, MAX_CT_RESPONSE_DOMAINS_PER_GROUP, MAX_CT_RESPONSE_HOSTNAMES_PER_GROUP, MAX_CT_RESPONSE_HOSTNAMES_PER_MATCH, MAX_CT_RESPONSE_RESULTS, MAX_CT_RESPONSE_TIMESTAMP_LENGTH, } from "./ct-response-bounds.mjs";
13
15
  const CRT_SH_TIMEOUT_MS = 20000;
14
- const MAX_RESULTS = 500;
16
+ const MAX_RESULTS = MAX_CT_RESPONSE_RESULTS;
15
17
  // crt.sh's response size scales with how many certificates ever matched the
16
18
  // keyword, not with MAX_RESULTS - a broad single-word keyword can have
17
19
  // millions of matching certificates, and unlike the domain-homepage fetch in
@@ -40,7 +42,10 @@ const MAX_TIMEOUT_RETRIES = 1;
40
42
  const MAX_CT_ROWS = 50_000;
41
43
  // Structured-match bounds.
42
44
  const MAX_MATCHES = MAX_RESULTS; // 500
43
- const MAX_HOSTNAMES_PER_MATCH = 50;
45
+ const MAX_HOSTNAMES_PER_MATCH = MAX_CT_RESPONSE_HOSTNAMES_PER_MATCH;
46
+ const MAX_CERTIFICATE_GROUPS = MAX_CT_RESPONSE_CERTIFICATE_GROUPS;
47
+ const MAX_DOMAINS_PER_CERTIFICATE_GROUP = MAX_CT_RESPONSE_DOMAINS_PER_GROUP;
48
+ const MAX_HOSTNAMES_PER_CERTIFICATE_GROUP = MAX_CT_RESPONSE_HOSTNAMES_PER_GROUP;
44
49
  const HOSTNAME_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/;
45
50
  // Maximum length for a digit-only certificate ID string before it is
46
51
  // rejected as overlong. Bounds decimal crt.sh record identifiers before
@@ -49,7 +54,7 @@ const MAX_ID_DIGITS = 32;
49
54
  // Maximum length for a serial_number string before it is rejected.
50
55
  const MAX_SERIAL_LENGTH = 128;
51
56
  // Maximum length for an entry_timestamp string before it is rejected.
52
- const MAX_TIMESTAMP_LENGTH = 64;
57
+ const MAX_TIMESTAMP_LENGTH = MAX_CT_RESPONSE_TIMESTAMP_LENGTH;
53
58
  function normalizeHostname(raw) {
54
59
  let h = raw.trim().toLowerCase();
55
60
  if (h.startsWith('*.'))
@@ -68,7 +73,7 @@ async function fetchCrtSh(keyword, attempt = 0, dependencies = {}) {
68
73
  let res;
69
74
  try {
70
75
  res = await fetcher(`https://crt.sh/?q=${encodeURIComponent(keyword)}&output=json`, {
71
- headers: { Accept: 'application/json', 'User-Agent': 'Mozilla/5.0 (compatible; DomainStatusChecker/1.0)' },
76
+ headers: whoisleuthRequestHeaders({ Accept: 'application/json' }),
72
77
  signal: controller.signal,
73
78
  });
74
79
  }
@@ -249,6 +254,7 @@ function validateEntryTimestamp(value) {
249
254
  * lastObservedAt: string | null,
250
255
  * certificateCount: number
251
256
  * }>,
257
+ * certificateGroups: CtCertificateGroup[],
252
258
  * truncated: boolean
253
259
  * }}
254
260
  */
@@ -268,14 +274,31 @@ function summarizeCtResults(rows) {
268
274
  // Map<registrableDomain, { hostnames: Set, certIds: Set,
269
275
  // firstObservedAt: string|null, lastObservedAt: string|null }>
270
276
  const groupMap = new Map();
277
+ const certificateMap = new Map();
271
278
  for (let i = 0; i < rows.length; i++) {
272
279
  const row = rows[i];
273
280
  if (!row || typeof row !== 'object')
274
281
  continue;
275
282
  const certId = resolveCertId(row, i);
276
283
  // Collect hostnames from name_value and common_name.
277
- const blob = `${row.name_value || ''}\n${row.common_name || ''}`;
284
+ const rawBlob = `${row.name_value || ''}\n${row.common_name || ''}`;
285
+ const blob = rawBlob;
278
286
  const ts = validateEntryTimestamp(row.entry_timestamp);
287
+ let certificate = certificateMap.get(certId);
288
+ if (!certificate) {
289
+ certificate = {
290
+ domains: new Set(),
291
+ hostnames: new Set(),
292
+ observedAt: ts,
293
+ wildcardObserved: false,
294
+ };
295
+ certificateMap.set(certId, certificate);
296
+ }
297
+ else if (ts !== null && (certificate.observedAt === null || ts > certificate.observedAt)) {
298
+ certificate.observedAt = ts;
299
+ }
300
+ if (rawBlob.split('\n').some((line) => line.trim().startsWith('*.')))
301
+ certificate.wildcardObserved = true;
279
302
  for (const line of blob.split('\n')) {
280
303
  const host = normalizeHostname(line);
281
304
  if (!host)
@@ -293,6 +316,8 @@ function summarizeCtResults(rows) {
293
316
  }
294
317
  if (!regDomain)
295
318
  continue;
319
+ certificate.hostnames.add(host);
320
+ certificate.domains.add(regDomain);
296
321
  let group = groupMap.get(regDomain);
297
322
  if (!group) {
298
323
  group = {
@@ -348,11 +373,37 @@ function summarizeCtResults(rows) {
348
373
  }
349
374
  return a.domain.localeCompare(b.domain);
350
375
  });
376
+ const certificateGroups = [...certificateMap.entries()]
377
+ .map(([certificateKey, group]) => ({
378
+ certificateKey: certificateKey.slice(0, 180),
379
+ domains: [...group.domains].sort().slice(0, MAX_DOMAINS_PER_CERTIFICATE_GROUP),
380
+ hostnames: [...group.hostnames].sort().slice(0, MAX_HOSTNAMES_PER_CERTIFICATE_GROUP),
381
+ observedAt: group.observedAt,
382
+ wildcardObserved: group.wildcardObserved,
383
+ domainCount: group.domains.size,
384
+ hostnameCount: group.hostnames.size,
385
+ }))
386
+ .filter((group) => group.domains.length > 0)
387
+ .sort((left, right) => (right.domainCount - left.domainCount
388
+ || right.hostnameCount - left.hostnameCount
389
+ || (right.observedAt ?? '').localeCompare(left.observedAt ?? '')
390
+ || left.certificateKey.localeCompare(right.certificateKey)));
391
+ const certificateGroupsTruncated = certificateGroups.length > MAX_CERTIFICATE_GROUPS
392
+ || certificateGroups.some((group) => (group.domainCount > MAX_DOMAINS_PER_CERTIFICATE_GROUP
393
+ || group.hostnameCount > MAX_HOSTNAMES_PER_CERTIFICATE_GROUP));
351
394
  const matchTruncated = matches.length > MAX_MATCHES;
352
395
  const perMatchTruncated = matches.some((m) => (groupMap.get(m.domain)?.hostnames.size || 0) > MAX_HOSTNAMES_PER_MATCH);
353
396
  return {
354
397
  domains: sortedLegacy.slice(0, MAX_RESULTS),
355
398
  matches: matches.slice(0, MAX_MATCHES),
399
+ certificateGroups: certificateGroups.slice(0, MAX_CERTIFICATE_GROUPS).map((group) => ({
400
+ certificateKey: group.certificateKey,
401
+ domains: group.domains,
402
+ hostnames: group.hostnames,
403
+ observedAt: group.observedAt,
404
+ wildcardObserved: group.wildcardObserved,
405
+ })),
406
+ certificateGroupsTruncated,
356
407
  truncated: legacyTruncated || matchTruncated || perMatchTruncated,
357
408
  };
358
409
  }
@@ -364,12 +415,12 @@ async function searchCertificateTransparency(keyword, dependencies = {}) {
364
415
  const trimmed = normalizeCtQuery(keyword);
365
416
  if (!trimmed) {
366
417
  return {
367
- domains: [], certCount: 0, truncated: false, matches: [],
418
+ domains: [], certCount: 0, truncated: false, matches: [], certificateGroups: [], certificateGroupsTruncated: false,
368
419
  observation: createObservation({
369
420
  status: 'success', observedAt: new Date().toISOString(), source: 'certificate_transparency',
370
421
  durationMs: Date.now() - startedAt, complete: true, truncated: false,
371
422
  limitations: ['Certificate Transparency observations indicate public certificate logging, not current site activity or maliciousness.'],
372
- diagnostics: { certificateRows: 0, matches: 0 },
423
+ diagnostics: { certificateRows: 0, matches: 0, certificateGroups: 0 },
373
424
  }),
374
425
  };
375
426
  }
@@ -379,14 +430,17 @@ async function searchCertificateTransparency(keyword, dependencies = {}) {
379
430
  certCount: data.length,
380
431
  ...summary,
381
432
  observation: createObservation({
382
- status: summary.truncated ? 'partial' : 'success',
433
+ status: summary.truncated || summary.certificateGroupsTruncated ? 'partial' : 'success',
383
434
  observedAt: new Date().toISOString(),
384
435
  source: 'certificate_transparency',
385
436
  durationMs: Date.now() - startedAt,
386
- complete: !summary.truncated,
387
- truncated: summary.truncated,
388
- limitations: ['Certificate Transparency observations indicate public certificate logging, not current site activity or maliciousness.'],
389
- diagnostics: { certificateRows: data.length, matches: summary.matches.length },
437
+ complete: !summary.truncated && !summary.certificateGroupsTruncated,
438
+ truncated: summary.truncated || summary.certificateGroupsTruncated,
439
+ limitations: [
440
+ 'Certificate Transparency observations indicate public certificate logging, not current site activity or maliciousness.',
441
+ ...(summary.certificateGroupsTruncated ? ['The optional certificate-group projection was capped independently of the registrable-domain result set.'] : []),
442
+ ],
443
+ diagnostics: { certificateRows: data.length, matches: summary.matches.length, certificateGroups: summary.certificateGroups.length, certificateGroupsTruncated: summary.certificateGroupsTruncated },
390
444
  }),
391
445
  };
392
446
  }
@@ -13,6 +13,7 @@ const MAX_NAMESERVERS = 16;
13
13
  const MAX_ERROR_LENGTH = 180;
14
14
  const DNS_DELEGATION_TIMEOUT_MS = 2200;
15
15
  const LAME_CODES = new Set(['ENOTAUTH', 'EREFUSED']);
16
+ const MAX_SOA_VALUE = 0xffff_ffff;
16
17
  function record(value) {
17
18
  return value && typeof value === 'object' && !Array.isArray(value)
18
19
  ? value
@@ -57,6 +58,27 @@ function errorDetail(error) {
57
58
  .trim()
58
59
  .slice(0, MAX_ERROR_LENGTH) || 'DNS query failed';
59
60
  }
61
+ function soaProjection(value) {
62
+ const source = record(value);
63
+ const nsname = hostname(source.nsname);
64
+ const hostmaster = hostname(source.hostmaster);
65
+ const number = (field) => (typeof field === 'number'
66
+ && Number.isSafeInteger(field)
67
+ && field >= 0
68
+ && field <= MAX_SOA_VALUE
69
+ ? field
70
+ : null);
71
+ const projection = {
72
+ nsname,
73
+ hostmaster,
74
+ serial: number(source.serial),
75
+ refresh: number(source.refresh),
76
+ retry: number(source.retry),
77
+ expire: number(source.expire),
78
+ minttl: number(source.minttl),
79
+ };
80
+ return Object.values(projection).every((field) => field === null) ? null : projection;
81
+ }
60
82
  function withTimeout(promise, timeoutMs) {
61
83
  return new Promise((resolve, reject) => {
62
84
  const timer = setTimeout(() => reject(Object.assign(new Error('DNS query timed out'), { code: 'ETIMEOUT' })), timeoutMs);
@@ -85,16 +107,20 @@ async function defaultAuthorityQuery(input) {
85
107
  ? soa.reason
86
108
  : null;
87
109
  if (reason !== null) {
110
+ const soaValue = soa.status === 'fulfilled' ? soaProjection(soa.value) : null;
88
111
  return {
89
112
  nameservers: nameservers.status === 'fulfilled' ? nameservers.value : [],
90
- soaPrimary: soa.status === 'fulfilled' ? soa.value.nsname : null,
113
+ soaPrimary: soaValue?.nsname ?? null,
114
+ soa: soaValue,
91
115
  errorCode: errorCode(reason),
92
116
  error: errorDetail(reason),
93
117
  };
94
118
  }
119
+ const soaValue = soa.status === 'fulfilled' ? soaProjection(soa.value) : null;
95
120
  return {
96
121
  nameservers: nameservers.status === 'fulfilled' ? nameservers.value : [],
97
- soaPrimary: soa.status === 'fulfilled' ? soa.value.nsname : null,
122
+ soaPrimary: soaValue?.nsname ?? null,
123
+ soa: soaValue,
98
124
  errorCode: null,
99
125
  error: null,
100
126
  };
@@ -201,8 +227,10 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
201
227
  const code = boundedText(query.errorCode, 40);
202
228
  const error = boundedText(query.error, MAX_ERROR_LENGTH);
203
229
  const nameservers = hostnames(query.nameservers);
204
- const soaPrimary = hostname(query.soaPrimary);
205
- const hasAnswer = nameservers.length > 0 || soaPrimary !== null;
230
+ const soa = soaProjection(query.soa)
231
+ ?? soaProjection({ nsname: query.soaPrimary });
232
+ const soaPrimary = soa?.nsname ?? hostname(query.soaPrimary);
233
+ const hasAnswer = nameservers.length > 0 || soaPrimary !== null || soa !== null;
206
234
  const state = error && hasAnswer
207
235
  ? 'partial'
208
236
  : code && LAME_CODES.has(code)
@@ -210,7 +238,7 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
210
238
  : error || !hasAnswer
211
239
  ? 'unreachable'
212
240
  : 'success';
213
- return { address, state, nameservers, soaPrimary, errorCode: code, error };
241
+ return { address, state, nameservers, soaPrimary, soa, errorCode: code, error };
214
242
  }
215
243
  catch (error) {
216
244
  const code = errorCode(error);
@@ -219,6 +247,7 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
219
247
  state: LAME_CODES.has(code) ? 'lame' : 'unreachable',
220
248
  nameservers: [],
221
249
  soaPrimary: null,
250
+ soa: null,
222
251
  errorCode: code || null,
223
252
  error: errorDetail(error),
224
253
  };
@@ -240,6 +269,7 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
240
269
  : 'unreachable',
241
270
  nameservers: retained?.nameservers ?? [],
242
271
  soaPrimary: retained?.soaPrimary ?? null,
272
+ soa: retained?.soa ?? null,
243
273
  attempts,
244
274
  };
245
275
  }));
@@ -250,6 +280,18 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
250
280
  const authoritySets = successfulAuthorities.map((authority) => authority.nameservers);
251
281
  const inconsistentAuthoritySets = authoritySets.some((set) => (!sameSet(set, authoritySets[0] ?? [])
252
282
  || (parentNameservers.length > 0 && !sameSet(set, parentNameservers))));
283
+ const retainedSoa = authorities
284
+ .filter((authority) => authority.state === 'success' || authority.state === 'partial')
285
+ .map((authority) => ({ nameserver: authority.nameserver, soa: authority.soa }))
286
+ .filter((entry) => entry.soa !== null);
287
+ const soaSerials = [...new Set(retainedSoa
288
+ .map((entry) => entry.soa.serial)
289
+ .filter((serial) => serial !== null))];
290
+ const soaPrimaries = [...new Set(retainedSoa
291
+ .map((entry) => entry.soa.nsname)
292
+ .filter((primary) => primary !== null))];
293
+ const soaSerialConflict = soaSerials.length > 1;
294
+ const soaPrimaryConflict = soaPrimaries.length > 1;
253
295
  const inBailiwick = candidates.filter((nameserver) => (nameserver === domain || nameserver.endsWith(`.${domain}`)));
254
296
  const missingGlue = inBailiwick.filter((nameserver) => (!(registry.nameserverDetails.find((item) => item.name === nameserver)?.addresses.length)));
255
297
  const dnssecMismatch = (registry.delegationSigned === true && registry.dsRecordCount === 0) || (registry.delegationSigned === false && registry.dsRecordCount > 0);
@@ -278,6 +320,15 @@ async function collectDnsDelegationHealth(domainValue, parentQueryValue, options
278
320
  : inconsistentAuthoritySets
279
321
  ? 'Direct nameserver answers are inconsistent'
280
322
  : 'Direct nameserver answers agree with the observed parent view', successfulAuthorities.map((authority) => `${authority.nameserver}: ${authority.nameservers.join(', ') || 'no NS answer'}`).join(' | ') || 'No direct answers.', 'Align the NS set served by every authority with the intended parent delegation before cutover.'),
323
+ finding('authority_soa_consistency', 'Authoritative SOA consistency', retainedSoa.length < 2 || soaSerials.length < 1 || soaPrimaries.length < 1
324
+ ? 'unknown'
325
+ : soaSerialConflict || soaPrimaryConflict ? 'warning' : 'healthy', retainedSoa.length < 2
326
+ ? 'Fewer than two authority SOA observations were available'
327
+ : soaSerialConflict
328
+ ? 'Authoritative servers published different SOA serials'
329
+ : soaPrimaryConflict
330
+ ? 'Authoritative servers published different SOA primary nameservers'
331
+ : 'Observed authority SOA serials and primary nameservers agree', retainedSoa.map((entry) => (`${entry.nameserver}: serial ${entry.soa.serial ?? 'unavailable'}, primary ${entry.soa.nsname ?? 'unavailable'}, refresh ${entry.soa.refresh ?? 'unavailable'}, retry ${entry.soa.retry ?? 'unavailable'}, expire ${entry.soa.expire ?? 'unavailable'}`)).join(' | ') || 'No complete SOA observation was available.', 'Confirm zone propagation and the intended SOA primary before relying on a DNS change or secondary-server state.'),
281
332
  finding('in_bailiwick_glue', 'In-bailiwick glue', !inBailiwick.length ? 'healthy' : missingGlue.length ? 'warning' : 'healthy', !inBailiwick.length
282
333
  ? 'No in-bailiwick nameserver requires registry glue'
283
334
  : missingGlue.length