@slicedearth/whoisleuth-cli 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/arguments.mjs +21 -5
- package/cli/artifact-validation/case-response.mjs +771 -0
- package/cli/artifact-validation/investigation-capsule.mjs +426 -0
- package/cli/artifact-validation/lookup-evidence.mjs +718 -0
- package/cli/artifact-validation/signed-review.mjs +598 -0
- package/cli/artifact-validation/structure-primitives.mjs +105 -0
- package/cli/artifact-verify.mjs +1 -1
- package/cli/assurance-command-runner.mjs +324 -0
- package/cli/command-reference.mjs +1175 -843
- package/cli/completion.mjs +56 -9
- package/cli/discriminated-command-handlers.mjs +11 -0
- package/cli/doctor.mjs +2 -2
- package/cli/evidence-command-runner.mjs +41 -31
- package/cli/export-evidence.mjs +1 -1
- package/cli/fail-policy.mjs +33 -8
- package/cli/formatters/terminal-command-formats.mjs +437 -0
- package/cli/formatters/terminal-lookup.mjs +758 -0
- package/cli/formatters/terminal-metadata.mjs +58 -0
- package/cli/formatters/terminal-shared.mjs +83 -0
- package/cli/formatters/terminal.mjs +5 -1232
- package/cli/history-command-runner.mjs +132 -0
- package/cli/inline-command-families.mjs +45 -0
- package/cli/inline-command-runner.mjs +55 -0
- package/cli/investigation-plan.mjs +9 -2
- package/cli/investigation-run.mjs +114 -9
- package/cli/lookup-brief.mjs +141 -23
- package/cli/lookup-browser-view.mjs +289 -0
- package/cli/lookup-browser.mjs +3 -288
- package/cli/mail-review.mjs +2 -6
- package/cli/offline-artifact-validation.mjs +19 -0
- package/cli/review-command-runner.mjs +269 -0
- package/cli/runner.mjs +15 -972
- package/cli/saved-lookup.mjs +19 -2
- package/cli/support-command-runner.mjs +229 -0
- package/cli/workflow-command-runner.mjs +129 -0
- package/docs/cli-reference.md +16 -9
- package/docs/cli.md +4 -2
- package/lib/availability.mjs +12 -9
- package/lib/bounded-contract-normalizers.mjs +2 -9
- package/lib/browser-library-profile.mjs +1 -2
- package/lib/client-behavior-profile.mjs +1 -2
- package/lib/credential-surface-profile.mjs +1 -1
- package/lib/domain-posture-analysis.mjs +10 -1
- package/lib/domain-posture.mjs +51 -23
- package/lib/evidence-export.mjs +46 -5
- package/lib/evidence-report-markdown.mjs +4 -0
- package/lib/evidence-report.mjs +50 -0
- package/lib/generated/cisa-kev-catalog.mjs +34 -4
- package/lib/generated/registrar-standing-catalogue.mjs +367 -0
- package/lib/json-record.mjs +12 -0
- package/lib/lookup-child-profile-contract.mjs +858 -0
- package/lib/lookup-contract-primitives.mjs +98 -0
- package/lib/lookup-diagnostics.mjs +96 -0
- package/lib/lookup-network-evidence-bounds.mjs +1 -0
- package/lib/lookup-response.mjs +367 -0
- package/lib/lookup.mjs +28 -443
- package/lib/page-role-profile.mjs +1 -3
- package/lib/registrar-standing-catalogue-contract.mjs +70 -0
- package/lib/registrar-standing-contract.mjs +300 -0
- package/lib/registrar-standing.mjs +269 -0
- package/lib/registry-insights.mjs +2 -8
- package/lib/risk-scoring.mjs +26 -10
- package/lib/sslbl-certificates.generated.mjs +138 -136
- package/lib/structured-data-identity.mjs +1 -3
- package/lib/technology-evidence-role.mjs +51 -0
- package/lib/threat-intelligence-runtime.mjs +3 -3
- package/lib/threat-intelligence-types.mjs +13 -0
- package/lib/threatfox-intelligence.mjs +1 -3
- package/lib/tls-intelligence.mjs +1 -2
- package/lib/urlhaus-intelligence.mjs +1 -3
- package/lib/urlscan-intelligence.mjs +1 -3
- package/lib/website-security-posture.mjs +1 -2
- package/lib/website-technology.mjs +72 -23
- package/lib/whois-common-formats.mjs +276 -0
- package/lib/whois-dialects.mjs +534 -0
- package/lib/whois-parser-support.mjs +305 -0
- package/lib/whois-parser.mjs +135 -0
- package/lib/whois.mjs +6 -1187
- package/package.json +2 -2
- package/packages/cases/case-evidence-model.mjs +9 -0
- package/packages/cases/case-migration-model.mjs +17 -11
- package/packages/cases/case-response-model.mjs +13 -9
- package/packages/cases/case-response-packet.mjs +297 -112
- package/packages/comparison/page-similarity.mjs +23 -0
- package/packages/contracts/case-portability.mjs +225 -61
- package/packages/contracts/cli-lookup.mjs +4 -3
- package/packages/contracts/domain-control-manifest.mjs +1 -1
- package/packages/contracts/investigation-portability.mjs +1 -1
- package/packages/contracts/risk-calibration.mjs +2 -2
- package/packages/contracts/workspace-portability.mjs +34 -6
- package/packages/investigation/evidence-coverage-ledger.mjs +21 -0
- package/packages/investigation/lookup-claim-readiness.mjs +2 -2
- package/packages/workspace/website-snapshot-model.mjs +79 -6
- package/third-party-notices.txt +1 -1
- package/cli/artifact-structure.mjs +0 -2507
package/lib/lookup.mjs
CHANGED
|
@@ -6,136 +6,21 @@
|
|
|
6
6
|
// just performed. This module fetches each registration source once and feeds
|
|
7
7
|
// those exact results into availability classification.
|
|
8
8
|
import { fetchRdapRecord, fetchRegistrarRdapRecord } from "./rdap.mjs";
|
|
9
|
-
import { buildWhoisChain
|
|
10
|
-
import { OPERATION_BUDGET_ERROR_CODE } from "./operation-budget.mjs";
|
|
9
|
+
import { buildWhoisChain } from "./whois.mjs";
|
|
11
10
|
import { checkDomainAvailability } from "./availability.mjs";
|
|
12
11
|
import { abortable } from "./abort.mjs";
|
|
13
|
-
import { collectDnsIntelligence, collectReverseDnsIntelligence,
|
|
12
|
+
import { collectDnsIntelligence, collectReverseDnsIntelligence, } from "./dns-intelligence.mjs";
|
|
14
13
|
import { createSelectedDnsResolvers } from "./dns-resolver-selection.mjs";
|
|
15
14
|
import { collectObservedNetworkContext } from "./observed-network-context.mjs";
|
|
16
|
-
import { collectSecurityTxt
|
|
17
|
-
import {
|
|
18
|
-
import { FEATURE_DISABLED_ERROR_CODE, featureDecision, networkFeaturePolicy } from "./feature-policy.mjs";
|
|
15
|
+
import { collectSecurityTxt } from "./security-txt.mjs";
|
|
16
|
+
import { featureDecision, networkFeaturePolicy } from "./feature-policy.mjs";
|
|
19
17
|
import { lookupUrlscanDomain } from "./urlscan-intelligence.mjs";
|
|
20
18
|
import { lookupUrlhausDomain } from "./urlhaus-intelligence.mjs";
|
|
21
19
|
import { lookupThreatfoxDomain } from "./threatfox-intelligence.mjs";
|
|
22
|
-
import { THREATFOX_PROVIDER, URLHAUS_PROVIDER, URLSCAN_PROVIDER, } from "./lookup-threat-provider-inventory.mjs";
|
|
23
|
-
import { createThreatIntelligenceResult } from "./threat-intelligence-contract.mjs";
|
|
24
|
-
import { THREAT_INTELLIGENCE_ENVELOPE_VERSION } from "./threat-intelligence-types.mjs";
|
|
25
|
-
import { buildRegistryInsights } from "./registry-insights.mjs";
|
|
26
20
|
import { inspectSslblCertificate } from "./sslbl-intelligence.mjs";
|
|
27
21
|
import { normalizeLookupSourceSettlement, plannedLookupProgressSources, } from "./lookup-source-progress.mjs";
|
|
28
|
-
import {
|
|
29
|
-
|
|
30
|
-
const LOOKUP_LEGACY_DIAGNOSTICS_VERSION = 7;
|
|
31
|
-
const LOOKUP_TIMING_VERSION = 1;
|
|
32
|
-
const MAX_LOOKUP_TIMING_MS = 120_000;
|
|
33
|
-
const LOOKUP_TIMING_SOURCE_ORDER = Object.freeze([
|
|
34
|
-
'rdap',
|
|
35
|
-
'whois',
|
|
36
|
-
'domain_evidence',
|
|
37
|
-
'reverse_dns',
|
|
38
|
-
'registrar_rdap',
|
|
39
|
-
'network_context',
|
|
40
|
-
'security_txt',
|
|
41
|
-
'external_intelligence',
|
|
42
|
-
'malware_host_intelligence',
|
|
43
|
-
'malware_ioc_intelligence',
|
|
44
|
-
]);
|
|
45
|
-
const LOOKUP_ERROR_CODES = Object.freeze({
|
|
46
|
-
AUTH_REQUIRED: 'AUTH_REQUIRED',
|
|
47
|
-
RATE_LIMITED: 'RATE_LIMITED',
|
|
48
|
-
CONCURRENCY_LIMITED: OPERATION_BUDGET_ERROR_CODE,
|
|
49
|
-
FEATURE_DISABLED: FEATURE_DISABLED_ERROR_CODE,
|
|
50
|
-
MISSING_QUERY: 'MISSING_QUERY',
|
|
51
|
-
INVALID_QUERY: 'INVALID_QUERY',
|
|
52
|
-
LOOKUP_FAILED: 'LOOKUP_FAILED',
|
|
53
|
-
RDAP_UPSTREAM_FAILED: 'RDAP_UPSTREAM_FAILED',
|
|
54
|
-
RDAP_UNSUPPORTED: 'RDAP_UNSUPPORTED',
|
|
55
|
-
WHOIS_UPSTREAM_FAILED: 'WHOIS_UPSTREAM_FAILED',
|
|
56
|
-
AVAILABILITY_CHECK_FAILED: 'AVAILABILITY_CHECK_FAILED',
|
|
57
|
-
});
|
|
58
|
-
function errorMessage(err, fallback) {
|
|
59
|
-
const message = err && typeof err === 'object' && 'message' in err
|
|
60
|
-
? err.message
|
|
61
|
-
: null;
|
|
62
|
-
return String(message || fallback);
|
|
63
|
-
}
|
|
64
|
-
function boundedSourceDetail(err, fallback) {
|
|
65
|
-
return errorMessage(err, fallback)
|
|
66
|
-
.replace(/[\u0000-\u001f\u007f]+/g, ' ')
|
|
67
|
-
.replace(/\s+/g, ' ')
|
|
68
|
-
.trim()
|
|
69
|
-
.slice(0, 240) || fallback;
|
|
70
|
-
}
|
|
71
|
-
function withoutNestedPublicationMetadata(value) {
|
|
72
|
-
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
73
|
-
return value;
|
|
74
|
-
const output = { ...value };
|
|
75
|
-
delete output.publicationMetadata;
|
|
76
|
-
return output;
|
|
77
|
-
}
|
|
78
|
-
function withoutNestedDeliveryMetadata(value) {
|
|
79
|
-
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
80
|
-
return value;
|
|
81
|
-
const output = { ...value };
|
|
82
|
-
if (output.response && typeof output.response === 'object' && !Array.isArray(output.response)) {
|
|
83
|
-
const response = { ...output.response };
|
|
84
|
-
delete response.deliveryMetadata;
|
|
85
|
-
output.response = response;
|
|
86
|
-
}
|
|
87
|
-
return output;
|
|
88
|
-
}
|
|
89
|
-
function boundedTimingMs(value) {
|
|
90
|
-
if (!Number.isFinite(value))
|
|
91
|
-
return 0;
|
|
92
|
-
return Math.min(MAX_LOOKUP_TIMING_MS, Math.max(0, Math.round(value)));
|
|
93
|
-
}
|
|
94
|
-
function createLookupTimingTracker(enabled, now) {
|
|
95
|
-
const lookupStartedAt = now();
|
|
96
|
-
const entries = new Map();
|
|
97
|
-
function measure(source, operation) {
|
|
98
|
-
const sourceStartedAt = now();
|
|
99
|
-
return Promise.resolve()
|
|
100
|
-
.then(operation)
|
|
101
|
-
.then((value) => {
|
|
102
|
-
const finishedAt = now();
|
|
103
|
-
entries.set(source, {
|
|
104
|
-
source,
|
|
105
|
-
outcome: 'fulfilled',
|
|
106
|
-
durationMs: boundedTimingMs(finishedAt - sourceStartedAt),
|
|
107
|
-
completedAfterMs: boundedTimingMs(finishedAt - lookupStartedAt),
|
|
108
|
-
});
|
|
109
|
-
return value;
|
|
110
|
-
}, (error) => {
|
|
111
|
-
const finishedAt = now();
|
|
112
|
-
entries.set(source, {
|
|
113
|
-
source,
|
|
114
|
-
outcome: 'rejected',
|
|
115
|
-
durationMs: boundedTimingMs(finishedAt - sourceStartedAt),
|
|
116
|
-
completedAfterMs: boundedTimingMs(finishedAt - lookupStartedAt),
|
|
117
|
-
});
|
|
118
|
-
throw error;
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
return {
|
|
122
|
-
measure(source, operation) {
|
|
123
|
-
return enabled ? measure(source, operation) : Promise.resolve().then(operation);
|
|
124
|
-
},
|
|
125
|
-
snapshot() {
|
|
126
|
-
if (!enabled)
|
|
127
|
-
return null;
|
|
128
|
-
const totalMs = boundedTimingMs(now() - lookupStartedAt);
|
|
129
|
-
return {
|
|
130
|
-
version: LOOKUP_TIMING_VERSION,
|
|
131
|
-
totalMs,
|
|
132
|
-
sources: LOOKUP_TIMING_SOURCE_ORDER
|
|
133
|
-
.map((source) => entries.get(source))
|
|
134
|
-
.filter((entry) => Boolean(entry)),
|
|
135
|
-
};
|
|
136
|
-
},
|
|
137
|
-
};
|
|
138
|
-
}
|
|
22
|
+
import { buildUnifiedLookupResponse } from "./lookup-response.mjs";
|
|
23
|
+
import { createLookupTimingTracker } from "./lookup-diagnostics.mjs";
|
|
139
24
|
async function runUnifiedLookup(classified, options = {}) {
|
|
140
25
|
const fetchRdap = options.fetchRdapRecord || fetchRdapRecord;
|
|
141
26
|
const fetchRegistrarRdap = options.fetchRegistrarRdapRecord || fetchRegistrarRdapRecord;
|
|
@@ -286,330 +171,30 @@ async function runUnifiedLookup(classified, options = {}) {
|
|
|
286
171
|
void Promise.resolve(sourcePromise).then((value) => notify(source, 'fulfilled', value), (error) => notify(source, 'rejected', error));
|
|
287
172
|
}
|
|
288
173
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
whoisPromise,
|
|
174
|
+
return buildUnifiedLookupResponse({
|
|
175
|
+
availabilityEnabled,
|
|
292
176
|
availabilityPromise,
|
|
293
|
-
|
|
294
|
-
|
|
177
|
+
classified,
|
|
178
|
+
compact,
|
|
179
|
+
dnsIntelligenceEnabled,
|
|
180
|
+
fast,
|
|
181
|
+
inspectCertificateWarningData,
|
|
295
182
|
networkContextPromise,
|
|
183
|
+
options,
|
|
184
|
+
rdapEnabled,
|
|
185
|
+
rdapPromise,
|
|
186
|
+
registrarRdapPromise,
|
|
187
|
+
reverseDnsEligible,
|
|
188
|
+
reverseDnsPromise,
|
|
296
189
|
securityTxtPromise,
|
|
297
|
-
|
|
298
|
-
urlhausIntelligencePromise,
|
|
190
|
+
skipWhois,
|
|
299
191
|
threatfoxIntelligencePromise,
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
: rdapResult.status === 'rejected' && Array.isArray(rdapResult.reason?.attempts)
|
|
307
|
-
? rdapResult.reason.attempts
|
|
308
|
-
: [];
|
|
309
|
-
let registrarRdap = null;
|
|
310
|
-
const registryRdapUsable = rdapRecord && rdapRecord.upstreamStatus === 200 && rdapRecord.parsed;
|
|
311
|
-
if (classified.type === 'domain' && rdapEnabled && !compact && registryRdapUsable) {
|
|
312
|
-
if (fast) {
|
|
313
|
-
registrarRdap = {
|
|
314
|
-
status: 'skipped',
|
|
315
|
-
detail: 'Registrar RDAP is omitted in fast RDAP-only mode.',
|
|
316
|
-
endpoint: null,
|
|
317
|
-
transportSecurity: null,
|
|
318
|
-
upstreamStatus: null,
|
|
319
|
-
fetchedAt: null,
|
|
320
|
-
attempt: null,
|
|
321
|
-
};
|
|
322
|
-
}
|
|
323
|
-
else if (registrarRdapResult.status === 'fulfilled') {
|
|
324
|
-
registrarRdap = registrarRdapResult.value;
|
|
325
|
-
}
|
|
326
|
-
else {
|
|
327
|
-
registrarRdap = registrarRdapResult.reason?.registrarRdap || {
|
|
328
|
-
status: 'error',
|
|
329
|
-
detail: boundedSourceDetail(registrarRdapResult.reason, 'Registrar RDAP lookup failed'),
|
|
330
|
-
endpoint: null,
|
|
331
|
-
transportSecurity: null,
|
|
332
|
-
upstreamStatus: null,
|
|
333
|
-
fetchedAt: null,
|
|
334
|
-
attempt: null,
|
|
335
|
-
};
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
const rdap = !rdapEnabled
|
|
339
|
-
? { skipped: true, detail: 'RDAP is disabled by deployment policy.' }
|
|
340
|
-
: rdapRecord
|
|
341
|
-
? { ...rdapRecord, ...(registrarRdap ? { registrarRdap } : {}) }
|
|
342
|
-
: {
|
|
343
|
-
error: rdapResult.status === 'rejected'
|
|
344
|
-
? boundedSourceDetail(rdapResult.reason, 'RDAP lookup failed')
|
|
345
|
-
: 'No RDAP registry found for this query via IANA bootstrap',
|
|
346
|
-
attempts: rdapAttempts,
|
|
347
|
-
};
|
|
348
|
-
let whois;
|
|
349
|
-
if (!whoisEnabled) {
|
|
350
|
-
whois = { skipped: true, detail: 'WHOIS is disabled by deployment policy.' };
|
|
351
|
-
}
|
|
352
|
-
else if (skipWhois) {
|
|
353
|
-
whois = { skipped: true, detail: 'WHOIS is omitted in fast RDAP-only mode.' };
|
|
354
|
-
}
|
|
355
|
-
else if (Array.isArray(whoisChain)) {
|
|
356
|
-
whois = { chain: whoisChain, parsed: parseWhoisChain(whoisChain) };
|
|
357
|
-
}
|
|
358
|
-
else {
|
|
359
|
-
whois = {
|
|
360
|
-
error: whoisResult.status === 'rejected'
|
|
361
|
-
? boundedSourceDetail(whoisResult.reason, 'WHOIS lookup failed')
|
|
362
|
-
: 'WHOIS returned no referral chain',
|
|
363
|
-
};
|
|
364
|
-
}
|
|
365
|
-
let availability = { applicable: false, type: classified.type };
|
|
366
|
-
if (classified.type === 'domain') {
|
|
367
|
-
if (!availabilityEnabled) {
|
|
368
|
-
availability = {
|
|
369
|
-
applicable: true,
|
|
370
|
-
domain: classified.value,
|
|
371
|
-
state: 'unknown',
|
|
372
|
-
confidence: 'low',
|
|
373
|
-
disabled: true,
|
|
374
|
-
detail: 'Availability analysis is disabled by deployment policy.',
|
|
375
|
-
};
|
|
376
|
-
}
|
|
377
|
-
else if (availabilityResult.status === 'fulfilled') {
|
|
378
|
-
const result = availabilityResult.value;
|
|
379
|
-
availability = {
|
|
380
|
-
applicable: true,
|
|
381
|
-
domain: classified.value,
|
|
382
|
-
inputHostname: classified.inputHostname,
|
|
383
|
-
registrableDomain: classified.registrableDomain,
|
|
384
|
-
isSubdomain: classified.isSubdomain,
|
|
385
|
-
...result,
|
|
386
|
-
};
|
|
387
|
-
}
|
|
388
|
-
else {
|
|
389
|
-
availability = {
|
|
390
|
-
applicable: true,
|
|
391
|
-
domain: classified.value,
|
|
392
|
-
state: 'unknown',
|
|
393
|
-
confidence: 'low',
|
|
394
|
-
detail: errorMessage(availabilityResult.reason, 'Availability lookup failed'),
|
|
395
|
-
};
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
const rdapStatus = !rdapEnabled
|
|
399
|
-
? 'disabled'
|
|
400
|
-
: rdapResult.status === 'rejected'
|
|
401
|
-
? 'error'
|
|
402
|
-
: !rdapRecord
|
|
403
|
-
? 'unsupported'
|
|
404
|
-
: rdapRecord.upstreamStatus === 404 ? 'not_found' : 'success';
|
|
405
|
-
const whoisStatus = !whoisEnabled
|
|
406
|
-
? 'disabled'
|
|
407
|
-
: skipWhois
|
|
408
|
-
? 'skipped'
|
|
409
|
-
: whoisResult.status === 'rejected' || !Array.isArray(whoisChain)
|
|
410
|
-
? 'error'
|
|
411
|
-
: whoisChain.length === 1
|
|
412
|
-
? 'unsupported'
|
|
413
|
-
: whois.parsed && whois.parsed.chainStatus === 'complete' ? 'complete' : 'partial';
|
|
414
|
-
const availabilityStatus = classified.type !== 'domain'
|
|
415
|
-
? 'not_applicable'
|
|
416
|
-
: !availabilityEnabled ? 'disabled'
|
|
417
|
-
: availabilityResult.status === 'rejected' ? 'error' : 'complete';
|
|
418
|
-
const reverseDns = reverseDnsEligible
|
|
419
|
-
? !dnsIntelligenceEnabled
|
|
420
|
-
? skippedReverseDnsIntelligence()
|
|
421
|
-
: reverseDnsResult.status === 'fulfilled' && reverseDnsResult.value
|
|
422
|
-
? reverseDnsResult.value
|
|
423
|
-
: failedReverseDnsIntelligence(reverseDnsResult.status === 'rejected'
|
|
424
|
-
? reverseDnsResult.reason
|
|
425
|
-
: 'Reverse DNS lookup returned no result')
|
|
426
|
-
: null;
|
|
427
|
-
// This is static access-policy context only. It performs no network work and
|
|
428
|
-
// is deliberately excluded from compact Bulk responses and from every
|
|
429
|
-
// availability input.
|
|
430
|
-
const registryAccess = classified.type === 'domain' && !compact
|
|
431
|
-
? registryAccessDiagnosticFor(classified.value)
|
|
432
|
-
: null;
|
|
433
|
-
const networkContext = networkContextPromise && networkContextResult.status === 'fulfilled'
|
|
434
|
-
? networkContextResult.value
|
|
435
|
-
: null;
|
|
436
|
-
const securityTxt = securityTxtPromise
|
|
437
|
-
? securityTxtResult.status === 'fulfilled'
|
|
438
|
-
? securityTxtResult.value
|
|
439
|
-
: securityTxtUnavailable(classified.inputHostname, securityTxtResult.reason)
|
|
440
|
-
: null;
|
|
441
|
-
// SSLBL comparison is an exact, local, deep-only enrichment over the leaf
|
|
442
|
-
// certificate already observed by the bounded TLS collector. It performs no
|
|
443
|
-
// request, never contributes to availability, and is omitted from compact
|
|
444
|
-
// Bulk and monitoring responses.
|
|
445
|
-
const sslbl = classified.type === 'domain' && !fast && !compact
|
|
446
|
-
? inspectCertificateWarningData(availability && typeof availability.tls === 'object' ? availability.tls : null, {
|
|
447
|
-
...(options.sslblSnapshot === undefined ? {} : { snapshot: options.sslblSnapshot }),
|
|
448
|
-
...(options.sslblNow === undefined ? {} : { now: options.sslblNow }),
|
|
449
|
-
})
|
|
450
|
-
: null;
|
|
451
|
-
const networkContextRdap = networkContext && typeof networkContext.rdap === 'object' && networkContext.rdap
|
|
452
|
-
? networkContext.rdap
|
|
453
|
-
: null;
|
|
454
|
-
const networkContextEndpoint = networkContext && typeof networkContext.endpoint === 'object' && networkContext.endpoint
|
|
455
|
-
? networkContext.endpoint
|
|
456
|
-
: null;
|
|
457
|
-
const lookupTiming = timing.snapshot();
|
|
458
|
-
const diagnostics = {
|
|
459
|
-
version: lookupTiming ? LOOKUP_DIAGNOSTICS_VERSION : LOOKUP_LEGACY_DIAGNOSTICS_VERSION,
|
|
460
|
-
...(lookupTiming ? { timing: lookupTiming } : {}),
|
|
461
|
-
...(registryAccess ? { registryAccess } : {}),
|
|
462
|
-
rdap: {
|
|
463
|
-
status: rdapStatus,
|
|
464
|
-
errorCode: rdapStatus === 'disabled'
|
|
465
|
-
? LOOKUP_ERROR_CODES.FEATURE_DISABLED
|
|
466
|
-
: rdapStatus === 'error'
|
|
467
|
-
? LOOKUP_ERROR_CODES.RDAP_UPSTREAM_FAILED
|
|
468
|
-
: rdapStatus === 'unsupported' ? LOOKUP_ERROR_CODES.RDAP_UNSUPPORTED : null,
|
|
469
|
-
endpoint: rdapRecord ? rdapRecord.rdapServer || null : null,
|
|
470
|
-
transportSecurity: rdapRecord ? rdapRecord.transportSecurity || null : null,
|
|
471
|
-
httpStatus: rdapRecord ? rdapRecord.upstreamStatus ?? null : null,
|
|
472
|
-
fetchedAt: rdapRecord ? rdapRecord.fetchedAt || null : null,
|
|
473
|
-
attempts: rdapAttempts,
|
|
474
|
-
...(registrarRdap ? {
|
|
475
|
-
registrar: {
|
|
476
|
-
status: registrarRdap.status,
|
|
477
|
-
endpoint: registrarRdap.endpoint || null,
|
|
478
|
-
transportSecurity: registrarRdap.transportSecurity || null,
|
|
479
|
-
httpStatus: registrarRdap.upstreamStatus ?? null,
|
|
480
|
-
fetchedAt: registrarRdap.fetchedAt || null,
|
|
481
|
-
attempt: registrarRdap.attempt || null,
|
|
482
|
-
},
|
|
483
|
-
} : {}),
|
|
484
|
-
},
|
|
485
|
-
whois: {
|
|
486
|
-
status: whoisStatus,
|
|
487
|
-
errorCode: whoisStatus === 'disabled'
|
|
488
|
-
? LOOKUP_ERROR_CODES.FEATURE_DISABLED
|
|
489
|
-
: whoisStatus === 'error' ? LOOKUP_ERROR_CODES.WHOIS_UPSTREAM_FAILED : null,
|
|
490
|
-
queriedAt: Array.isArray(whoisChain) && whoisChain[0] ? whoisChain[0].queriedAt || null : null,
|
|
491
|
-
authoritativeHop: whois.parsed ? whois.parsed.authoritativeHop || null : null,
|
|
492
|
-
failedHop: whois.parsed ? whois.parsed.failedHop || null : null,
|
|
493
|
-
conflictingHop: whois.parsed ? whois.parsed.conflictingHop || null : null,
|
|
494
|
-
},
|
|
495
|
-
availability: {
|
|
496
|
-
status: availabilityStatus,
|
|
497
|
-
errorCode: availabilityStatus === 'disabled'
|
|
498
|
-
? LOOKUP_ERROR_CODES.FEATURE_DISABLED
|
|
499
|
-
: availabilityStatus === 'error' ? LOOKUP_ERROR_CODES.AVAILABILITY_CHECK_FAILED : null,
|
|
500
|
-
resultState: availability.applicable === true ? availability.state || 'unknown' : null,
|
|
501
|
-
},
|
|
502
|
-
...(reverseDns ? {
|
|
503
|
-
reverseDns: {
|
|
504
|
-
status: reverseDns.status,
|
|
505
|
-
observedAt: reverseDns.observedAt,
|
|
506
|
-
complete: reverseDns.complete,
|
|
507
|
-
truncated: reverseDns.truncated,
|
|
508
|
-
},
|
|
509
|
-
} : {}),
|
|
510
|
-
...(networkContext ? {
|
|
511
|
-
network: {
|
|
512
|
-
status: networkContext.status,
|
|
513
|
-
address: networkContextEndpoint?.address || null,
|
|
514
|
-
family: networkContextEndpoint?.family || null,
|
|
515
|
-
addressSource: networkContextEndpoint?.selectedFrom || null,
|
|
516
|
-
endpoint: networkContextRdap?.endpoint || null,
|
|
517
|
-
transportSecurity: networkContextRdap?.transportSecurity || null,
|
|
518
|
-
httpStatus: networkContextRdap?.httpStatus ?? null,
|
|
519
|
-
fetchedAt: networkContextRdap?.fetchedAt || null,
|
|
520
|
-
attempts: networkContextRdap?.attempts || [],
|
|
521
|
-
},
|
|
522
|
-
} : {}),
|
|
523
|
-
...(securityTxt ? {
|
|
524
|
-
securityTxt: {
|
|
525
|
-
status: securityTxt.status,
|
|
526
|
-
state: securityTxt.state,
|
|
527
|
-
endpoint: securityTxt.finalUrl,
|
|
528
|
-
httpStatus: securityTxt.httpStatus,
|
|
529
|
-
observedAt: securityTxt.observedAt,
|
|
530
|
-
complete: securityTxt.complete,
|
|
531
|
-
truncated: securityTxt.truncated,
|
|
532
|
-
},
|
|
533
|
-
} : {}),
|
|
534
|
-
...(sslbl ? {
|
|
535
|
-
sslbl: {
|
|
536
|
-
status: sslbl.status,
|
|
537
|
-
verdict: sslbl.verdict,
|
|
538
|
-
observedAt: sslbl.observedAt,
|
|
539
|
-
complete: sslbl.complete,
|
|
540
|
-
snapshotUpdatedAt: sslbl.snapshot.sourceUpdatedAt,
|
|
541
|
-
},
|
|
542
|
-
} : {}),
|
|
543
|
-
};
|
|
544
|
-
// Bulk triage only consumes the derived availability evidence and source
|
|
545
|
-
// diagnostics. Omitting raw RDAP JSON and multi-hop WHOIS bodies from that
|
|
546
|
-
// opt-in response prevents large scans from downloading and retaining the
|
|
547
|
-
// same registry payloads the backend already used to build `availability`.
|
|
548
|
-
if (compact) {
|
|
549
|
-
const bulkComparison = !fast
|
|
550
|
-
? buildBulkComparisonEvidence(availability)
|
|
551
|
-
: null;
|
|
552
|
-
const { credentialSurfaceProfile: _credentialSurfaceProfile, structuredDataIdentity: _structuredDataIdentity, technologyProfile: _technologyProfile, pageRoleProfile: _pageRoleProfile, clientBehaviorProfile: _clientBehaviorProfile, securityPosture: _securityPosture, pageIdentity: richPageIdentity, http: richHttp, ...compactAvailability } = availability;
|
|
553
|
-
return {
|
|
554
|
-
availability: {
|
|
555
|
-
...compactAvailability,
|
|
556
|
-
...(richPageIdentity !== undefined ? { pageIdentity: withoutNestedPublicationMetadata(richPageIdentity) } : {}),
|
|
557
|
-
...(richHttp !== undefined ? { http: withoutNestedDeliveryMetadata(richHttp) } : {}),
|
|
558
|
-
...(bulkComparison ? { bulkComparison } : {}),
|
|
559
|
-
},
|
|
560
|
-
diagnostics,
|
|
561
|
-
};
|
|
562
|
-
}
|
|
563
|
-
const targetDomain = classified.registrableDomain || classified.value;
|
|
564
|
-
const threatIntelligenceProviders = [];
|
|
565
|
-
if (urlscanIntelligencePromise) {
|
|
566
|
-
threatIntelligenceProviders.push(urlscanIntelligenceResult.status === 'fulfilled' && urlscanIntelligenceResult.value
|
|
567
|
-
? urlscanIntelligenceResult.value
|
|
568
|
-
: createThreatIntelligenceResult(URLSCAN_PROVIDER, { type: 'domain', value: targetDomain }, { state: 'error', detail: 'Archived provider intelligence could not be completed.' }));
|
|
569
|
-
}
|
|
570
|
-
if (urlhausIntelligencePromise) {
|
|
571
|
-
threatIntelligenceProviders.push(urlhausIntelligenceResult.status === 'fulfilled' && urlhausIntelligenceResult.value
|
|
572
|
-
? urlhausIntelligenceResult.value
|
|
573
|
-
: createThreatIntelligenceResult(URLHAUS_PROVIDER, { type: 'domain', value: targetDomain }, { state: 'error', detail: 'Malware-host intelligence could not be completed.' }));
|
|
574
|
-
}
|
|
575
|
-
if (threatfoxIntelligencePromise) {
|
|
576
|
-
threatIntelligenceProviders.push(threatfoxIntelligenceResult.status === 'fulfilled' && threatfoxIntelligenceResult.value
|
|
577
|
-
? threatfoxIntelligenceResult.value
|
|
578
|
-
: createThreatIntelligenceResult(THREATFOX_PROVIDER, { type: 'domain', value: targetDomain }, { state: 'error', detail: 'Malware-IOC intelligence could not be completed.' }));
|
|
579
|
-
}
|
|
580
|
-
const threatIntelligence = threatIntelligenceProviders.length
|
|
581
|
-
? { version: THREAT_INTELLIGENCE_ENVELOPE_VERSION, providers: threatIntelligenceProviders }
|
|
582
|
-
: null;
|
|
583
|
-
const registrarRdapParsed = registrarRdap
|
|
584
|
-
&& typeof registrarRdap.parsed === 'object'
|
|
585
|
-
&& registrarRdap.parsed
|
|
586
|
-
&& !Array.isArray(registrarRdap.parsed)
|
|
587
|
-
? registrarRdap.parsed
|
|
588
|
-
: null;
|
|
589
|
-
const registryInsights = classified.type === 'domain' && !fast && !compact
|
|
590
|
-
? buildRegistryInsights({
|
|
591
|
-
rdapParsed: rdapRecord?.parsed,
|
|
592
|
-
rdapStatus,
|
|
593
|
-
rdapFetchedAt: rdapRecord?.fetchedAt,
|
|
594
|
-
whoisParsed: whois.parsed,
|
|
595
|
-
whoisStatus,
|
|
596
|
-
whoisQueriedAt: Array.isArray(whoisChain) && whoisChain[0] ? whoisChain[0].queriedAt : null,
|
|
597
|
-
registrarRdapParsed,
|
|
598
|
-
registrarRdapStatus: registrarRdap?.status,
|
|
599
|
-
registrarRdapFetchedAt: registrarRdap?.fetchedAt,
|
|
600
|
-
})
|
|
601
|
-
: null;
|
|
602
|
-
return {
|
|
603
|
-
rdap,
|
|
604
|
-
whois,
|
|
605
|
-
availability,
|
|
606
|
-
diagnostics,
|
|
607
|
-
...(registryInsights ? { registryInsights } : {}),
|
|
608
|
-
...(reverseDns ? { reverseDns } : {}),
|
|
609
|
-
...(networkContext ? { networkContext } : {}),
|
|
610
|
-
...(securityTxt ? { securityTxt } : {}),
|
|
611
|
-
...(sslbl ? { sslbl } : {}),
|
|
612
|
-
...(threatIntelligence ? { threatIntelligence } : {}),
|
|
613
|
-
};
|
|
192
|
+
timing,
|
|
193
|
+
urlhausIntelligencePromise,
|
|
194
|
+
urlscanIntelligencePromise,
|
|
195
|
+
whoisEnabled,
|
|
196
|
+
whoisPromise,
|
|
197
|
+
});
|
|
614
198
|
}
|
|
615
|
-
export { runUnifiedLookup
|
|
199
|
+
export { runUnifiedLookup };
|
|
200
|
+
export { LOOKUP_DIAGNOSTICS_VERSION, LOOKUP_LEGACY_DIAGNOSTICS_VERSION, LOOKUP_TIMING_VERSION, MAX_LOOKUP_TIMING_MS, LOOKUP_ERROR_CODES, } from "./lookup-diagnostics.mjs";
|
|
@@ -3,10 +3,8 @@
|
|
|
3
3
|
// descriptions only; it does not retain page text, arbitrary attributes, URLs,
|
|
4
4
|
// or matched markup, and it never treats a role as proof of intent or safety.
|
|
5
5
|
import { createObservation } from "../packages/evidence/observation.mjs";
|
|
6
|
+
import { MAX_PAGE_ROLE_EVIDENCE, MAX_PAGE_ROLE_FINDINGS, PAGE_ROLE_PROFILE_VERSION, } from "./lookup-child-profile-contract.mjs";
|
|
6
7
|
import { analyzeStaticHtml, } from "./static-html-analysis.mjs";
|
|
7
|
-
const PAGE_ROLE_PROFILE_VERSION = 1;
|
|
8
|
-
const MAX_PAGE_ROLE_FINDINGS = 4;
|
|
9
|
-
const MAX_PAGE_ROLE_EVIDENCE = 4;
|
|
10
8
|
const MAX_ROLE_TITLE_CHARS = 200;
|
|
11
9
|
const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f]/;
|
|
12
10
|
function boundedLowercase(value, maximum) {
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { defineSchemaCompatibility } from "../packages/contracts/schema-compatibility.mjs";
|
|
2
|
+
export const REGISTRAR_STANDING_CATALOGUE_SCHEMA = 'whoisleuth.registrar-standing-catalogue';
|
|
3
|
+
export const REGISTRAR_STANDING_CATALOGUE_VERSION = 1;
|
|
4
|
+
export const MAX_REGISTRAR_STANDING_CATALOGUE_BYTES = 512 * 1024;
|
|
5
|
+
export const REGISTRAR_STANDING_MAX_AGE_DAYS = 14;
|
|
6
|
+
export const REGISTRAR_STANDING_MAX_FUTURE_SKEW_MS = 5 * 60 * 1_000;
|
|
7
|
+
export const IANA_REGISTRAR_SOURCE_URL = 'https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv';
|
|
8
|
+
export const ICANN_COMPLIANCE_SOURCE_URL = 'https://www.icann.org/compliance/notices';
|
|
9
|
+
export const REGISTRAR_STANDING_AUDIT_SCHEMA = 'whoisleuth.registrar-standing-drift-audit';
|
|
10
|
+
export const REGISTRAR_STANDING_AUDIT_VERSION = 1;
|
|
11
|
+
export const MAX_REGISTRAR_STANDING_AUDIT_BYTES = 32 * 1024;
|
|
12
|
+
export function registrarStandingOfficialSourceUrl(value, source, noticeId = null) {
|
|
13
|
+
if (typeof value !== 'string' || value.length > 2_048)
|
|
14
|
+
return null;
|
|
15
|
+
try {
|
|
16
|
+
const parsed = new URL(value);
|
|
17
|
+
if (parsed.protocol !== 'https:'
|
|
18
|
+
|| parsed.username
|
|
19
|
+
|| parsed.password
|
|
20
|
+
|| parsed.port
|
|
21
|
+
|| parsed.search
|
|
22
|
+
|| parsed.hash)
|
|
23
|
+
return null;
|
|
24
|
+
if (source === 'iana_catalogue') {
|
|
25
|
+
return parsed.href === IANA_REGISTRAR_SOURCE_URL ? parsed.href : null;
|
|
26
|
+
}
|
|
27
|
+
if (source === 'icann_index') {
|
|
28
|
+
return parsed.href === ICANN_COMPLIANCE_SOURCE_URL ? parsed.href : null;
|
|
29
|
+
}
|
|
30
|
+
const identifier = typeof noticeId === 'string'
|
|
31
|
+
? /^notice-([1-9]\d{0,7})$/u.exec(noticeId)?.[1]
|
|
32
|
+
: null;
|
|
33
|
+
return identifier
|
|
34
|
+
&& parsed.hostname === 'www.icann.org'
|
|
35
|
+
&& new RegExp(`^/uploads/compliance_notice/attachment/${identifier}/[A-Za-z0-9._~-]+\\.pdf$`, 'u').test(parsed.pathname)
|
|
36
|
+
? parsed.href
|
|
37
|
+
: null;
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export const REGISTRAR_STANDING_CATALOGUE_COMPATIBILITY = defineSchemaCompatibility({
|
|
44
|
+
id: 'derived.registrar-standing-catalogue',
|
|
45
|
+
kind: 'derived',
|
|
46
|
+
schema: REGISTRAR_STANDING_CATALOGUE_SCHEMA,
|
|
47
|
+
currentVersion: REGISTRAR_STANDING_CATALOGUE_VERSION,
|
|
48
|
+
supportedVersions: [REGISTRAR_STANDING_CATALOGUE_VERSION],
|
|
49
|
+
acceptsUnversionedLegacy: false,
|
|
50
|
+
futureVersionBehavior: 'reject',
|
|
51
|
+
migration: 'exact_current_only',
|
|
52
|
+
writeSemantics: 'normalized_rewrite',
|
|
53
|
+
byteBudget: MAX_REGISTRAR_STANDING_CATALOGUE_BYTES,
|
|
54
|
+
owner: 'lib/registrar-standing-catalogue-contract.mts',
|
|
55
|
+
note: 'Checked-in target-free IANA accreditation and current-year ICANN compliance projection refreshed only by the bounded maintainer command.',
|
|
56
|
+
});
|
|
57
|
+
export const REGISTRAR_STANDING_AUDIT_COMPATIBILITY = defineSchemaCompatibility({
|
|
58
|
+
id: 'maintainer.registrar-standing-drift-audit',
|
|
59
|
+
kind: 'cli_document',
|
|
60
|
+
schema: REGISTRAR_STANDING_AUDIT_SCHEMA,
|
|
61
|
+
currentVersion: REGISTRAR_STANDING_AUDIT_VERSION,
|
|
62
|
+
supportedVersions: [REGISTRAR_STANDING_AUDIT_VERSION],
|
|
63
|
+
acceptsUnversionedLegacy: false,
|
|
64
|
+
futureVersionBehavior: 'not_applicable',
|
|
65
|
+
migration: 'read_only',
|
|
66
|
+
writeSemantics: 'read_only',
|
|
67
|
+
byteBudget: MAX_REGISTRAR_STANDING_AUDIT_BYTES,
|
|
68
|
+
owner: 'lib/registrar-standing-catalogue-contract.mts',
|
|
69
|
+
note: 'Target-free two-source drift report that compares bounded normalized official projections without modifying the checked-in catalogue.',
|
|
70
|
+
});
|