@slicedearth/whoisleuth-cli 1.36.1 → 1.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/bin/whoisleuth.mjs +52 -24
- package/cli/archive-inspect.mjs +37 -2
- package/cli/arguments.mjs +493 -157
- package/cli/artifact-verify.mjs +91 -15
- package/cli/bulk-checkpoint.mjs +7 -11
- package/cli/bulk-command-runner.mjs +19 -2
- package/cli/case-pack.mjs +136 -0
- package/cli/ci-report.mjs +73 -0
- package/cli/collection-preflight.mjs +59 -0
- package/cli/command-reference.mjs +395 -0
- package/cli/compare.mjs +1 -1
- package/cli/completion.mjs +49 -11
- package/cli/config-profile.mjs +180 -0
- package/cli/ct-event-intake.mjs +159 -0
- package/cli/discovery-scan-command-runner.mjs +15 -1
- package/cli/doctor.mjs +11 -5
- package/cli/domain-control-monitor.mjs +104 -0
- package/cli/domain-control-observations.mjs +235 -0
- package/cli/evidence-command-arguments.mjs +12 -1
- package/cli/evidence-command-runner.mjs +3 -2
- package/cli/evidence-signing.mjs +4 -4
- package/cli/external-observation-mapping.mjs +162 -0
- package/cli/fail-policy.mjs +64 -0
- package/cli/formatters/html.mjs +2 -2
- package/cli/formatters/json.mjs +13 -1
- package/cli/formatters/terminal.mjs +7 -3
- package/cli/investigation-manifest.mjs +113 -0
- package/cli/investigation-plan.mjs +2 -2
- package/cli/investigation-run.mjs +112 -0
- package/cli/local-mmdb-review.mjs +76 -0
- package/cli/lookalike-calibration.mjs +121 -0
- package/cli/lookup-brief.mjs +116 -0
- package/cli/lookup-command-runner.mjs +12 -2
- package/cli/lookup-diff.mjs +17 -19
- package/cli/lookup-plan.mjs +3 -3
- package/cli/lookup-reconcile.mjs +141 -0
- package/cli/lookup-timeline.mjs +1 -1
- package/cli/mail-review.mjs +1 -1
- package/cli/network-command-runner.mjs +4 -1
- package/cli/offline-evidence-review.mjs +125 -10
- package/cli/open-asset-model-bridge.mjs +169 -0
- package/cli/registry-cohort.mjs +121 -0
- package/cli/registry-doctor.mjs +220 -0
- package/cli/registry-fixture-scaffold.mjs +73 -0
- package/cli/risk-calibration.mjs +117 -9
- package/cli/runner.mjs +561 -280
- package/cli/saved-lookup.mjs +2 -2
- package/cli/sharing-review.mjs +202 -0
- package/cli/source-reliability.mjs +1 -1
- package/docs/cli-reference.md +1336 -0
- package/docs/cli.md +259 -940
- package/frontend/src/lib/analysis/acquisition-decision-packet.js +1 -1
- package/frontend/src/lib/analysis/acquisition-due-diligence.js +7 -7
- package/frontend/src/lib/analysis/brand-profile-model.js +46 -4
- package/frontend/src/lib/analysis/bulk-domain-comparison.js +12 -12
- package/frontend/src/lib/analysis/bulk-mail-exposure.js +1 -1
- package/frontend/src/lib/analysis/bulk-session-model.js +7 -2
- package/frontend/src/lib/analysis/case-evidence-model.js +26 -4
- package/frontend/src/lib/analysis/case-migration-model.js +6 -1
- package/frontend/src/lib/analysis/case-record-contracts.js +4 -2
- package/frontend/src/lib/analysis/case-record-core.js +5 -1
- package/frontend/src/lib/analysis/case-record-operations.js +6 -1
- package/frontend/src/lib/analysis/case-report.js +573 -0
- package/frontend/src/lib/analysis/case-response-model.js +30 -1
- package/frontend/src/lib/analysis/case-response-packet.js +7 -7
- package/frontend/src/lib/analysis/detection-rule-model.js +36 -0
- package/frontend/src/lib/analysis/evidence-coverage-ledger.js +328 -0
- package/frontend/src/lib/analysis/external-findings-import.js +320 -0
- package/frontend/src/lib/analysis/investigation-capsule.js +105 -0
- package/frontend/src/lib/analysis/lookup-asset-graph.js +806 -0
- package/frontend/src/lib/analysis/lookup-decision-support.js +487 -0
- package/frontend/src/lib/analysis/lookup-investigation-brief.js +153 -0
- package/frontend/src/lib/analysis/lookup-presentation.js +64 -0
- package/frontend/src/lib/analysis/lookup-response.js +2 -0
- package/frontend/src/lib/analysis/lookup-source-refresh.js +304 -0
- package/frontend/src/lib/analysis/lookup-summary-model.js +398 -0
- package/frontend/src/lib/analysis/page-similarity.js +1 -1
- package/frontend/src/lib/analysis/relationship-evidence.js +3 -3
- package/frontend/src/lib/analysis/relationship-observation-model.js +4 -4
- package/frontend/src/lib/analysis/scoring.js +9 -89
- package/frontend/src/lib/analysis/shortlist-model.js +8 -4
- package/frontend/src/lib/analysis/visualization-bounds.js +26 -0
- package/frontend/src/lib/analysis/visualization-force-graph.js +352 -0
- package/frontend/src/lib/analysis/visualization-matrix.js +88 -0
- package/frontend/src/lib/analysis/visualization-models.js +369 -0
- package/frontend/src/lib/analysis/visualization-time-series.js +144 -0
- package/frontend/src/lib/analysis/workspace-archive-crypto.js +1 -1
- package/frontend/src/lib/analysis/workspace-archive.js +4 -2
- package/lib/abusech-auth.mjs +1 -1
- package/lib/analyst-taxonomy.mjs +40 -0
- package/lib/availability.mjs +2 -2
- package/lib/bounded-contract-normalizers.mjs +33 -0
- package/lib/browser-library-profile.mjs +15 -2
- package/lib/certificate-extension-profile.mjs +71 -0
- package/lib/credential-surface-profile.mjs +1 -1
- package/lib/dns-convergence-review.mjs +183 -0
- package/lib/dns-delegation-health.mjs +196 -3
- package/lib/domain-assurance.mjs +368 -0
- package/lib/domain-change-packet.mjs +133 -0
- package/lib/domain-change-review.mjs +307 -0
- package/lib/domain-control-flight-recorder.mjs +268 -0
- package/lib/domain-control-manifest.mjs +50 -7
- package/lib/domain-portfolio-review.mjs +151 -0
- package/lib/domain-posture.mjs +9 -9
- package/lib/encrypted-dns-contract.mjs +1 -1
- package/lib/evidence-export.mjs +1 -1
- package/lib/evidence-report-markdown.mjs +1 -1
- package/lib/evidence-report.mjs +1 -1
- package/lib/generated/cisa-kev-catalog.mjs +1668 -0
- package/lib/html-signals.mjs +5 -11
- package/lib/ip-prefix.mjs +34 -1
- package/lib/local-geoip-evidence.mjs +2 -2
- package/lib/lookup-response-contract.mjs +346 -0
- package/lib/nameserver-preflight-review.mjs +159 -0
- package/lib/observed-network-context.mjs +1 -1
- package/lib/opportunity-scoring.mjs +107 -0
- package/lib/page-fingerprints.mjs +1 -1
- package/lib/page-language-signals.mjs +124 -0
- package/lib/rdap-capabilities.mjs +6 -6
- package/lib/rdap-search-workbench.mjs +96 -6
- package/lib/registry-access.mjs +1 -1
- package/lib/registry-capability-catalogue.mjs +4 -4
- package/lib/registry-insights.mjs +6 -6
- package/lib/risk-scoring.mjs +247 -83
- package/lib/scoring-evidence-quality.mjs +107 -0
- package/lib/security-txt.mjs +3 -3
- package/lib/static-html-analysis.mjs +29 -1
- package/lib/threat-intelligence-manifests.mjs +1 -1
- package/lib/threatfox-intelligence.mjs +1 -1
- package/lib/tls-intelligence.mjs +13 -3
- package/lib/trust-store-comparison.mjs +146 -0
- package/lib/urlhaus-intelligence.mjs +1 -1
- package/lib/website-security-posture.mjs +5 -5
- package/lib/website-technology.mjs +25 -11
- package/lib/whois-chain.mjs +1 -1
- package/lib/zone-intent-review.mjs +534 -0
- package/package.json +5 -1
- package/third-party-notices.txt +911 -19
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { buildScoreEvidenceQuality, } from "./scoring-evidence-quality.mjs";
|
|
2
|
+
export const OPPORTUNITY_MODEL_VERSION = 2;
|
|
3
|
+
export const OPPORTUNITY_REVIEW_THRESHOLD = 70;
|
|
4
|
+
const STATE_BASE = Object.freeze({
|
|
5
|
+
available: 82,
|
|
6
|
+
for_sale: 70,
|
|
7
|
+
expiring: 52,
|
|
8
|
+
registered: 22,
|
|
9
|
+
});
|
|
10
|
+
const STATE_LABELS = Object.freeze({
|
|
11
|
+
available: 'available',
|
|
12
|
+
for_sale: 'registered with a sale signal',
|
|
13
|
+
expiring: 'registered in a lifecycle transition',
|
|
14
|
+
registered: 'registered',
|
|
15
|
+
});
|
|
16
|
+
function finite(value) {
|
|
17
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
18
|
+
}
|
|
19
|
+
function formatAge(days) {
|
|
20
|
+
if (days < 60)
|
|
21
|
+
return `${Math.max(0, Math.round(days))}d old`;
|
|
22
|
+
const years = days / 365.25;
|
|
23
|
+
return years < 1 ? `${Math.round(days / 30)}mo old` : `${years.toFixed(1)}y old`;
|
|
24
|
+
}
|
|
25
|
+
function addFactor(factors, dimensions, family, label, delta) {
|
|
26
|
+
factors.push({ family, label, delta });
|
|
27
|
+
dimensions.set(family, (dimensions.get(family) ?? 0) + delta);
|
|
28
|
+
}
|
|
29
|
+
export function normalizeOpportunityModelVersion(value) {
|
|
30
|
+
return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 && value <= 1000 ? value : null;
|
|
31
|
+
}
|
|
32
|
+
// Opportunity is acquisition readiness, not price, value, eligibility, or a
|
|
33
|
+
// prediction that a registration will lapse. It uses only bounded evidence
|
|
34
|
+
// already collected for the current lookup.
|
|
35
|
+
export function explainOpportunityScore(value) {
|
|
36
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
37
|
+
return null;
|
|
38
|
+
const input = value;
|
|
39
|
+
const state = input.availability ?? input.state;
|
|
40
|
+
if (typeof state !== 'string' || !Object.hasOwn(STATE_BASE, state))
|
|
41
|
+
return null;
|
|
42
|
+
const factors = [];
|
|
43
|
+
const dimensions = new Map();
|
|
44
|
+
addFactor(factors, dimensions, 'registration', `Base readiness for “${STATE_LABELS[state] ?? state}”`, STATE_BASE[state] ?? 0);
|
|
45
|
+
if (input.confidence === 'high')
|
|
46
|
+
addFactor(factors, dimensions, 'registration', 'High-confidence registration decision', 8);
|
|
47
|
+
else if (input.confidence === 'medium')
|
|
48
|
+
addFactor(factors, dimensions, 'registration', 'Medium-confidence registration decision', 3);
|
|
49
|
+
else if (input.confidence === 'low')
|
|
50
|
+
addFactor(factors, dimensions, 'registration', 'Low-confidence registration decision', -8);
|
|
51
|
+
if (state === 'for_sale')
|
|
52
|
+
addFactor(factors, dimensions, 'listing', 'Bounded sale signal observed; listing remains unverified', 6);
|
|
53
|
+
else if (input.activityStatus === 'parked')
|
|
54
|
+
addFactor(factors, dimensions, 'listing', 'Parked page observed without a verified sale offer', 3);
|
|
55
|
+
else if (state === 'registered' && input.activityStatus === 'active') {
|
|
56
|
+
addFactor(factors, dimensions, 'listing', 'Active website suggests an in-use registration', -12);
|
|
57
|
+
}
|
|
58
|
+
if (input.hasPublicRegistrantContact === true) {
|
|
59
|
+
addFactor(factors, dimensions, 'contactability', 'Public registrant contact route observed; acquisition suitability is not established', 10);
|
|
60
|
+
}
|
|
61
|
+
const expiresInDays = finite(input.expiresInDays);
|
|
62
|
+
if (state === 'registered' && expiresInDays !== null && expiresInDays >= 0 && expiresInDays < 30) {
|
|
63
|
+
addFactor(factors, dimensions, 'lifecycle', 'Published expiry is within 30 days; release is not implied', 4);
|
|
64
|
+
}
|
|
65
|
+
if (typeof input.privacyProtected === 'boolean') {
|
|
66
|
+
addFactor(factors, dimensions, 'contactability', 'Registration privacy is neutral; it does not establish contactability', 0);
|
|
67
|
+
}
|
|
68
|
+
const ageDays = finite(input.domainAgeDays);
|
|
69
|
+
if (ageDays !== null && ageDays >= 0) {
|
|
70
|
+
addFactor(factors, dimensions, 'registration', `Domain age (${formatAge(ageDays)}) is context only`, 0);
|
|
71
|
+
}
|
|
72
|
+
const rawScore = Math.round(factors.reduce((total, factor) => total + factor.delta, 0));
|
|
73
|
+
const score = Math.max(0, Math.min(100, rawScore));
|
|
74
|
+
const dimensionLabels = {
|
|
75
|
+
registration: 'Registration certainty',
|
|
76
|
+
listing: 'Listing readiness',
|
|
77
|
+
contactability: 'Contactability',
|
|
78
|
+
lifecycle: 'Lifecycle timing',
|
|
79
|
+
};
|
|
80
|
+
const observedFamilies = [...dimensions.keys()];
|
|
81
|
+
const evidenceQuality = buildScoreEvidenceQuality(input, {
|
|
82
|
+
expectedDepth: 'fast',
|
|
83
|
+
observedFamilies,
|
|
84
|
+
expectedFamilies: ['registration'],
|
|
85
|
+
});
|
|
86
|
+
return {
|
|
87
|
+
modelVersion: OPPORTUNITY_MODEL_VERSION,
|
|
88
|
+
score,
|
|
89
|
+
rawScore,
|
|
90
|
+
capped: score !== rawScore,
|
|
91
|
+
factors,
|
|
92
|
+
dimensions: [...dimensions.entries()].map(([id, contribution]) => ({ id, label: dimensionLabels[id], contribution })),
|
|
93
|
+
evidenceQuality,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
export function computeOpportunityScore(value) {
|
|
97
|
+
return explainOpportunityScore(value)?.score ?? null;
|
|
98
|
+
}
|
|
99
|
+
export function opportunityTone(score) {
|
|
100
|
+
if (score === null)
|
|
101
|
+
return 'neutral';
|
|
102
|
+
if (score >= OPPORTUNITY_REVIEW_THRESHOLD)
|
|
103
|
+
return 'good';
|
|
104
|
+
if (score >= 40)
|
|
105
|
+
return 'neutral';
|
|
106
|
+
return 'warn';
|
|
107
|
+
}
|
|
@@ -430,7 +430,7 @@ function createPageFingerprints(html, options = {}) {
|
|
|
430
430
|
|| parsedStructure.structureLimitReached || text?.truncated === true || forms?.truncated === true
|
|
431
431
|
|| resourceHosts.truncated || identifiers.truncated;
|
|
432
432
|
const limitations = [
|
|
433
|
-
'Fingerprints
|
|
433
|
+
'Fingerprints summarise capped static HTML and are comparison aids, not cryptographic proof of page authorship or intent.',
|
|
434
434
|
'Visible-text SimHash is a fuzzy similarity fingerprint and must not be treated as a cryptographic digest.',
|
|
435
435
|
'DOM-structure SimHash is derived from a standards-compliant static token stream and is a review aid, not proof that two pages share an author or template.',
|
|
436
436
|
];
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// Reviewed, bounded language packs for static homepage text already retained by
|
|
2
|
+
// Deep Lookup. Matching emits fixed labels only: no page phrase, surrounding
|
|
3
|
+
// text, or arbitrary markup is returned or stored.
|
|
4
|
+
import { analyzeStaticHtml, MAX_STATIC_HTML_CHARS, } from "./static-html-analysis.mjs";
|
|
5
|
+
export const PAGE_LANGUAGE_SIGNAL_VERSION = 1;
|
|
6
|
+
export const MAX_PAGE_LANGUAGE_SIGNAL_CHARS = MAX_STATIC_HTML_CHARS;
|
|
7
|
+
const CONTROL_CHARACTER_RE = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u;
|
|
8
|
+
const CATEGORY_LABELS = Object.freeze({
|
|
9
|
+
account_verification: 'account-verification',
|
|
10
|
+
account_restriction: 'account-restriction',
|
|
11
|
+
payment_update: 'payment-update',
|
|
12
|
+
recovery_secret: 'wallet or recovery-secret',
|
|
13
|
+
urgent_action: 'urgent-action',
|
|
14
|
+
});
|
|
15
|
+
const PAGE_LANGUAGE_PACKS = Object.freeze([
|
|
16
|
+
{
|
|
17
|
+
id: 'en-reviewed-v1', language: 'en', label: 'English', patterns: [
|
|
18
|
+
{ category: 'account_verification', expression: /(?:verify your account|confirm your identity|unusual (?:sign-?in|login) activity|click here to (?:verify|confirm|update|restore)|confirm your password)/iu },
|
|
19
|
+
{ category: 'account_restriction', expression: /(?:account has been (?:suspended|limited|locked|restricted)|your account will be (?:suspended|closed|locked|terminated)|re-?activate your account|unauthorized access detected|your password (?:has expired|will expire soon))/iu },
|
|
20
|
+
{ category: 'urgent_action', expression: /(?:security alert|immediate action required)/iu },
|
|
21
|
+
{ category: 'payment_update', expression: /update your (?:payment|billing) (?:information|details)/iu },
|
|
22
|
+
{ category: 'recovery_secret', expression: /(?:(?:connect|verify) your wallet|(?:enter|verify) your (?:seed|recovery) phrase|enter your private key)/iu },
|
|
23
|
+
],
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
id: 'es-reviewed-v1', language: 'es', label: 'Spanish', patterns: [
|
|
27
|
+
{ category: 'account_verification', expression: /(?:verifica (?:tu|su) cuenta|confirma (?:tu|su) identidad)/iu },
|
|
28
|
+
{ category: 'account_restriction', expression: /(?:tu|su) cuenta (?:ha sido|será) (?:suspendida|bloqueada|limitada|restringida)/iu },
|
|
29
|
+
{ category: 'urgent_action', expression: /(?:acción inmediata requerida|alerta de seguridad)/iu },
|
|
30
|
+
{ category: 'payment_update', expression: /actualiza (?:tu|su) (?:información|datos) de pago/iu },
|
|
31
|
+
{ category: 'recovery_secret', expression: /(?:introduce|ingrese) (?:tu|su) frase de recuperación/iu },
|
|
32
|
+
],
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
id: 'fr-reviewed-v1', language: 'fr', label: 'French', patterns: [
|
|
36
|
+
{ category: 'account_verification', expression: /(?:vérifiez votre compte|confirmez votre identité|veuillez confirmer votre identité)/iu },
|
|
37
|
+
{ category: 'account_restriction', expression: /votre compte (?:a été|sera) (?:suspendu|bloqué|limité|restreint)/iu },
|
|
38
|
+
{ category: 'urgent_action', expression: /(?:action immédiate requise|alerte de sécurité)/iu },
|
|
39
|
+
{ category: 'payment_update', expression: /mettez à jour vos (?:informations|coordonnées) de paiement/iu },
|
|
40
|
+
{ category: 'recovery_secret', expression: /saisissez votre phrase de récupération/iu },
|
|
41
|
+
],
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
id: 'de-reviewed-v1', language: 'de', label: 'German', patterns: [
|
|
45
|
+
{ category: 'account_verification', expression: /(?:bestätigen sie ihr konto|bestätigen sie ihre identität)/iu },
|
|
46
|
+
{ category: 'account_restriction', expression: /ihr konto wurde (?:gesperrt|eingeschränkt|suspendiert)/iu },
|
|
47
|
+
{ category: 'urgent_action', expression: /(?:sofortiges handeln erforderlich|sicherheitswarnung)/iu },
|
|
48
|
+
{ category: 'payment_update', expression: /aktualisieren sie ihre zahlungsdaten/iu },
|
|
49
|
+
{ category: 'recovery_secret', expression: /geben sie ihre wiederherstellungsphrase ein/iu },
|
|
50
|
+
],
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
id: 'pt-reviewed-v1', language: 'pt', label: 'Portuguese', patterns: [
|
|
54
|
+
{ category: 'account_verification', expression: /(?:verifique sua conta|confirme sua identidade)/iu },
|
|
55
|
+
{ category: 'account_restriction', expression: /sua conta (?:foi|será) (?:suspensa|bloqueada|limitada|restrita)/iu },
|
|
56
|
+
{ category: 'urgent_action', expression: /(?:ação imediata necessária|alerta de segurança)/iu },
|
|
57
|
+
{ category: 'payment_update', expression: /atualize suas informações de pagamento/iu },
|
|
58
|
+
{ category: 'recovery_secret', expression: /insira sua frase de recuperação/iu },
|
|
59
|
+
],
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
id: 'it-reviewed-v1', language: 'it', label: 'Italian', patterns: [
|
|
63
|
+
{ category: 'account_verification', expression: /(?:verifica il tuo account|conferma la tua identità)/iu },
|
|
64
|
+
{ category: 'account_restriction', expression: /il tuo account (?:è stato|sarà) (?:sospeso|bloccato|limitato)/iu },
|
|
65
|
+
{ category: 'urgent_action', expression: /(?:azione immediata richiesta|avviso di sicurezza)/iu },
|
|
66
|
+
{ category: 'payment_update', expression: /aggiorna (?:i tuoi dati|le tue informazioni) di pagamento/iu },
|
|
67
|
+
{ category: 'recovery_secret', expression: /inserisci la (?:tua )?frase di recupero/iu },
|
|
68
|
+
],
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
id: 'nl-reviewed-v1', language: 'nl', label: 'Dutch', patterns: [
|
|
72
|
+
{ category: 'account_verification', expression: /(?:verifieer uw account|bevestig uw identiteit)/iu },
|
|
73
|
+
{ category: 'account_restriction', expression: /uw account (?:is|wordt) (?:geblokkeerd|beperkt|opgeschort)/iu },
|
|
74
|
+
{ category: 'urgent_action', expression: /(?:onmiddellijke actie vereist|beveiligingswaarschuwing)/iu },
|
|
75
|
+
{ category: 'payment_update', expression: /werk uw betalingsgegevens bij/iu },
|
|
76
|
+
{ category: 'recovery_secret', expression: /voer uw herstelzin in/iu },
|
|
77
|
+
],
|
|
78
|
+
},
|
|
79
|
+
]);
|
|
80
|
+
function preferredPacks(documentLanguage) {
|
|
81
|
+
if (typeof documentLanguage !== 'string')
|
|
82
|
+
return PAGE_LANGUAGE_PACKS;
|
|
83
|
+
const primary = documentLanguage.trim().toLowerCase().split('-', 1)[0];
|
|
84
|
+
if (!primary)
|
|
85
|
+
return PAGE_LANGUAGE_PACKS;
|
|
86
|
+
const declared = PAGE_LANGUAGE_PACKS.filter((pack) => pack.language === primary);
|
|
87
|
+
return declared.length
|
|
88
|
+
? Object.freeze([...declared, ...PAGE_LANGUAGE_PACKS.filter((pack) => pack.language !== primary)])
|
|
89
|
+
: PAGE_LANGUAGE_PACKS;
|
|
90
|
+
}
|
|
91
|
+
export function detectPageLanguageSignal(input, documentLanguage, suppliedAnalysis) {
|
|
92
|
+
if (typeof input !== 'string' || !input || input.length > MAX_PAGE_LANGUAGE_SIGNAL_CHARS || CONTROL_CHARACTER_RE.test(input)) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
const analysis = suppliedAnalysis ?? analyzeStaticHtml(input, { includeVisibleText: true });
|
|
96
|
+
if (analysis.inputLimitReached || analysis.visibleTextLimitReached)
|
|
97
|
+
return null;
|
|
98
|
+
const visibleText = analysis.visibleText
|
|
99
|
+
.replace(/\s+/gu, ' ')
|
|
100
|
+
.trim();
|
|
101
|
+
if (!visibleText)
|
|
102
|
+
return null;
|
|
103
|
+
for (const pack of preferredPacks(documentLanguage)) {
|
|
104
|
+
for (const pattern of pack.patterns) {
|
|
105
|
+
if (!pattern.expression.test(visibleText))
|
|
106
|
+
continue;
|
|
107
|
+
return Object.freeze({
|
|
108
|
+
version: PAGE_LANGUAGE_SIGNAL_VERSION,
|
|
109
|
+
packId: pack.id,
|
|
110
|
+
language: pack.language,
|
|
111
|
+
category: pattern.category,
|
|
112
|
+
label: `Reviewed ${pack.label} ${CATEGORY_LABELS[pattern.category]} language`,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
export function pageLanguagePackCatalogue() {
|
|
119
|
+
return Object.freeze(PAGE_LANGUAGE_PACKS.map((pack) => Object.freeze({
|
|
120
|
+
id: pack.id,
|
|
121
|
+
language: pack.language,
|
|
122
|
+
categories: Object.freeze([...new Set(pack.patterns.map((pattern) => pattern.category))]),
|
|
123
|
+
})));
|
|
124
|
+
}
|
|
@@ -44,7 +44,7 @@ const EXTENSION_CATALOG = Object.freeze({
|
|
|
44
44
|
farv1: {
|
|
45
45
|
label: 'Federated authentication',
|
|
46
46
|
category: 'standard',
|
|
47
|
-
capability: 'Federated
|
|
47
|
+
capability: 'Federated authorisation profile declaration.',
|
|
48
48
|
},
|
|
49
49
|
fred: {
|
|
50
50
|
label: 'Registry-system extension',
|
|
@@ -111,7 +111,7 @@ const EXTENSION_CATALOG = Object.freeze({
|
|
|
111
111
|
paging: {
|
|
112
112
|
label: 'Result paging',
|
|
113
113
|
category: 'standard',
|
|
114
|
-
capability: 'Paged search-result
|
|
114
|
+
capability: 'Paged search-result behaviour.',
|
|
115
115
|
},
|
|
116
116
|
platformns: {
|
|
117
117
|
label: 'Nameserver platform extension',
|
|
@@ -146,17 +146,17 @@ const EXTENSION_CATALOG = Object.freeze({
|
|
|
146
146
|
rirsearch1: {
|
|
147
147
|
label: 'Address-registry search',
|
|
148
148
|
category: 'standard',
|
|
149
|
-
capability: 'Versioned network and autonomous-system search
|
|
149
|
+
capability: 'Versioned network and autonomous-system search behaviour.',
|
|
150
150
|
},
|
|
151
151
|
sorting: {
|
|
152
152
|
label: 'Result sorting',
|
|
153
153
|
category: 'standard',
|
|
154
|
-
capability: 'Sorted search-result
|
|
154
|
+
capability: 'Sorted search-result behaviour.',
|
|
155
155
|
},
|
|
156
156
|
subsetting: {
|
|
157
157
|
label: 'Partial response',
|
|
158
158
|
category: 'standard',
|
|
159
|
-
capability: 'Field-subset response
|
|
159
|
+
capability: 'Field-subset response behaviour.',
|
|
160
160
|
},
|
|
161
161
|
ttl0: {
|
|
162
162
|
label: 'DNS TTL extension',
|
|
@@ -270,7 +270,7 @@ export function inspectRdapCapabilities(parsedValue, status) {
|
|
|
270
270
|
.map((entry) => entry.identifier)
|
|
271
271
|
.slice(0, MAX_RDAP_UNKNOWN_CAPABILITIES);
|
|
272
272
|
const limitations = [
|
|
273
|
-
'Declarations describe what the server claims for this response; they do not prove that an operation is
|
|
273
|
+
'Declarations describe what the server claims for this response; they do not prove that an operation is authorised, implemented correctly, complete, or available anonymously.',
|
|
274
274
|
'Unknown identifiers remain visible and neutral. Catalogue recognition never affects availability or Risk scoring.',
|
|
275
275
|
];
|
|
276
276
|
if (state === 'unavailable') {
|
|
@@ -2,10 +2,16 @@ const RDAP_SEARCH_WORKBENCH_SCHEMA = 'whoisleuth.rdap-search-workbench';
|
|
|
2
2
|
const RDAP_SEARCH_WORKBENCH_VERSION = 1;
|
|
3
3
|
const MAX_HELP_ENTRIES = 64;
|
|
4
4
|
const MAX_QUERY_VALUE_LENGTH = 254;
|
|
5
|
-
const SEARCHABLE_RESOURCE_TYPES = new Set(['domains', 'nameservers', 'entities'
|
|
6
|
-
const RELATED_RESOURCE_TYPES = new Set(['
|
|
5
|
+
const SEARCHABLE_RESOURCE_TYPES = new Set(['domains', 'nameservers', 'entities']);
|
|
6
|
+
const RELATED_RESOURCE_TYPES = new Set(['entity']);
|
|
7
7
|
const RECOGNIZED_PROPERTIES = new Set(['handle', 'fn', 'email', 'role']);
|
|
8
8
|
const SENSITIVE_PROPERTIES = new Set(['fn', 'email']);
|
|
9
|
+
const REGISTERED_PROPERTY_PATHS = Object.freeze({
|
|
10
|
+
email: Object.freeze(["$.entities[*].vcardArray[1][?(@[0]=='email')][3]"]),
|
|
11
|
+
fn: Object.freeze(["$.entities[*].vcardArray[1][?(@[0]=='fn')][3]"]),
|
|
12
|
+
handle: Object.freeze(['$.entities[*].handle']),
|
|
13
|
+
role: Object.freeze(['$.entities[*].roles']),
|
|
14
|
+
});
|
|
9
15
|
function record(value) {
|
|
10
16
|
return value && typeof value === 'object' && !Array.isArray(value)
|
|
11
17
|
? value
|
|
@@ -136,14 +142,13 @@ function planRdapReverseSearch(help, requested) {
|
|
|
136
142
|
limitation: 'The supplied help response did not advertise this exact reverse-search tuple.',
|
|
137
143
|
});
|
|
138
144
|
}
|
|
139
|
-
const parameter = `${relatedResourceType}_${property}`;
|
|
140
145
|
const contactDisclosure = capability.disclosure === 'contact';
|
|
141
146
|
return Object.freeze({
|
|
142
147
|
schema: RDAP_SEARCH_WORKBENCH_SCHEMA,
|
|
143
148
|
version: RDAP_SEARCH_WORKBENCH_VERSION,
|
|
144
149
|
state: 'ready',
|
|
145
|
-
requestPath: `/${searchableResourceType}`,
|
|
146
|
-
query: Object.freeze({ [
|
|
150
|
+
requestPath: `/${searchableResourceType}/reverse_search/${relatedResourceType}`,
|
|
151
|
+
query: Object.freeze({ [property]: value }),
|
|
147
152
|
disclosure: Object.freeze({
|
|
148
153
|
class: capability.disclosure,
|
|
149
154
|
summary: contactDisclosure
|
|
@@ -154,4 +159,89 @@ function planRdapReverseSearch(help, requested) {
|
|
|
154
159
|
limitation: 'This is an offline request plan. It neither sends the query nor establishes result authority, completeness, or currentness.',
|
|
155
160
|
});
|
|
156
161
|
}
|
|
157
|
-
|
|
162
|
+
function inspectRdapReverseSearchResponse(value, requestedProperties = []) {
|
|
163
|
+
const document = record(value);
|
|
164
|
+
if (!document) {
|
|
165
|
+
return Object.freeze({
|
|
166
|
+
schema: RDAP_SEARCH_WORKBENCH_SCHEMA,
|
|
167
|
+
version: RDAP_SEARCH_WORKBENCH_VERSION,
|
|
168
|
+
state: 'invalid',
|
|
169
|
+
mappings: Object.freeze([]),
|
|
170
|
+
missingProperties: Object.freeze([]),
|
|
171
|
+
rejectedCount: 0,
|
|
172
|
+
truncated: false,
|
|
173
|
+
limitations: Object.freeze(['The reverse-search response was not an object.']),
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
const requested = [...new Set(requestedProperties
|
|
177
|
+
.map(boundedToken)
|
|
178
|
+
.filter((property) => Boolean(property)))]
|
|
179
|
+
.slice(0, RECOGNIZED_PROPERTIES.size);
|
|
180
|
+
const raw = document.reverse_search_properties_mapping;
|
|
181
|
+
if (!Array.isArray(raw)) {
|
|
182
|
+
return Object.freeze({
|
|
183
|
+
schema: RDAP_SEARCH_WORKBENCH_SCHEMA,
|
|
184
|
+
version: RDAP_SEARCH_WORKBENCH_VERSION,
|
|
185
|
+
state: 'invalid',
|
|
186
|
+
mappings: Object.freeze([]),
|
|
187
|
+
missingProperties: Object.freeze(requested),
|
|
188
|
+
rejectedCount: raw === undefined ? 0 : 1,
|
|
189
|
+
truncated: false,
|
|
190
|
+
limitations: Object.freeze([
|
|
191
|
+
'The response did not provide a valid reverse-search property mapping, so its result fields cannot be reconciled with the requested predicates.',
|
|
192
|
+
]),
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
const mappings = [];
|
|
196
|
+
const seen = new Set();
|
|
197
|
+
let rejectedCount = 0;
|
|
198
|
+
const truncated = raw.length > MAX_HELP_ENTRIES;
|
|
199
|
+
for (const entry of raw.slice(0, MAX_HELP_ENTRIES)) {
|
|
200
|
+
const candidate = record(entry);
|
|
201
|
+
const property = boundedToken(candidate?.property);
|
|
202
|
+
const propertyPath = typeof candidate?.propertyPath === 'string'
|
|
203
|
+
? candidate.propertyPath.trim().slice(0, 512)
|
|
204
|
+
: '';
|
|
205
|
+
if (!property || !propertyPath || /[\x00-\x1f\x7f]/u.test(propertyPath)) {
|
|
206
|
+
rejectedCount += 1;
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
const key = `${property}\u0000${propertyPath}`;
|
|
210
|
+
if (seen.has(key))
|
|
211
|
+
continue;
|
|
212
|
+
seen.add(key);
|
|
213
|
+
mappings.push(Object.freeze({
|
|
214
|
+
property,
|
|
215
|
+
propertyPath,
|
|
216
|
+
state: REGISTERED_PROPERTY_PATHS[property]?.includes(propertyPath)
|
|
217
|
+
? 'registered'
|
|
218
|
+
: 'unrecognized',
|
|
219
|
+
}));
|
|
220
|
+
}
|
|
221
|
+
if (truncated)
|
|
222
|
+
rejectedCount += raw.length - MAX_HELP_ENTRIES;
|
|
223
|
+
const mappedProperties = new Set(mappings.map((mapping) => mapping.property));
|
|
224
|
+
const missingProperties = requested.filter((property) => !mappedProperties.has(property));
|
|
225
|
+
const limitations = [];
|
|
226
|
+
if (missingProperties.length) {
|
|
227
|
+
limitations.push(`The response omitted mappings for requested properties: ${missingProperties.join(', ')}.`);
|
|
228
|
+
}
|
|
229
|
+
if (mappings.some((mapping) => mapping.state === 'unrecognized')) {
|
|
230
|
+
limitations.push('At least one mapping differs from the locally reviewed IANA registration and requires manual specification review.');
|
|
231
|
+
}
|
|
232
|
+
if (rejectedCount)
|
|
233
|
+
limitations.push(`${rejectedCount} malformed or excess mapping entr${rejectedCount === 1 ? 'y was' : 'ies were'} not used.`);
|
|
234
|
+
if (!mappings.length)
|
|
235
|
+
limitations.push('No usable reverse-search property mapping was retained.');
|
|
236
|
+
return Object.freeze({
|
|
237
|
+
schema: RDAP_SEARCH_WORKBENCH_SCHEMA,
|
|
238
|
+
version: RDAP_SEARCH_WORKBENCH_VERSION,
|
|
239
|
+
state: mappings.length && !limitations.length ? 'complete' : mappings.length ? 'partial' : 'invalid',
|
|
240
|
+
mappings: Object.freeze(mappings),
|
|
241
|
+
missingProperties: Object.freeze(missingProperties),
|
|
242
|
+
rejectedCount,
|
|
243
|
+
truncated,
|
|
244
|
+
limitations: Object.freeze(limitations),
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
export { MAX_HELP_ENTRIES, MAX_QUERY_VALUE_LENGTH, RDAP_SEARCH_WORKBENCH_SCHEMA, RDAP_SEARCH_WORKBENCH_VERSION, normalizeRdapSearchHelp, planRdapReverseSearch, inspectRdapReverseSearchResponse, };
|
package/lib/registry-access.mjs
CHANGED
|
@@ -3,7 +3,7 @@ const REGISTRY_ACCESS_PROFILE_LABELS = Object.freeze({
|
|
|
3
3
|
'iana-referral': 'IANA referral discovery',
|
|
4
4
|
'no-iana-service': 'No service published by IANA',
|
|
5
5
|
'registry-policy-restricted': 'Registry policy restricted',
|
|
6
|
-
'source-ip-authorization-required': 'Source-IP
|
|
6
|
+
'source-ip-authorization-required': 'Source-IP authorisation required',
|
|
7
7
|
});
|
|
8
8
|
const WHOIS_REGISTRY_ACCESS_PROFILES = new Set([
|
|
9
9
|
'iana-referral',
|
|
@@ -51,9 +51,9 @@ const REGISTRY_STANDARDS_COVERAGE_SNAPSHOT = Object.freeze({
|
|
|
51
51
|
]),
|
|
52
52
|
interpretation: 'This official-source snapshot describes published service coverage at the verification date. It does not test current reachability or decide registration, availability, ownership, safety, or maliciousness.',
|
|
53
53
|
});
|
|
54
|
-
const DISCOVERY_LIMITATION = 'IANA discovery is available, but no suffix-specific query, encoding, or parser
|
|
54
|
+
const DISCOVERY_LIMITATION = 'IANA discovery is available, but no suffix-specific query, encoding, or parser behaviour is fixture-verified.';
|
|
55
55
|
const FIXTURE_LIMITATION = 'Synthetic fixtures verify the current parser profile; they do not prove current live-registry reachability, policy, or field publication.';
|
|
56
|
-
const ES_ACCESS_LIMITATION = 'The registry WHOIS service requires advance source-IP
|
|
56
|
+
const ES_ACCESS_LIMITATION = 'The registry WHOIS service requires advance source-IP authorisation. A failed or unavailable query is not evidence that the domain is unregistered.';
|
|
57
57
|
const SWITCH_ACCESS_LIMITATION = 'The registry directs public domain searches to its official lookup. Its non-standard-port Domain Check is not integrated, and IANA publishes no RDAP service. Missing registry data is not evidence that the domain is unregistered.';
|
|
58
58
|
const VN_ACCESS_LIMITATION = 'IANA publishes no domain WHOIS or RDAP service for this suffix. The official browser lookup is not integrated, and missing registry data is not evidence that the domain is unregistered.';
|
|
59
59
|
const UK_TRANSITION_LIMITATION = 'Synthetic fixtures verify the documented sectioned port-43 response while that WHOIS service is phased out. RDAP remains the preferred registry source, and fixture coverage does not prove current reachability or field publication.';
|
|
@@ -64,8 +64,8 @@ const SHARED_WHOIS_NO_RDAP_LIMITATION = 'Synthetic fixture reuse is limited to a
|
|
|
64
64
|
const WHOIS_ONLY_FIXTURE_LIMITATION = 'Synthetic fixtures verify the current WHOIS parser profile. IANA publishes no RDAP bootstrap service for this suffix, and missing registry data is not evidence that the domain is unregistered.';
|
|
65
65
|
const NEGATIVE_ONLY_FIXTURE_LIMITATION = 'Synthetic fixtures verify only the authoritative no-record dialect; registered-field compatibility is not claimed, and fixture coverage does not prove current live-registry reachability or policy.';
|
|
66
66
|
const WHOIS_ONLY_NEGATIVE_FIXTURE_LIMITATION = 'Synthetic fixtures verify only the authoritative WHOIS no-record dialect; registered-field compatibility is not claimed, IANA publishes no RDAP bootstrap service, and missing registry data is not evidence that the domain is unregistered.';
|
|
67
|
-
const UNVERIFIED_WHOIS_REFERRAL_LIMITATION = 'IANA publishes a domain WHOIS referral, but its response
|
|
68
|
-
const UNVERIFIED_WHOIS_WITH_RDAP_LIMITATION = 'IANA publishes domain WHOIS and RDAP discovery, but the WHOIS response
|
|
67
|
+
const UNVERIFIED_WHOIS_REFERRAL_LIMITATION = 'IANA publishes a domain WHOIS referral, but its response behaviour is not fixture-verified. A failed, restricted, prohibited, reserved, or otherwise inconclusive response is not evidence that the domain is unregistered, and IANA publishes no RDAP bootstrap service for this suffix.';
|
|
68
|
+
const UNVERIFIED_WHOIS_WITH_RDAP_LIMITATION = 'IANA publishes domain WHOIS and RDAP discovery, but the WHOIS response behaviour is not fixture-verified. A failed, restricted, prohibited, reserved, or otherwise inconclusive WHOIS response is not evidence that the domain is unregistered; RDAP remains separately attributed.';
|
|
69
69
|
const NORID_CLOSED_SUFFIX_LIMITATION = 'The registry has not opened this suffix for registrations, and IANA publishes no domain WHOIS or RDAP service. Missing registry data is contextual only and must not be interpreted as a live availability result.';
|
|
70
70
|
const MIL_ACCESS_LIMITATION = 'IANA publishes no public domain WHOIS or RDAP service for this sponsored suffix. Missing registry data is contextual only and is not evidence that a name is unregistered or available.';
|
|
71
71
|
const ARPA_ACCESS_LIMITATION = 'This infrastructure suffix is not ordinary public registration space. IANA publishes WHOIS metadata but no RDAP bootstrap service, and missing or non-domain-shaped data must not be interpreted as registration availability.';
|
|
@@ -83,7 +83,7 @@ function disclosure(source, parsedValue, status) {
|
|
|
83
83
|
return {
|
|
84
84
|
source,
|
|
85
85
|
state: 'unavailable',
|
|
86
|
-
detail: 'The source did not provide a usable
|
|
86
|
+
detail: 'The source did not provide a usable normalised publication.',
|
|
87
87
|
};
|
|
88
88
|
}
|
|
89
89
|
const parsed = record(parsedValue);
|
|
@@ -211,11 +211,11 @@ function publicationDiagnostic(source, parsedValue, status, observedAt) {
|
|
|
211
211
|
if (parsed.serverTruncated === true)
|
|
212
212
|
issues.push('The RDAP server declared a truncated response.');
|
|
213
213
|
if (parsed.entitiesTruncated === true)
|
|
214
|
-
issues.push('The
|
|
214
|
+
issues.push('The normalised entity inventory was capped.');
|
|
215
215
|
if (parsed.statusesTruncated === true)
|
|
216
|
-
issues.push('The
|
|
216
|
+
issues.push('The normalised status inventory was capped.');
|
|
217
217
|
if (strings(parsed.fieldsTruncated, 20).length)
|
|
218
|
-
issues.push('One or more
|
|
218
|
+
issues.push('One or more normalised WHOIS fields were capped.');
|
|
219
219
|
if (parsed.chainStatus === 'partial')
|
|
220
220
|
issues.push('The WHOIS referral chain was incomplete or conflicting.');
|
|
221
221
|
const conformance = strings(parsed.conformance, 30);
|
|
@@ -306,9 +306,9 @@ export function buildRegistryInsights(input) {
|
|
|
306
306
|
: unavailableCount
|
|
307
307
|
? 'The comparison is partial because at least one publication was unavailable or incomplete.'
|
|
308
308
|
: sourceOnlyCount || redactedCount
|
|
309
|
-
? 'The sources differ in publication or disclosure without a
|
|
309
|
+
? 'The sources differ in publication or disclosure without a normalised conflict.'
|
|
310
310
|
: counts.equivalent
|
|
311
|
-
? 'The comparable fields are
|
|
311
|
+
? 'The comparable fields are normalised as equivalent.'
|
|
312
312
|
: 'No comparable fields were available.',
|
|
313
313
|
},
|
|
314
314
|
publications: [
|