@slicedearth/whoisleuth-cli 2.5.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +1 -1
- package/cli/artifact-validation/case-response.mjs +51 -8
- package/cli/artifact-validation/lookup-evidence.mjs +15 -1
- package/cli/artifact-validation/signed-review.mjs +2 -51
- package/cli/artifact-verify.mjs +9 -2
- package/cli/bulk-command-runner.mjs +4 -6
- package/cli/case-command.mjs +79 -3
- package/cli/case-pack.mjs +7 -2
- package/cli/command-definition.mjs +2 -1
- package/cli/discovery-scan-command-runner.mjs +4 -6
- package/cli/domain-feed.mjs +61 -0
- package/cli/formatters/json.mjs +1 -13
- package/cli/indicator-set-command.mjs +1 -2
- package/cli/intake-command.mjs +43 -3
- package/cli/investigation-artifacts.mjs +2 -3
- package/cli/investigation-package-review.mjs +2 -0
- package/cli/local-mmdb-review.mjs +27 -4
- package/cli/lookup-command-runner.mjs +1 -0
- package/cli/offline-evidence-review.mjs +51 -5
- package/cli/review-arguments.mjs +3 -2
- package/cli/review-command-definitions.mjs +14 -8
- package/cli/review-command-runner.mjs +6 -3
- package/cli/risk-calibration.mjs +6 -4
- package/cli/selected-input-worker.mjs +12 -2
- package/cli/support-command-runner.mjs +3 -3
- package/cli/watchlist-review.mjs +43 -0
- package/cli/workflow-arguments.mjs +14 -0
- package/cli/workflow-command-definitions.mjs +26 -1
- package/cli/workflow-command-runner.mjs +32 -0
- package/docs/cli.md +99 -2
- package/docs/contextual-reviews.md +28 -5
- package/docs/infrastructure-observations.md +126 -0
- package/docs/offline-intake-context.md +213 -0
- package/lib/credential-form-attribution.mjs +136 -0
- package/lib/credential-surface-profile.mjs +6 -2
- package/lib/dns-delegation-health.mjs +23 -7
- package/lib/dns-intelligence.mjs +18 -4
- package/lib/domain-posture-analysis.mjs +53 -2
- package/lib/domain-posture-parsers.mjs +9 -0
- package/lib/domain-posture.mjs +29 -10
- package/lib/evidence-export-privacy.mjs +5 -2
- package/lib/evidence-report.mjs +1 -0
- package/lib/favicon.mjs +13 -4
- package/lib/html-signals.mjs +6 -0
- package/lib/lookup-child-profile-contract.mjs +28 -11
- package/lib/lookup-response-contract.mjs +1 -0
- package/lib/operation-budget.mjs +4 -0
- package/lib/service-binding-dns.mjs +19 -2
- package/lib/sslbl-certificates.generated.mjs +138 -137
- package/lib/static-html-analysis.mjs +32 -4
- package/lib/technology-evidence-role.mjs +1 -51
- package/lib/tls-intelligence.mjs +30 -5
- package/lib/website-enrichment.mjs +39 -23
- package/lib/whois-chain.mjs +12 -3
- package/lib/whois-common-formats.mjs +3 -1
- package/lib/whois-contacts.mjs +13 -2
- package/package.json +7 -5
- package/packages/analysis/risk-scoring.mjs +12 -9
- package/packages/cases/case-containment-handoff.mjs +195 -0
- package/packages/cases/case-evidence-links.mjs +16 -0
- package/packages/cases/case-evidence-model.mjs +14 -3
- package/packages/cases/case-object-coverage.mjs +33 -0
- package/packages/cases/case-packet-correction.mjs +106 -0
- package/packages/cases/case-recheck-model.mjs +35 -5
- package/packages/cases/case-record-operations.mjs +74 -9
- package/packages/cases/case-record-projection.mjs +15 -3
- package/packages/cases/case-record-version-input.mjs +23 -1
- package/packages/cases/case-report.mjs +30 -0
- package/packages/cases/case-requested-evidence.mjs +1 -1
- package/packages/cases/case-response-actions.mjs +57 -1
- package/packages/cases/case-response-model.mjs +7 -0
- package/packages/cases/case-response-object.mjs +119 -0
- package/packages/cases/case-response-outcomes.mjs +81 -14
- package/packages/cases/case-response-packet-render.mjs +22 -3
- package/packages/cases/case-response-packet.mjs +30 -2
- package/packages/cases/case-response-review-inputs.mjs +48 -7
- package/packages/cases/case-workflow-metadata.mjs +1 -9
- package/packages/contracts/analyst-interchange.mjs +7 -5
- package/packages/contracts/candidate-watch-review.mjs +54 -0
- package/packages/contracts/case-portability.mjs +260 -42
- package/packages/contracts/cli-command-semantics.mjs +3 -0
- package/packages/contracts/context-review.mjs +2 -1
- package/packages/contracts/external-observation-interchange.mjs +96 -28
- package/packages/contracts/lookup-evidence.mjs +3 -2
- package/packages/contracts/message-intake.mjs +7 -1
- package/packages/contracts/risk-calibration.mjs +44 -29
- package/packages/contracts/schema-lifecycle.mjs +3 -1
- package/packages/contracts/web-capture.mjs +1 -1
- package/packages/contracts/workspace-portability.mjs +86 -16
- package/packages/evidence/collection-quality.mjs +1 -1
- package/packages/evidence/credential-form-attribution.mjs +60 -0
- package/packages/evidence/image-regions.mjs +15 -0
- package/packages/evidence/record-identity.mjs +60 -16
- package/packages/evidence/technology-evidence-role.mjs +51 -0
- package/packages/interchange/external-findings-import.mjs +32 -17
- package/packages/interchange/json-output.mjs +14 -0
- package/packages/interchange/managed-indicator-export.mjs +3 -2
- package/packages/interchange/managed-indicator-set.mjs +13 -0
- package/packages/investigation/acquisition-decision-packet.mjs +78 -0
- package/packages/investigation/context-review.mjs +3 -3
- package/packages/investigation/document-intake.mjs +9 -3
- package/packages/investigation/domain-history-review.mjs +43 -11
- package/packages/investigation/domain-transition-review.mjs +2 -0
- package/packages/investigation/infrastructure-observation.mjs +234 -0
- package/packages/investigation/intake-context.mjs +284 -0
- package/packages/investigation/intake-indicators.mjs +116 -0
- package/packages/investigation/intake-phones.mjs +155 -0
- package/packages/investigation/intake-report.mjs +2 -1
- package/packages/investigation/investigation-manifest.mjs +33 -7
- package/packages/investigation/link-intake.mjs +38 -8
- package/packages/investigation/message-intake.mjs +55 -15
- package/packages/monitoring/domain-feed.mjs +321 -0
- package/packages/workspace/brand-candidate-workflow.mjs +351 -0
- package/packages/workspace/brand-keyword-campaign.mjs +231 -0
- package/packages/workspace/brand-profile-model.mjs +46 -2
- package/packages/workspace/bulk-session-model.mjs +4 -0
- package/packages/workspace/candidate-watch-handoff.mjs +195 -0
- package/packages/workspace/detection-rule-model.mjs +10 -2
- package/packages/workspace/shortlist-model.mjs +10 -2
- package/packages/workspace/watchlist-history.mjs +31 -4
- package/packages/workspace/watchlist-store.mjs +136 -7
- package/third-party-notices.txt +29 -29
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import { htmlTreeEvents, isHtmlElement, parseBoundedHtml, MAX_STATIC_HTML_CHARS, MAX_STATIC_HTML_TAGS, MAX_STATIC_HTML_TREE_EVENTS, } from "./bounded-html-document.mjs";
|
|
5
5
|
import { MAX_CSP_META_POLICIES, MAX_RESPONSE_POLICY_HEADER_BYTES, } from "./response-policy.mjs";
|
|
6
6
|
import { MAX_FAVICON_BYTES, MAX_FAVICON_CANDIDATES } from "./outbound-request-bounds.mjs";
|
|
7
|
+
import { CREDENTIAL_CATEGORY_NAMES, MAX_CREDENTIAL_FORMS, MAX_CREDENTIAL_INPUTS } from "../packages/evidence/credential-form-attribution.mjs";
|
|
7
8
|
import { MAX_PAGE_PUBLICATION_DECLARATIONS, MAX_PAGE_PUBLICATION_META_ELEMENTS, MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES, } from "./homepage-metadata-contract.mjs";
|
|
8
9
|
const MAX_STATIC_STRUCTURE_TOKENS = MAX_STATIC_HTML_TREE_EVENTS;
|
|
9
10
|
const MAX_TAG_LENGTH = 4_096;
|
|
@@ -14,8 +15,8 @@ const MAX_SCRIPT_MEDIA_TYPE_LENGTH = 120;
|
|
|
14
15
|
const MAX_INLINE_SCRIPT_TOTAL_CHARS = 65_536;
|
|
15
16
|
const MAX_INLINE_SCRIPT_CHARS = MAX_INLINE_SCRIPT_TOTAL_CHARS;
|
|
16
17
|
const MAX_STATIC_VISIBLE_TEXT_CHARS = MAX_STATIC_HTML_CHARS;
|
|
17
|
-
const MAX_STATIC_FORMS =
|
|
18
|
-
const MAX_STATIC_INPUTS =
|
|
18
|
+
const MAX_STATIC_FORMS = MAX_CREDENTIAL_FORMS;
|
|
19
|
+
const MAX_STATIC_INPUTS = MAX_CREDENTIAL_INPUTS;
|
|
19
20
|
const MAX_STATIC_PUBLICATION_META_ELEMENTS = MAX_PAGE_PUBLICATION_META_ELEMENTS;
|
|
20
21
|
const MAX_STATIC_PUBLICATION_DECLARATIONS = MAX_PAGE_PUBLICATION_DECLARATIONS;
|
|
21
22
|
const MAX_STATIC_ROBOTS_DIRECTIVES = MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES;
|
|
@@ -197,6 +198,29 @@ function stylesheetCandidate(attributes) {
|
|
|
197
198
|
truncated: rel.truncated || href.truncated || disabled.truncated || media.truncated,
|
|
198
199
|
};
|
|
199
200
|
}
|
|
201
|
+
export function staticControlDisabled(elements, index, firstLegends) {
|
|
202
|
+
const hasDisabled = (element) => element.attributes.some(attribute => attribute.name === 'disabled');
|
|
203
|
+
if (hasDisabled(elements[index]))
|
|
204
|
+
return true;
|
|
205
|
+
for (let parent = elements[index]?.parent ?? null; parent !== null; parent = elements[parent]?.parent ?? null) {
|
|
206
|
+
const fieldset = elements[parent];
|
|
207
|
+
if (!fieldset.html || fieldset.name !== 'fieldset' || !hasDisabled(fieldset))
|
|
208
|
+
continue;
|
|
209
|
+
const legend = firstLegends.get(parent);
|
|
210
|
+
let underLegend = false;
|
|
211
|
+
for (let current = index; current !== null; current = elements[current]?.parent ?? null) {
|
|
212
|
+
if (current === legend) {
|
|
213
|
+
underLegend = true;
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
if (current === parent)
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
if (!underLegend)
|
|
220
|
+
return true;
|
|
221
|
+
}
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
200
224
|
function analyzeStaticHtml(value, options = {}) {
|
|
201
225
|
const parsed = parseBoundedHtml(value);
|
|
202
226
|
const { inputLimitReached } = parsed;
|
|
@@ -209,6 +233,7 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
209
233
|
const iconCounts = [0, 0];
|
|
210
234
|
const tokens = [];
|
|
211
235
|
const elementStack = [];
|
|
236
|
+
const firstLegends = new Map();
|
|
212
237
|
const visibleTextParts = [];
|
|
213
238
|
const structureTokens = [];
|
|
214
239
|
const scripts = [];
|
|
@@ -386,6 +411,8 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
386
411
|
};
|
|
387
412
|
elementStack.push(elements.length);
|
|
388
413
|
elements.push(element);
|
|
414
|
+
if (htmlElement && tagName === 'legend' && element.parent !== null && !firstLegends.has(element.parent))
|
|
415
|
+
firstLegends.set(element.parent, elements.length - 1);
|
|
389
416
|
appendToken({ kind: 'start', element });
|
|
390
417
|
// Foreign drawing attributes are not HTML technology, form or resource
|
|
391
418
|
// inputs. Their per-element omission still reaches fingerprinting, but
|
|
@@ -628,7 +655,8 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
628
655
|
}
|
|
629
656
|
else {
|
|
630
657
|
forms.inputsObserved += 1;
|
|
631
|
-
const categories =
|
|
658
|
+
const categories = staticControlDisabled(elements, elements.length - 1, firstLegends)
|
|
659
|
+
? { values: [], truncated: false } : inputCategories(token.attrs);
|
|
632
660
|
if (categories.truncated)
|
|
633
661
|
forms.truncated = true;
|
|
634
662
|
if (categories.values.length)
|
|
@@ -716,4 +744,4 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
716
744
|
inlineCharactersExamined,
|
|
717
745
|
};
|
|
718
746
|
}
|
|
719
|
-
export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_REFERENCE_LENGTH, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_PUBLICATION_META_ELEMENTS, MAX_STATIC_PUBLICATION_DECLARATIONS, MAX_STATIC_ROBOTS_DIRECTIVES, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, analyzeStaticHtml, };
|
|
747
|
+
export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_REFERENCE_LENGTH, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_PUBLICATION_META_ELEMENTS, MAX_STATIC_PUBLICATION_DECLARATIONS, MAX_STATIC_ROBOTS_DIRECTIVES, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, analyzeStaticHtml, inputCategories as classifyStaticCredentialInput, };
|
|
@@ -1,51 +1 @@
|
|
|
1
|
-
|
|
2
|
-
'observed_edge',
|
|
3
|
-
'application_platform',
|
|
4
|
-
'framework_runtime',
|
|
5
|
-
'embedded_dependency',
|
|
6
|
-
]);
|
|
7
|
-
const TECHNOLOGY_EVIDENCE_ROLE_SET = new Set(TECHNOLOGY_EVIDENCE_ROLE_ORDER);
|
|
8
|
-
function record(value) {
|
|
9
|
-
return value && typeof value === 'object' && !Array.isArray(value)
|
|
10
|
-
? value
|
|
11
|
-
: {};
|
|
12
|
-
}
|
|
13
|
-
function technologyEvidenceRoles(value) {
|
|
14
|
-
const finding = record(value);
|
|
15
|
-
const roles = new Set();
|
|
16
|
-
let hasExplicitRole = false;
|
|
17
|
-
for (const role of Array.isArray(finding.roles) ? finding.roles.slice(0, 4) : []) {
|
|
18
|
-
if (typeof role === 'string' && TECHNOLOGY_EVIDENCE_ROLE_SET.has(role)) {
|
|
19
|
-
roles.add(role);
|
|
20
|
-
hasExplicitRole = true;
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
const evidenceValues = Array.isArray(finding.evidence) ? finding.evidence.slice(0, 4) : [];
|
|
24
|
-
for (const evidenceValue of evidenceValues) {
|
|
25
|
-
const evidence = record(evidenceValue);
|
|
26
|
-
if (typeof evidence.role === 'string'
|
|
27
|
-
&& TECHNOLOGY_EVIDENCE_ROLE_SET.has(evidence.role)) {
|
|
28
|
-
roles.add(evidence.role);
|
|
29
|
-
hasExplicitRole = true;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
if (!hasExplicitRole) {
|
|
33
|
-
const resourceOriginOnly = evidenceValues.length > 0
|
|
34
|
-
&& evidenceValues.every((evidence) => (String(record(evidence).source).toLowerCase() === 'resource origin'));
|
|
35
|
-
if (evidenceValues.some((evidence) => String(record(evidence).source).toLowerCase() === 'resource origin')) {
|
|
36
|
-
roles.add('embedded_dependency');
|
|
37
|
-
}
|
|
38
|
-
if (!resourceOriginOnly) {
|
|
39
|
-
const category = String(finding.category).toLowerCase();
|
|
40
|
-
if (category === 'delivery platform')
|
|
41
|
-
roles.add('observed_edge');
|
|
42
|
-
else if (['application runtime', 'web framework', 'static site generator', 'web server'].includes(category)) {
|
|
43
|
-
roles.add('framework_runtime');
|
|
44
|
-
}
|
|
45
|
-
else
|
|
46
|
-
roles.add('application_platform');
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
return TECHNOLOGY_EVIDENCE_ROLE_ORDER.filter((role) => roles.has(role));
|
|
50
|
-
}
|
|
51
|
-
export { TECHNOLOGY_EVIDENCE_ROLE_ORDER, technologyEvidenceRoles, };
|
|
1
|
+
export { TECHNOLOGY_EVIDENCE_ROLE_ORDER, technologyEvidenceRoles, } from "../packages/evidence/technology-evidence-role.mjs";
|
package/lib/tls-intelligence.mjs
CHANGED
|
@@ -705,6 +705,7 @@ function skippedTlsObservation(detail = 'TLS evidence collection is disabled by
|
|
|
705
705
|
};
|
|
706
706
|
}
|
|
707
707
|
async function collectTlsIntelligence(hostname, options = {}) {
|
|
708
|
+
options.signal?.throwIfAborted();
|
|
708
709
|
const normalizedHostname = normalizeTlsHostname(hostname);
|
|
709
710
|
const now = options.now || Date.now;
|
|
710
711
|
const observedAt = options.observedAt || (() => new Date().toISOString());
|
|
@@ -725,9 +726,11 @@ async function collectTlsIntelligence(hostname, options = {}) {
|
|
|
725
726
|
resolutionDeadline = setTimer(() => reject(new Error('TLS resolution timed out')), timeoutMs);
|
|
726
727
|
}),
|
|
727
728
|
]);
|
|
729
|
+
options.signal?.throwIfAborted();
|
|
728
730
|
records = normalizePublicAddressRecords(resolved);
|
|
729
731
|
}
|
|
730
732
|
catch (error) {
|
|
733
|
+
options.signal?.throwIfAborted();
|
|
731
734
|
return failedTlsObservation(error, { sniHost: normalizedHostname, observedAt: observedAt(), durationMs: now() - started });
|
|
732
735
|
}
|
|
733
736
|
finally {
|
|
@@ -755,18 +758,29 @@ async function collectTlsIntelligence(hostname, options = {}) {
|
|
|
755
758
|
durationMs: now() - started,
|
|
756
759
|
});
|
|
757
760
|
}
|
|
758
|
-
return new Promise((resolve) => {
|
|
761
|
+
return new Promise((resolve, reject) => {
|
|
759
762
|
let socket;
|
|
760
763
|
let deadline;
|
|
761
764
|
let settled = false;
|
|
762
|
-
const
|
|
763
|
-
if (settled)
|
|
764
|
-
return;
|
|
765
|
-
settled = true;
|
|
765
|
+
const cleanup = () => {
|
|
766
766
|
if (deadline !== undefined)
|
|
767
767
|
clearTimer(deadline);
|
|
768
|
+
options.signal?.removeEventListener('abort', cancelled);
|
|
768
769
|
if (socket && typeof socket.destroy === 'function')
|
|
769
770
|
socket.destroy();
|
|
771
|
+
};
|
|
772
|
+
const cancelled = () => {
|
|
773
|
+
if (settled)
|
|
774
|
+
return;
|
|
775
|
+
settled = true;
|
|
776
|
+
cleanup();
|
|
777
|
+
reject(options.signal.reason);
|
|
778
|
+
};
|
|
779
|
+
const finish = (result) => {
|
|
780
|
+
if (settled)
|
|
781
|
+
return;
|
|
782
|
+
settled = true;
|
|
783
|
+
cleanup();
|
|
770
784
|
resolve(result);
|
|
771
785
|
};
|
|
772
786
|
const fail = (error) => finish(failedTlsObservation(error, {
|
|
@@ -776,6 +790,11 @@ async function collectTlsIntelligence(hostname, options = {}) {
|
|
|
776
790
|
observedAt: observedAt(),
|
|
777
791
|
durationMs: now() - started,
|
|
778
792
|
}));
|
|
793
|
+
options.signal?.addEventListener('abort', cancelled, { once: true });
|
|
794
|
+
if (options.signal?.aborted) {
|
|
795
|
+
cancelled();
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
779
798
|
try {
|
|
780
799
|
socket = connect({
|
|
781
800
|
host: selected.address,
|
|
@@ -787,6 +806,8 @@ async function collectTlsIntelligence(hostname, options = {}) {
|
|
|
787
806
|
checkServerIdentity: () => undefined,
|
|
788
807
|
ALPNProtocols: ['h2', 'http/1.1'],
|
|
789
808
|
}, () => {
|
|
809
|
+
if (settled)
|
|
810
|
+
return;
|
|
790
811
|
try {
|
|
791
812
|
const peerCertificate = socket.getPeerCertificate(true);
|
|
792
813
|
let hostnameError = null;
|
|
@@ -816,6 +837,10 @@ async function collectTlsIntelligence(hostname, options = {}) {
|
|
|
816
837
|
fail(error);
|
|
817
838
|
}
|
|
818
839
|
});
|
|
840
|
+
if (settled) {
|
|
841
|
+
socket.destroy();
|
|
842
|
+
return;
|
|
843
|
+
}
|
|
819
844
|
socket.once('error', fail);
|
|
820
845
|
deadline = setTimer(() => fail(new Error('TLS handshake timed out')), remainingMs);
|
|
821
846
|
if (settled && deadline !== undefined)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// Bounded website and network enrichment, independent of registration authority.
|
|
2
2
|
import { capturedWebCollectionQuality } from "../packages/evidence/collection-quality.mjs";
|
|
3
|
+
import { isValidAsciiHostname } from "../packages/contracts/domain-name.mjs";
|
|
3
4
|
import { recordOrEmpty as errorRecord } from "./json-record.mjs";
|
|
4
5
|
import { safeFetchDetailed, readTextCapped } from "./safe-fetch.mjs";
|
|
5
6
|
import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
|
|
@@ -21,26 +22,34 @@ import { HOMEPAGE_FETCH_TIMEOUT_MS, MAX_HOMEPAGE_BYTES, } from "./outbound-reque
|
|
|
21
22
|
// what's checkable without credentials is the parking/landing-page
|
|
22
23
|
// nameservers and homepage copy these services actually use, which is
|
|
23
24
|
// broader coverage of the same signal, not a live cross-marketplace lookup.
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
/
|
|
33
|
-
|
|
34
|
-
/afternic/
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
25
|
+
const PARKING_NS_DOMAINS = [
|
|
26
|
+
'sedoparking.com',
|
|
27
|
+
'sedo.com',
|
|
28
|
+
'above.com',
|
|
29
|
+
'bodis.com',
|
|
30
|
+
'parkingcrew.net',
|
|
31
|
+
'dan.com',
|
|
32
|
+
'hugedomains.com',
|
|
33
|
+
// https://www.atom.com/help/article/how-to-verify-your-standard-premium-domains
|
|
34
|
+
'squadhelp.com',
|
|
35
|
+
// https://blog.afternic.com/lander-selection-redefining-your-use-of-afternic-landing-pages/
|
|
36
|
+
'afternic.com',
|
|
37
|
+
'voodoo.com',
|
|
38
|
+
'fabulous.com',
|
|
39
|
+
'smartname.com',
|
|
40
|
+
'domainsponsor.com',
|
|
41
|
+
'undeveloped.com',
|
|
42
|
+
'trafficz.com',
|
|
43
|
+
'dsredirection.com',
|
|
43
44
|
];
|
|
45
|
+
function isParkingNameserver(value) {
|
|
46
|
+
const hostname = value.trim().replace(/\.$/u, '');
|
|
47
|
+
// Validate before case folding: non-ASCII letters can lowercase to ASCII.
|
|
48
|
+
if (!isValidAsciiHostname(hostname))
|
|
49
|
+
return false;
|
|
50
|
+
const normalised = hostname.toLowerCase();
|
|
51
|
+
return PARKING_NS_DOMAINS.some(domain => normalised === domain || normalised.endsWith(`.${domain}`));
|
|
52
|
+
}
|
|
44
53
|
const FOR_SALE_PATH_RE = /\/(?:premium-)?domains?-for-sale(?:\/|$)/i;
|
|
45
54
|
function forSaleRedirectSignal(httpObservation) {
|
|
46
55
|
if (!httpObservation || typeof httpObservation !== 'object')
|
|
@@ -180,6 +189,7 @@ function deriveWebsiteActivity(homepageStatus, hasFavicon, alreadyParked = false
|
|
|
180
189
|
return 'unreachable';
|
|
181
190
|
}
|
|
182
191
|
async function enrichWebsite(context, options, featurePolicy) {
|
|
192
|
+
options.signal?.throwIfAborted();
|
|
183
193
|
const { domain, observationHostname, selectedUrl, registration, registryDnsEvidence, nameservers, dnssec } = context;
|
|
184
194
|
const websiteProbeEnabled = featureDecision('website_probe', featurePolicy).enabled;
|
|
185
195
|
const collectDns = options.collectDnsIntelligence || collectDnsIntelligence;
|
|
@@ -194,7 +204,7 @@ async function enrichWebsite(context, options, featurePolicy) {
|
|
|
194
204
|
// deep mode only), and check for a configured mail exchanger as a
|
|
195
205
|
// phishing-risk signal (a lookalike domain that can receive/send mail is
|
|
196
206
|
// capable of running credential-harvesting or BEC campaigns).
|
|
197
|
-
const nsSignal = registration.state === 'registered' ? nameservers.find(
|
|
207
|
+
const nsSignal = registration.state === 'registered' ? nameservers.find(isParkingNameserver) : undefined;
|
|
198
208
|
let forSaleSignal = nsSignal ? `parking nameserver (${nsSignal})` : null;
|
|
199
209
|
let activityStatus = nsSignal && observationHostname === domain ? 'parked' : 'unknown';
|
|
200
210
|
// Homepage + bounded DNS evidence resolve in parallel; the favicon fetch is sequenced after
|
|
@@ -202,7 +212,7 @@ async function enrichWebsite(context, options, featurePolicy) {
|
|
|
202
212
|
// sites serve no /favicon.ico and only point to a CDN PNG this way). One
|
|
203
213
|
// extra round-trip on the already-slow deep path, in exchange for finding
|
|
204
214
|
// favicons the bare /favicon.ico probe would miss.
|
|
205
|
-
const
|
|
215
|
+
const pending = [
|
|
206
216
|
websiteProbeEnabled ? fetchHomepageForDomain(observationHostname, {
|
|
207
217
|
...(selectedUrl ? { selectedUrl } : {}), ...(options.signal ? { signal: options.signal } : {}),
|
|
208
218
|
}).catch((err) => ({
|
|
@@ -221,6 +231,7 @@ async function enrichWebsite(context, options, featurePolicy) {
|
|
|
221
231
|
}),
|
|
222
232
|
dnsIntelligenceEnabled
|
|
223
233
|
? collectDns(observationHostname, {
|
|
234
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
224
235
|
...(options.dnsResolvers ? { resolvers: options.dnsResolvers } : {}),
|
|
225
236
|
includeExtendedContext: options.includeExtendedDnsContext === true,
|
|
226
237
|
includeInheritedCaa: options.includeInheritedCaa === true,
|
|
@@ -232,9 +243,11 @@ async function enrichWebsite(context, options, featurePolicy) {
|
|
|
232
243
|
includeInheritedCaa: options.includeInheritedCaa === true,
|
|
233
244
|
})),
|
|
234
245
|
tlsIntelligenceEnabled
|
|
235
|
-
? collectTls(observationHostname)
|
|
246
|
+
? collectTls(observationHostname, { ...(options.signal ? { signal: options.signal } : {}) })
|
|
236
247
|
: Promise.resolve(skippedTlsObservation()),
|
|
237
|
-
]
|
|
248
|
+
];
|
|
249
|
+
const [homepage, dnsIntelligence, tlsIntelligence] = await Promise.all(pending)
|
|
250
|
+
.finally(() => Promise.allSettled(pending));
|
|
238
251
|
options.signal?.throwIfAborted();
|
|
239
252
|
const page = homepage.text;
|
|
240
253
|
const pageBaseUrl = homepage.analysisBaseUrl ?? (typeof homepage.http?.finalUrl === 'string' ? homepage.http.finalUrl : `https://${observationHostname}/`);
|
|
@@ -243,6 +256,7 @@ async function enrichWebsite(context, options, featurePolicy) {
|
|
|
243
256
|
domainSaleSignal: null,
|
|
244
257
|
pageTitle: null,
|
|
245
258
|
hasPasswordField: false,
|
|
259
|
+
hasExternalPasswordForm: null,
|
|
246
260
|
phishingLanguageMatch: null,
|
|
247
261
|
hasExternalFormAction: null,
|
|
248
262
|
externalAssetHosts: [],
|
|
@@ -312,8 +326,10 @@ async function enrichWebsite(context, options, featurePolicy) {
|
|
|
312
326
|
const favicon = websiteProbeEnabled
|
|
313
327
|
? await fetchFaviconForDomain(observationHostname, {
|
|
314
328
|
baseUrl: pageBaseUrl, ...(faviconEvidence ? { htmlAnalysis: faviconEvidence } : {}),
|
|
329
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
315
330
|
}).catch(() => null)
|
|
316
331
|
: null;
|
|
332
|
+
options.signal?.throwIfAborted();
|
|
317
333
|
const faviconHash = favicon ? favicon.hash : null;
|
|
318
334
|
const faviconPHash = favicon ? favicon.phash : null;
|
|
319
335
|
const responsePolicy = qualifyResponsePolicyWithCspMeta(homepage.responsePolicy, htmlSignals.cspMetaPolicy);
|
package/lib/whois-chain.mjs
CHANGED
|
@@ -146,7 +146,16 @@ export async function buildWhoisChainUncached(query, options = {}) {
|
|
|
146
146
|
return chain;
|
|
147
147
|
}
|
|
148
148
|
export async function buildWhoisChain(query, options = {}) {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
149
|
+
let ownedCollection;
|
|
150
|
+
try {
|
|
151
|
+
return await cached(`whois:${query.toLowerCase()}`, () => {
|
|
152
|
+
ownedCollection = buildWhoisChainUncached(query, options);
|
|
153
|
+
return ownedCollection;
|
|
154
|
+
}, options.signal);
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
// Cache cancellation rejects promptly, but a started transport still belongs
|
|
158
|
+
// to this lookup lease. Drain it before its caller can release that lease.
|
|
159
|
+
await ownedCollection?.catch(() => { });
|
|
160
|
+
}
|
|
152
161
|
}
|
|
@@ -245,7 +245,9 @@ function applyWhoisCommonFormats(text, isRootHop, dialects, context) {
|
|
|
245
245
|
// sometimes just a hostname, sometimes "hostname ip.addr" - rather
|
|
246
246
|
// than a per-line "Name Server:" label. Only meaningful on non-root hops.
|
|
247
247
|
if (!isRootHop && nameservers.size === 0) {
|
|
248
|
-
|
|
248
|
+
// Whitespace after a colon belongs to that colon: two independent runs
|
|
249
|
+
// around an optional colon allow quadratic backtracking on invalid lines.
|
|
250
|
+
const headerMatch = text.match(/^[ \t*]*(?:Name ?[Ss]ervers|Domain Servers|DNS servers)[ \t.]*(?::[ \t]*)?$/mi);
|
|
249
251
|
if (headerMatch) {
|
|
250
252
|
let found = 0;
|
|
251
253
|
for (const line of text.slice((headerMatch.index ?? 0) + headerMatch[0].length).split('\n')) {
|
package/lib/whois-contacts.mjs
CHANGED
|
@@ -30,7 +30,9 @@ export function resolveIsnicRole(text, handle) {
|
|
|
30
30
|
if (!handle)
|
|
31
31
|
return null;
|
|
32
32
|
const escaped = handle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
33
|
-
|
|
33
|
+
// Capture the whole role value, including leading whitespace. A separate
|
|
34
|
+
// whitespace run overlaps this capture and backtracks when the handle fails.
|
|
35
|
+
const headerMatch = text.match(new RegExp(`^[ \\t]*role:(.+)\\r?\\n[ \\t]*nic-hdl:[ \\t]*${escaped}[ \\t]*$`, 'im'));
|
|
34
36
|
if (!headerMatch)
|
|
35
37
|
return null;
|
|
36
38
|
const lines = text
|
|
@@ -143,7 +145,7 @@ export function parseIndentedContactBlock(text, headerRe) {
|
|
|
143
145
|
if (lines.length === 0)
|
|
144
146
|
return null;
|
|
145
147
|
const remaining = [...lines];
|
|
146
|
-
const emailIndex = remaining.findIndex(
|
|
148
|
+
const emailIndex = remaining.findIndex(isIndentedContactEmail);
|
|
147
149
|
const email = emailIndex !== -1
|
|
148
150
|
? remaining.splice(emailIndex, 1)[0]
|
|
149
151
|
: null;
|
|
@@ -161,3 +163,12 @@ export function parseIndentedContactBlock(text, headerRe) {
|
|
|
161
163
|
truncated: allLines.length > lines.length,
|
|
162
164
|
};
|
|
163
165
|
}
|
|
166
|
+
// Preserve the legacy contact classifier (not a mailbox validator), without
|
|
167
|
+
// repeatedly trying a different separator in a long, dot-heavy failed match.
|
|
168
|
+
function isIndentedContactEmail(line) {
|
|
169
|
+
const at = line.indexOf('@');
|
|
170
|
+
if (at <= 0 || line.indexOf('@', at + 1) !== -1 || /\s/.test(line))
|
|
171
|
+
return false;
|
|
172
|
+
const dot = line.indexOf('.', at + 2);
|
|
173
|
+
return dot !== -1 && dot < line.length - 1;
|
|
174
|
+
}
|
package/package.json
CHANGED
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"bugs": {
|
|
34
34
|
"url": "https://github.com/slicedearth/whoisleuth/issues"
|
|
35
35
|
},
|
|
36
|
-
"version": "2.
|
|
36
|
+
"version": "2.6.0",
|
|
37
37
|
"dependencies": {
|
|
38
38
|
"@nuintun/qrcode": "5.0.3",
|
|
39
39
|
"@peculiar/x509": "2.1.0",
|
|
@@ -41,12 +41,12 @@
|
|
|
41
41
|
"fflate": "0.8.3",
|
|
42
42
|
"maxmind": "5.0.7",
|
|
43
43
|
"parse5": "8.0.1",
|
|
44
|
-
"pdfjs-dist": "6.
|
|
45
|
-
"postal-mime": "
|
|
44
|
+
"pdfjs-dist": "6.4.299",
|
|
45
|
+
"postal-mime": "4.0.4",
|
|
46
46
|
"reflect-metadata": "0.2.2",
|
|
47
47
|
"saxes": "6.0.0",
|
|
48
|
-
"tldts": "7.4.
|
|
49
|
-
"undici": "8.11.
|
|
48
|
+
"tldts": "7.4.16",
|
|
49
|
+
"undici": "8.11.2"
|
|
50
50
|
},
|
|
51
51
|
"files": [
|
|
52
52
|
"bin/**/*.mjs",
|
|
@@ -58,6 +58,8 @@
|
|
|
58
58
|
"docs/cli.md",
|
|
59
59
|
"docs/cli-reference.md",
|
|
60
60
|
"docs/contextual-reviews.md",
|
|
61
|
+
"docs/offline-intake-context.md",
|
|
62
|
+
"docs/infrastructure-observations.md",
|
|
61
63
|
"DISCLOSURE",
|
|
62
64
|
"LICENSE",
|
|
63
65
|
"NOTICE",
|
|
@@ -6,20 +6,20 @@ const STATE_LABELS = Object.freeze({
|
|
|
6
6
|
expiring: 'expiring/pending delete',
|
|
7
7
|
registered: 'registered',
|
|
8
8
|
});
|
|
9
|
-
// Version
|
|
10
|
-
//
|
|
11
|
-
|
|
12
|
-
// operational properties remain supporting context, and missing evidence never
|
|
13
|
-
// contributes points.
|
|
14
|
-
export const RISK_MODEL_VERSION = 8;
|
|
9
|
+
// Version 9 requires form-scoped password/destination evidence. Older page-wide
|
|
10
|
+
// flags never establish that a password control belongs to an external form.
|
|
11
|
+
export const RISK_MODEL_VERSION = 9;
|
|
15
12
|
export const RISK_REVIEW_THRESHOLD = 70;
|
|
16
13
|
const CURRENT_RISK_MODEL = Object.freeze({
|
|
17
14
|
modelVersion: RISK_MODEL_VERSION,
|
|
18
15
|
includePageBaselineMatch: false,
|
|
16
|
+
requireFormAttribution: true,
|
|
19
17
|
});
|
|
18
|
+
const RISK_MODEL_V8 = Object.freeze({ modelVersion: 8, includePageBaselineMatch: false, requireFormAttribution: false });
|
|
20
19
|
const RISK_MODEL_V7 = Object.freeze({
|
|
21
20
|
modelVersion: 7,
|
|
22
21
|
includePageBaselineMatch: true,
|
|
22
|
+
requireFormAttribution: false,
|
|
23
23
|
});
|
|
24
24
|
const RISK_STATE_BASE = Object.freeze({
|
|
25
25
|
registered: 6,
|
|
@@ -104,7 +104,7 @@ function scoreQuality(input) {
|
|
|
104
104
|
if ([input.faviconMatch, input.faviconNearMatch, input.reusesOfficialAssets, input.pageBaselineMatch].some(hasBoolean)) {
|
|
105
105
|
observedFamilies.push('brand-presentation');
|
|
106
106
|
}
|
|
107
|
-
if ([input.hasPasswordField, input.hasExternalFormAction].some(hasBoolean) || typeof input.phishingLanguageMatch === 'string') {
|
|
107
|
+
if ([input.hasPasswordField, input.hasExternalFormAction, input.hasExternalPasswordForm].some(hasBoolean) || typeof input.phishingLanguageMatch === 'string') {
|
|
108
108
|
observedFamilies.push('credential-lure');
|
|
109
109
|
}
|
|
110
110
|
if (calibrateExternalIntelligenceRisk(input.threatIntelligence, input.domain).eligibleProviderCount > 0) {
|
|
@@ -159,8 +159,8 @@ function explainRiskScoreInternal(input, excludedFamily, model) {
|
|
|
159
159
|
if (typeof input.phishingLanguageMatch === 'string' && input.phishingLanguageMatch.trim()) {
|
|
160
160
|
add('credential-lure', 'Suspicious urgency language observed', 8);
|
|
161
161
|
}
|
|
162
|
-
if (input.hasPasswordField === true && input.hasExternalFormAction === true) {
|
|
163
|
-
add('credential-lure', 'Password form submits to an external origin', 10);
|
|
162
|
+
if (model.requireFormAttribution ? input.hasExternalPasswordForm === true : input.hasPasswordField === true && input.hasExternalFormAction === true) {
|
|
163
|
+
add('credential-lure', model.requireFormAttribution ? 'Password form declares an external destination' : 'Password form submits to an external origin', 10);
|
|
164
164
|
}
|
|
165
165
|
else if (input.hasPasswordField === true) {
|
|
166
166
|
add('credential-lure', 'Login/password form present', 5);
|
|
@@ -254,6 +254,9 @@ export function computeRiskScore(input) {
|
|
|
254
254
|
export function explainRiskScoreV7(input) {
|
|
255
255
|
return explainRiskScoreInternal(input, null, RISK_MODEL_V7);
|
|
256
256
|
}
|
|
257
|
+
export function explainRiskScoreV8(input) {
|
|
258
|
+
return explainRiskScoreInternal(input, null, RISK_MODEL_V8);
|
|
259
|
+
}
|
|
257
260
|
// Retained only for deterministic offline comparison of reviewed calibration
|
|
258
261
|
// datasets. Runtime lookups always use the current model above.
|
|
259
262
|
export function explainRiskScoreV6(input) {
|