@slicedearth/whoisleuth-cli 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +6 -0
- package/README.md +20 -15
- package/bin/whoisleuth.mjs +8 -1
- package/cli/archive-content-digest.mjs +14 -0
- package/cli/archive-inspect.mjs +20 -53
- package/cli/arguments.mjs +454 -1748
- package/cli/artifact-validation/case-response.mjs +66 -35
- package/cli/artifact-validation/investigation-capsule.mjs +52 -31
- package/cli/artifact-validation/lookup-evidence.mjs +12 -6
- package/cli/artifact-validation/signed-review.mjs +112 -72
- package/cli/artifact-validation/structure-primitives.mjs +1 -105
- package/cli/artifact-verify.mjs +90 -24
- package/cli/assurance-command-runner.mjs +39 -4
- package/cli/bagit-folder.mjs +57 -0
- package/cli/bagit-review.mjs +23 -0
- package/cli/bounded-json.mjs +1 -1
- package/cli/bulk-checkpoint.mjs +34 -3
- package/cli/bulk-command-runner.mjs +2 -0
- package/cli/bulk-output.mjs +29 -3
- package/cli/case-command.mjs +195 -0
- package/cli/case-pack.mjs +66 -111
- package/cli/command-argument-grammar.mjs +215 -0
- package/cli/command-reference.mjs +342 -285
- package/cli/completion.mjs +303 -80
- package/cli/config-profile.mjs +7 -5
- package/cli/ct-event-intake.mjs +12 -12
- package/cli/discover.mjs +4 -3
- package/cli/discovery-observation-snapshot.mjs +59 -18
- package/cli/discovery-scan-command-runner.mjs +21 -0
- package/cli/discovery-scan.mjs +13 -4
- package/cli/discovery-workflow.mjs +1 -1
- package/cli/domain-control-monitor.mjs +50 -20
- package/cli/domain-control-observations.mjs +149 -102
- package/cli/errors.mjs +33 -1
- package/cli/evidence-command-runner.mjs +16 -5
- package/cli/evidence-signing.mjs +2 -2
- package/cli/external-observation-mapping.mjs +7 -8
- package/cli/formatters/terminal-lookup.mjs +18 -46
- package/cli/formatters/terminal-shared.mjs +5 -3
- package/cli/formatters/terminal-technology.mjs +114 -0
- package/cli/inline-command-families.mjs +2 -0
- package/cli/input.mjs +116 -0
- package/cli/interactive-launcher.mjs +2 -160
- package/cli/interchange-report.mjs +18 -8
- package/cli/investigation-artifacts.mjs +176 -0
- package/cli/investigation-checkpoint.mjs +10 -0
- package/cli/investigation-folder.mjs +114 -0
- package/cli/investigation-manifest.mjs +1 -112
- package/cli/investigation-package-review.mjs +103 -0
- package/cli/investigation-plan.mjs +32 -19
- package/cli/investigation-run.mjs +188 -33
- package/cli/local-document-checkpoint.mjs +170 -0
- package/cli/lookalike-calibration.mjs +7 -2
- package/cli/lookup-brief.mjs +4 -2
- package/cli/lookup-browser-view.mjs +1 -1
- package/cli/lookup-browser.mjs +5 -12
- package/cli/lookup-command-runner.mjs +17 -6
- package/cli/lookup-diff.mjs +37 -3
- package/cli/lookup-plan.mjs +11 -5
- package/cli/lookup-reconcile.mjs +10 -8
- package/cli/mail-header-review.mjs +467 -0
- package/cli/mail-review.mjs +66 -14
- package/cli/manual.mjs +1 -0
- package/cli/network-command-runner.mjs +2 -0
- package/cli/output-file.mjs +31 -5
- package/cli/registry-doctor.mjs +3 -1
- package/cli/registry-support.mjs +1 -1
- package/cli/retained-artifact-diff.mjs +2 -2
- package/cli/review-command-runner.mjs +70 -0
- package/cli/risk-calibration.mjs +19 -17
- package/cli/runner.mjs +62 -7
- package/cli/sharing-review.mjs +21 -34
- package/cli/signer-trust.mjs +104 -0
- package/cli/source-reliability.mjs +74 -83
- package/cli/terminal-input.mjs +168 -0
- package/cli/workflow-command-runner.mjs +68 -13
- package/docs/cli-reference.md +77 -24
- package/docs/cli.md +300 -25
- package/lib/abort.mjs +4 -1
- package/lib/availability.mjs +161 -112
- package/lib/bounded-file.mjs +3 -1
- package/lib/bounded-html-document.mjs +125 -0
- package/lib/bounded-json.mjs +51 -8
- package/lib/browser-library-profile.mjs +42 -51
- package/lib/browser-library-worker.mjs +112 -0
- package/lib/client-behavior-profile.mjs +3 -3
- package/lib/ct-search.mjs +11 -4
- package/lib/dns-convergence-review.mjs +1 -0
- package/lib/dns-delegation-health.mjs +24 -14
- package/lib/dns-inheritance-review.mjs +217 -0
- package/lib/dns-intelligence.mjs +73 -26
- package/lib/dns-mx.mjs +1 -1
- package/lib/dnssec-chain-validation.mjs +6 -0
- package/lib/dnssec-evidence-validation.mjs +11 -3
- package/lib/domain-change-packet.mjs +49 -16
- package/lib/domain-change-review.mjs +14 -8
- package/lib/domain-control-flight-recorder.mjs +221 -117
- package/lib/domain-control-manifest.mjs +129 -69
- package/lib/domain-posture-analysis.mjs +20 -7
- package/lib/domain-posture-parsers.mjs +201 -8
- package/lib/domain-posture.mjs +147 -29
- package/lib/evidence-export-privacy.mjs +561 -0
- package/lib/evidence-export.mjs +11 -543
- package/lib/evidence-report.mjs +5 -3
- package/lib/favicon.mjs +117 -41
- package/lib/generated/cisa-kev-catalog.mjs +12 -4
- package/lib/generated/registrar-standing-catalogue.mjs +7 -7
- package/lib/generated/retire-browser-catalog.mjs +2335 -940
- package/lib/hostname.mjs +2 -38
- package/lib/html-signals.mjs +111 -108
- package/lib/http-evidence-bounds.mjs +2 -1
- package/lib/http-intelligence.mjs +39 -26
- package/lib/local-geoip-evidence.mjs +19 -3
- package/lib/lookup-cache.mjs +34 -11
- package/lib/lookup-child-profile-contract.mjs +24 -15
- package/lib/lookup-diagnostics.mjs +24 -1
- package/lib/lookup-progress.mjs +30 -13
- package/lib/lookup-response-contract.mjs +1048 -0
- package/lib/lookup-response.mjs +8 -10
- package/lib/lookup-source-progress.mjs +40 -14
- package/lib/lookup.mjs +48 -27
- package/lib/nameserver-preflight-review.mjs +1 -0
- package/lib/observed-network-context.mjs +43 -27
- package/lib/outbound-request-bounds.mjs +4 -1
- package/lib/page-fingerprints.mjs +69 -133
- package/lib/page-role-profile.mjs +12 -8
- package/lib/perceptual-hash.mjs +43 -14
- package/lib/rdap-bootstrap.mjs +72 -22
- package/lib/rdap-capabilities.mjs +1 -1
- package/lib/rdap-client.mjs +15 -7
- package/lib/rdap-normalization.mjs +33 -18
- package/lib/rdap-search-workbench.mjs +1 -1
- package/lib/rdap-transport.mjs +47 -20
- package/lib/rdap-types.mjs +2 -1
- package/lib/rdap.mjs +18 -3
- package/lib/registrable-domain.mjs +21 -1
- package/lib/registrar-standing-contract.mjs +1 -0
- package/lib/registry-capabilities.mjs +5 -1
- package/lib/registry-capability-catalogue.mjs +2 -1
- package/lib/registry-dates.mjs +2 -190
- package/lib/registry-insights.mjs +85 -34
- package/lib/response-policy.mjs +86 -48
- package/lib/risk-calibration-summary.mjs +18 -6
- package/lib/rpki-evidence.mjs +8 -5
- package/lib/safe-fetch.mjs +54 -47
- package/lib/security-txt.mjs +42 -7
- package/lib/service-binding-dns.mjs +8 -0
- package/lib/smtp-transport-review.mjs +27 -15
- package/lib/sslbl-certificates.generated.mjs +136 -135
- package/lib/static-html-analysis.mjs +178 -151
- package/lib/structured-data-identity.mjs +1 -0
- package/lib/technology-markup.mjs +100 -0
- package/lib/threat-intelligence-manifests.mjs +11 -139
- package/lib/threat-intelligence-runtime.mjs +24 -52
- package/lib/threat-intelligence-targets.mjs +6 -34
- package/lib/threat-intelligence-types.mjs +39 -0
- package/lib/typosquat-generator.mjs +19 -15
- package/lib/urlhaus-intelligence.mjs +8 -3
- package/lib/urlscan-intelligence.mjs +1 -1
- package/lib/website-security-posture.mjs +30 -6
- package/lib/website-technology.mjs +153 -45
- package/lib/whois-authority.mjs +14 -2
- package/lib/whois-chain.mjs +9 -2
- package/lib/whois-common-formats.mjs +5 -5
- package/lib/whois-contracts.mjs +2 -1
- package/lib/whois-dialects.mjs +2 -2
- package/lib/whois-parser-support.mjs +8 -1
- package/lib/whois-transport.mjs +23 -7
- package/lib/zone-intent-review.mjs +16 -6
- package/package.json +2 -1
- package/packages/cases/case-attachment-model.mjs +83 -0
- package/packages/cases/case-evidence-model.mjs +156 -62
- package/packages/cases/case-export-input.mjs +53 -0
- package/packages/cases/case-migration-model.mjs +127 -44
- package/packages/cases/case-model.mjs +1 -0
- package/packages/cases/case-recheck-model.mjs +91 -0
- package/packages/cases/case-record-contracts.mjs +5 -20
- package/packages/cases/case-record-core.mjs +12 -22
- package/packages/cases/case-record-decisions.mjs +72 -0
- package/packages/cases/case-record-model.mjs +1 -0
- package/packages/cases/case-record-operations.mjs +95 -31
- package/packages/cases/case-record-projection.mjs +221 -0
- package/packages/cases/case-report.mjs +70 -54
- package/packages/cases/case-response-actions.mjs +654 -0
- package/packages/cases/case-response-model.mjs +64 -1115
- package/packages/cases/case-response-outcomes.mjs +317 -0
- package/packages/cases/case-response-packet-render.mjs +153 -0
- package/packages/cases/case-response-packet-types.mjs +1 -0
- package/packages/cases/case-response-packet-vocabulary.mjs +48 -0
- package/packages/cases/case-response-packet.mjs +74 -572
- package/packages/cases/case-response-records.mjs +94 -0
- package/packages/cases/case-response-review-inputs.mjs +371 -0
- package/packages/cases/case-response-values.mjs +85 -0
- package/packages/cases/case-review-package.mjs +45 -0
- package/packages/cases/case-selection.mjs +46 -0
- package/packages/cases/case-storage-model.mjs +31 -2
- package/packages/cases/response-route-freshness.mjs +16 -0
- package/packages/comparison/capture-context.mjs +61 -0
- package/packages/comparison/comparison-ledger-bulk.mjs +13 -11
- package/packages/comparison/page-similarity.mjs +5 -2
- package/packages/comparison/relationship-evidence.mjs +85 -8
- package/packages/comparison/relationship-provenance.mjs +93 -0
- package/packages/contracts/analyst-interchange.mjs +216 -0
- package/packages/contracts/analyst-review-state-contract.mjs +11 -0
- package/packages/contracts/analyst-review-state.mjs +5 -4
- package/packages/contracts/case-portability.mjs +366 -83
- package/packages/contracts/case-views-contract.mjs +12 -0
- package/packages/contracts/case-views.mjs +26 -0
- package/packages/contracts/cli-command-semantics.mjs +94 -0
- package/packages/contracts/cli-lookup.mjs +0 -1
- package/packages/contracts/domain-control-flight-recorder.mjs +80 -42
- package/packages/contracts/domain-control-manifest.mjs +127 -73
- package/packages/contracts/domain-control-monitor.mjs +48 -45
- package/packages/contracts/domain-control-review.mjs +179 -121
- package/packages/contracts/domain-name.mjs +21 -0
- package/packages/contracts/external-observation-interchange.mjs +2 -1
- package/packages/contracts/extracted-domain-lifecycle.mjs +11 -32
- package/packages/contracts/http-url.mjs +2 -0
- package/packages/contracts/investigation-package-limits.mjs +24 -0
- package/packages/contracts/investigation-portability.mjs +56 -27
- package/packages/contracts/investigation-run.mjs +16 -0
- package/packages/contracts/ip-address.mjs +34 -0
- package/packages/contracts/lookup-evidence.mjs +35 -0
- package/packages/contracts/offline-comparison.mjs +1 -2
- package/packages/contracts/page-fingerprints.mjs +15 -0
- package/packages/contracts/risk-calibration.mjs +15 -11
- package/packages/contracts/schema-compatibility.mjs +1 -1
- package/packages/contracts/schema-lifecycle.mjs +83 -153
- package/packages/contracts/selected-file-limits.mjs +6 -0
- package/packages/contracts/vulnerability-identifiers.mjs +5 -0
- package/packages/contracts/web-capture.mjs +10 -0
- package/packages/contracts/workspace-portability.mjs +183 -211
- package/packages/evidence/artifact-integrity.mjs +8 -2
- package/packages/evidence/artifact-structure.mjs +105 -0
- package/packages/evidence/decision-fact.mjs +4 -1
- package/packages/evidence/domain-control-runtime.mjs +201 -33
- package/packages/evidence/domain-posture-context.mjs +79 -0
- package/packages/evidence/image-regions.mjs +56 -0
- package/packages/evidence/latest-observations.mjs +28 -0
- package/packages/evidence/lookup-target.mjs +104 -0
- package/packages/evidence/observation.mjs +16 -1
- package/packages/evidence/passphrase-encryption.mjs +44 -0
- package/packages/evidence/record-identity.mjs +72 -0
- package/packages/evidence/registry-dates.mjs +190 -0
- package/packages/evidence/retained-file.mjs +69 -0
- package/packages/evidence/review-age.mjs +14 -0
- package/packages/interchange/bagit.mjs +342 -0
- package/packages/interchange/bounded-zip-extraction.mjs +278 -0
- package/packages/interchange/crc32.mjs +17 -0
- package/packages/interchange/external-findings-import.mjs +92 -51
- package/packages/interchange/web-capture-import.mjs +382 -0
- package/packages/investigation/bulk-mail-exposure.mjs +2 -1
- package/packages/investigation/investigation-bagit.mjs +14 -0
- package/packages/investigation/investigation-capsule.mjs +33 -5
- package/packages/investigation/investigation-manifest.mjs +200 -0
- package/packages/investigation/investigation-package-crypto.mjs +98 -0
- package/packages/investigation/investigation-package.mjs +200 -0
- package/packages/investigation/lookup-asset-graph-inputs.mjs +38 -0
- package/packages/investigation/lookup-asset-graph.mjs +193 -165
- package/packages/investigation/lookup-claim-readiness.mjs +18 -0
- package/packages/investigation/lookup-investigation-brief.mjs +3 -3
- package/packages/investigation/lookup-task-guidance.mjs +58 -0
- package/packages/monitoring/analyst-review-state.mjs +38 -90
- package/packages/workspace/brand-profile-model.mjs +276 -22
- package/packages/workspace/bulk-columns.mjs +17 -0
- package/packages/workspace/bulk-review-model.mjs +23 -11
- package/packages/workspace/bulk-session-model.mjs +100 -24
- package/packages/workspace/bulk-sort.mjs +1 -0
- package/packages/workspace/case-views.mjs +102 -0
- package/packages/workspace/detection-rule-model.mjs +70 -22
- package/packages/workspace/hostile-input.mjs +29 -8
- package/packages/workspace/investigation-template-model.mjs +19 -4
- package/packages/workspace/page-baseline.mjs +18 -15
- package/packages/workspace/relationship-observation-model.mjs +41 -23
- package/packages/workspace/watchlist-history.mjs +2 -2
- package/packages/workspace/watchlist-store.mjs +8 -2
- package/packages/workspace/website-snapshot-model.mjs +164 -26
- package/packages/workspace/workspace-archive-crypto.mjs +41 -61
- package/packages/workspace/workspace-archive.mjs +87 -21
- package/third-party-notices.txt +29 -2
- package/cli/evidence-command-arguments.mjs +0 -156
- package/packages/investigation/bulk-review-export.mjs +0 -63
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// Native HTML tree construction over an already-captured body. Resource limits
|
|
2
|
+
// are enforced while nodes are created, including implied/reconstructed nodes;
|
|
3
|
+
// a post-parse traversal limit alone would not bound hostile parser work.
|
|
4
|
+
import { parse, defaultTreeAdapter as adapter } from 'parse5';
|
|
5
|
+
import { MAX_HOMEPAGE_BYTES } from "./outbound-request-bounds.mjs";
|
|
6
|
+
import { MAX_STATIC_HTML_TAGS, MAX_STATIC_HTML_NODES, MAX_STATIC_HTML_DEPTH, } from "../packages/contracts/page-fingerprints.mjs";
|
|
7
|
+
export { MAX_STATIC_HTML_TAGS, MAX_STATIC_HTML_NODES, MAX_STATIC_HTML_DEPTH, MAX_STATIC_HTML_TREE_EVENTS, } from "../packages/contracts/page-fingerprints.mjs";
|
|
8
|
+
// UTF-8 decoding cannot produce more UTF-16 code units than captured bytes.
|
|
9
|
+
// Direct string callers share this ceiling; tree construction remains bounded
|
|
10
|
+
// independently of the source size.
|
|
11
|
+
export const MAX_STATIC_HTML_CHARS = MAX_HOMEPAGE_BYTES;
|
|
12
|
+
const MAX_CONSTRUCTED_ATTRIBUTE_CHARS = MAX_STATIC_HTML_CHARS * 4;
|
|
13
|
+
const HTML_NAMESPACE = 'http://www.w3.org/1999/xhtml';
|
|
14
|
+
export function parseBoundedHtml(value) {
|
|
15
|
+
const supplied = typeof value === 'string' ? value : '';
|
|
16
|
+
const stop = new Error('Static HTML construction limit reached.');
|
|
17
|
+
let document;
|
|
18
|
+
let nodes = 0;
|
|
19
|
+
let elements = 0;
|
|
20
|
+
let depth = 0;
|
|
21
|
+
let attributeCharacters = 0;
|
|
22
|
+
let constructionLimitReached = false;
|
|
23
|
+
function reserveNode() {
|
|
24
|
+
if (nodes >= MAX_STATIC_HTML_NODES)
|
|
25
|
+
throw stop;
|
|
26
|
+
nodes += 1;
|
|
27
|
+
}
|
|
28
|
+
function reserveAttributes(attributes) {
|
|
29
|
+
for (const attribute of attributes) {
|
|
30
|
+
attributeCharacters += attribute.name.length + attribute.value.length;
|
|
31
|
+
if (attributeCharacters > MAX_CONSTRUCTED_ATTRIBUTE_CHARS)
|
|
32
|
+
throw stop;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
const boundedAdapter = {
|
|
36
|
+
...adapter,
|
|
37
|
+
createDocument() {
|
|
38
|
+
reserveNode();
|
|
39
|
+
document = adapter.createDocument();
|
|
40
|
+
return document;
|
|
41
|
+
},
|
|
42
|
+
createDocumentFragment() { reserveNode(); return adapter.createDocumentFragment(); },
|
|
43
|
+
createElement(name, namespace, attributes) {
|
|
44
|
+
// Allow the three mandatory document elements in addition to source tags.
|
|
45
|
+
// Reconstructed formatting elements also consume this finite allowance.
|
|
46
|
+
if (elements >= MAX_STATIC_HTML_TAGS + 3)
|
|
47
|
+
throw stop;
|
|
48
|
+
elements += 1;
|
|
49
|
+
reserveAttributes(attributes);
|
|
50
|
+
reserveNode();
|
|
51
|
+
return adapter.createElement(name, namespace, attributes);
|
|
52
|
+
},
|
|
53
|
+
createCommentNode(text) { reserveNode(); return adapter.createCommentNode(text); },
|
|
54
|
+
insertText(parent, text) {
|
|
55
|
+
const previous = parent.childNodes.at(-1);
|
|
56
|
+
if (!previous || !adapter.isTextNode(previous))
|
|
57
|
+
reserveNode();
|
|
58
|
+
adapter.insertText(parent, text);
|
|
59
|
+
},
|
|
60
|
+
insertTextBefore(parent, text, reference) {
|
|
61
|
+
const previous = parent.childNodes[parent.childNodes.indexOf(reference) - 1];
|
|
62
|
+
if (!previous || !adapter.isTextNode(previous))
|
|
63
|
+
reserveNode();
|
|
64
|
+
adapter.insertTextBefore(parent, text, reference);
|
|
65
|
+
},
|
|
66
|
+
setDocumentType(parent, name, publicId, systemId) {
|
|
67
|
+
if (!parent.childNodes.some((node) => node.nodeName === '#documentType'))
|
|
68
|
+
reserveNode();
|
|
69
|
+
adapter.setDocumentType(parent, name, publicId, systemId);
|
|
70
|
+
},
|
|
71
|
+
adoptAttributes(element, attributes) {
|
|
72
|
+
reserveAttributes(attributes);
|
|
73
|
+
adapter.adoptAttributes(element, attributes);
|
|
74
|
+
},
|
|
75
|
+
onItemPush() {
|
|
76
|
+
depth += 1;
|
|
77
|
+
if (depth > MAX_STATIC_HTML_DEPTH)
|
|
78
|
+
throw stop;
|
|
79
|
+
},
|
|
80
|
+
onItemPop() { depth -= 1; },
|
|
81
|
+
};
|
|
82
|
+
try {
|
|
83
|
+
parse(supplied.slice(0, MAX_STATIC_HTML_CHARS), {
|
|
84
|
+
treeAdapter: boundedAdapter,
|
|
85
|
+
sourceCodeLocationInfo: true,
|
|
86
|
+
scriptingEnabled: true,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
if (error !== stop)
|
|
91
|
+
throw error;
|
|
92
|
+
constructionLimitReached = true;
|
|
93
|
+
}
|
|
94
|
+
// Parser or adapter failures are not silently re-labelled as empty evidence.
|
|
95
|
+
if (!document)
|
|
96
|
+
throw new Error('Static HTML document was not initialised.');
|
|
97
|
+
return { document, inputLimitReached: supplied.length > MAX_STATIC_HTML_CHARS, constructionLimitReached };
|
|
98
|
+
}
|
|
99
|
+
export function isHtmlElement(node) {
|
|
100
|
+
return adapter.isElementNode(node) && node.namespaceURI === HTML_NAMESPACE;
|
|
101
|
+
}
|
|
102
|
+
// Iterative native-tree order, without entering inert template fragments.
|
|
103
|
+
// Foreign elements are retained for structure, not interpreted as HTML forms.
|
|
104
|
+
export function* htmlTreeEvents(document) {
|
|
105
|
+
const pending = [{ node: document, exit: false }];
|
|
106
|
+
while (pending.length) {
|
|
107
|
+
const { node, exit } = pending.pop();
|
|
108
|
+
if (adapter.isElementNode(node)) {
|
|
109
|
+
yield { kind: exit ? 'end' : 'start', element: node };
|
|
110
|
+
if (exit)
|
|
111
|
+
continue;
|
|
112
|
+
pending.push({ node, exit: true });
|
|
113
|
+
if (isHtmlElement(node) && node.tagName === 'template')
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
else if (adapter.isTextNode(node)) {
|
|
117
|
+
yield { kind: 'text', text: node.value };
|
|
118
|
+
}
|
|
119
|
+
if ('childNodes' in node) {
|
|
120
|
+
for (let index = node.childNodes.length - 1; index >= 0; index -= 1) {
|
|
121
|
+
pending.push({ node: node.childNodes[index], exit: false });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
package/lib/bounded-json.mjs
CHANGED
|
@@ -3,6 +3,23 @@ export const MAX_BOUNDED_JSON_KEYS = 50_000;
|
|
|
3
3
|
export const MAX_BOUNDED_JSON_VALUES = 100_000;
|
|
4
4
|
export const MAX_BOUNDED_JSON_CONTAINER_ITEMS = 10_000;
|
|
5
5
|
export const UNSAFE_JSON_OBJECT_KEYS = Object.freeze(['__proto__']);
|
|
6
|
+
/** Aggregate work allowed by a separately enforced UTF-8 byte budget. */
|
|
7
|
+
export function boundedJsonLimitsForBytes(maximumBytes) {
|
|
8
|
+
if (!Number.isSafeInteger(maximumBytes) || maximumBytes < 1) {
|
|
9
|
+
throw new TypeError('The JSON byte budget must be a positive safe integer.');
|
|
10
|
+
}
|
|
11
|
+
// A member needs at least five bytes (comma, two quotes, colon, value).
|
|
12
|
+
// Values need at least two bytes each, apart from a scalar root or the last
|
|
13
|
+
// array item. These upper bounds do not depend on a model's current fields.
|
|
14
|
+
// Nesting and individual-container limits remain independently bounded.
|
|
15
|
+
return {
|
|
16
|
+
maximumKeys: Math.floor(maximumBytes / 5),
|
|
17
|
+
maximumValues: Math.ceil(maximumBytes / 2),
|
|
18
|
+
// Decoded text is a lower bound on its UTF-8 JSON representation. Checking
|
|
19
|
+
// it before stringify also prevents an oversized string allocation.
|
|
20
|
+
maximumStringCodeUnits: maximumBytes,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
6
23
|
function syntaxError() {
|
|
7
24
|
throw new TypeError('Artefact input is not valid JSON.');
|
|
8
25
|
}
|
|
@@ -16,7 +33,14 @@ export function assertBoundedJsonStructure(value, label = 'JSON value', limits =
|
|
|
16
33
|
const maximumContainerItems = limits.maximumContainerItems ?? MAX_BOUNDED_JSON_CONTAINER_ITEMS;
|
|
17
34
|
let keys = 0;
|
|
18
35
|
let values = 0;
|
|
36
|
+
let stringCodeUnits = 0;
|
|
19
37
|
const ancestors = new Set();
|
|
38
|
+
const text = (value) => {
|
|
39
|
+
stringCodeUnits += value.length;
|
|
40
|
+
if (limits.maximumStringCodeUnits !== undefined && stringCodeUnits > limits.maximumStringCodeUnits) {
|
|
41
|
+
throw new TypeError(`${label} exceeds the aggregate text limit.`);
|
|
42
|
+
}
|
|
43
|
+
};
|
|
20
44
|
const visit = (current, depth) => {
|
|
21
45
|
values += 1;
|
|
22
46
|
if (values > maximumValues) {
|
|
@@ -25,7 +49,11 @@ export function assertBoundedJsonStructure(value, label = 'JSON value', limits =
|
|
|
25
49
|
if (depth > maximumDepth) {
|
|
26
50
|
throw new TypeError(`${label} exceeds the ${maximumDepth}-level nesting limit.`);
|
|
27
51
|
}
|
|
28
|
-
if (
|
|
52
|
+
if (typeof current === 'string') {
|
|
53
|
+
text(current);
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
if (current === null || typeof current === 'boolean')
|
|
29
57
|
return;
|
|
30
58
|
if (typeof current === 'number') {
|
|
31
59
|
if (!Number.isFinite(current))
|
|
@@ -91,6 +119,7 @@ export function assertBoundedJsonStructure(value, label = 'JSON value', limits =
|
|
|
91
119
|
for (const key of objectKeys) {
|
|
92
120
|
if (!isSafeJsonObjectKey(key))
|
|
93
121
|
throw new TypeError(`${label} contains an unsafe object key.`);
|
|
122
|
+
text(key);
|
|
94
123
|
visit(descriptors[key].value, depth + 1);
|
|
95
124
|
}
|
|
96
125
|
}
|
|
@@ -98,7 +127,7 @@ export function assertBoundedJsonStructure(value, label = 'JSON value', limits =
|
|
|
98
127
|
};
|
|
99
128
|
visit(value, 0);
|
|
100
129
|
}
|
|
101
|
-
export function scanBoundedJson(raw, limits = {}) {
|
|
130
|
+
export function scanBoundedJson(raw, limits = {}, observeString) {
|
|
102
131
|
const maximumDepth = limits.maximumDepth ?? MAX_BOUNDED_JSON_DEPTH;
|
|
103
132
|
const maximumKeys = limits.maximumKeys ?? MAX_BOUNDED_JSON_KEYS;
|
|
104
133
|
const maximumValues = limits.maximumValues ?? MAX_BOUNDED_JSON_VALUES;
|
|
@@ -106,9 +135,14 @@ export function scanBoundedJson(raw, limits = {}) {
|
|
|
106
135
|
let index = 0;
|
|
107
136
|
let keys = 0;
|
|
108
137
|
let values = 0;
|
|
138
|
+
let stringCodeUnits = 0;
|
|
109
139
|
const whitespace = () => {
|
|
110
|
-
while (index < raw.length
|
|
140
|
+
while (index < raw.length) {
|
|
141
|
+
const code = raw.charCodeAt(index);
|
|
142
|
+
if (code !== 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d)
|
|
143
|
+
break;
|
|
111
144
|
index += 1;
|
|
145
|
+
}
|
|
112
146
|
};
|
|
113
147
|
const stringToken = () => {
|
|
114
148
|
const start = index;
|
|
@@ -117,12 +151,18 @@ export function scanBoundedJson(raw, limits = {}) {
|
|
|
117
151
|
const character = raw[index];
|
|
118
152
|
if (character === '"') {
|
|
119
153
|
index += 1;
|
|
154
|
+
let value;
|
|
120
155
|
try {
|
|
121
|
-
|
|
156
|
+
value = JSON.parse(raw.slice(start, index));
|
|
122
157
|
}
|
|
123
158
|
catch {
|
|
124
159
|
syntaxError();
|
|
125
160
|
}
|
|
161
|
+
stringCodeUnits += value.length;
|
|
162
|
+
if (limits.maximumStringCodeUnits !== undefined && stringCodeUnits > limits.maximumStringCodeUnits) {
|
|
163
|
+
throw new TypeError('Artefact JSON exceeds the aggregate text limit.');
|
|
164
|
+
}
|
|
165
|
+
return { value, offset: start };
|
|
126
166
|
}
|
|
127
167
|
if (character === '\\') {
|
|
128
168
|
index += 1;
|
|
@@ -160,7 +200,7 @@ export function scanBoundedJson(raw, limits = {}) {
|
|
|
160
200
|
}
|
|
161
201
|
index += match[0].length;
|
|
162
202
|
};
|
|
163
|
-
const value = (depth) => {
|
|
203
|
+
const value = (depth, propertyKey) => {
|
|
164
204
|
values += 1;
|
|
165
205
|
if (values > maximumValues) {
|
|
166
206
|
throw new TypeError(`Artefact JSON exceeds the ${maximumValues}-value limit.`);
|
|
@@ -171,7 +211,8 @@ export function scanBoundedJson(raw, limits = {}) {
|
|
|
171
211
|
whitespace();
|
|
172
212
|
const character = raw[index];
|
|
173
213
|
if (character === '"') {
|
|
174
|
-
stringToken();
|
|
214
|
+
const parsedString = stringToken();
|
|
215
|
+
observeString?.({ ...parsedString, kind: 'value', ...(propertyKey === undefined ? {} : { propertyKey }) });
|
|
175
216
|
return;
|
|
176
217
|
}
|
|
177
218
|
if (character === '{') {
|
|
@@ -185,7 +226,8 @@ export function scanBoundedJson(raw, limits = {}) {
|
|
|
185
226
|
while (index < raw.length) {
|
|
186
227
|
if (raw[index] !== '"')
|
|
187
228
|
syntaxError();
|
|
188
|
-
const
|
|
229
|
+
const parsedKey = stringToken();
|
|
230
|
+
const key = parsedKey.value;
|
|
189
231
|
if (seen.size >= maximumContainerItems) {
|
|
190
232
|
throw new TypeError(`Artefact JSON contains a container with more than ${maximumContainerItems} items.`);
|
|
191
233
|
}
|
|
@@ -198,11 +240,12 @@ export function scanBoundedJson(raw, limits = {}) {
|
|
|
198
240
|
if (!isSafeJsonObjectKey(key))
|
|
199
241
|
throw new TypeError('Artefact JSON contains an unsafe object key.');
|
|
200
242
|
seen.add(key);
|
|
243
|
+
observeString?.({ ...parsedKey, kind: 'key' });
|
|
201
244
|
whitespace();
|
|
202
245
|
if (raw[index] !== ':')
|
|
203
246
|
syntaxError();
|
|
204
247
|
index += 1;
|
|
205
|
-
value(depth + 1);
|
|
248
|
+
value(depth + 1, key);
|
|
206
249
|
whitespace();
|
|
207
250
|
if (raw[index] === '}') {
|
|
208
251
|
index += 1;
|
|
@@ -5,9 +5,11 @@
|
|
|
5
5
|
// are treated as untrusted inputs and are not retained in the result.
|
|
6
6
|
import { createHash } from 'node:crypto';
|
|
7
7
|
import { Worker } from 'node:worker_threads';
|
|
8
|
+
import { createInlineLibraryScanner } from "./browser-library-worker.mjs";
|
|
8
9
|
import { RETIRE_BROWSER_CATALOG } from "./generated/retire-browser-catalog.mjs";
|
|
9
10
|
import { CISA_KEV_CATALOG } from "./generated/cisa-kev-catalog.mjs";
|
|
10
|
-
import { createObservation } from "../packages/evidence/observation.mjs";
|
|
11
|
+
import { createObservation, normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
|
|
12
|
+
import { isCveIdentifier } from "../packages/contracts/vulnerability-identifiers.mjs";
|
|
11
13
|
import { BROWSER_LIBRARY_PROFILE_VERSION, MAX_LIBRARY_FINDINGS, } from "./lookup-child-profile-contract.mjs";
|
|
12
14
|
import { MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_STATIC_HTML_CHARS, analyzeStaticHtml, } from "./static-html-analysis.mjs";
|
|
13
15
|
// Catalogue expressions are reviewed third-party inputs. Keep every individual
|
|
@@ -25,7 +27,6 @@ const MAX_MATCHES_PER_PATTERN = 4;
|
|
|
25
27
|
const MAX_VERSION_LENGTH = 64;
|
|
26
28
|
const COMPONENT_RE = /^[a-z0-9._-]{1,80}$/i;
|
|
27
29
|
const VERSION_RE = /^[0-9][0-9.a-z_-]{0,63}$/i;
|
|
28
|
-
const CVE_RE = /^CVE-[0-9X-]+$/;
|
|
29
30
|
const GHSA_RE = /^GHSA-[A-Z0-9-]+$/;
|
|
30
31
|
const CWE_RE = /^CWE-[0-9]+$/;
|
|
31
32
|
const SEVERITY_WEIGHT = Object.freeze({
|
|
@@ -37,6 +38,9 @@ const SEVERITY_WEIGHT = Object.freeze({
|
|
|
37
38
|
});
|
|
38
39
|
const CATALOG_COMPONENTS = RETIRE_BROWSER_CATALOG.components;
|
|
39
40
|
const KNOWN_EXPLOITED_IDENTIFIERS = new Set(CISA_KEV_CATALOG.identifiers);
|
|
41
|
+
const KNOWN_EXPLOITED_RELEASED_AT = normalizeExplicitIsoTimestamp(CISA_KEV_CATALOG.releasedAt);
|
|
42
|
+
if (KNOWN_EXPLOITED_RELEASED_AT === null)
|
|
43
|
+
throw new Error('Pinned known-exploited catalogue has an invalid release timestamp.');
|
|
40
44
|
const INLINE_EXTRACTOR_CATALOGUE = Object.freeze(Object.entries(CATALOG_COMPONENTS).map(([component, value]) => {
|
|
41
45
|
const extractors = record(record(value).extractors);
|
|
42
46
|
return Object.freeze({
|
|
@@ -95,10 +99,16 @@ parentPort.on('message', (job) => {
|
|
|
95
99
|
} catch {
|
|
96
100
|
Atomics.store(control, 0, -1);
|
|
97
101
|
}
|
|
98
|
-
|
|
102
|
+
parentPort.postMessage({ id: job.id });
|
|
99
103
|
});
|
|
100
104
|
`;
|
|
101
|
-
|
|
105
|
+
const scanInlineLibrary = createInlineLibraryScanner(() => new Worker(INLINE_REGEX_WORKER_SOURCE, { eval: true, workerData: { catalogue: INLINE_EXTRACTOR_CATALOGUE } }), {
|
|
106
|
+
maximumCharacters: MAX_INLINE_LIBRARY_SCAN_TOTAL_CHARS + MAX_SCRIPT_ELEMENTS,
|
|
107
|
+
outputBytes: MAX_INLINE_LIBRARY_WORKER_BYTES,
|
|
108
|
+
deadlineMs: MAX_INLINE_LIBRARY_SCAN_MS,
|
|
109
|
+
// One worker/output buffer, with at most sixteen bounded pending inputs.
|
|
110
|
+
pending: 16,
|
|
111
|
+
});
|
|
102
112
|
function record(value) {
|
|
103
113
|
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
104
114
|
? value
|
|
@@ -218,48 +228,15 @@ function scanFilename(detected, reference) {
|
|
|
218
228
|
}
|
|
219
229
|
}
|
|
220
230
|
}
|
|
221
|
-
function scanInlineSignatures(detected, value) {
|
|
231
|
+
async function scanInlineSignatures(detected, value, signal) {
|
|
222
232
|
if (!value)
|
|
223
233
|
return { timedOut: false, unavailable: false };
|
|
224
|
-
const
|
|
225
|
-
|
|
226
|
-
|
|
234
|
+
const result = await scanInlineLibrary(value, signal);
|
|
235
|
+
signal?.throwIfAborted();
|
|
236
|
+
if (result.output === null)
|
|
237
|
+
return result;
|
|
227
238
|
try {
|
|
228
|
-
|
|
229
|
-
const createdWorker = new Worker(INLINE_REGEX_WORKER_SOURCE, {
|
|
230
|
-
eval: true,
|
|
231
|
-
workerData: { catalogue: INLINE_EXTRACTOR_CATALOGUE },
|
|
232
|
-
});
|
|
233
|
-
inlineRegexWorker = createdWorker;
|
|
234
|
-
createdWorker.unref();
|
|
235
|
-
createdWorker.once('exit', () => {
|
|
236
|
-
if (inlineRegexWorker === createdWorker)
|
|
237
|
-
inlineRegexWorker = null;
|
|
238
|
-
});
|
|
239
|
-
createdWorker.once('error', () => {
|
|
240
|
-
if (inlineRegexWorker === createdWorker)
|
|
241
|
-
inlineRegexWorker = null;
|
|
242
|
-
});
|
|
243
|
-
}
|
|
244
|
-
inlineRegexWorker.postMessage({ value, control: controlBuffer, output: outputBuffer });
|
|
245
|
-
}
|
|
246
|
-
catch {
|
|
247
|
-
void inlineRegexWorker?.terminate().catch(() => { });
|
|
248
|
-
inlineRegexWorker = null;
|
|
249
|
-
return { timedOut: false, unavailable: true };
|
|
250
|
-
}
|
|
251
|
-
const wait = Atomics.wait(control, 0, 0, MAX_INLINE_LIBRARY_SCAN_MS);
|
|
252
|
-
const length = Atomics.load(control, 0);
|
|
253
|
-
if (wait === 'timed-out') {
|
|
254
|
-
void inlineRegexWorker?.terminate().catch(() => { });
|
|
255
|
-
inlineRegexWorker = null;
|
|
256
|
-
return { timedOut: true, unavailable: false };
|
|
257
|
-
}
|
|
258
|
-
if (length < 1 || length > MAX_INLINE_LIBRARY_WORKER_BYTES) {
|
|
259
|
-
return { timedOut: false, unavailable: true };
|
|
260
|
-
}
|
|
261
|
-
try {
|
|
262
|
-
const parsed = JSON.parse(Buffer.from(new Uint8Array(outputBuffer, 0, length)).toString('utf8'));
|
|
239
|
+
const parsed = JSON.parse(result.output);
|
|
263
240
|
if (!Array.isArray(parsed))
|
|
264
241
|
return { timedOut: false, unavailable: true };
|
|
265
242
|
for (const item of parsed.slice(0, MAX_LIBRARY_FINDINGS * 4)) {
|
|
@@ -311,13 +288,20 @@ function findingFromDetection(detected) {
|
|
|
311
288
|
let highestSeverity = null;
|
|
312
289
|
const identifiers = new Set();
|
|
313
290
|
const weaknesses = new Set();
|
|
291
|
+
let omittedCveIdentifiers = 0;
|
|
314
292
|
for (const vulnerability of vulnerabilities) {
|
|
315
293
|
const candidateSeverity = severity(vulnerability.severity);
|
|
316
294
|
if (candidateSeverity
|
|
317
295
|
&& (!highestSeverity || SEVERITY_WEIGHT[candidateSeverity] > SEVERITY_WEIGHT[highestSeverity]))
|
|
318
296
|
highestSeverity = candidateSeverity;
|
|
319
297
|
const vulnerabilityIdentifiers = record(vulnerability.identifiers);
|
|
320
|
-
|
|
298
|
+
omittedCveIdentifiers += typeof vulnerability.omittedCveIdentifiers === 'number'
|
|
299
|
+
&& Number.isSafeInteger(vulnerability.omittedCveIdentifiers) && vulnerability.omittedCveIdentifiers > 0
|
|
300
|
+
? vulnerability.omittedCveIdentifiers : 0;
|
|
301
|
+
const suppliedCve = Array.isArray(vulnerabilityIdentifiers.CVE) ? vulnerabilityIdentifiers.CVE : [];
|
|
302
|
+
const validCve = suppliedCve.filter(isCveIdentifier);
|
|
303
|
+
omittedCveIdentifiers += suppliedCve.length - validCve.length;
|
|
304
|
+
for (const cve of validCve.slice(0, 16)) {
|
|
321
305
|
if (identifiers.size < MAX_ADVISORY_IDENTIFIERS)
|
|
322
306
|
identifiers.add(cve);
|
|
323
307
|
}
|
|
@@ -332,7 +316,7 @@ function findingFromDetection(detected) {
|
|
|
332
316
|
}
|
|
333
317
|
const advisoryIdentifiers = [...identifiers].sort();
|
|
334
318
|
const knownExploitedIdentifiers = advisoryIdentifiers.filter((identifier) => KNOWN_EXPLOITED_IDENTIFIERS.has(identifier));
|
|
335
|
-
|
|
319
|
+
const finding = {
|
|
336
320
|
id: detected.component,
|
|
337
321
|
name: detected.component,
|
|
338
322
|
apparentVersion: detected.version,
|
|
@@ -344,8 +328,10 @@ function findingFromDetection(detected) {
|
|
|
344
328
|
knownExploitedIdentifiers,
|
|
345
329
|
weaknessClasses: [...weaknesses].sort(),
|
|
346
330
|
};
|
|
331
|
+
return { finding, omittedCveIdentifiers };
|
|
347
332
|
}
|
|
348
|
-
function analyzeBrowserLibraries(input = {}) {
|
|
333
|
+
async function analyzeBrowserLibraries(input = {}) {
|
|
334
|
+
input.signal?.throwIfAborted();
|
|
349
335
|
const htmlAnalysis = input.htmlAnalysis ?? analyzeStaticHtml(input.html);
|
|
350
336
|
const detected = new Map();
|
|
351
337
|
let referencesExamined = 0;
|
|
@@ -369,10 +355,12 @@ function analyzeBrowserLibraries(input = {}) {
|
|
|
369
355
|
inlineSignatureSamples.push(signatureScan.signatureContent);
|
|
370
356
|
}
|
|
371
357
|
}
|
|
372
|
-
const inlineSignatureResult = scanInlineSignatures(detected, inlineSignatureSamples.join('\u0000'));
|
|
358
|
+
const inlineSignatureResult = await scanInlineSignatures(detected, inlineSignatureSamples.join('\u0000'), input.signal);
|
|
359
|
+
input.signal?.throwIfAborted();
|
|
373
360
|
const inlineSignatureUnavailable = inlineSignatureResult.timedOut || inlineSignatureResult.unavailable;
|
|
374
|
-
const
|
|
375
|
-
|
|
361
|
+
const projected = [...detected.values()].map(findingFromDetection);
|
|
362
|
+
const omittedCveIdentifiers = projected.reduce((total, entry) => total + entry.omittedCveIdentifiers, 0);
|
|
363
|
+
const allFindings = projected.map((entry) => entry.finding)
|
|
376
364
|
.sort((left, right) => ((right.highestSeverity ? SEVERITY_WEIGHT[right.highestSeverity] : -1)
|
|
377
365
|
- (left.highestSeverity ? SEVERITY_WEIGHT[left.highestSeverity] : -1)
|
|
378
366
|
|| right.advisoryCount - left.advisoryCount
|
|
@@ -386,7 +374,8 @@ function analyzeBrowserLibraries(input = {}) {
|
|
|
386
374
|
|| htmlAnalysis.inlineLimitReached
|
|
387
375
|
|| inlineSignatureLimitReached
|
|
388
376
|
|| inlineSignatureUnavailable
|
|
389
|
-
|| findingLimitReached
|
|
377
|
+
|| findingLimitReached
|
|
378
|
+
|| omittedCveIdentifiers > 0;
|
|
390
379
|
const limitations = [
|
|
391
380
|
'Library versions are inferred from passive static signatures and may be absent, transformed, or misleading.',
|
|
392
381
|
'A catalogue advisory match identifies a component version associated with a published advisory; it does not establish reachability or exploitability.',
|
|
@@ -411,6 +400,8 @@ function analyzeBrowserLibraries(input = {}) {
|
|
|
411
400
|
limitations.push('Passive library signature matching was unavailable in its isolated worker; hash evidence was still evaluated.');
|
|
412
401
|
if (findingLimitReached)
|
|
413
402
|
limitations.push(`Only the first ${MAX_LIBRARY_FINDINGS} library findings were retained.`);
|
|
403
|
+
if (omittedCveIdentifiers)
|
|
404
|
+
limitations.push(`${omittedCveIdentifiers} supplied CVE identifier ${omittedCveIdentifiers === 1 ? 'entry was' : 'entries were'} omitted from matching catalogue advisories because of invalid syntax or a source limit. Advisory matches are still counted.`);
|
|
414
405
|
return {
|
|
415
406
|
profileVersion: BROWSER_LIBRARY_PROFILE_VERSION,
|
|
416
407
|
catalog: {
|
|
@@ -421,7 +412,7 @@ function analyzeBrowserLibraries(input = {}) {
|
|
|
421
412
|
knownExploitedCatalog: {
|
|
422
413
|
name: 'CISA KEV',
|
|
423
414
|
version: CISA_KEV_CATALOG.catalogVersion,
|
|
424
|
-
releasedAt:
|
|
415
|
+
releasedAt: KNOWN_EXPLOITED_RELEASED_AT,
|
|
425
416
|
},
|
|
426
417
|
...createObservation({
|
|
427
418
|
status: truncated ? 'partial' : 'success',
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/** One bounded matcher worker. Queue deadlines include waiting; cancellation never stops a different scan. */
|
|
2
|
+
export function createInlineLibraryScanner(createWorker, limits) {
|
|
3
|
+
let worker = null;
|
|
4
|
+
let active = null;
|
|
5
|
+
let closing = false;
|
|
6
|
+
let nextId = 0;
|
|
7
|
+
const queue = [];
|
|
8
|
+
const pending = new Set();
|
|
9
|
+
function retireWorker() {
|
|
10
|
+
if (!worker)
|
|
11
|
+
return;
|
|
12
|
+
const retired = worker;
|
|
13
|
+
worker = null;
|
|
14
|
+
closing = true;
|
|
15
|
+
void Promise.resolve().then(() => retired.terminate()).catch(() => { }).finally(() => { closing = false; pump(); });
|
|
16
|
+
}
|
|
17
|
+
function pump() {
|
|
18
|
+
if (active || closing)
|
|
19
|
+
return;
|
|
20
|
+
const job = queue.shift();
|
|
21
|
+
if (!job) {
|
|
22
|
+
worker?.unref();
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
active = job;
|
|
26
|
+
let current;
|
|
27
|
+
const control = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
|
|
28
|
+
const output = new Uint8Array(new SharedArrayBuffer(limits.outputBytes));
|
|
29
|
+
const cleanup = job.cleanup;
|
|
30
|
+
const failed = () => settle(job, { timedOut: false, unavailable: true, output: null }, true);
|
|
31
|
+
const message = (value) => {
|
|
32
|
+
if (!value || typeof value !== 'object' || !('id' in value) || value.id !== job.id)
|
|
33
|
+
return;
|
|
34
|
+
const length = Atomics.load(control, 0);
|
|
35
|
+
if (length < 1 || length > output.byteLength) {
|
|
36
|
+
failed();
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
settle(job, { timedOut: false, unavailable: false, output: Buffer.from(output.subarray(0, length)).toString('utf8') });
|
|
40
|
+
};
|
|
41
|
+
job.cleanup = () => {
|
|
42
|
+
cleanup();
|
|
43
|
+
current?.off('message', message);
|
|
44
|
+
current?.off('error', failed);
|
|
45
|
+
current?.off('exit', failed);
|
|
46
|
+
};
|
|
47
|
+
try {
|
|
48
|
+
if (!worker) {
|
|
49
|
+
const created = createWorker();
|
|
50
|
+
worker = created;
|
|
51
|
+
const idleStopped = () => { if (!active && worker === created)
|
|
52
|
+
worker = null; };
|
|
53
|
+
created.on('error', idleStopped);
|
|
54
|
+
created.on('exit', idleStopped);
|
|
55
|
+
}
|
|
56
|
+
current = worker;
|
|
57
|
+
if (active !== job) {
|
|
58
|
+
retireWorker();
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
current.ref();
|
|
62
|
+
current.on('message', message);
|
|
63
|
+
current.on('error', failed);
|
|
64
|
+
current.on('exit', failed);
|
|
65
|
+
current.postMessage({ id: job.id, value: job.value, control: control.buffer, output: output.buffer });
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
failed();
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function settle(job, result, retire = false, aborted = false) {
|
|
72
|
+
if (!pending.delete(job))
|
|
73
|
+
return;
|
|
74
|
+
job.cleanup();
|
|
75
|
+
if (active === job) {
|
|
76
|
+
active = null;
|
|
77
|
+
if (retire)
|
|
78
|
+
retireWorker();
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
const index = queue.indexOf(job);
|
|
82
|
+
if (index >= 0)
|
|
83
|
+
queue.splice(index, 1);
|
|
84
|
+
}
|
|
85
|
+
if (aborted)
|
|
86
|
+
job.reject(job.signal?.reason ?? new DOMException('Aborted', 'AbortError'));
|
|
87
|
+
else
|
|
88
|
+
job.resolve(result);
|
|
89
|
+
queueMicrotask(pump);
|
|
90
|
+
}
|
|
91
|
+
return function scan(value, signal) {
|
|
92
|
+
signal?.throwIfAborted();
|
|
93
|
+
if (!value)
|
|
94
|
+
return Promise.resolve({ timedOut: false, unavailable: false, output: '[]' });
|
|
95
|
+
if (value.length > limits.maximumCharacters || pending.size >= limits.pending) {
|
|
96
|
+
return Promise.resolve({ timedOut: false, unavailable: true, output: null });
|
|
97
|
+
}
|
|
98
|
+
return new Promise((resolve, reject) => {
|
|
99
|
+
const job = { id: ++nextId, value, ...(signal ? { signal } : {}), resolve, reject, cleanup: () => { } };
|
|
100
|
+
const timer = setTimeout(() => settle(job, { timedOut: true, unavailable: false, output: null }, true), limits.deadlineMs);
|
|
101
|
+
const abort = () => settle(job, { timedOut: false, unavailable: true, output: null }, true, true);
|
|
102
|
+
job.cleanup = () => { clearTimeout(timer); signal?.removeEventListener('abort', abort); };
|
|
103
|
+
pending.add(job);
|
|
104
|
+
queue.push(job);
|
|
105
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
106
|
+
if (signal?.aborted)
|
|
107
|
+
abort();
|
|
108
|
+
else
|
|
109
|
+
pump();
|
|
110
|
+
});
|
|
111
|
+
};
|
|
112
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Static client-side behaviour indicators derived from the existing bounded
|
|
2
|
-
// HTML
|
|
2
|
+
// HTML parse. Scripts are never fetched or executed. Only fixed
|
|
3
3
|
// indicator identifiers, descriptions, evidence classes, and bounded counts
|
|
4
4
|
// leave this module; script references and contents are discarded.
|
|
5
5
|
import { createObservation } from "../packages/evidence/observation.mjs";
|
|
@@ -20,7 +20,6 @@ function patternCount(value, pattern) {
|
|
|
20
20
|
}
|
|
21
21
|
function analyzeClientBehavior(input = {}) {
|
|
22
22
|
const htmlAnalysis = input.htmlAnalysis ?? analyzeStaticHtml(input.html);
|
|
23
|
-
const markup = htmlAnalysis.markup;
|
|
24
23
|
const inlineScripts = htmlAnalysis.scripts
|
|
25
24
|
.filter((script) => script.reference === null)
|
|
26
25
|
.map((script) => script.inlineContent)
|
|
@@ -31,7 +30,8 @@ function analyzeClientBehavior(input = {}) {
|
|
|
31
30
|
return;
|
|
32
31
|
indicators.push({ id, label, evidenceClass, occurrences: boundedOccurrences(occurrences), explanation });
|
|
33
32
|
}
|
|
34
|
-
add('inline_event_handlers', 'Inline event handlers', 'static_markup',
|
|
33
|
+
add('inline_event_handlers', 'Inline event handlers', 'static_markup', htmlAnalysis.elements.reduce((count, element) => count
|
|
34
|
+
+ element.attributes.filter((attribute) => /^on[a-z]{2,24}$/u.test(attribute.name)).length, 0), 'Inline HTML event-handler attributes were observed.');
|
|
35
35
|
add('client_navigation', 'Client-side navigation', 'inline_script', patternCount(inlineScripts, /\b(?:history\.(?:pushState|replaceState)|location\.(?:assign|replace)|window\.location)\b/gu), 'Inline script contains a client-side navigation API.');
|
|
36
36
|
add('form_interception', 'Form interception', 'inline_script', patternCount(inlineScripts, /(?:addEventListener\s*\(\s*['"]submit['"]|\.onsubmit\s*=|preventDefault\s*\()/gu), 'Inline script contains a form-submission or default-action interception marker.');
|
|
37
37
|
add('service_worker', 'Service worker registration', 'inline_script', patternCount(inlineScripts, /\bnavigator\.serviceWorker\.register\s*\(/gu), 'Inline script contains a service-worker registration call.');
|
package/lib/ct-search.mjs
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// earliest public signal a new lookalike exists at all. Shared by the
|
|
8
8
|
// Express server and the Netlify Functions.
|
|
9
9
|
import { parse } from 'tldts';
|
|
10
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
10
11
|
import { normalizeCtQuery } from "./ct-query.mjs";
|
|
11
12
|
import { safeFetch, readTextCapped } from "./safe-fetch.mjs";
|
|
12
13
|
import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
|
|
@@ -71,23 +72,26 @@ function normalizeHostname(raw) {
|
|
|
71
72
|
h = h.slice(2);
|
|
72
73
|
return HOSTNAME_RE.test(h) && isValidAsciiHostname(h, { requireLowercase: true }) ? h : null;
|
|
73
74
|
}
|
|
74
|
-
function delay(ms) {
|
|
75
|
-
return
|
|
75
|
+
function delay(ms, signal) {
|
|
76
|
+
return sleep(ms, undefined, signal ? { signal } : {});
|
|
76
77
|
}
|
|
77
78
|
async function fetchCrtSh(keyword, attempt = 0, dependencies = {}) {
|
|
79
|
+
dependencies.signal?.throwIfAborted();
|
|
78
80
|
const fetcher = dependencies.fetcher || safeFetch;
|
|
79
81
|
const wait = dependencies.delay || delay;
|
|
80
82
|
const controller = new AbortController();
|
|
83
|
+
const signal = dependencies.signal ? AbortSignal.any([dependencies.signal, controller.signal]) : controller.signal;
|
|
81
84
|
const timeout = setTimeout(() => controller.abort(), CRT_SH_TIMEOUT_MS);
|
|
82
85
|
try {
|
|
83
86
|
let res;
|
|
84
87
|
try {
|
|
85
88
|
res = await fetcher(`https://crt.sh/?q=${encodeURIComponent(keyword)}&output=json`, {
|
|
86
89
|
headers: whoisleuthRequestHeaders({ Accept: 'application/json' }),
|
|
87
|
-
signal
|
|
90
|
+
signal,
|
|
88
91
|
}, 0);
|
|
89
92
|
}
|
|
90
93
|
catch (err) {
|
|
94
|
+
dependencies.signal?.throwIfAborted();
|
|
91
95
|
if (!(err instanceof Error) || err.name !== 'AbortError')
|
|
92
96
|
throw err;
|
|
93
97
|
if (attempt < MAX_TIMEOUT_RETRIES)
|
|
@@ -101,7 +105,8 @@ async function fetchCrtSh(keyword, attempt = 0, dependencies = {}) {
|
|
|
101
105
|
// undici's own idle-timeout eventually notices.
|
|
102
106
|
await res.body?.cancel().catch(() => { });
|
|
103
107
|
if (RETRYABLE_STATUSES.has(res.status) && attempt < MAX_RETRIES) {
|
|
104
|
-
await wait(RETRY_DELAY_MS * (attempt + 1));
|
|
108
|
+
await wait(RETRY_DELAY_MS * (attempt + 1), dependencies.signal);
|
|
109
|
+
dependencies.signal?.throwIfAborted();
|
|
105
110
|
return fetchCrtSh(keyword, attempt + 1, dependencies);
|
|
106
111
|
}
|
|
107
112
|
throw new Error(RETRYABLE_STATUSES.has(res.status)
|
|
@@ -109,6 +114,7 @@ async function fetchCrtSh(keyword, attempt = 0, dependencies = {}) {
|
|
|
109
114
|
: `crt.sh returned ${res.status}`);
|
|
110
115
|
}
|
|
111
116
|
const { text, truncated } = await readTextCapped(res, CRT_SH_MAX_BYTES, { fatalUtf8: true });
|
|
117
|
+
signal.throwIfAborted();
|
|
112
118
|
if (truncated) {
|
|
113
119
|
throw new Error(`crt.sh returned more than ${CRT_SH_MAX_BYTES / (1024 * 1024)}MB of results for "${keyword}" - try a narrower/more specific keyword.`);
|
|
114
120
|
}
|
|
@@ -464,6 +470,7 @@ function summarizeCtResults(rows) {
|
|
|
464
470
|
// Production search
|
|
465
471
|
// ---------------------------------------------------------------------------
|
|
466
472
|
async function searchCertificateTransparency(keyword, dependencies = {}) {
|
|
473
|
+
dependencies.signal?.throwIfAborted();
|
|
467
474
|
const startedAt = Date.now();
|
|
468
475
|
const trimmed = normalizeCtQuery(keyword);
|
|
469
476
|
if (!trimmed) {
|
|
@@ -160,6 +160,7 @@ export function reviewDnsConvergence(inputRaw, generatedAtValue = new Date().toI
|
|
|
160
160
|
});
|
|
161
161
|
});
|
|
162
162
|
const reasons = Object.freeze([
|
|
163
|
+
...(rows.length === 0 ? ['No comparable owner and record-type scope was supplied or observed.'] : []),
|
|
163
164
|
...rows.filter((row) => row.state !== 'converged').map((row) => `${row.owner} ${row.type} is ${row.state}.`),
|
|
164
165
|
...latestByObserver.filter((item) => item.state !== 'observed').map((item) => `${item.observer} latest evidence is ${item.state}.`),
|
|
165
166
|
].slice(0, 200));
|