@slicedearth/whoisleuth-cli 1.39.1 → 1.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -4
- package/bin/whoisleuth.mjs +31 -9
- package/cli/archive-inspect.mjs +37 -2
- package/cli/arguments.mjs +361 -256
- package/cli/artifact-verify.mjs +35 -1
- package/cli/bulk-checkpoint.mjs +72 -39
- package/cli/bulk-command-runner.mjs +34 -4
- package/cli/bulk.mjs +18 -7
- package/cli/case-pack.mjs +136 -0
- package/cli/ci-report.mjs +73 -0
- package/cli/collection-preflight.mjs +59 -0
- package/cli/command-reference.mjs +395 -0
- package/cli/completion.mjs +67 -13
- package/cli/config-profile.mjs +183 -0
- package/cli/ct-event-intake.mjs +159 -0
- package/cli/discovery-observation-snapshot.mjs +8 -14
- package/cli/discovery-scan-command-runner.mjs +29 -2
- package/cli/discovery-snapshot.mjs +8 -15
- package/cli/domain-control-monitor.mjs +104 -0
- package/cli/domain-control-observations.mjs +235 -0
- package/cli/errors.mjs +7 -1
- package/cli/evidence-command-arguments.mjs +11 -0
- package/cli/evidence-command-runner.mjs +1 -0
- package/cli/external-observation-mapping.mjs +162 -0
- package/cli/fail-policy.mjs +64 -0
- package/cli/formatters/html.mjs +1 -1
- package/cli/formatters/json.mjs +12 -4
- package/cli/formatters/terminal.mjs +1 -1
- package/cli/http.mjs +2 -1
- package/cli/investigation-manifest.mjs +113 -0
- package/cli/investigation-run.mjs +112 -0
- package/cli/lookup-brief.mjs +116 -0
- package/cli/lookup-command-runner.mjs +8 -1
- package/cli/lookup-diff.mjs +17 -19
- package/cli/lookup-reconcile.mjs +1 -1
- package/cli/mail-review.mjs +5 -5
- package/cli/manual.mjs +1 -1
- package/cli/network-command-runner.mjs +4 -1
- package/cli/offline-evidence-review.mjs +105 -8
- package/cli/open-asset-model-bridge.mjs +169 -0
- package/cli/registry-cohort.mjs +121 -0
- package/cli/registry-doctor.mjs +106 -11
- package/cli/registry-fixture-scaffold.mjs +74 -0
- package/cli/runner.mjs +388 -315
- package/docs/cli-reference.md +266 -21
- package/docs/cli.md +43 -7
- package/frontend/src/lib/analysis/acquisition-due-diligence.js +2 -2
- package/frontend/src/lib/analysis/brand-profile-model.js +50 -6
- package/frontend/src/lib/analysis/case-evidence-model.js +1 -1
- package/frontend/src/lib/analysis/case-record-core.js +1 -1
- package/frontend/src/lib/analysis/case-record-operations.js +1 -1
- package/frontend/src/lib/analysis/case-report.js +573 -0
- package/frontend/src/lib/analysis/case-response-model.js +30 -1
- package/frontend/src/lib/analysis/detection-rule-model.js +36 -0
- package/frontend/src/lib/analysis/external-findings-import.js +320 -0
- package/frontend/src/lib/analysis/lookup-asset-graph.js +8 -8
- package/frontend/src/lib/analysis/lookup-decision-support.js +9 -2
- package/frontend/src/lib/analysis/lookup-presentation.js +14 -30
- package/frontend/src/lib/analysis/lookup-source-refresh.js +36 -63
- package/frontend/src/lib/analysis/scoring.js +1 -1
- package/frontend/src/lib/analysis/utils.js +1 -1
- package/frontend/src/lib/analysis/visualization-models.js +2 -3
- package/frontend/src/lib/analysis/workspace-archive-crypto.js +1 -4
- package/frontend/src/lib/analysis/workspace-archive.js +5 -7
- package/lib/abusech-auth.mjs +1 -1
- package/lib/availability.mjs +1 -1
- package/lib/bounded-contract-normalizers.mjs +33 -0
- package/lib/bounded-file.mjs +69 -0
- package/lib/bounded-json-response.mjs +157 -0
- package/lib/browser-library-profile.mjs +167 -38
- package/lib/dns-convergence-review.mjs +183 -0
- package/lib/dns-delegation-health.mjs +196 -3
- package/lib/domain-change-packet.mjs +133 -0
- package/lib/domain-change-review.mjs +307 -0
- package/lib/domain-control-flight-recorder.mjs +268 -0
- package/lib/domain-portfolio-review.mjs +151 -0
- package/lib/evidence-export.mjs +1 -1
- package/lib/html-signals.mjs +5 -11
- package/lib/lookup-cache.mjs +5 -1
- package/lib/nameserver-preflight-review.mjs +159 -0
- package/lib/page-language-signals.mjs +124 -0
- package/lib/rdap-transport.mjs +6 -6
- package/lib/risk-scoring.mjs +42 -1
- package/lib/safe-fetch.mjs +37 -32
- package/lib/static-html-analysis.mjs +29 -1
- package/lib/threat-intelligence-runtime.mjs +1 -2
- package/lib/trust-store-comparison.mjs +146 -0
- package/lib/website-technology.mjs +22 -12
- package/lib/zone-intent-review.mjs +534 -0
- package/package.json +1 -1
package/lib/safe-fetch.mjs
CHANGED
|
@@ -128,21 +128,26 @@ function isPrivateIpv6(ip) {
|
|
|
128
128
|
return true; // ::1 (loopback)
|
|
129
129
|
if (g0.startsWith('ff'))
|
|
130
130
|
return true; // multicast ff00::/8
|
|
131
|
-
// IPv4-mapped
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
|
|
135
|
-
if ([g0, g1, g2, g3, g4].every((g) => g === '0000') && (g5 === 'ffff' || g5 === '0000')) {
|
|
131
|
+
// IPv4-mapped addresses put the embedded address in the last two groups.
|
|
132
|
+
// Accept only a mapped public IPv4 target. The deprecated IPv4-compatible
|
|
133
|
+
// form is rejected below rather than treated as a routable IPv6 address.
|
|
134
|
+
if ([g0, g1, g2, g3, g4].every((g) => g === '0000') && g5 === 'ffff') {
|
|
136
135
|
const embedded = groupsToIpv4(g6, g7);
|
|
137
136
|
if (embedded && isPrivateIpv4(embedded))
|
|
138
137
|
return true;
|
|
138
|
+
if (embedded)
|
|
139
|
+
return false;
|
|
139
140
|
}
|
|
141
|
+
if ([g0, g1, g2, g3, g4, g5].every((g) => g === '0000'))
|
|
142
|
+
return true;
|
|
140
143
|
// NAT64 well-known prefix (64:ff9b::/96) - same tail embedding, different
|
|
141
144
|
// prefix; some IPv6-only networks synthesize these to reach IPv4 hosts.
|
|
142
145
|
if (g0 === '0064' && g1 === 'ff9b' && [g2, g3, g4, g5].every((g) => g === '0000')) {
|
|
143
146
|
const embedded = groupsToIpv4(g6, g7);
|
|
144
147
|
if (embedded && isPrivateIpv4(embedded))
|
|
145
148
|
return true;
|
|
149
|
+
if (embedded)
|
|
150
|
+
return false;
|
|
146
151
|
}
|
|
147
152
|
// RFC 8215 local-use NAT64 prefix (64:ff9b:1::/48). Unlike the well-known
|
|
148
153
|
// /96 above, operators may choose how IPv4 bits are embedded within this
|
|
@@ -150,21 +155,13 @@ function isPrivateIpv6(ip) {
|
|
|
150
155
|
// rather than attempting to decode one layout and leaving others open.
|
|
151
156
|
if (g0 === '0064' && g1 === 'ff9b' && g2 === '0001')
|
|
152
157
|
return true;
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
// obfuscated client IPv4 address. Decode both so a transition address
|
|
161
|
-
// cannot smuggle a private/loopback target past the IPv4 guard.
|
|
162
|
-
if (g0 === '2001' && g1 === '0000') {
|
|
163
|
-
const server = groupsToIpv4(g2, g3);
|
|
164
|
-
const client = groupsToIpv4(((~parseInt(g6, 16)) & 0xffff).toString(16), ((~parseInt(g7, 16)) & 0xffff).toString(16));
|
|
165
|
-
if ((server && isPrivateIpv4(server)) || (client && isPrivateIpv4(client)))
|
|
166
|
-
return true;
|
|
167
|
-
}
|
|
158
|
+
// Deprecated transition mechanisms are not direct global-unicast targets.
|
|
159
|
+
// Reject their entire ranges instead of attempting to classify embedded
|
|
160
|
+
// addresses whose relays and routing behaviour are outside this boundary.
|
|
161
|
+
if (g0 === '2002')
|
|
162
|
+
return true; // 6to4 2002::/16
|
|
163
|
+
if (g0 === '2001' && g1 === '0000')
|
|
164
|
+
return true; // Teredo 2001::/32
|
|
168
165
|
if (/^fe[89ab][0-9a-f]$/.test(g0))
|
|
169
166
|
return true; // link-local fe80::/10
|
|
170
167
|
if (/^fe[cdef][0-9a-f]$/.test(g0))
|
|
@@ -175,11 +172,20 @@ function isPrivateIpv6(ip) {
|
|
|
175
172
|
return true; // discard-only 100::/64
|
|
176
173
|
if (g0 === '2001' && g1 === '0db8')
|
|
177
174
|
return true; // documentation 2001:db8::/32
|
|
175
|
+
if (g0 === '3fff' && parseInt(g1, 16) <= 0x0fff)
|
|
176
|
+
return true; // documentation 3fff::/20
|
|
178
177
|
if (g0 === '2001' && g1 === '0002' && g2 === '0000')
|
|
179
178
|
return true; // benchmarking 2001:2::/48
|
|
180
179
|
if (g0 === '2001' && /^00[1-2][0-9a-f]$/.test(g1))
|
|
181
180
|
return true; // ORCHID/ORCHIDv2
|
|
182
|
-
|
|
181
|
+
// Ordinary publicly routed IPv6 unicast addresses are allocated from
|
|
182
|
+
// 2000::/3. Treat everything outside that allocation as reserved unless it
|
|
183
|
+
// matched one of the explicitly supported public transition forms above.
|
|
184
|
+
// An allow-list boundary is intentional here: newly written or unassigned
|
|
185
|
+
// address space must not silently become an outbound SSRF target merely
|
|
186
|
+
// because it was absent from this file's special-purpose exclusions.
|
|
187
|
+
const firstGroup = parseInt(g0, 16);
|
|
188
|
+
return !Number.isInteger(firstGroup) || firstGroup < 0x2000 || firstGroup > 0x3fff;
|
|
183
189
|
}
|
|
184
190
|
function isPrivateAddress(ip) {
|
|
185
191
|
const version = net.isIP(ip);
|
|
@@ -373,13 +379,12 @@ async function readTextCapped(res, maxBytes, options = {}) {
|
|
|
373
379
|
bytesRead,
|
|
374
380
|
...(hasher ? { sha256: hasher.digest('hex') } : {}),
|
|
375
381
|
});
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
hasher?.update(captured);
|
|
381
|
-
return result(new TextDecoder().decode(captured), full.length > maxBytes, captured.length);
|
|
382
|
+
if (res.body === null)
|
|
383
|
+
return result('', false, 0);
|
|
384
|
+
if (typeof res.body?.getReader !== 'function') {
|
|
385
|
+
throw new Error('Response body does not expose a bounded readable stream');
|
|
382
386
|
}
|
|
387
|
+
const reader = res.body.getReader();
|
|
383
388
|
const decoder = new TextDecoder();
|
|
384
389
|
let text = '';
|
|
385
390
|
let received = 0;
|
|
@@ -413,12 +418,12 @@ async function readTextCapped(res, maxBytes, options = {}) {
|
|
|
413
418
|
// text (a favicon image) - collects raw bytes into a Buffer instead of
|
|
414
419
|
// decoding as UTF-8, which would corrupt non-text bytes.
|
|
415
420
|
async function readBytesCapped(res, maxBytes) {
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
return { bytes, truncated: buf.length > maxBytes, bytesRead: bytes.length };
|
|
421
|
+
if (res.body === null)
|
|
422
|
+
return { bytes: Buffer.alloc(0), truncated: false, bytesRead: 0 };
|
|
423
|
+
if (typeof res.body?.getReader !== 'function') {
|
|
424
|
+
throw new Error('Response body does not expose a bounded readable stream');
|
|
421
425
|
}
|
|
426
|
+
const reader = res.body.getReader();
|
|
422
427
|
const chunks = [];
|
|
423
428
|
let received = 0;
|
|
424
429
|
let truncated = false;
|
|
@@ -15,6 +15,7 @@ const MAX_SCRIPT_REFERENCE_LENGTH = 2_048;
|
|
|
15
15
|
const MAX_SCRIPT_MEDIA_TYPE_LENGTH = 120;
|
|
16
16
|
const MAX_INLINE_SCRIPT_CHARS = 32_768;
|
|
17
17
|
const MAX_INLINE_SCRIPT_TOTAL_CHARS = 65_536;
|
|
18
|
+
const MAX_STATIC_VISIBLE_TEXT_CHARS = MAX_STATIC_HTML_CHARS;
|
|
18
19
|
const MAX_STATIC_FORMS = 50;
|
|
19
20
|
const MAX_STATIC_INPUTS = 500;
|
|
20
21
|
const MAX_FORM_ATTRIBUTE_LENGTH = 2_048;
|
|
@@ -48,6 +49,7 @@ const VOID_TAGS = new Set([
|
|
|
48
49
|
'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta',
|
|
49
50
|
'param', 'source', 'track', 'wbr',
|
|
50
51
|
]);
|
|
52
|
+
const NON_VISIBLE_TEXT_TAGS = new Set(['script', 'style', 'template']);
|
|
51
53
|
function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LENGTH) {
|
|
52
54
|
const attribute = attributes
|
|
53
55
|
.slice(0, MAX_ATTRIBUTES_PER_TAG)
|
|
@@ -190,7 +192,9 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
190
192
|
const inputLimitReached = supplied.length > MAX_STATIC_HTML_CHARS;
|
|
191
193
|
const html = supplied.slice(0, MAX_STATIC_HTML_CHARS);
|
|
192
194
|
const baseUrl = safeBaseUrl(options.baseUrl);
|
|
195
|
+
const includeVisibleText = options.includeVisibleText === true;
|
|
193
196
|
const markup = [];
|
|
197
|
+
const visibleTextParts = [];
|
|
194
198
|
const structureTokens = [];
|
|
195
199
|
const scripts = [];
|
|
196
200
|
const cspMetaPolicies = [];
|
|
@@ -211,9 +215,12 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
211
215
|
let structureLimitReached = false;
|
|
212
216
|
let scriptLimitReached = false;
|
|
213
217
|
let inlineLimitReached = false;
|
|
218
|
+
let visibleTextLimitReached = false;
|
|
214
219
|
let cspMetaLimitReached = false;
|
|
215
220
|
let insideExplicitHead = false;
|
|
216
221
|
let scriptElementSeen = false;
|
|
222
|
+
let nonVisibleTextDepth = 0;
|
|
223
|
+
let visibleTextCharacters = 0;
|
|
217
224
|
function appendInlineScript(chars) {
|
|
218
225
|
if (!activeInlineScript || !chars)
|
|
219
226
|
return;
|
|
@@ -234,12 +241,26 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
234
241
|
}
|
|
235
242
|
structureTokens.push(value);
|
|
236
243
|
}
|
|
244
|
+
function appendVisibleText(chars) {
|
|
245
|
+
if (!includeVisibleText || nonVisibleTextDepth > 0 || !chars)
|
|
246
|
+
return;
|
|
247
|
+
const remaining = MAX_STATIC_VISIBLE_TEXT_CHARS - visibleTextCharacters;
|
|
248
|
+
const retainedLength = Math.max(0, Math.min(chars.length, remaining));
|
|
249
|
+
if (retainedLength > 0) {
|
|
250
|
+
visibleTextParts.push(chars.slice(0, retainedLength));
|
|
251
|
+
visibleTextCharacters += retainedLength;
|
|
252
|
+
}
|
|
253
|
+
if (retainedLength < chars.length)
|
|
254
|
+
visibleTextLimitReached = true;
|
|
255
|
+
}
|
|
237
256
|
const handler = {
|
|
238
257
|
onStartTag(token) {
|
|
239
258
|
const tagName = token.tagName.toLowerCase();
|
|
240
259
|
const rawTextMode = RAW_TEXT_MODES[tagName];
|
|
241
260
|
if (!token.selfClosing && rawTextMode !== undefined)
|
|
242
261
|
tokenizer.state = rawTextMode;
|
|
262
|
+
if (!token.selfClosing && NON_VISIBLE_TEXT_TAGS.has(tagName))
|
|
263
|
+
nonVisibleTextDepth += 1;
|
|
243
264
|
if (tagsExamined >= MAX_STATIC_HTML_TAGS) {
|
|
244
265
|
tagLimitReached = true;
|
|
245
266
|
activeInlineScript = null;
|
|
@@ -332,15 +353,20 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
332
353
|
insideExplicitHead = false;
|
|
333
354
|
if (!VOID_TAGS.has(tagName))
|
|
334
355
|
appendStructureToken(`/${tagName}`);
|
|
356
|
+
if (NON_VISIBLE_TEXT_TAGS.has(tagName) && nonVisibleTextDepth > 0)
|
|
357
|
+
nonVisibleTextDepth -= 1;
|
|
335
358
|
},
|
|
336
359
|
onCharacter(token) {
|
|
337
360
|
appendInlineScript(token.chars);
|
|
361
|
+
appendVisibleText(token.chars);
|
|
338
362
|
},
|
|
339
363
|
onNullCharacter(token) {
|
|
340
364
|
appendInlineScript(token.chars);
|
|
365
|
+
appendVisibleText(token.chars);
|
|
341
366
|
},
|
|
342
367
|
onWhitespaceCharacter(token) {
|
|
343
368
|
appendInlineScript(token.chars);
|
|
369
|
+
appendVisibleText(token.chars);
|
|
344
370
|
},
|
|
345
371
|
onComment() { },
|
|
346
372
|
onDoctype() { },
|
|
@@ -350,6 +376,7 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
350
376
|
tokenizer.write(html, true);
|
|
351
377
|
return {
|
|
352
378
|
markup: markup.join('\n'),
|
|
379
|
+
visibleText: visibleTextParts.join(''),
|
|
353
380
|
structureTokens,
|
|
354
381
|
scripts,
|
|
355
382
|
cspMetaPolicies,
|
|
@@ -360,8 +387,9 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
360
387
|
structureLimitReached,
|
|
361
388
|
scriptLimitReached,
|
|
362
389
|
inlineLimitReached,
|
|
390
|
+
visibleTextLimitReached,
|
|
363
391
|
tagsExamined,
|
|
364
392
|
inlineCharactersExamined,
|
|
365
393
|
};
|
|
366
394
|
}
|
|
367
|
-
export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };
|
|
395
|
+
export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// output budgets before producing separately attributed normalized evidence.
|
|
6
6
|
import { createObservation } from "./observation.mjs";
|
|
7
7
|
import { assertCuratedConnectorDefinition, assertThreatIntelligenceProvider, } from "./threat-intelligence-definition-registry.mjs";
|
|
8
|
-
import { boundedHttpsUrl,
|
|
8
|
+
import { boundedHttpsUrl, boundedString, isRecord, isoTimestamp, strictBoundedString, } from "./bounded-contract-normalizers.mjs";
|
|
9
9
|
import { CURATED_CONNECTOR_ENTITY_TYPES, MAX_THREAT_INTELLIGENCE_URL_LENGTH, normalizeCuratedConnectorTarget, normalizeThreatIntelligenceTarget, } from "./threat-intelligence-targets.mjs";
|
|
10
10
|
const THREAT_INTELLIGENCE_CONTRACT_VERSION = 1;
|
|
11
11
|
const THREAT_INTELLIGENCE_SCHEMA = 'whoisleuth.threat-intelligence-result';
|
|
@@ -78,7 +78,6 @@ const CONNECTOR_RELATIONSHIP_ENDPOINTS = Object.freeze({
|
|
|
78
78
|
});
|
|
79
79
|
// createObservation() retains at most 40 source characters, so provider IDs use
|
|
80
80
|
// the same ceiling and can never be truncated at the provenance boundary.
|
|
81
|
-
const MAX_PROVIDER_ID_LENGTH = 40;
|
|
82
81
|
const MAX_PROVIDER_LABEL_LENGTH = 100;
|
|
83
82
|
const MAX_DETAIL_LENGTH = 500;
|
|
84
83
|
const MAX_FINDING_ID_LENGTH = 160;
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { enumValue, exactKeys, requireBoundedString, requireIsoTimestamp, requireRecord, } from "./bounded-contract-normalizers.mjs";
|
|
2
|
+
export const TRUST_STORE_COMPARISON_INPUT_SCHEMA = 'whoisleuth.trust-store-comparison.input';
|
|
3
|
+
export const TRUST_STORE_COMPARISON_REVIEW_SCHEMA = 'whoisleuth.trust-store-comparison.review';
|
|
4
|
+
export const TRUST_STORE_COMPARISON_REVIEW_VERSION = 1;
|
|
5
|
+
export const MAX_TRUST_STORE_COMPARISON_STORES = 16;
|
|
6
|
+
export const MAX_TRUST_STORE_ANCHORS = 2_048;
|
|
7
|
+
export const MAX_TRUST_STORE_CHAIN_CERTIFICATES = 16;
|
|
8
|
+
const ROOT_KEYS = new Set(['schema', 'version', 'certificate', 'stores']);
|
|
9
|
+
const CERTIFICATE_KEYS = new Set([
|
|
10
|
+
'source', 'observedAt', 'state', 'chainTruncated',
|
|
11
|
+
'fingerprintsSha256', 'runtimeAuthorisation',
|
|
12
|
+
]);
|
|
13
|
+
const STORE_KEYS = new Set(['name', 'version', 'source', 'reviewedAt', 'state', 'anchorsSha256']);
|
|
14
|
+
const EVIDENCE_STATES = new Set(['observed', 'partial', 'unavailable']);
|
|
15
|
+
const RUNTIME_AUTHORISATION = new Set(['authorised', 'unauthorised', 'unknown']);
|
|
16
|
+
const SHA256_RE = /^[a-f0-9]{64}$/u;
|
|
17
|
+
function digest(value, label) {
|
|
18
|
+
const normalised = requireBoundedString(value, label, 64).toLowerCase();
|
|
19
|
+
if (!SHA256_RE.test(normalised))
|
|
20
|
+
throw new TypeError(`${label} must be a SHA-256 hexadecimal digest.`);
|
|
21
|
+
return normalised;
|
|
22
|
+
}
|
|
23
|
+
function digests(value, label, maximum, sort = true) {
|
|
24
|
+
if (!Array.isArray(value) || value.length > maximum) {
|
|
25
|
+
throw new TypeError(`${label} must contain no more than ${maximum} SHA-256 digests.`);
|
|
26
|
+
}
|
|
27
|
+
const values = [...new Set(value.map((entry, index) => digest(entry, `${label}[${index}]`)))];
|
|
28
|
+
return Object.freeze(sort ? values.sort() : values);
|
|
29
|
+
}
|
|
30
|
+
export function compareTrustStoreEvidence(inputRaw, generatedAtValue = new Date().toISOString()) {
|
|
31
|
+
const input = requireRecord(inputRaw, 'Trust-store comparison input');
|
|
32
|
+
if (input.schema !== TRUST_STORE_COMPARISON_INPUT_SCHEMA || input.version !== 1) {
|
|
33
|
+
throw new TypeError(`Trust-store comparison input must use ${TRUST_STORE_COMPARISON_INPUT_SCHEMA} version 1.`);
|
|
34
|
+
}
|
|
35
|
+
exactKeys(input, ROOT_KEYS, 'Trust-store comparison input');
|
|
36
|
+
const certificateInput = requireRecord(input.certificate, 'certificate');
|
|
37
|
+
exactKeys(certificateInput, CERTIFICATE_KEYS, 'certificate');
|
|
38
|
+
const certificateState = enumValue(certificateInput.state, EVIDENCE_STATES, 'certificate.state');
|
|
39
|
+
if (typeof certificateInput.chainTruncated !== 'boolean')
|
|
40
|
+
throw new TypeError('certificate.chainTruncated must be a Boolean.');
|
|
41
|
+
const certificateFingerprints = digests(certificateInput.fingerprintsSha256, 'certificate.fingerprintsSha256', MAX_TRUST_STORE_CHAIN_CERTIFICATES, false);
|
|
42
|
+
if (certificateState === 'unavailable' && certificateFingerprints.length) {
|
|
43
|
+
throw new TypeError('An unavailable certificate observation cannot contain fingerprints.');
|
|
44
|
+
}
|
|
45
|
+
if (certificateState !== 'unavailable' && !certificateFingerprints.length) {
|
|
46
|
+
throw new TypeError('An observed or partial certificate observation requires at least one fingerprint.');
|
|
47
|
+
}
|
|
48
|
+
const certificate = Object.freeze({
|
|
49
|
+
source: requireBoundedString(certificateInput.source, 'certificate.source', 240),
|
|
50
|
+
observedAt: requireIsoTimestamp(certificateInput.observedAt, 'certificate.observedAt'),
|
|
51
|
+
state: certificateState,
|
|
52
|
+
chainTruncated: certificateInput.chainTruncated,
|
|
53
|
+
leafSha256: certificateFingerprints[0] ?? null,
|
|
54
|
+
fingerprintsSha256: certificateFingerprints,
|
|
55
|
+
runtimeAuthorisation: enumValue(certificateInput.runtimeAuthorisation, RUNTIME_AUTHORISATION, 'certificate.runtimeAuthorisation'),
|
|
56
|
+
});
|
|
57
|
+
if (!Array.isArray(input.stores) || input.stores.length < 1 || input.stores.length > MAX_TRUST_STORE_COMPARISON_STORES) {
|
|
58
|
+
throw new TypeError(`stores must contain between 1 and ${MAX_TRUST_STORE_COMPARISON_STORES} entries.`);
|
|
59
|
+
}
|
|
60
|
+
const storeKeys = new Set();
|
|
61
|
+
const stores = input.stores.map((raw, index) => {
|
|
62
|
+
const store = requireRecord(raw, `stores[${index}]`);
|
|
63
|
+
exactKeys(store, STORE_KEYS, `stores[${index}]`);
|
|
64
|
+
const name = requireBoundedString(store.name, `stores[${index}].name`, 120);
|
|
65
|
+
const version = requireBoundedString(store.version, `stores[${index}].version`, 120);
|
|
66
|
+
const key = `${name}\u0000${version}`;
|
|
67
|
+
if (storeKeys.has(key))
|
|
68
|
+
throw new TypeError(`stores[${index}] repeats a store name and version.`);
|
|
69
|
+
storeKeys.add(key);
|
|
70
|
+
const storeState = enumValue(store.state, EVIDENCE_STATES, `stores[${index}].state`);
|
|
71
|
+
const anchors = digests(store.anchorsSha256, `stores[${index}].anchorsSha256`, MAX_TRUST_STORE_ANCHORS);
|
|
72
|
+
if (storeState === 'unavailable' && anchors.length)
|
|
73
|
+
throw new TypeError(`stores[${index}] cannot contain anchors when unavailable.`);
|
|
74
|
+
if (storeState === 'observed' && !anchors.length)
|
|
75
|
+
throw new TypeError(`stores[${index}] requires at least one anchor when observed.`);
|
|
76
|
+
return Object.freeze({
|
|
77
|
+
name,
|
|
78
|
+
version,
|
|
79
|
+
source: requireBoundedString(store.source, `stores[${index}].source`, 240),
|
|
80
|
+
reviewedAt: requireIsoTimestamp(store.reviewedAt, `stores[${index}].reviewedAt`),
|
|
81
|
+
state: storeState,
|
|
82
|
+
anchors,
|
|
83
|
+
});
|
|
84
|
+
}).sort((left, right) => left.name.localeCompare(right.name) || left.version.localeCompare(right.version));
|
|
85
|
+
const chainSet = new Set(certificate.fingerprintsSha256);
|
|
86
|
+
const comparisons = stores.map((store) => {
|
|
87
|
+
const matchedAnchorSha256 = store.anchors.filter((anchor) => chainSet.has(anchor));
|
|
88
|
+
const completeInputs = certificate.state === 'observed'
|
|
89
|
+
&& certificate.chainTruncated === false
|
|
90
|
+
&& store.state === 'observed';
|
|
91
|
+
const state = matchedAnchorSha256.length
|
|
92
|
+
? 'anchor_observed'
|
|
93
|
+
: completeInputs
|
|
94
|
+
? 'not_observed'
|
|
95
|
+
: 'inconclusive';
|
|
96
|
+
return Object.freeze({
|
|
97
|
+
store: Object.freeze({
|
|
98
|
+
name: store.name,
|
|
99
|
+
version: store.version,
|
|
100
|
+
source: store.source,
|
|
101
|
+
reviewedAt: store.reviewedAt,
|
|
102
|
+
state: store.state,
|
|
103
|
+
anchorCount: store.anchors.length,
|
|
104
|
+
}),
|
|
105
|
+
state,
|
|
106
|
+
matchedAnchorSha256: Object.freeze(matchedAnchorSha256),
|
|
107
|
+
explanation: state === 'anchor_observed'
|
|
108
|
+
? 'At least one supplied chain fingerprint exactly matched a supplied trust-anchor fingerprint.'
|
|
109
|
+
: state === 'not_observed'
|
|
110
|
+
? 'No supplied chain fingerprint matched this complete supplied trust-store snapshot.'
|
|
111
|
+
: 'Certificate-chain or trust-store evidence was incomplete or unavailable, so non-observation cannot be established.',
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
const incomplete = certificate.state !== 'observed'
|
|
115
|
+
|| certificate.chainTruncated
|
|
116
|
+
|| stores.some((store) => store.state !== 'observed');
|
|
117
|
+
return Object.freeze({
|
|
118
|
+
schema: TRUST_STORE_COMPARISON_REVIEW_SCHEMA,
|
|
119
|
+
version: TRUST_STORE_COMPARISON_REVIEW_VERSION,
|
|
120
|
+
generatedAt: requireIsoTimestamp(generatedAtValue, 'generatedAt'),
|
|
121
|
+
state: incomplete ? 'partial' : 'complete',
|
|
122
|
+
certificate: Object.freeze({
|
|
123
|
+
source: certificate.source,
|
|
124
|
+
observedAt: certificate.observedAt,
|
|
125
|
+
state: certificate.state,
|
|
126
|
+
chainTruncated: certificate.chainTruncated,
|
|
127
|
+
leafSha256: certificate.leafSha256,
|
|
128
|
+
chainFingerprintCount: certificate.fingerprintsSha256.length,
|
|
129
|
+
runtimeAuthorisation: certificate.runtimeAuthorisation,
|
|
130
|
+
}),
|
|
131
|
+
comparisons: Object.freeze(comparisons),
|
|
132
|
+
counts: Object.freeze({
|
|
133
|
+
stores: comparisons.length,
|
|
134
|
+
anchorObserved: comparisons.filter((comparison) => comparison.state === 'anchor_observed').length,
|
|
135
|
+
notObserved: comparisons.filter((comparison) => comparison.state === 'not_observed').length,
|
|
136
|
+
inconclusive: comparisons.filter((comparison) => comparison.state === 'inconclusive').length,
|
|
137
|
+
}),
|
|
138
|
+
limitations: Object.freeze([
|
|
139
|
+
'This local comparison uses only supplied SHA-256 fingerprints and makes no TLS, certificate-authority, operating-system, or trust-store request.',
|
|
140
|
+
'An exact anchor fingerprint intersection does not build or validate a certificate path, check signatures, names, validity, constraints, revocation, policy, or application-specific trust.',
|
|
141
|
+
'A complete supplied chain can omit its root certificate. A not-observed result therefore does not mean the runtime, browser, operating system, or another client would reject the certificate.',
|
|
142
|
+
'Trust stores differ by product, platform, version, policy, update state, and local administration. The analyst-supplied source and version remain part of every comparison.',
|
|
143
|
+
'Runtime authorisation is preserved as separate context and is not overridden by this fingerprint comparison.',
|
|
144
|
+
]),
|
|
145
|
+
});
|
|
146
|
+
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
import { analyzeBrowserLibraries } from "./browser-library-profile.mjs";
|
|
8
8
|
import { createObservation } from "./observation.mjs";
|
|
9
9
|
import { MAX_STATIC_HTML_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, } from "./static-html-analysis.mjs";
|
|
10
|
-
const TECHNOLOGY_PROFILE_VERSION =
|
|
10
|
+
const TECHNOLOGY_PROFILE_VERSION = 10;
|
|
11
11
|
const MAX_TECHNOLOGY_HTML_CHARS = MAX_STATIC_HTML_CHARS;
|
|
12
12
|
const MAX_TECHNOLOGY_TAG_LENGTH = MAX_TAG_LENGTH;
|
|
13
13
|
const MAX_TECHNOLOGY_FINDINGS = 24;
|
|
@@ -40,15 +40,17 @@ function normalizedResourceHosts(value) {
|
|
|
40
40
|
}
|
|
41
41
|
return hosts;
|
|
42
42
|
}
|
|
43
|
-
const
|
|
43
|
+
const PASSIVE_TECHNOLOGY_HEADER_NAMES = Object.freeze([
|
|
44
44
|
'cf-ray',
|
|
45
45
|
'x-drupal-cache',
|
|
46
|
-
'x-
|
|
46
|
+
'x-served-by',
|
|
47
|
+
'x-nf-request-id',
|
|
47
48
|
'x-powered-by',
|
|
48
49
|
'x-shopify-stage',
|
|
49
50
|
'x-sorting-hat-podid',
|
|
50
51
|
'x-vercel-id',
|
|
51
52
|
]);
|
|
53
|
+
const PASSIVE_HEADER_NAMES = new Set(PASSIVE_TECHNOLOGY_HEADER_NAMES);
|
|
52
54
|
function normalizedResponseHeaders(value) {
|
|
53
55
|
const input = value && typeof value === 'object' && !Array.isArray(value)
|
|
54
56
|
? value
|
|
@@ -148,7 +150,9 @@ const TECHNOLOGY_SIGNATURES = [
|
|
|
148
150
|
},
|
|
149
151
|
{
|
|
150
152
|
id: 'craft-cms', name: 'Craft CMS', category: 'content management',
|
|
151
|
-
evidence: [
|
|
153
|
+
evidence: [
|
|
154
|
+
responseHeaderEvidence('x-powered-by', /(?:^|,\s*)craft cms(?:\s|$|\/)/i, 'The passive X-Powered-By response header identifies Craft CMS.'),
|
|
155
|
+
],
|
|
152
156
|
},
|
|
153
157
|
{
|
|
154
158
|
id: 'typo3', name: 'TYPO3 CMS', category: 'content management',
|
|
@@ -198,17 +202,21 @@ const TECHNOLOGY_SIGNATURES = [
|
|
|
198
202
|
},
|
|
199
203
|
{
|
|
200
204
|
id: 'opencart', name: 'OpenCart', category: 'commerce',
|
|
201
|
-
evidence: [
|
|
205
|
+
evidence: [
|
|
206
|
+
htmlEvidence(['index.php?route=common/home', 'image/catalog/opencart-logo.png'], 'Static markup contains OpenCart routing or default asset conventions.'),
|
|
207
|
+
],
|
|
202
208
|
},
|
|
203
209
|
{
|
|
204
210
|
id: 'prestashop', name: 'PrestaShop', category: 'commerce',
|
|
205
|
-
evidence: [
|
|
211
|
+
evidence: [
|
|
212
|
+
htmlEvidence(['/modules/ps_'], 'Static resource paths use PrestaShop module conventions.'),
|
|
213
|
+
],
|
|
206
214
|
},
|
|
207
215
|
{
|
|
208
216
|
id: 'wix', name: 'Wix', category: 'site builder',
|
|
209
217
|
requiresNonResourceEvidence: true,
|
|
210
218
|
evidence: [
|
|
211
|
-
generatorEvidence(/^wix(?:\s|$)/i, 'Generator metadata identifies Wix.'),
|
|
219
|
+
generatorEvidence(/^wix(?:\.com)?(?:\s|$)/i, 'Generator metadata identifies Wix.'),
|
|
212
220
|
htmlEvidence(['data-mesh-id='], 'Static markup contains a Wix-specific document attribute.', 'medium'),
|
|
213
221
|
resourceEvidence(['static.parastorage.com', 'wixstatic.com'], 'A retained resource origin uses Wix delivery infrastructure.'),
|
|
214
222
|
],
|
|
@@ -242,7 +250,7 @@ const TECHNOLOGY_SIGNATURES = [
|
|
|
242
250
|
id: 'weebly', name: 'Weebly', category: 'site builder',
|
|
243
251
|
requiresNonResourceEvidence: true,
|
|
244
252
|
evidence: [
|
|
245
|
-
|
|
253
|
+
htmlEvidence(['id="wsite-base-style"', "id='wsite-base-style'", 'title="wsite-theme-css"', "title='wsite-theme-css'"], 'Static markup contains Weebly-specific theme attributes.'),
|
|
246
254
|
resourceEvidence(['editmysite.com'], 'A retained resource origin uses Weebly delivery infrastructure.'),
|
|
247
255
|
],
|
|
248
256
|
},
|
|
@@ -310,13 +318,15 @@ const TECHNOLOGY_SIGNATURES = [
|
|
|
310
318
|
{
|
|
311
319
|
id: 'cloudfront', name: 'Amazon CloudFront', category: 'delivery platform',
|
|
312
320
|
evidence: [
|
|
313
|
-
serverEvidence(/^cloudfront(?:\s|$|\/)/i, 'The selected response server header identifies Amazon CloudFront.'),
|
|
314
321
|
resourceEvidence(['cloudfront.net'], 'A retained resource origin uses Amazon CloudFront delivery infrastructure.'),
|
|
315
322
|
],
|
|
316
323
|
},
|
|
317
324
|
{
|
|
318
325
|
id: 'netlify', name: 'Netlify', category: 'delivery platform',
|
|
319
|
-
evidence: [
|
|
326
|
+
evidence: [
|
|
327
|
+
serverEvidence(/^netlify(?:\s|$|\/)/i, 'The selected response server header identifies Netlify.'),
|
|
328
|
+
responseHeaderEvidence('x-nf-request-id', null, 'A Netlify request identifier response header was observed.', 'medium'),
|
|
329
|
+
],
|
|
320
330
|
},
|
|
321
331
|
{
|
|
322
332
|
id: 'vercel', name: 'Vercel', category: 'delivery platform',
|
|
@@ -327,7 +337,7 @@ const TECHNOLOGY_SIGNATURES = [
|
|
|
327
337
|
},
|
|
328
338
|
{
|
|
329
339
|
id: 'fastly', name: 'Fastly', category: 'delivery platform',
|
|
330
|
-
evidence: [responseHeaderEvidence('x-
|
|
340
|
+
evidence: [responseHeaderEvidence('x-served-by', /(?:^|,\s*)cache-[a-z0-9-]+-[a-z]{3}(?:\s*,|$)/i, 'The passive X-Served-By response header contains a Fastly cache-node identifier.', 'medium')],
|
|
331
341
|
},
|
|
332
342
|
{
|
|
333
343
|
id: 'nginx', name: 'nginx', category: 'web server',
|
|
@@ -434,4 +444,4 @@ function analyzeWebsiteTechnology(input = {}) {
|
|
|
434
444
|
browserLibraryProfile,
|
|
435
445
|
};
|
|
436
446
|
}
|
|
437
|
-
export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, };
|
|
447
|
+
export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, PASSIVE_TECHNOLOGY_HEADER_NAMES, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, };
|