@slicedearth/whoisleuth-cli 1.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/LICENSE +661 -0
  2. package/LICENSES/Retire.js-Apache-2.0.txt +199 -0
  3. package/NOTICE +76 -0
  4. package/README.md +84 -0
  5. package/TRADEMARKS.md +41 -0
  6. package/bin/whoisleuth.mjs +35 -0
  7. package/cli/archive-inspect.mjs +198 -0
  8. package/cli/arguments.mjs +1022 -0
  9. package/cli/artifact-verify.mjs +210 -0
  10. package/cli/bulk-checkpoint.mjs +218 -0
  11. package/cli/bulk-command-runner.mjs +109 -0
  12. package/cli/bulk-output.mjs +112 -0
  13. package/cli/bulk.mjs +133 -0
  14. package/cli/compare.mjs +238 -0
  15. package/cli/completion.mjs +175 -0
  16. package/cli/csv.mjs +1 -0
  17. package/cli/discover.mjs +50 -0
  18. package/cli/discovery-command-runner.mjs +36 -0
  19. package/cli/discovery-observation-snapshot.mjs +315 -0
  20. package/cli/discovery-scan-command-runner.mjs +141 -0
  21. package/cli/discovery-scan.mjs +325 -0
  22. package/cli/discovery-snapshot.mjs +133 -0
  23. package/cli/discovery-workflow.mjs +73 -0
  24. package/cli/doctor.mjs +137 -0
  25. package/cli/errors.mjs +21 -0
  26. package/cli/evidence-command-arguments.mjs +145 -0
  27. package/cli/evidence-command-runner.mjs +106 -0
  28. package/cli/evidence-signing.mjs +215 -0
  29. package/cli/exit-codes.mjs +9 -0
  30. package/cli/export-evidence.mjs +28 -0
  31. package/cli/formatters/evidence-report.mjs +3 -0
  32. package/cli/formatters/html.mjs +69 -0
  33. package/cli/formatters/json.mjs +135 -0
  34. package/cli/formatters/markdown.mjs +3 -0
  35. package/cli/formatters/terminal.mjs +658 -0
  36. package/cli/http.mjs +27 -0
  37. package/cli/lookup-command-runner.mjs +87 -0
  38. package/cli/lookup-diff.mjs +171 -0
  39. package/cli/mail-review.mjs +204 -0
  40. package/cli/manual.mjs +34 -0
  41. package/cli/network-command-runner.mjs +88 -0
  42. package/cli/output-file.mjs +97 -0
  43. package/cli/page-compare.mjs +123 -0
  44. package/cli/posture.mjs +20 -0
  45. package/cli/progress-events.mjs +53 -0
  46. package/cli/progress.mjs +56 -0
  47. package/cli/registry-access.mjs +3 -0
  48. package/cli/registry-support.mjs +120 -0
  49. package/cli/risk-calibration.mjs +294 -0
  50. package/cli/runner-types.mjs +1 -0
  51. package/cli/runner.mjs +726 -0
  52. package/cli/saved-lookup.mjs +87 -0
  53. package/cli/source-reliability.mjs +727 -0
  54. package/cli/strict-exit.mjs +55 -0
  55. package/cli/terminal-presentation.mjs +147 -0
  56. package/docs/cli.md +827 -0
  57. package/frontend/src/lib/analysis/acquisition-decision-packet.js +93 -0
  58. package/frontend/src/lib/analysis/acquisition-due-diligence.js +391 -0
  59. package/frontend/src/lib/analysis/artifact-integrity.js +21 -0
  60. package/frontend/src/lib/analysis/brand-profile-model.js +421 -0
  61. package/frontend/src/lib/analysis/bulk-domain-comparison.js +246 -0
  62. package/frontend/src/lib/analysis/bulk-mail-exposure.js +192 -0
  63. package/frontend/src/lib/analysis/bulk-retry-plan.js +86 -0
  64. package/frontend/src/lib/analysis/bulk-review-export.js +62 -0
  65. package/frontend/src/lib/analysis/bulk-review-model.js +266 -0
  66. package/frontend/src/lib/analysis/bulk-session-model.js +515 -0
  67. package/frontend/src/lib/analysis/bulk-sort.js +49 -0
  68. package/frontend/src/lib/analysis/campaign-model.js +265 -0
  69. package/frontend/src/lib/analysis/case-evidence-model.js +642 -0
  70. package/frontend/src/lib/analysis/case-migration-model.js +248 -0
  71. package/frontend/src/lib/analysis/case-model.js +5 -0
  72. package/frontend/src/lib/analysis/case-record-contracts.js +44 -0
  73. package/frontend/src/lib/analysis/case-record-core.js +232 -0
  74. package/frontend/src/lib/analysis/case-record-model.js +7 -0
  75. package/frontend/src/lib/analysis/case-record-operations.js +224 -0
  76. package/frontend/src/lib/analysis/case-response-model.js +640 -0
  77. package/frontend/src/lib/analysis/case-response-packet.js +573 -0
  78. package/frontend/src/lib/analysis/case-storage-model.js +93 -0
  79. package/frontend/src/lib/analysis/detection-rule-model.js +306 -0
  80. package/frontend/src/lib/analysis/dns-record-normalization.js +11 -0
  81. package/frontend/src/lib/analysis/http-summary.js +165 -0
  82. package/frontend/src/lib/analysis/investigation-guide.js +500 -0
  83. package/frontend/src/lib/analysis/investigation-template-model.js +144 -0
  84. package/frontend/src/lib/analysis/page-baseline.js +272 -0
  85. package/frontend/src/lib/analysis/page-similarity.js +179 -0
  86. package/frontend/src/lib/analysis/relationship-evidence.js +265 -0
  87. package/frontend/src/lib/analysis/relationship-observation-model.js +423 -0
  88. package/frontend/src/lib/analysis/scoring.js +135 -0
  89. package/frontend/src/lib/analysis/shortlist-model.js +209 -0
  90. package/frontend/src/lib/analysis/utils.js +265 -0
  91. package/frontend/src/lib/analysis/watchlist-history.js +643 -0
  92. package/frontend/src/lib/analysis/watchlist-store.js +127 -0
  93. package/frontend/src/lib/analysis/website-snapshot-model.js +302 -0
  94. package/frontend/src/lib/analysis/workspace-archive-crypto.js +282 -0
  95. package/frontend/src/lib/analysis/workspace-archive.js +472 -0
  96. package/lib/abort.mjs +26 -0
  97. package/lib/abusech-auth.mjs +19 -0
  98. package/lib/availability.mjs +716 -0
  99. package/lib/bounded-contract-normalizers.mjs +61 -0
  100. package/lib/browser-library-profile.mjs +310 -0
  101. package/lib/bulk-comparison-evidence.mjs +83 -0
  102. package/lib/classify.mjs +93 -0
  103. package/lib/client-behavior-profile.mjs +83 -0
  104. package/lib/credential-surface-profile.mjs +66 -0
  105. package/lib/csv.mjs +23 -0
  106. package/lib/ct-query.mjs +27 -0
  107. package/lib/ct-search.mjs +393 -0
  108. package/lib/distributed-operation-budget.mjs +461 -0
  109. package/lib/dns-delegation-health.mjs +344 -0
  110. package/lib/dns-intelligence.mjs +739 -0
  111. package/lib/dns-mx.mjs +24 -0
  112. package/lib/dns-resolver-selection.mjs +33 -0
  113. package/lib/domain-posture-analysis.mjs +254 -0
  114. package/lib/domain-posture-parsers.mjs +358 -0
  115. package/lib/domain-posture.mjs +745 -0
  116. package/lib/error-detail.mjs +10 -0
  117. package/lib/evidence-export.mjs +431 -0
  118. package/lib/evidence-report-markdown.mjs +71 -0
  119. package/lib/evidence-report.mjs +391 -0
  120. package/lib/external-intelligence-risk.mjs +108 -0
  121. package/lib/favicon.mjs +146 -0
  122. package/lib/feature-policy.mjs +70 -0
  123. package/lib/generated/retire-browser-catalog.mjs +8724 -0
  124. package/lib/generated/unicode-confusables-17.mjs +93 -0
  125. package/lib/hostname.mjs +38 -0
  126. package/lib/html-signals.mjs +680 -0
  127. package/lib/http-intelligence.mjs +232 -0
  128. package/lib/idn-confusable-policy.mjs +86 -0
  129. package/lib/idn-confusables.mjs +465 -0
  130. package/lib/lookup-cache.mjs +130 -0
  131. package/lib/lookup-progress.mjs +291 -0
  132. package/lib/lookup-source-progress.mjs +109 -0
  133. package/lib/lookup-threat-provider-inventory.mjs +13 -0
  134. package/lib/lookup.mjs +592 -0
  135. package/lib/observation.mjs +114 -0
  136. package/lib/observed-network-context.mjs +355 -0
  137. package/lib/operation-budget.mjs +489 -0
  138. package/lib/page-fingerprints.mjs +481 -0
  139. package/lib/page-role-profile.mjs +117 -0
  140. package/lib/perceptual-hash.mjs +451 -0
  141. package/lib/rdap-attempts.mjs +34 -0
  142. package/lib/rdap-bootstrap.mjs +173 -0
  143. package/lib/rdap-capabilities.mjs +292 -0
  144. package/lib/rdap-client.mjs +125 -0
  145. package/lib/rdap-normalization.mjs +691 -0
  146. package/lib/rdap-registrar.mjs +196 -0
  147. package/lib/rdap-transport.mjs +39 -0
  148. package/lib/rdap-types.mjs +1 -0
  149. package/lib/rdap-validation.mjs +94 -0
  150. package/lib/rdap.mjs +16 -0
  151. package/lib/registry-access.mjs +32 -0
  152. package/lib/registry-capabilities.mjs +117 -0
  153. package/lib/registry-capability-catalogue.mjs +1206 -0
  154. package/lib/registry-comparison.mjs +318 -0
  155. package/lib/registry-dates.mjs +190 -0
  156. package/lib/registry-insights.mjs +325 -0
  157. package/lib/response-policy.mjs +256 -0
  158. package/lib/risk-scoring.mjs +164 -0
  159. package/lib/safe-fetch.mjs +446 -0
  160. package/lib/security-txt.mjs +359 -0
  161. package/lib/service-binding-dns.mjs +608 -0
  162. package/lib/sslbl-certificates.generated.mjs +143 -0
  163. package/lib/sslbl-intelligence.mjs +241 -0
  164. package/lib/static-html-analysis.mjs +340 -0
  165. package/lib/structured-data-identity.mjs +323 -0
  166. package/lib/threat-intelligence-contract.mjs +5 -0
  167. package/lib/threat-intelligence-definition-registry.mjs +34 -0
  168. package/lib/threat-intelligence-fixtures.mjs +82 -0
  169. package/lib/threat-intelligence-manifests.mjs +512 -0
  170. package/lib/threat-intelligence-runtime.mjs +662 -0
  171. package/lib/threat-intelligence-targets.mjs +168 -0
  172. package/lib/threat-intelligence-types.mjs +3 -0
  173. package/lib/threatfox-intelligence.mjs +307 -0
  174. package/lib/tls-intelligence.mjs +814 -0
  175. package/lib/typosquat-generator.mjs +884 -0
  176. package/lib/urlhaus-intelligence.mjs +361 -0
  177. package/lib/urlscan-intelligence.mjs +349 -0
  178. package/lib/website-security-posture.mjs +325 -0
  179. package/lib/website-technology.mjs +437 -0
  180. package/lib/whois-authority.mjs +98 -0
  181. package/lib/whois-chain.mjs +261 -0
  182. package/lib/whois-contacts.mjs +163 -0
  183. package/lib/whois-contracts.mjs +3 -0
  184. package/lib/whois-normalization.mjs +27 -0
  185. package/lib/whois-transport.mjs +100 -0
  186. package/lib/whois-values.mjs +92 -0
  187. package/lib/whois.mjs +1187 -0
  188. package/package.json +55 -0
  189. package/third-party-notices.txt +5571 -0
@@ -0,0 +1,241 @@
1
+ // Exact local matching against the generated SSLBL certificate snapshot.
2
+ // Lookup never contacts SSLBL. A stale snapshot can preserve a historical
3
+ // positive match, but a stale miss remains inconclusive.
4
+ import { SSLBL_CERTIFICATE_SNAPSHOT } from "./sslbl-certificates.generated.mjs";
5
+ import { createHash } from 'node:crypto';
6
+ export const SSLBL_INTELLIGENCE_VERSION = 1;
7
+ export const SSLBL_SNAPSHOT_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000;
8
+ export const SSLBL_SNAPSHOT_EXPIRED_AGE_MS = 45 * 24 * 60 * 60 * 1000;
9
+ export const MAX_SSLBL_SNAPSHOT_ENTRIES = 50_000;
10
+ export const MAX_SSLBL_FINGERPRINT_CHUNKS = 1_000;
11
+ export const SSLBL_SOURCE_PAGE = 'https://sslbl.abuse.ch/blacklist/';
12
+ const SHA1_RE = /^[a-f0-9]{40}$/u;
13
+ const SHA256_RE = /^[a-f0-9]{64}$/u;
14
+ const SNAPSHOT_SCHEMA = 'whoisleuth.sslbl-certificate-snapshot';
15
+ const SNAPSHOT_SOURCE = 'https://sslbl.abuse.ch/blacklist/sslblacklist.csv';
16
+ function record(value) {
17
+ return value && typeof value === 'object' && !Array.isArray(value)
18
+ ? value
19
+ : {};
20
+ }
21
+ function timestamp(value) {
22
+ if (typeof value !== 'string' || value.length > 64)
23
+ return null;
24
+ const parsed = Date.parse(value);
25
+ return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
26
+ }
27
+ function normalizedFingerprint(value) {
28
+ if (typeof value !== 'string')
29
+ return null;
30
+ const normalized = value.replaceAll(':', '').trim().toLowerCase();
31
+ return SHA1_RE.test(normalized) ? normalized : null;
32
+ }
33
+ function fingerprintDigest(fingerprints) {
34
+ return createHash('sha256')
35
+ .update([...fingerprints].sort().join('\n'), 'utf8')
36
+ .digest('hex');
37
+ }
38
+ function validateSnapshot(value) {
39
+ const candidate = record(value);
40
+ const sourceUpdatedAt = timestamp(candidate.sourceUpdatedAt);
41
+ const generatedAt = timestamp(candidate.generatedAt);
42
+ if (candidate.schema !== SNAPSHOT_SCHEMA
43
+ || candidate.version !== 1
44
+ || candidate.source !== SNAPSHOT_SOURCE
45
+ || !sourceUpdatedAt
46
+ || !generatedAt
47
+ || Date.parse(sourceUpdatedAt) > Date.parse(generatedAt)
48
+ || typeof candidate.sourceDigestSha256 !== 'string'
49
+ || !SHA256_RE.test(candidate.sourceDigestSha256)
50
+ || typeof candidate.entriesDigestSha256 !== 'string'
51
+ || !SHA256_RE.test(candidate.entriesDigestSha256)
52
+ || !Number.isSafeInteger(candidate.entryCount)
53
+ || Number(candidate.entryCount) < 1
54
+ || Number(candidate.entryCount) > MAX_SSLBL_SNAPSHOT_ENTRIES
55
+ || !Array.isArray(candidate.fingerprintChunks)
56
+ || candidate.fingerprintChunks.length < 1
57
+ || candidate.fingerprintChunks.length > MAX_SSLBL_FINGERPRINT_CHUNKS) {
58
+ return null;
59
+ }
60
+ const fingerprints = new Set();
61
+ for (const chunk of candidate.fingerprintChunks) {
62
+ if (typeof chunk !== 'string' || chunk.length > 4_000 || !chunk.endsWith('\n'))
63
+ return null;
64
+ for (const line of chunk.split('\n')) {
65
+ if (!line)
66
+ continue;
67
+ if (!SHA1_RE.test(line) || fingerprints.has(line))
68
+ return null;
69
+ fingerprints.add(line);
70
+ if (fingerprints.size > MAX_SSLBL_SNAPSHOT_ENTRIES)
71
+ return null;
72
+ }
73
+ }
74
+ if (fingerprints.size !== candidate.entryCount
75
+ || fingerprintDigest(fingerprints) !== candidate.entriesDigestSha256) {
76
+ return null;
77
+ }
78
+ return {
79
+ snapshot: candidate,
80
+ fingerprints,
81
+ };
82
+ }
83
+ let builtInSnapshotValidation;
84
+ function validatedSnapshot(value) {
85
+ if (value !== undefined)
86
+ return validateSnapshot(value);
87
+ if (builtInSnapshotValidation === undefined) {
88
+ builtInSnapshotValidation = validateSnapshot(SSLBL_CERTIFICATE_SNAPSHOT);
89
+ }
90
+ return builtInSnapshotValidation;
91
+ }
92
+ function effectiveNow(value) {
93
+ const candidate = value instanceof Date
94
+ ? value.getTime()
95
+ : typeof value === 'number'
96
+ ? value
97
+ : typeof value === 'string'
98
+ ? Date.parse(value)
99
+ : Date.now();
100
+ return Number.isFinite(candidate) ? candidate : Date.now();
101
+ }
102
+ export function sslblSnapshotHealth(options = {}) {
103
+ const validated = validatedSnapshot(options.snapshot);
104
+ if (!validated) {
105
+ return Object.freeze({
106
+ state: 'invalid',
107
+ sourceUpdatedAt: null,
108
+ generatedAt: null,
109
+ ageSeconds: null,
110
+ entryCount: null,
111
+ digestSha256: null,
112
+ detail: 'The local SSLBL certificate snapshot is missing or failed validation.',
113
+ });
114
+ }
115
+ const now = effectiveNow(options.now);
116
+ const sourceUpdatedAt = timestamp(validated.snapshot.sourceUpdatedAt);
117
+ const generatedAt = timestamp(validated.snapshot.generatedAt);
118
+ if (!sourceUpdatedAt || !generatedAt || Date.parse(sourceUpdatedAt) > now) {
119
+ return Object.freeze({
120
+ state: 'invalid',
121
+ sourceUpdatedAt,
122
+ generatedAt,
123
+ ageSeconds: null,
124
+ entryCount: validated.snapshot.entryCount,
125
+ digestSha256: validated.snapshot.entriesDigestSha256,
126
+ detail: 'The local SSLBL certificate snapshot has invalid or future-dated timestamps.',
127
+ });
128
+ }
129
+ const ageMs = now - Date.parse(sourceUpdatedAt);
130
+ const state = ageMs > SSLBL_SNAPSHOT_EXPIRED_AGE_MS
131
+ ? 'expired'
132
+ : ageMs > SSLBL_SNAPSHOT_MAX_AGE_MS
133
+ ? 'stale'
134
+ : 'current';
135
+ return Object.freeze({
136
+ state,
137
+ sourceUpdatedAt,
138
+ generatedAt,
139
+ ageSeconds: Math.floor(ageMs / 1000),
140
+ entryCount: validated.snapshot.entryCount,
141
+ digestSha256: validated.snapshot.entriesDigestSha256,
142
+ detail: state === 'current'
143
+ ? 'The local SSLBL certificate snapshot is current.'
144
+ : state === 'stale'
145
+ ? 'The local SSLBL certificate snapshot is stale; positive matches remain review leads and misses are inconclusive.'
146
+ : 'The local SSLBL certificate snapshot is too old to support a current comparison.',
147
+ });
148
+ }
149
+ function tlsFingerprint(value) {
150
+ const tls = record(value);
151
+ const certificate = record(tls.certificate);
152
+ return normalizedFingerprint(certificate.fingerprintSha1);
153
+ }
154
+ function unavailable(observedAt, fingerprintSha1, detail) {
155
+ return Object.freeze({
156
+ sslblVersion: SSLBL_INTELLIGENCE_VERSION,
157
+ source: 'sslbl',
158
+ status: 'unavailable',
159
+ verdict: 'inconclusive',
160
+ complete: false,
161
+ observedAt,
162
+ fingerprintSha1,
163
+ referenceUrl: null,
164
+ snapshot: Object.freeze({
165
+ sourceUpdatedAt: null,
166
+ generatedAt: null,
167
+ ageSeconds: null,
168
+ entryCount: null,
169
+ digestSha256: null,
170
+ }),
171
+ detail,
172
+ limitations: Object.freeze([
173
+ 'Unavailable certificate warning data is not evidence that the observed certificate is absent from SSLBL or safe.',
174
+ 'SSLBL associates listed certificate fingerprints with observed botnet command-and-control activity; a match does not establish current domain ownership, intent, or activity.',
175
+ ]),
176
+ });
177
+ }
178
+ export function inspectSslblCertificate(tls, options = {}) {
179
+ const nowValue = effectiveNow(options.now);
180
+ const observedAt = new Date(nowValue).toISOString();
181
+ const fingerprintSha1 = tlsFingerprint(tls);
182
+ if (!fingerprintSha1) {
183
+ return unavailable(observedAt, null, 'The deep TLS observation did not retain a usable leaf-certificate SHA-1 fingerprint.');
184
+ }
185
+ const validated = validatedSnapshot(options.snapshot);
186
+ if (!validated) {
187
+ return unavailable(observedAt, fingerprintSha1, 'The local SSLBL certificate snapshot is missing or failed validation.');
188
+ }
189
+ const sourceUpdatedAt = timestamp(validated.snapshot.sourceUpdatedAt);
190
+ const generatedAt = timestamp(validated.snapshot.generatedAt);
191
+ if (!sourceUpdatedAt || !generatedAt) {
192
+ return unavailable(observedAt, fingerprintSha1, 'The local SSLBL certificate snapshot has invalid timestamps.');
193
+ }
194
+ if (Date.parse(sourceUpdatedAt) > nowValue) {
195
+ return unavailable(observedAt, fingerprintSha1, 'The local SSLBL certificate snapshot is future-dated and cannot support a current comparison.');
196
+ }
197
+ const ageMs = nowValue - Date.parse(sourceUpdatedAt);
198
+ if (ageMs > SSLBL_SNAPSHOT_EXPIRED_AGE_MS) {
199
+ return unavailable(observedAt, fingerprintSha1, 'The local SSLBL certificate snapshot is too old to support a current comparison.');
200
+ }
201
+ const listed = validated.fingerprints.has(fingerprintSha1);
202
+ const stale = ageMs > SSLBL_SNAPSHOT_MAX_AGE_MS;
203
+ const verdict = listed
204
+ ? 'listed'
205
+ : stale
206
+ ? 'inconclusive'
207
+ : 'not_listed';
208
+ const referenceUrl = listed
209
+ ? `https://sslbl.abuse.ch/ssl-certificates/sha1/${fingerprintSha1}/`
210
+ : null;
211
+ return Object.freeze({
212
+ sslblVersion: SSLBL_INTELLIGENCE_VERSION,
213
+ source: 'sslbl',
214
+ status: stale ? 'stale' : 'success',
215
+ verdict,
216
+ complete: !stale,
217
+ observedAt,
218
+ fingerprintSha1,
219
+ referenceUrl,
220
+ snapshot: Object.freeze({
221
+ sourceUpdatedAt,
222
+ generatedAt,
223
+ ageSeconds: Math.floor(ageMs / 1000),
224
+ entryCount: validated.snapshot.entryCount,
225
+ digestSha256: validated.snapshot.entriesDigestSha256,
226
+ }),
227
+ detail: listed
228
+ ? stale
229
+ ? 'The observed leaf certificate appears in the stale local SSLBL snapshot. Verify the current provider record before acting.'
230
+ : 'The observed leaf certificate appears in the local SSLBL certificate snapshot.'
231
+ : stale
232
+ ? 'The stale local snapshot contains no exact match, so the result is inconclusive.'
233
+ : 'No exact fingerprint match was found in the current local SSLBL certificate snapshot.',
234
+ limitations: Object.freeze([
235
+ 'SSLBL matching uses the exact observed leaf-certificate SHA-1 fingerprint and makes no network request.',
236
+ 'No match is not evidence that the domain, certificate, or service is safe.',
237
+ 'Certificates can be reused across unrelated domains or infrastructure; a match does not prove ownership, intent, or current activity.',
238
+ ...(stale ? ['The snapshot is older than the current-evidence threshold and should be refreshed.'] : []),
239
+ ]),
240
+ });
241
+ }
@@ -0,0 +1,340 @@
1
+ // One bounded tokenization pass over the already-captured homepage body. The
2
+ // parse5 tokenizer applies HTML tokenization rules without constructing a DOM,
3
+ // so deeply nested hostile markup cannot create an unbounded element tree.
4
+ // Only normalized start-tag markup and capped script indicators are retained.
5
+ import { Tokenizer, TokenizerMode } from 'parse5';
6
+ const MAX_STATIC_HTML_CHARS = 300_000;
7
+ const MAX_STATIC_HTML_TAGS = 8_192;
8
+ const MAX_STATIC_STRUCTURE_TOKENS = 4_096;
9
+ const MAX_TECHNOLOGY_TAGS = 2_048;
10
+ const MAX_TAG_LENGTH = 4_096;
11
+ const MAX_ATTRIBUTES_PER_TAG = 128;
12
+ const MAX_SCRIPT_ELEMENTS = 64;
13
+ const MAX_SCRIPT_REFERENCE_LENGTH = 2_048;
14
+ const MAX_SCRIPT_MEDIA_TYPE_LENGTH = 120;
15
+ const MAX_INLINE_SCRIPT_CHARS = 32_768;
16
+ const MAX_INLINE_SCRIPT_TOTAL_CHARS = 65_536;
17
+ const MAX_STATIC_FORMS = 50;
18
+ const MAX_STATIC_INPUTS = 500;
19
+ const MAX_FORM_ATTRIBUTE_LENGTH = 2_048;
20
+ const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f]/;
21
+ const PAYMENT_AUTOCOMPLETE_TOKENS = new Set([
22
+ 'cc-name',
23
+ 'cc-given-name',
24
+ 'cc-additional-name',
25
+ 'cc-family-name',
26
+ 'cc-number',
27
+ 'cc-exp',
28
+ 'cc-exp-month',
29
+ 'cc-exp-year',
30
+ 'cc-csc',
31
+ 'cc-type',
32
+ 'transaction-currency',
33
+ 'transaction-amount',
34
+ ]);
35
+ const RAW_TEXT_MODES = Object.freeze({
36
+ iframe: TokenizerMode.RAWTEXT,
37
+ noembed: TokenizerMode.RAWTEXT,
38
+ noframes: TokenizerMode.RAWTEXT,
39
+ script: TokenizerMode.SCRIPT_DATA,
40
+ style: TokenizerMode.RAWTEXT,
41
+ template: TokenizerMode.RAWTEXT,
42
+ textarea: TokenizerMode.RCDATA,
43
+ title: TokenizerMode.RCDATA,
44
+ xmp: TokenizerMode.RAWTEXT,
45
+ });
46
+ const VOID_TAGS = new Set([
47
+ 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta',
48
+ 'param', 'source', 'track', 'wbr',
49
+ ]);
50
+ function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LENGTH) {
51
+ const attribute = attributes
52
+ .slice(0, MAX_ATTRIBUTES_PER_TAG)
53
+ .find((candidate) => candidate.name.toLowerCase() === name);
54
+ if (!attribute)
55
+ return { present: false, value: null, truncated: attributes.length > MAX_ATTRIBUTES_PER_TAG };
56
+ if (attribute.value.length > maximumLength || CONTROL_CHARACTER_RE.test(attribute.value)) {
57
+ return { present: true, value: null, truncated: true };
58
+ }
59
+ return {
60
+ present: true,
61
+ value: attribute.value.trim().toLowerCase(),
62
+ truncated: attributes.length > MAX_ATTRIBUTES_PER_TAG,
63
+ };
64
+ }
65
+ function safeBaseUrl(value) {
66
+ if (typeof value !== 'string' || value.length > MAX_FORM_ATTRIBUTE_LENGTH || CONTROL_CHARACTER_RE.test(value))
67
+ return null;
68
+ try {
69
+ const parsed = new URL(value);
70
+ return ['http:', 'https:'].includes(parsed.protocol) && !parsed.username && !parsed.password && parsed.hostname
71
+ ? parsed
72
+ : null;
73
+ }
74
+ catch {
75
+ return null;
76
+ }
77
+ }
78
+ function formMethod(attributes) {
79
+ const method = attributeValue(attributes, 'method', 24);
80
+ if (!method.present && method.truncated)
81
+ return { value: 'other', truncated: true };
82
+ if (!method.present || method.value === '')
83
+ return { value: 'missing', truncated: method.truncated };
84
+ if (method.value === null)
85
+ return { value: 'other', truncated: true };
86
+ return {
87
+ value: ['get', 'post', 'dialog'].includes(method.value)
88
+ ? method.value
89
+ : 'other',
90
+ truncated: method.truncated,
91
+ };
92
+ }
93
+ function formAction(attributes, baseUrl) {
94
+ const action = attributeValue(attributes, 'action');
95
+ if (!action.present && action.truncated) {
96
+ return { relationship: 'unclassified', cleartext: false, truncated: true };
97
+ }
98
+ if (!action.present || action.value === '') {
99
+ return { relationship: 'missing', cleartext: false, truncated: action.truncated };
100
+ }
101
+ if (action.value === null) {
102
+ return { relationship: 'unclassified', cleartext: false, truncated: true };
103
+ }
104
+ if (!baseUrl) {
105
+ return { relationship: 'unclassified', cleartext: false, truncated: action.truncated };
106
+ }
107
+ try {
108
+ const parsed = new URL(action.value, baseUrl);
109
+ if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password || !parsed.hostname) {
110
+ return { relationship: 'unclassified', cleartext: false, truncated: action.truncated };
111
+ }
112
+ return {
113
+ relationship: parsed.origin === baseUrl.origin ? 'sameOrigin' : 'external',
114
+ cleartext: parsed.protocol === 'http:',
115
+ truncated: action.truncated,
116
+ };
117
+ }
118
+ catch {
119
+ return { relationship: 'unclassified', cleartext: false, truncated: action.truncated };
120
+ }
121
+ }
122
+ function inputCategories(attributes) {
123
+ if (attributes.length > MAX_ATTRIBUTES_PER_TAG)
124
+ return { values: [], truncated: true };
125
+ const type = attributeValue(attributes, 'type', 40);
126
+ const autocomplete = attributeValue(attributes, 'autocomplete', 160);
127
+ const disabled = attributeValue(attributes, 'disabled', 10).present;
128
+ const truncated = type.truncated || autocomplete.truncated;
129
+ if (disabled || type.value === 'hidden')
130
+ return { values: [], truncated };
131
+ const normalizedType = type.value || 'text';
132
+ const autocompleteTokens = (autocomplete.value || '').split(/\s+/u).filter(Boolean).slice(0, 16);
133
+ const categories = new Set();
134
+ if (normalizedType === 'password' || autocompleteTokens.some((token) => ['current-password', 'new-password'].includes(token))) {
135
+ categories.add('password');
136
+ }
137
+ if (normalizedType === 'email' || autocompleteTokens.includes('email'))
138
+ categories.add('email');
139
+ if (autocompleteTokens.includes('username'))
140
+ categories.add('username');
141
+ if (autocompleteTokens.includes('one-time-code'))
142
+ categories.add('one_time_code');
143
+ if (autocompleteTokens.some((token) => PAYMENT_AUTOCOMPLETE_TOKENS.has(token)))
144
+ categories.add('payment');
145
+ return { values: [...categories], truncated };
146
+ }
147
+ function scriptReference(attributes) {
148
+ for (const attribute of attributes.slice(0, MAX_ATTRIBUTES_PER_TAG)) {
149
+ if (attribute.name.toLowerCase() !== 'src')
150
+ continue;
151
+ if (attribute.value.length === 0
152
+ || attribute.value.length > MAX_SCRIPT_REFERENCE_LENGTH
153
+ || CONTROL_CHARACTER_RE.test(attribute.value))
154
+ return null;
155
+ return attribute.value;
156
+ }
157
+ return null;
158
+ }
159
+ function scriptMediaType(attributes) {
160
+ for (const attribute of attributes.slice(0, MAX_ATTRIBUTES_PER_TAG)) {
161
+ if (attribute.name.toLowerCase() !== 'type')
162
+ continue;
163
+ if (attribute.value.length === 0
164
+ || attribute.value.length > MAX_SCRIPT_MEDIA_TYPE_LENGTH
165
+ || CONTROL_CHARACTER_RE.test(attribute.value))
166
+ return null;
167
+ return attribute.value.trim().toLowerCase().split(';', 1)[0] || null;
168
+ }
169
+ return null;
170
+ }
171
+ function serializedStartTag(tagName, attributes) {
172
+ let serialized = `<${tagName.toLowerCase()}`;
173
+ let limitReached = attributes.length > MAX_ATTRIBUTES_PER_TAG;
174
+ for (const attribute of attributes.slice(0, MAX_ATTRIBUTES_PER_TAG)) {
175
+ const candidate = ` ${attribute.name.toLowerCase()}="${attribute.value.toLowerCase()}"`;
176
+ if (serialized.length + candidate.length + 1 > MAX_TAG_LENGTH) {
177
+ limitReached = true;
178
+ break;
179
+ }
180
+ serialized += candidate;
181
+ }
182
+ return {
183
+ markup: serialized.length + 1 <= MAX_TAG_LENGTH ? `${serialized}>` : null,
184
+ limitReached,
185
+ };
186
+ }
187
+ function analyzeStaticHtml(value, options = {}) {
188
+ const supplied = typeof value === 'string' ? value : '';
189
+ const inputLimitReached = supplied.length > MAX_STATIC_HTML_CHARS;
190
+ const html = supplied.slice(0, MAX_STATIC_HTML_CHARS);
191
+ const baseUrl = safeBaseUrl(options.baseUrl);
192
+ const markup = [];
193
+ const structureTokens = [];
194
+ const scripts = [];
195
+ const forms = {
196
+ formsObserved: 0,
197
+ inputsObserved: 0,
198
+ classifiedInputs: 0,
199
+ categories: { password: 0, email: 0, username: 0, one_time_code: 0, payment: 0 },
200
+ methods: { missing: 0, get: 0, post: 0, dialog: 0, other: 0 },
201
+ actions: { sameOrigin: 0, external: 0, missing: 0, cleartext: 0, unclassified: 0 },
202
+ truncated: false,
203
+ };
204
+ let tokenizer;
205
+ let activeInlineScript = null;
206
+ let tagsExamined = 0;
207
+ let inlineCharactersExamined = 0;
208
+ let tagLimitReached = false;
209
+ let structureLimitReached = false;
210
+ let scriptLimitReached = false;
211
+ let inlineLimitReached = false;
212
+ function appendInlineScript(chars) {
213
+ if (!activeInlineScript || !chars)
214
+ return;
215
+ const perScriptRemaining = MAX_INLINE_SCRIPT_CHARS - activeInlineScript.inlineContent.length;
216
+ const totalRemaining = MAX_INLINE_SCRIPT_TOTAL_CHARS - inlineCharactersExamined;
217
+ const retainedLength = Math.max(0, Math.min(chars.length, perScriptRemaining, totalRemaining));
218
+ if (retainedLength > 0) {
219
+ activeInlineScript.inlineContent += chars.slice(0, retainedLength);
220
+ inlineCharactersExamined += retainedLength;
221
+ }
222
+ if (retainedLength < chars.length)
223
+ inlineLimitReached = true;
224
+ }
225
+ function appendStructureToken(value) {
226
+ if (structureTokens.length >= MAX_STATIC_STRUCTURE_TOKENS) {
227
+ structureLimitReached = true;
228
+ return;
229
+ }
230
+ structureTokens.push(value);
231
+ }
232
+ const handler = {
233
+ onStartTag(token) {
234
+ const tagName = token.tagName.toLowerCase();
235
+ const rawTextMode = RAW_TEXT_MODES[tagName];
236
+ if (!token.selfClosing && rawTextMode !== undefined)
237
+ tokenizer.state = rawTextMode;
238
+ if (tagsExamined >= MAX_STATIC_HTML_TAGS) {
239
+ tagLimitReached = true;
240
+ activeInlineScript = null;
241
+ return;
242
+ }
243
+ tagsExamined += 1;
244
+ appendStructureToken(token.selfClosing || VOID_TAGS.has(tagName) ? `${tagName}/` : tagName);
245
+ if (markup.length < MAX_TECHNOLOGY_TAGS) {
246
+ const serialized = serializedStartTag(tagName, token.attrs);
247
+ if (serialized.limitReached)
248
+ tagLimitReached = true;
249
+ if (serialized.markup)
250
+ markup.push(serialized.markup);
251
+ }
252
+ else {
253
+ tagLimitReached = true;
254
+ }
255
+ if (tagName === 'form') {
256
+ if (forms.formsObserved >= MAX_STATIC_FORMS) {
257
+ forms.truncated = true;
258
+ }
259
+ else {
260
+ forms.formsObserved += 1;
261
+ const method = formMethod(token.attrs);
262
+ forms.methods[method.value] += 1;
263
+ const action = formAction(token.attrs, baseUrl);
264
+ forms.actions[action.relationship] += 1;
265
+ if (action.cleartext)
266
+ forms.actions.cleartext += 1;
267
+ if (method.truncated || action.truncated)
268
+ forms.truncated = true;
269
+ }
270
+ }
271
+ else if (tagName === 'input') {
272
+ if (forms.inputsObserved >= MAX_STATIC_INPUTS) {
273
+ forms.truncated = true;
274
+ }
275
+ else {
276
+ forms.inputsObserved += 1;
277
+ const categories = inputCategories(token.attrs);
278
+ if (categories.truncated)
279
+ forms.truncated = true;
280
+ if (categories.values.length)
281
+ forms.classifiedInputs += 1;
282
+ for (const category of categories.values)
283
+ forms.categories[category] += 1;
284
+ }
285
+ }
286
+ if (tagName !== 'script') {
287
+ activeInlineScript = null;
288
+ return;
289
+ }
290
+ if (scripts.length >= MAX_SCRIPT_ELEMENTS) {
291
+ scriptLimitReached = true;
292
+ activeInlineScript = null;
293
+ return;
294
+ }
295
+ const reference = scriptReference(token.attrs);
296
+ const script = {
297
+ reference,
298
+ inlineContent: '',
299
+ mediaType: scriptMediaType(token.attrs),
300
+ };
301
+ scripts.push(script);
302
+ activeInlineScript = reference ? null : script;
303
+ },
304
+ onEndTag(token) {
305
+ const tagName = token.tagName.toLowerCase();
306
+ if (tagName === 'script')
307
+ activeInlineScript = null;
308
+ if (!VOID_TAGS.has(tagName))
309
+ appendStructureToken(`/${tagName}`);
310
+ },
311
+ onCharacter(token) {
312
+ appendInlineScript(token.chars);
313
+ },
314
+ onNullCharacter(token) {
315
+ appendInlineScript(token.chars);
316
+ },
317
+ onWhitespaceCharacter(token) {
318
+ appendInlineScript(token.chars);
319
+ },
320
+ onComment() { },
321
+ onDoctype() { },
322
+ onEof() { },
323
+ };
324
+ tokenizer = new Tokenizer({ sourceCodeLocationInfo: false }, handler);
325
+ tokenizer.write(html, true);
326
+ return {
327
+ markup: markup.join('\n'),
328
+ structureTokens,
329
+ scripts,
330
+ forms,
331
+ inputLimitReached,
332
+ tagLimitReached,
333
+ structureLimitReached,
334
+ scriptLimitReached,
335
+ inlineLimitReached,
336
+ tagsExamined,
337
+ inlineCharactersExamined,
338
+ };
339
+ }
340
+ export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };