@slicedearth/whoisleuth-cli 2.3.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. package/NOTICE +6 -0
  2. package/README.md +19 -15
  3. package/bin/whoisleuth.mjs +8 -1
  4. package/cli/archive-content-digest.mjs +14 -0
  5. package/cli/archive-inspect.mjs +20 -53
  6. package/cli/arguments.mjs +68 -6
  7. package/cli/artifact-validation/case-response.mjs +47 -41
  8. package/cli/artifact-validation/investigation-capsule.mjs +52 -31
  9. package/cli/artifact-validation/lookup-evidence.mjs +12 -6
  10. package/cli/artifact-validation/signed-review.mjs +83 -61
  11. package/cli/artifact-validation/structure-primitives.mjs +1 -105
  12. package/cli/artifact-verify.mjs +90 -24
  13. package/cli/assurance-command-runner.mjs +39 -4
  14. package/cli/bagit-folder.mjs +57 -0
  15. package/cli/bagit-review.mjs +23 -0
  16. package/cli/bounded-json.mjs +1 -1
  17. package/cli/bulk-command-runner.mjs +2 -0
  18. package/cli/bulk-output.mjs +29 -3
  19. package/cli/case-command.mjs +195 -0
  20. package/cli/case-pack.mjs +39 -31
  21. package/cli/command-argument-grammar.mjs +4 -0
  22. package/cli/command-reference.mjs +145 -75
  23. package/cli/completion.mjs +298 -79
  24. package/cli/config-profile.mjs +7 -5
  25. package/cli/ct-event-intake.mjs +12 -12
  26. package/cli/discover.mjs +4 -3
  27. package/cli/discovery-observation-snapshot.mjs +44 -16
  28. package/cli/discovery-scan-command-runner.mjs +2 -0
  29. package/cli/discovery-scan.mjs +7 -4
  30. package/cli/discovery-workflow.mjs +1 -1
  31. package/cli/domain-control-monitor.mjs +50 -20
  32. package/cli/domain-control-observations.mjs +149 -102
  33. package/cli/errors.mjs +33 -1
  34. package/cli/evidence-command-runner.mjs +16 -5
  35. package/cli/evidence-signing.mjs +2 -2
  36. package/cli/external-observation-mapping.mjs +7 -8
  37. package/cli/formatters/terminal-lookup.mjs +18 -46
  38. package/cli/formatters/terminal-shared.mjs +5 -3
  39. package/cli/formatters/terminal-technology.mjs +114 -0
  40. package/cli/inline-command-families.mjs +1 -0
  41. package/cli/interactive-launcher.mjs +2 -160
  42. package/cli/interchange-report.mjs +4 -8
  43. package/cli/investigation-artifacts.mjs +176 -0
  44. package/cli/investigation-checkpoint.mjs +10 -0
  45. package/cli/investigation-folder.mjs +114 -0
  46. package/cli/investigation-manifest.mjs +1 -112
  47. package/cli/investigation-package-review.mjs +103 -0
  48. package/cli/investigation-plan.mjs +32 -19
  49. package/cli/investigation-run.mjs +188 -33
  50. package/cli/local-document-checkpoint.mjs +170 -0
  51. package/cli/lookup-browser.mjs +5 -12
  52. package/cli/lookup-command-runner.mjs +17 -6
  53. package/cli/lookup-diff.mjs +37 -3
  54. package/cli/lookup-plan.mjs +11 -5
  55. package/cli/lookup-reconcile.mjs +10 -8
  56. package/cli/mail-review.mjs +66 -14
  57. package/cli/manual.mjs +1 -0
  58. package/cli/network-command-runner.mjs +2 -0
  59. package/cli/output-file.mjs +31 -5
  60. package/cli/registry-doctor.mjs +3 -1
  61. package/cli/registry-support.mjs +1 -1
  62. package/cli/retained-artifact-diff.mjs +2 -2
  63. package/cli/review-command-runner.mjs +45 -0
  64. package/cli/runner.mjs +55 -6
  65. package/cli/sharing-review.mjs +21 -34
  66. package/cli/signer-trust.mjs +104 -0
  67. package/cli/source-reliability.mjs +74 -83
  68. package/cli/terminal-input.mjs +168 -0
  69. package/cli/workflow-command-runner.mjs +68 -13
  70. package/docs/cli-reference.md +76 -30
  71. package/docs/cli.md +289 -25
  72. package/lib/abort.mjs +4 -1
  73. package/lib/availability.mjs +161 -112
  74. package/lib/bounded-file.mjs +3 -1
  75. package/lib/bounded-html-document.mjs +125 -0
  76. package/lib/bounded-json.mjs +43 -3
  77. package/lib/browser-library-profile.mjs +42 -51
  78. package/lib/browser-library-worker.mjs +112 -0
  79. package/lib/client-behavior-profile.mjs +3 -3
  80. package/lib/ct-search.mjs +11 -4
  81. package/lib/dns-delegation-health.mjs +24 -14
  82. package/lib/dns-inheritance-review.mjs +217 -0
  83. package/lib/dns-intelligence.mjs +36 -23
  84. package/lib/dns-mx.mjs +1 -1
  85. package/lib/dnssec-evidence-validation.mjs +11 -3
  86. package/lib/domain-change-packet.mjs +7 -2
  87. package/lib/domain-change-review.mjs +14 -8
  88. package/lib/domain-control-flight-recorder.mjs +219 -118
  89. package/lib/domain-control-manifest.mjs +117 -57
  90. package/lib/domain-posture-analysis.mjs +20 -7
  91. package/lib/domain-posture-parsers.mjs +88 -6
  92. package/lib/domain-posture.mjs +113 -26
  93. package/lib/evidence-export-privacy.mjs +561 -0
  94. package/lib/evidence-export.mjs +11 -592
  95. package/lib/evidence-report.mjs +3 -1
  96. package/lib/favicon.mjs +117 -41
  97. package/lib/generated/cisa-kev-catalog.mjs +12 -4
  98. package/lib/generated/registrar-standing-catalogue.mjs +7 -7
  99. package/lib/generated/retire-browser-catalog.mjs +2335 -940
  100. package/lib/hostname.mjs +2 -38
  101. package/lib/html-signals.mjs +82 -85
  102. package/lib/http-evidence-bounds.mjs +2 -1
  103. package/lib/http-intelligence.mjs +39 -26
  104. package/lib/lookup-cache.mjs +34 -11
  105. package/lib/lookup-child-profile-contract.mjs +24 -15
  106. package/lib/lookup-diagnostics.mjs +24 -1
  107. package/lib/lookup-progress.mjs +30 -13
  108. package/lib/lookup-response-contract.mjs +1048 -0
  109. package/lib/lookup-response.mjs +7 -10
  110. package/lib/lookup-source-progress.mjs +17 -14
  111. package/lib/lookup.mjs +48 -27
  112. package/lib/observed-network-context.mjs +43 -27
  113. package/lib/outbound-request-bounds.mjs +4 -1
  114. package/lib/page-fingerprints.mjs +69 -133
  115. package/lib/page-role-profile.mjs +12 -8
  116. package/lib/perceptual-hash.mjs +43 -14
  117. package/lib/rdap-bootstrap.mjs +72 -22
  118. package/lib/rdap-client.mjs +15 -7
  119. package/lib/rdap-normalization.mjs +33 -18
  120. package/lib/rdap-search-workbench.mjs +1 -1
  121. package/lib/rdap-transport.mjs +47 -20
  122. package/lib/rdap-types.mjs +2 -1
  123. package/lib/rdap.mjs +18 -3
  124. package/lib/registrable-domain.mjs +21 -1
  125. package/lib/registrar-standing-contract.mjs +1 -0
  126. package/lib/registry-capabilities.mjs +5 -1
  127. package/lib/registry-capability-catalogue.mjs +2 -1
  128. package/lib/registry-dates.mjs +2 -190
  129. package/lib/registry-insights.mjs +56 -18
  130. package/lib/response-policy.mjs +86 -48
  131. package/lib/risk-calibration-summary.mjs +17 -3
  132. package/lib/rpki-evidence.mjs +8 -5
  133. package/lib/safe-fetch.mjs +54 -47
  134. package/lib/security-txt.mjs +10 -7
  135. package/lib/service-binding-dns.mjs +8 -0
  136. package/lib/smtp-transport-review.mjs +27 -15
  137. package/lib/sslbl-certificates.generated.mjs +136 -135
  138. package/lib/static-html-analysis.mjs +117 -146
  139. package/lib/structured-data-identity.mjs +1 -0
  140. package/lib/technology-markup.mjs +100 -0
  141. package/lib/threat-intelligence-manifests.mjs +11 -139
  142. package/lib/threat-intelligence-runtime.mjs +24 -52
  143. package/lib/threat-intelligence-targets.mjs +6 -34
  144. package/lib/threat-intelligence-types.mjs +39 -0
  145. package/lib/typosquat-generator.mjs +19 -15
  146. package/lib/urlhaus-intelligence.mjs +8 -3
  147. package/lib/urlscan-intelligence.mjs +1 -1
  148. package/lib/website-security-posture.mjs +30 -6
  149. package/lib/website-technology.mjs +111 -71
  150. package/lib/whois-authority.mjs +14 -2
  151. package/lib/whois-chain.mjs +9 -2
  152. package/lib/whois-common-formats.mjs +5 -5
  153. package/lib/whois-contracts.mjs +2 -1
  154. package/lib/whois-dialects.mjs +2 -2
  155. package/lib/whois-parser-support.mjs +8 -1
  156. package/lib/whois-transport.mjs +23 -7
  157. package/package.json +2 -1
  158. package/packages/cases/case-attachment-model.mjs +83 -0
  159. package/packages/cases/case-evidence-model.mjs +156 -62
  160. package/packages/cases/case-export-input.mjs +53 -0
  161. package/packages/cases/case-migration-model.mjs +59 -25
  162. package/packages/cases/case-model.mjs +1 -0
  163. package/packages/cases/case-recheck-model.mjs +91 -0
  164. package/packages/cases/case-record-core.mjs +3 -2
  165. package/packages/cases/case-record-operations.mjs +89 -27
  166. package/packages/cases/case-record-projection.mjs +23 -20
  167. package/packages/cases/case-report.mjs +63 -52
  168. package/packages/cases/case-response-actions.mjs +654 -0
  169. package/packages/cases/case-response-model.mjs +41 -1129
  170. package/packages/cases/case-response-outcomes.mjs +317 -0
  171. package/packages/cases/case-response-packet-render.mjs +153 -0
  172. package/packages/cases/case-response-packet-types.mjs +1 -0
  173. package/packages/cases/case-response-packet-vocabulary.mjs +48 -0
  174. package/packages/cases/case-response-packet.mjs +38 -587
  175. package/packages/cases/case-response-records.mjs +94 -0
  176. package/packages/cases/case-response-review-inputs.mjs +371 -0
  177. package/packages/cases/case-response-values.mjs +85 -0
  178. package/packages/cases/case-review-package.mjs +45 -0
  179. package/packages/cases/case-selection.mjs +46 -0
  180. package/packages/cases/case-storage-model.mjs +25 -0
  181. package/packages/cases/response-route-freshness.mjs +16 -0
  182. package/packages/comparison/capture-context.mjs +61 -0
  183. package/packages/comparison/comparison-ledger-bulk.mjs +4 -1
  184. package/packages/comparison/page-similarity.mjs +2 -1
  185. package/packages/comparison/relationship-evidence.mjs +85 -8
  186. package/packages/comparison/relationship-provenance.mjs +93 -0
  187. package/packages/contracts/analyst-interchange.mjs +216 -0
  188. package/packages/contracts/analyst-review-state-contract.mjs +11 -0
  189. package/packages/contracts/analyst-review-state.mjs +5 -4
  190. package/packages/contracts/case-portability.mjs +279 -116
  191. package/packages/contracts/case-views-contract.mjs +12 -0
  192. package/packages/contracts/case-views.mjs +26 -0
  193. package/packages/contracts/cli-command-semantics.mjs +1 -0
  194. package/packages/contracts/cli-lookup.mjs +0 -1
  195. package/packages/contracts/domain-control-flight-recorder.mjs +80 -42
  196. package/packages/contracts/domain-control-manifest.mjs +127 -73
  197. package/packages/contracts/domain-control-monitor.mjs +48 -45
  198. package/packages/contracts/domain-control-review.mjs +179 -121
  199. package/packages/contracts/domain-name.mjs +21 -0
  200. package/packages/contracts/external-observation-interchange.mjs +2 -1
  201. package/packages/contracts/extracted-domain-lifecycle.mjs +11 -32
  202. package/packages/contracts/http-url.mjs +2 -0
  203. package/packages/contracts/investigation-package-limits.mjs +24 -0
  204. package/packages/contracts/investigation-portability.mjs +56 -27
  205. package/packages/contracts/investigation-run.mjs +16 -0
  206. package/packages/contracts/ip-address.mjs +34 -0
  207. package/packages/contracts/lookup-evidence.mjs +35 -0
  208. package/packages/contracts/offline-comparison.mjs +1 -2
  209. package/packages/contracts/page-fingerprints.mjs +15 -0
  210. package/packages/contracts/risk-calibration.mjs +0 -1
  211. package/packages/contracts/schema-compatibility.mjs +1 -1
  212. package/packages/contracts/schema-lifecycle.mjs +83 -153
  213. package/packages/contracts/selected-file-limits.mjs +6 -0
  214. package/packages/contracts/vulnerability-identifiers.mjs +5 -0
  215. package/packages/contracts/web-capture.mjs +10 -0
  216. package/packages/contracts/workspace-portability.mjs +160 -225
  217. package/packages/evidence/artifact-integrity.mjs +8 -2
  218. package/packages/evidence/artifact-structure.mjs +105 -0
  219. package/packages/evidence/decision-fact.mjs +4 -1
  220. package/packages/evidence/domain-control-runtime.mjs +155 -27
  221. package/packages/evidence/domain-posture-context.mjs +79 -0
  222. package/packages/evidence/image-regions.mjs +56 -0
  223. package/packages/evidence/latest-observations.mjs +28 -0
  224. package/packages/evidence/lookup-target.mjs +104 -0
  225. package/packages/evidence/observation.mjs +16 -1
  226. package/packages/evidence/passphrase-encryption.mjs +44 -0
  227. package/packages/evidence/record-identity.mjs +72 -0
  228. package/packages/evidence/registry-dates.mjs +190 -0
  229. package/packages/evidence/retained-file.mjs +69 -0
  230. package/packages/evidence/review-age.mjs +14 -0
  231. package/packages/interchange/bagit.mjs +342 -0
  232. package/packages/interchange/bounded-zip-extraction.mjs +278 -0
  233. package/packages/interchange/crc32.mjs +17 -0
  234. package/packages/interchange/external-findings-import.mjs +89 -49
  235. package/packages/interchange/web-capture-import.mjs +382 -0
  236. package/packages/investigation/bulk-mail-exposure.mjs +2 -1
  237. package/packages/investigation/investigation-bagit.mjs +14 -0
  238. package/packages/investigation/investigation-capsule.mjs +33 -5
  239. package/packages/investigation/investigation-manifest.mjs +200 -0
  240. package/packages/investigation/investigation-package-crypto.mjs +98 -0
  241. package/packages/investigation/investigation-package.mjs +200 -0
  242. package/packages/investigation/lookup-asset-graph-inputs.mjs +38 -0
  243. package/packages/investigation/lookup-asset-graph.mjs +161 -141
  244. package/packages/investigation/lookup-claim-readiness.mjs +18 -0
  245. package/packages/investigation/lookup-investigation-brief.mjs +3 -3
  246. package/packages/investigation/lookup-task-guidance.mjs +58 -0
  247. package/packages/monitoring/analyst-review-state.mjs +38 -90
  248. package/packages/workspace/brand-profile-model.mjs +158 -20
  249. package/packages/workspace/bulk-columns.mjs +17 -0
  250. package/packages/workspace/bulk-review-model.mjs +11 -5
  251. package/packages/workspace/bulk-session-model.mjs +95 -20
  252. package/packages/workspace/bulk-sort.mjs +1 -0
  253. package/packages/workspace/case-views.mjs +102 -0
  254. package/packages/workspace/hostile-input.mjs +29 -8
  255. package/packages/workspace/investigation-template-model.mjs +19 -4
  256. package/packages/workspace/page-baseline.mjs +18 -15
  257. package/packages/workspace/relationship-observation-model.mjs +41 -26
  258. package/packages/workspace/watchlist-history.mjs +2 -2
  259. package/packages/workspace/watchlist-store.mjs +8 -2
  260. package/packages/workspace/website-snapshot-model.mjs +70 -30
  261. package/packages/workspace/workspace-archive-crypto.mjs +41 -61
  262. package/packages/workspace/workspace-archive.mjs +85 -21
  263. package/third-party-notices.txt +29 -2
  264. package/packages/investigation/bulk-review-export.mjs +0 -63
@@ -1,21 +1,18 @@
1
- // One bounded tokenization pass over the already-captured homepage body. The
2
- // parse5 tokenizer applies HTML tokenization rules without constructing a DOM,
3
- // so deeply nested hostile markup cannot create an unbounded element tree.
4
- // Only normalized start-tag markup and capped script indicators are retained.
5
- import { Tokenizer, TokenizerMode } from 'parse5';
1
+ // One bounded native HTML parse over the already-captured homepage body.
2
+ // Consumers share element/attribute evidence instead of reparsing reconstructed
3
+ // markup. Only bounded transient projections leave this module, never a DOM.
4
+ import { htmlTreeEvents, isHtmlElement, parseBoundedHtml, MAX_STATIC_HTML_CHARS, MAX_STATIC_HTML_TAGS, MAX_STATIC_HTML_TREE_EVENTS, } from "./bounded-html-document.mjs";
6
5
  import { MAX_CSP_META_POLICIES, MAX_RESPONSE_POLICY_HEADER_BYTES, } from "./response-policy.mjs";
6
+ import { MAX_FAVICON_BYTES, MAX_FAVICON_CANDIDATES } from "./outbound-request-bounds.mjs";
7
7
  import { MAX_PAGE_PUBLICATION_DECLARATIONS, MAX_PAGE_PUBLICATION_META_ELEMENTS, MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES, } from "./homepage-metadata-contract.mjs";
8
- const MAX_STATIC_HTML_CHARS = 300_000;
9
- const MAX_STATIC_HTML_TAGS = 8_192;
10
- const MAX_STATIC_STRUCTURE_TOKENS = 4_096;
11
- const MAX_TECHNOLOGY_TAGS = 2_048;
8
+ const MAX_STATIC_STRUCTURE_TOKENS = MAX_STATIC_HTML_TREE_EVENTS;
12
9
  const MAX_TAG_LENGTH = 4_096;
13
10
  const MAX_ATTRIBUTES_PER_TAG = 128;
14
11
  const MAX_SCRIPT_ELEMENTS = 64;
15
12
  const MAX_SCRIPT_REFERENCE_LENGTH = 2_048;
16
13
  const MAX_SCRIPT_MEDIA_TYPE_LENGTH = 120;
17
- const MAX_INLINE_SCRIPT_CHARS = 32_768;
18
14
  const MAX_INLINE_SCRIPT_TOTAL_CHARS = 65_536;
15
+ const MAX_INLINE_SCRIPT_CHARS = MAX_INLINE_SCRIPT_TOTAL_CHARS;
19
16
  const MAX_STATIC_VISIBLE_TEXT_CHARS = MAX_STATIC_HTML_CHARS;
20
17
  const MAX_STATIC_FORMS = 50;
21
18
  const MAX_STATIC_INPUTS = 500;
@@ -39,29 +36,11 @@ const PAYMENT_AUTOCOMPLETE_TOKENS = new Set([
39
36
  'transaction-currency',
40
37
  'transaction-amount',
41
38
  ]);
42
- const RAW_TEXT_MODES = Object.freeze({
43
- iframe: TokenizerMode.RAWTEXT,
44
- noembed: TokenizerMode.RAWTEXT,
45
- noframes: TokenizerMode.RAWTEXT,
46
- script: TokenizerMode.SCRIPT_DATA,
47
- style: TokenizerMode.RAWTEXT,
48
- template: TokenizerMode.RAWTEXT,
49
- textarea: TokenizerMode.RCDATA,
50
- title: TokenizerMode.RCDATA,
51
- xmp: TokenizerMode.RAWTEXT,
52
- });
53
39
  const VOID_TAGS = new Set([
54
40
  'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta',
55
41
  'param', 'source', 'track', 'wbr',
56
42
  ]);
57
- const NON_VISIBLE_TEXT_TAGS = new Set(['script', 'style', 'template']);
58
- const HEAD_CONTEXT_TAGS = new Set([
59
- 'html', 'head', 'base', 'basefont', 'bgsound', 'link', 'meta', 'title',
60
- 'noscript', 'noframes', 'style', 'template', 'script',
61
- ]);
62
- const IMPLICIT_HEAD_TEXT_TAGS = new Set([
63
- 'title', 'noscript', 'noframes', 'style', 'template', 'script',
64
- ]);
43
+ const NON_VISIBLE_TEXT_TAGS = new Set(['head', 'script', 'style', 'template', 'noscript', 'iframe', 'noembed', 'noframes']);
65
44
  const ROBOTS_DIRECTIVES = new Set([
66
45
  'all', 'follow', 'index', 'max-image-preview', 'max-snippet', 'max-video-preview',
67
46
  'noarchive', 'nocache', 'nofollow', 'noimageindex', 'noindex', 'none', 'nositelinkssearchbox',
@@ -218,31 +197,18 @@ function stylesheetCandidate(attributes) {
218
197
  truncated: rel.truncated || href.truncated || disabled.truncated || media.truncated,
219
198
  };
220
199
  }
221
- function serializedStartTag(tagName, attributes) {
222
- let serialized = `<${tagName.toLowerCase()}`;
223
- let limitReached = attributes.length > MAX_ATTRIBUTES_PER_TAG;
224
- for (const attribute of attributes.slice(0, MAX_ATTRIBUTES_PER_TAG)) {
225
- const candidate = ` ${attribute.name.toLowerCase()}="${attribute.value.toLowerCase()}"`;
226
- if (serialized.length + candidate.length + 1 > MAX_TAG_LENGTH) {
227
- limitReached = true;
228
- break;
229
- }
230
- serialized += candidate;
231
- }
232
- return {
233
- markup: serialized.length + 1 <= MAX_TAG_LENGTH ? `${serialized}>` : null,
234
- limitReached,
235
- };
236
- }
237
200
  function analyzeStaticHtml(value, options = {}) {
238
- const supplied = typeof value === 'string' ? value : '';
239
- const inputLimitReached = supplied.length > MAX_STATIC_HTML_CHARS;
240
- const html = supplied.slice(0, MAX_STATIC_HTML_CHARS);
201
+ const parsed = parseBoundedHtml(value);
202
+ const { inputLimitReached } = parsed;
241
203
  const documentUrl = safeBaseUrl(options.baseUrl);
242
204
  let effectiveBaseUrl = documentUrl;
243
205
  let baseHrefState = 'absent';
244
206
  const includeVisibleText = options.includeVisibleText === true;
245
- const markup = [];
207
+ const elements = [];
208
+ const iconLinks = [];
209
+ const iconCounts = [0, 0];
210
+ const tokens = [];
211
+ const elementStack = [];
246
212
  const visibleTextParts = [];
247
213
  const structureTokens = [];
248
214
  const scripts = [];
@@ -298,11 +264,10 @@ function analyzeStaticHtml(value, options = {}) {
298
264
  };
299
265
  const robotDirectives = new Set();
300
266
  const twitterCardTypes = new Set();
301
- let tokenizer;
302
267
  let activeInlineScript = null;
303
268
  let tagsExamined = 0;
304
269
  let inlineCharactersExamined = 0;
305
- let tagLimitReached = false;
270
+ let tagLimitReached = parsed.constructionLimitReached;
306
271
  let structureLimitReached = false;
307
272
  let scriptLimitReached = false;
308
273
  let inlineLimitReached = false;
@@ -310,11 +275,9 @@ function analyzeStaticHtml(value, options = {}) {
310
275
  let cspMetaLimitReached = false;
311
276
  let publicationMetaElements = 0;
312
277
  let robotDirectiveTokens = 0;
313
- let publicationTagLimitReached = false;
278
+ let publicationTagLimitReached = parsed.constructionLimitReached;
314
279
  let insideExplicitHead = false;
315
- let implicitHeadTextDepth = 0;
316
- let headScopeClosed = false;
317
- let bodyContentStarted = false;
280
+ let insideHead = false;
318
281
  let scriptElementSeen = false;
319
282
  let nonVisibleTextDepth = 0;
320
283
  let visibleTextCharacters = 0;
@@ -332,9 +295,8 @@ function analyzeStaticHtml(value, options = {}) {
332
295
  function appendInlineScript(chars) {
333
296
  if (!activeInlineScript || !chars)
334
297
  return;
335
- const perScriptRemaining = MAX_INLINE_SCRIPT_CHARS - activeInlineScript.inlineContent.length;
336
298
  const totalRemaining = MAX_INLINE_SCRIPT_TOTAL_CHARS - inlineCharactersExamined;
337
- const retainedLength = Math.max(0, Math.min(chars.length, perScriptRemaining, totalRemaining));
299
+ const retainedLength = Math.max(0, Math.min(chars.length, totalRemaining));
338
300
  if (retainedLength > 0) {
339
301
  activeInlineScript.inlineContent += chars.slice(0, retainedLength);
340
302
  inlineCharactersExamined += retainedLength;
@@ -361,33 +323,96 @@ function analyzeStaticHtml(value, options = {}) {
361
323
  if (retainedLength < chars.length)
362
324
  visibleTextLimitReached = true;
363
325
  }
364
- const handler = {
365
- onStartTag(token) {
366
- const tagName = token.tagName.toLowerCase();
367
- const rawTextMode = RAW_TEXT_MODES[tagName];
368
- if (!token.selfClosing && rawTextMode !== undefined)
369
- tokenizer.state = rawTextMode;
370
- if (!token.selfClosing && NON_VISIBLE_TEXT_TAGS.has(tagName))
371
- nonVisibleTextDepth += 1;
372
- if (tagsExamined >= MAX_STATIC_HTML_TAGS) {
373
- tagLimitReached = true;
374
- publicationTagLimitReached = true;
326
+ function appendToken(token) {
327
+ if (tokens.length >= MAX_STATIC_STRUCTURE_TOKENS)
328
+ structureLimitReached = true;
329
+ else
330
+ tokens.push(token);
331
+ }
332
+ for (const event of htmlTreeEvents(parsed.document)) {
333
+ if (event.kind === 'text') {
334
+ appendInlineScript(event.text);
335
+ appendTitle(event.text);
336
+ appendVisibleText(event.text);
337
+ if (nonVisibleTextDepth === 0)
338
+ appendToken({ kind: 'text', value: event.text });
339
+ continue;
340
+ }
341
+ const node = event.element;
342
+ const tagName = node.tagName.toLowerCase();
343
+ const htmlElement = isHtmlElement(node);
344
+ if (event.kind === 'end') {
345
+ elementStack.pop();
346
+ if (tagName === 'script')
375
347
  activeInlineScript = null;
376
- return;
348
+ if (htmlElement && tagName === 'title')
349
+ titleDepth = 0;
350
+ if (htmlElement && tagName === 'head') {
351
+ insideHead = false;
352
+ insideExplicitHead = false;
353
+ }
354
+ if (!htmlElement || !VOID_TAGS.has(tagName)) {
355
+ appendStructureToken(`/${tagName}`);
356
+ appendToken({ kind: 'end', name: tagName });
377
357
  }
378
- tagsExamined += 1;
379
- appendStructureToken(token.selfClosing || VOID_TAGS.has(tagName) ? `${tagName}/` : tagName);
380
- if (tagName === 'head')
381
- insideExplicitHead = true;
382
- if (tagName === 'body' || (!insideExplicitHead && !HEAD_CONTEXT_TAGS.has(tagName))) {
383
- bodyContentStarted = true;
384
- headScopeClosed = true;
358
+ if (NON_VISIBLE_TEXT_TAGS.has(tagName))
359
+ nonVisibleTextDepth -= 1;
360
+ continue;
361
+ }
362
+ {
363
+ const token = { attrs: node.attrs };
364
+ if (NON_VISIBLE_TEXT_TAGS.has(tagName))
365
+ nonVisibleTextDepth += 1;
366
+ if (node.sourceCodeLocation?.startTag)
367
+ tagsExamined += 1;
368
+ appendStructureToken(htmlElement && VOID_TAGS.has(tagName) ? `${tagName}/` : tagName);
369
+ if (htmlElement && tagName === 'head') {
370
+ insideHead = true;
371
+ insideExplicitHead = Boolean(node.sourceCodeLocation?.startTag);
385
372
  }
386
- if (!insideExplicitHead && !headScopeClosed && !token.selfClosing && IMPLICIT_HEAD_TEXT_TAGS.has(tagName)) {
387
- implicitHeadTextDepth += 1;
373
+ const inPublicationHead = insideHead;
374
+ const attributes = [];
375
+ let attributeLength = 0;
376
+ for (const attribute of node.attrs) {
377
+ attributeLength += attribute.name.length + attribute.value.length;
378
+ if (attributes.length >= MAX_ATTRIBUTES_PER_TAG || attributeLength > MAX_TAG_LENGTH)
379
+ break;
380
+ attributes.push({ name: attribute.name, value: attribute.value });
388
381
  }
389
- const inPublicationHead = insideExplicitHead || (!headScopeClosed && !bodyContentStarted && HEAD_CONTEXT_TAGS.has(tagName));
390
- if (tagName === 'base' && inPublicationHead && baseHrefState === 'absent') {
382
+ const element = {
383
+ name: tagName, html: htmlElement, attributes,
384
+ attributesTruncated: attributes.length !== node.attrs.length,
385
+ parent: elementStack.at(-1) ?? null, inHead: insideHead,
386
+ };
387
+ elementStack.push(elements.length);
388
+ elements.push(element);
389
+ appendToken({ kind: 'start', element });
390
+ // Foreign drawing attributes are not HTML technology, form or resource
391
+ // inputs. Their per-element omission still reaches fingerprinting, but
392
+ // must not make unrelated HTML evidence incomplete.
393
+ if (htmlElement && element.attributesTruncated)
394
+ tagLimitReached = true;
395
+ if (!htmlElement)
396
+ continue;
397
+ if (tagName === 'link') {
398
+ const rel = node.attrs.find((attribute) => attribute.name === 'rel')?.value;
399
+ const href = node.attrs.find((attribute) => attribute.name === 'href')?.value.trim();
400
+ if (rel && rel.length <= 128 && href) {
401
+ const relations = rel.toLowerCase().split(/[\t\n\f\r ]+/u);
402
+ const priority = relations.includes('icon') ? 0
403
+ : relations.some((relation) => ['apple-touch-icon', 'apple-touch-icon-precomposed'].includes(relation)) ? 1 : null;
404
+ // Inline image bytes have their own bound; the general 4 KiB
405
+ // attribute projection must not silently discard supported icons.
406
+ const maxHref = /^data:/iu.test(href) ? MAX_FAVICON_BYTES * 3 + 128 : 2048;
407
+ if (priority !== null && iconCounts[priority] < MAX_FAVICON_CANDIDATES && href.length <= maxHref) {
408
+ iconLinks.push({ href, priority });
409
+ iconCounts[priority] += 1;
410
+ }
411
+ }
412
+ }
413
+ // The first document base applies even when it appears in the body;
414
+ // publication metadata's separate head-only policy does not govern URLs.
415
+ if (tagName === 'base' && baseHrefState === 'absent') {
391
416
  const href = attributeValue(token.attrs, 'href', MAX_FORM_ATTRIBUTE_LENGTH, false);
392
417
  if (href.truncated && !href.present) {
393
418
  effectiveBaseUrl = documentUrl;
@@ -408,7 +433,7 @@ function analyzeStaticHtml(value, options = {}) {
408
433
  }
409
434
  }
410
435
  }
411
- if (tagName === 'title' && titleDepth === 0 && titleParts.length === 0)
436
+ if (tagName === 'title' && titleParts.length === 0)
412
437
  titleDepth = 1;
413
438
  if (/^h[1-6]$/u.test(tagName)) {
414
439
  const key = tagName;
@@ -450,10 +475,10 @@ function analyzeStaticHtml(value, options = {}) {
450
475
  if (candidate.candidate)
451
476
  publicationMetadata.renderBlockingCandidates.stylesheet += 1;
452
477
  }
453
- if (tagName === 'meta' && insideExplicitHead) {
478
+ if (tagName === 'meta' && insideHead) {
454
479
  const httpEquiv = attributeValue(token.attrs, 'http-equiv', 64);
455
480
  if (httpEquiv.value === 'content-security-policy') {
456
- const content = attributeValue(token.attrs, 'content', MAX_RESPONSE_POLICY_HEADER_BYTES);
481
+ const content = attributeValue(token.attrs, 'content', MAX_RESPONSE_POLICY_HEADER_BYTES, false);
457
482
  if (content.value === null || !content.value) {
458
483
  cspMetaLimitReached = cspMetaLimitReached || content.present || content.truncated;
459
484
  }
@@ -584,16 +609,6 @@ function analyzeStaticHtml(value, options = {}) {
584
609
  }
585
610
  }
586
611
  }
587
- if (markup.length < MAX_TECHNOLOGY_TAGS) {
588
- const serialized = serializedStartTag(tagName, token.attrs);
589
- if (serialized.limitReached)
590
- tagLimitReached = true;
591
- if (serialized.markup)
592
- markup.push(serialized.markup);
593
- }
594
- else {
595
- tagLimitReached = true;
596
- }
597
612
  if (tagName === 'form') {
598
613
  if (forms.formsObserved >= MAX_STATIC_FORMS) {
599
614
  forms.truncated = true;
@@ -624,13 +639,13 @@ function analyzeStaticHtml(value, options = {}) {
624
639
  }
625
640
  if (tagName !== 'script') {
626
641
  activeInlineScript = null;
627
- return;
642
+ continue;
628
643
  }
629
644
  scriptElementSeen = true;
630
645
  if (scripts.length >= MAX_SCRIPT_ELEMENTS) {
631
646
  scriptLimitReached = true;
632
647
  activeInlineScript = null;
633
- return;
648
+ continue;
634
649
  }
635
650
  const reference = scriptReference(token.attrs);
636
651
  const script = {
@@ -640,54 +655,8 @@ function analyzeStaticHtml(value, options = {}) {
640
655
  };
641
656
  scripts.push(script);
642
657
  activeInlineScript = reference ? null : script;
643
- },
644
- onEndTag(token) {
645
- const tagName = token.tagName.toLowerCase();
646
- if (tagName === 'script')
647
- activeInlineScript = null;
648
- if (tagName === 'title' && titleDepth > 0)
649
- titleDepth = 0;
650
- if (tagName === 'head') {
651
- insideExplicitHead = false;
652
- headScopeClosed = true;
653
- }
654
- if (!insideExplicitHead && IMPLICIT_HEAD_TEXT_TAGS.has(tagName) && implicitHeadTextDepth > 0) {
655
- implicitHeadTextDepth -= 1;
656
- }
657
- if (!VOID_TAGS.has(tagName))
658
- appendStructureToken(`/${tagName}`);
659
- if (NON_VISIBLE_TEXT_TAGS.has(tagName) && nonVisibleTextDepth > 0)
660
- nonVisibleTextDepth -= 1;
661
- },
662
- onCharacter(token) {
663
- if (!insideExplicitHead && !headScopeClosed && implicitHeadTextDepth === 0 && token.chars.trim()) {
664
- bodyContentStarted = true;
665
- headScopeClosed = true;
666
- }
667
- appendInlineScript(token.chars);
668
- appendTitle(token.chars);
669
- appendVisibleText(token.chars);
670
- },
671
- onNullCharacter(token) {
672
- if (!insideExplicitHead && !headScopeClosed && implicitHeadTextDepth === 0) {
673
- bodyContentStarted = true;
674
- headScopeClosed = true;
675
- }
676
- appendInlineScript(token.chars);
677
- appendTitle(token.chars);
678
- appendVisibleText(token.chars);
679
- },
680
- onWhitespaceCharacter(token) {
681
- appendInlineScript(token.chars);
682
- appendTitle(token.chars);
683
- appendVisibleText(token.chars);
684
- },
685
- onComment() { },
686
- onDoctype() { },
687
- onEof() { },
688
- };
689
- tokenizer = new Tokenizer({ sourceCodeLocationInfo: false }, handler);
690
- tokenizer.write(html, true);
658
+ }
659
+ }
691
660
  for (const attributes of pendingFormActions) {
692
661
  const action = formAction(attributes, effectiveBaseUrl, documentUrl?.origin ?? null);
693
662
  forms.actions[action.relationship] += 1;
@@ -723,11 +692,13 @@ function analyzeStaticHtml(value, options = {}) {
723
692
  const value = titleParts.join('').replace(CONTROL_CHARACTER_RE_GLOBAL, ' ').replace(/\ufffd/gu, ' ').replace(/\s+/gu, ' ').trim();
724
693
  if (!value)
725
694
  return null;
726
- return value.length > 200 ? `${value.slice(0, 200)}…` : value;
695
+ return value.length > 200 ? `${value.slice(0, 199)}…` : value;
727
696
  })(),
728
697
  effectiveBaseUrl: effectiveBaseUrl?.toString() ?? null,
729
698
  baseHrefState,
730
- markup: markup.join('\n'),
699
+ elements,
700
+ iconLinks: iconLinks.sort((left, right) => left.priority - right.priority).slice(0, MAX_FAVICON_CANDIDATES),
701
+ tokens,
731
702
  visibleText: visibleTextParts.join('').replace(CONTROL_CHARACTER_RE_GLOBAL, ' ').replace(/\s+/gu, ' '),
732
703
  structureTokens,
733
704
  scripts,
@@ -745,4 +716,4 @@ function analyzeStaticHtml(value, options = {}) {
745
716
  inlineCharactersExamined,
746
717
  };
747
718
  }
748
- export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_PUBLICATION_META_ELEMENTS, MAX_STATIC_PUBLICATION_DECLARATIONS, MAX_STATIC_ROBOTS_DIRECTIVES, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };
719
+ export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_PUBLICATION_META_ELEMENTS, MAX_STATIC_PUBLICATION_DECLARATIONS, MAX_STATIC_ROBOTS_DIRECTIVES, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, analyzeStaticHtml, };
@@ -290,6 +290,7 @@ function analyzeStructuredDataIdentity(input = {}) {
290
290
  }
291
291
  const truncated = input.sourceTruncated === true
292
292
  || htmlAnalysis.inputLimitReached
293
+ || htmlAnalysis.tagLimitReached
293
294
  || htmlAnalysis.scriptLimitReached
294
295
  || htmlAnalysis.inlineLimitReached
295
296
  || limitReached;
@@ -0,0 +1,100 @@
1
+ const CONTROL_RE = /[\u0000-\u001f\u007f]/u;
2
+ export function attributeValue(element, name) {
3
+ return element.attributes.find((attribute) => attribute.name === name)?.value.toLowerCase() ?? '';
4
+ }
5
+ export function hasAttributes(element, ...names) {
6
+ return element.attributes.some((attribute) => names.includes(attribute.name));
7
+ }
8
+ function httpUrl(value, base) {
9
+ if (typeof value !== 'string' || value.length > 2048 || CONTROL_RE.test(value) || !value.trim())
10
+ return null;
11
+ try {
12
+ const url = new URL(value, base);
13
+ return ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password ? url : null;
14
+ }
15
+ catch {
16
+ return null;
17
+ }
18
+ }
19
+ export function createTechnologyMarkupContext(analysis, effectiveBaseUrl, documentOrigin) {
20
+ const elements = analysis.elements.filter((element) => element.html && element.name !== 'template');
21
+ const documentUrl = httpUrl(documentOrigin);
22
+ const suppliedBase = effectiveBaseUrl ?? analysis.effectiveBaseUrl;
23
+ const base = httpUrl(suppliedBase) ?? documentUrl;
24
+ // A reserved base permits legacy minimised relative references without
25
+ // attributing an absolute external URL to an unknown document origin.
26
+ const resolutionBase = base ?? new URL('https://document.invalid/');
27
+ const references = (element, names) => (element.attributes.flatMap((attribute) => {
28
+ if (!names.includes(attribute.name))
29
+ return [];
30
+ if (attribute.value.trim().startsWith('#'))
31
+ return [];
32
+ const url = httpUrl(attribute.value, resolutionBase);
33
+ if (!url)
34
+ return [];
35
+ const relative = !/^(?:[a-z][a-z0-9+.-]*:|\/\/)/iu.test(attribute.value.trim());
36
+ const embedded = documentUrl
37
+ ? url.origin !== documentUrl.origin
38
+ : Boolean(base || suppliedBase || documentOrigin) || !relative;
39
+ return [{ url, embedded }];
40
+ }));
41
+ const resources = [];
42
+ const navigation = [];
43
+ for (const element of elements) {
44
+ switch (element.name) {
45
+ case 'script':
46
+ case 'img':
47
+ case 'iframe':
48
+ case 'frame':
49
+ case 'source':
50
+ case 'audio':
51
+ case 'embed':
52
+ resources.push(...references(element, ['src']));
53
+ break;
54
+ case 'video':
55
+ resources.push(...references(element, ['src', 'poster']));
56
+ break;
57
+ case 'link': {
58
+ const relations = attributeValue(element, 'rel').trim().split(/\s+/u);
59
+ // Existing minimised inputs retain a link reference without its rel.
60
+ // Explicit document metadata or navigation relations are not assets.
61
+ if (!hasAttributes(element, 'rel') || relations.some((relation) => ['stylesheet', 'preload', 'modulepreload', 'icon'].includes(relation))) {
62
+ resources.push(...references(element, ['href']));
63
+ }
64
+ break;
65
+ }
66
+ case 'object':
67
+ resources.push(...references(element, ['data']));
68
+ break;
69
+ case 'input':
70
+ if (attributeValue(element, 'type') === 'image')
71
+ resources.push(...references(element, ['src']));
72
+ break;
73
+ case 'a':
74
+ case 'area':
75
+ navigation.push(...references(element, ['href']));
76
+ break;
77
+ }
78
+ }
79
+ return { elements, resources, navigation };
80
+ }
81
+ export function elementMarker(matches, reviewMarkup) {
82
+ return { matches: ({ elements }) => elements.some(matches), reviewMarkup };
83
+ }
84
+ export function resourceMarker(matches, reviewPath) {
85
+ const sharedOrigin = reviewPath.startsWith('https://');
86
+ return {
87
+ matches: ({ resources }) => resources.some(({ url }) => matches(url)),
88
+ embedded: ({ resources }) => sharedOrigin || !resources.some(({ url, embedded }) => !embedded && matches(url)),
89
+ reviewMarkup: `<link href="${reviewPath}">`,
90
+ embeddedReviewMarkup: `<link href="${sharedOrigin ? reviewPath : `https://embedded.invalid${reviewPath}`}">`,
91
+ };
92
+ }
93
+ export function navigationMarker(matches, reviewPath) {
94
+ return {
95
+ matches: ({ navigation }) => navigation.some(({ url }) => matches(url)),
96
+ embedded: ({ navigation }) => !navigation.some(({ url, embedded }) => !embedded && matches(url)),
97
+ reviewMarkup: `<a href="${reviewPath}"></a>`,
98
+ embeddedReviewMarkup: `<a href="https://embedded.invalid/${reviewPath}"></a>`,
99
+ };
100
+ }