@slicedearth/whoisleuth-cli 2.3.0 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +8 -2
- package/README.md +19 -15
- package/bin/whoisleuth.mjs +8 -1
- package/cli/archive-content-digest.mjs +14 -0
- package/cli/archive-inspect.mjs +20 -53
- package/cli/argument-values.mjs +55 -0
- package/cli/arguments.mjs +24 -408
- package/cli/artifact-validation/case-response.mjs +65 -44
- package/cli/artifact-validation/investigation-capsule.mjs +52 -31
- package/cli/artifact-validation/lookup-evidence.mjs +15 -9
- package/cli/artifact-validation/signed-review.mjs +136 -68
- package/cli/artifact-validation/structure-primitives.mjs +1 -105
- package/cli/artifact-verify.mjs +92 -24
- package/cli/assurance-arguments.mjs +56 -0
- package/cli/assurance-command-definitions.mjs +179 -0
- package/cli/assurance-command-runner.mjs +40 -5
- package/cli/bagit-folder.mjs +57 -0
- package/cli/bagit-review.mjs +23 -0
- package/cli/bounded-json.mjs +1 -1
- package/cli/bulk-checkpoint.mjs +1 -1
- package/cli/bulk-command-runner.mjs +4 -2
- package/cli/bulk-output.mjs +29 -3
- package/cli/bulk.mjs +2 -3
- package/cli/case-command.mjs +212 -0
- package/cli/case-pack.mjs +50 -32
- package/cli/ci-report.mjs +6 -3
- package/cli/collection-arguments.mjs +123 -0
- package/cli/collection-command-definitions.mjs +330 -0
- package/cli/collection-preflight.mjs +8 -1
- package/cli/command-argument-grammar.mjs +8 -206
- package/cli/command-definition.mjs +329 -0
- package/cli/command-reference.mjs +94 -1357
- package/cli/command-seeds.mjs +18 -0
- package/cli/compare.mjs +1 -1
- package/cli/completion.mjs +298 -79
- package/cli/config-profile.mjs +8 -6
- package/cli/ct-event-intake.mjs +16 -16
- package/cli/discover.mjs +5 -4
- package/cli/discovery-observation-snapshot.mjs +46 -18
- package/cli/discovery-scan-command-runner.mjs +5 -3
- package/cli/discovery-scan.mjs +7 -4
- package/cli/discovery-snapshot.mjs +2 -2
- package/cli/discovery-workflow.mjs +5 -5
- package/cli/domain-control-monitor.mjs +50 -20
- package/cli/domain-control-observations.mjs +151 -103
- package/cli/errors.mjs +37 -6
- package/cli/evidence-arguments.mjs +46 -0
- package/cli/evidence-command-definitions.mjs +69 -0
- package/cli/evidence-command-runner.mjs +18 -8
- package/cli/evidence-signing.mjs +5 -4
- package/cli/external-observation-mapping.mjs +9 -10
- package/cli/formatters/terminal-command-formats.mjs +6 -0
- package/cli/formatters/terminal-lookup.mjs +24 -46
- package/cli/formatters/terminal-shared.mjs +5 -3
- package/cli/formatters/terminal-technology.mjs +114 -0
- package/cli/history-arguments.mjs +39 -0
- package/cli/history-command-definitions.mjs +88 -0
- package/cli/indicator-set-command.mjs +33 -0
- package/cli/inline-command-families.mjs +6 -46
- package/cli/inline-command-runner.mjs +4 -31
- package/cli/input.mjs +2 -2
- package/cli/intake-command.mjs +107 -0
- package/cli/interactive-launcher.mjs +2 -160
- package/cli/interchange-report.mjs +8 -10
- package/cli/investigation-artifacts.mjs +176 -0
- package/cli/investigation-checkpoint.mjs +10 -0
- package/cli/investigation-folder.mjs +114 -0
- package/cli/investigation-manifest.mjs +1 -112
- package/cli/investigation-package-review.mjs +103 -0
- package/cli/investigation-plan.mjs +38 -147
- package/cli/investigation-recipes.mjs +177 -0
- package/cli/investigation-run.mjs +256 -35
- package/cli/local-document-checkpoint.mjs +170 -0
- package/cli/local-mmdb-review.mjs +197 -17
- package/cli/lookalike-calibration.mjs +1 -1
- package/cli/lookup-brief.mjs +2 -2
- package/cli/lookup-browser.mjs +5 -12
- package/cli/lookup-command-runner.mjs +33 -39
- package/cli/lookup-diff.mjs +47 -12
- package/cli/lookup-plan.mjs +16 -14
- package/cli/lookup-reconcile.mjs +15 -12
- package/cli/lookup-timeline.mjs +7 -6
- package/cli/mail-header-review.mjs +24 -247
- package/cli/mail-review.mjs +75 -21
- package/cli/manual.mjs +1 -0
- package/cli/network-arguments.mjs +51 -0
- package/cli/network-command-definitions.mjs +130 -0
- package/cli/network-command-runner.mjs +45 -31
- package/cli/offline-evidence-review.mjs +39 -6
- package/cli/output-file.mjs +31 -5
- package/cli/posture.mjs +1 -1
- package/cli/registry-doctor.mjs +3 -1
- package/cli/registry-fixture-scaffold.mjs +1 -1
- package/cli/registry-support.mjs +1 -1
- package/cli/retained-artifact-diff.mjs +10 -9
- package/cli/review-arguments.mjs +131 -0
- package/cli/review-command-definitions.mjs +273 -0
- package/cli/review-command-runner.mjs +104 -149
- package/cli/risk-calibration.mjs +8 -8
- package/cli/runner.mjs +74 -37
- package/cli/saved-lookup.mjs +2 -2
- package/cli/selected-input-worker.mjs +63 -0
- package/cli/sharing-review.mjs +8 -80
- package/cli/signer-trust.mjs +104 -0
- package/cli/source-reliability.mjs +75 -84
- package/cli/support-arguments.mjs +49 -0
- package/cli/support-command-definitions.mjs +209 -0
- package/cli/support-command-runner.mjs +2 -2
- package/cli/terminal-input.mjs +168 -0
- package/cli/workflow-arguments.mjs +86 -0
- package/cli/workflow-command-definitions.mjs +78 -0
- package/cli/workflow-command-runner.mjs +75 -13
- package/docs/cli-reference.md +103 -30
- package/docs/cli.md +510 -25
- package/docs/contextual-reviews.md +155 -0
- package/lib/abort.mjs +4 -1
- package/lib/analyst-taxonomy.mjs +2 -40
- package/lib/application-version.mjs +1 -1
- package/lib/availability.mjs +27 -754
- package/lib/bounded-contract-normalizers.mjs +4 -0
- package/lib/bounded-file.mjs +3 -1
- package/lib/bounded-html-document.mjs +125 -0
- package/lib/bounded-json.mjs +2 -290
- package/lib/browser-library-profile.mjs +99 -130
- package/lib/browser-library-worker.mjs +112 -0
- package/lib/candidate-provenance-bounds.mjs +2 -0
- package/lib/classify.mjs +1 -1
- package/lib/client-behavior-profile.mjs +3 -3
- package/lib/ct-query.mjs +2 -27
- package/lib/ct-response-bounds.mjs +2 -9
- package/lib/ct-search.mjs +89 -46
- package/lib/dmarc-discovery.mjs +62 -0
- package/lib/dns-convergence-review.mjs +21 -8
- package/lib/dns-delegation-health.mjs +36 -22
- package/lib/dns-inheritance-review.mjs +207 -0
- package/lib/dns-intelligence.mjs +44 -29
- package/lib/dns-mx.mjs +1 -1
- package/lib/dns-query-coverage.mjs +40 -0
- package/lib/dnssec-evidence-validation.mjs +11 -3
- package/lib/domain-assurance.mjs +1 -1
- package/lib/domain-change-packet.mjs +18 -14
- package/lib/domain-change-review.mjs +40 -16
- package/lib/domain-control-flight-recorder.mjs +301 -124
- package/lib/domain-control-manifest.mjs +126 -58
- package/lib/domain-posture-analysis.mjs +41 -11
- package/lib/domain-posture-parsers.mjs +88 -6
- package/lib/domain-posture.mjs +120 -32
- package/lib/domain-registration.mjs +392 -0
- package/lib/evidence-export-privacy.mjs +568 -0
- package/lib/evidence-export.mjs +12 -593
- package/lib/evidence-report.mjs +7 -2
- package/lib/external-intelligence-risk.mjs +2 -124
- package/lib/favicon.mjs +117 -41
- package/lib/generated/cisa-kev-catalog.mjs +50 -4
- package/lib/generated/registrar-standing-catalogue.mjs +18 -10
- package/lib/generated/retire-browser-catalog.mjs +2742 -1034
- package/lib/generated/{unicode-confusables-17.mjs → unicode-confusables.mjs} +33 -33
- package/lib/html-signals.mjs +87 -85
- package/lib/http-evidence-bounds.mjs +2 -7
- package/lib/http-intelligence.mjs +52 -27
- package/lib/idn-confusable-policy.mjs +6 -4
- package/lib/idn-confusables.mjs +1 -1
- package/lib/interchange-fidelity-registry.mjs +10 -0
- package/lib/lookup-cache.mjs +34 -11
- package/lib/lookup-child-profile-contract.mjs +24 -15
- package/lib/lookup-contract-primitives.mjs +1 -1
- package/lib/lookup-diagnostics.mjs +2 -1
- package/lib/lookup-network-evidence-bounds.mjs +1 -10
- package/lib/lookup-progress.mjs +79 -13
- package/lib/lookup-response-contract.mjs +1077 -0
- package/lib/lookup-response.mjs +11 -24
- package/lib/lookup-source-progress.mjs +24 -35
- package/lib/lookup.mjs +49 -28
- package/lib/observed-network-context.mjs +43 -27
- package/lib/opportunity-scoring.mjs +2 -107
- package/lib/outbound-identity.mjs +1 -1
- package/lib/outbound-request-bounds.mjs +4 -1
- package/lib/page-fingerprints.mjs +87 -134
- package/lib/page-role-profile.mjs +12 -8
- package/lib/perceptual-hash-comparison.mjs +2 -34
- package/lib/perceptual-hash.mjs +79 -24
- package/lib/portable-generator.mjs +2 -29
- package/lib/project-metadata.mjs +2 -8
- package/lib/rdap-bootstrap.mjs +72 -22
- package/lib/rdap-client.mjs +26 -14
- package/lib/rdap-normalization.mjs +33 -18
- package/lib/rdap-search-workbench.mjs +1 -1
- package/lib/rdap-transport.mjs +47 -20
- package/lib/rdap-types.mjs +2 -1
- package/lib/rdap.mjs +18 -3
- package/lib/registrable-domain.mjs +2 -36
- package/lib/registrar-standing-contract.mjs +2 -1
- package/lib/registrar-standing.mjs +9 -13
- package/lib/registry-capabilities.mjs +7 -3
- package/lib/registry-capability-catalogue.mjs +29 -20
- package/lib/registry-dates.mjs +2 -190
- package/lib/registry-insights.mjs +56 -18
- package/lib/response-policy.mjs +86 -48
- package/lib/risk-calibration-summary.mjs +2 -338
- package/lib/risk-scoring.mjs +2 -345
- package/lib/rpki-evidence.mjs +8 -5
- package/lib/safe-fetch.mjs +54 -47
- package/lib/scoring-evidence-quality.mjs +2 -107
- package/lib/security-txt.mjs +10 -7
- package/lib/semantic-version.mjs +2 -54
- package/lib/service-binding-dns.mjs +35 -0
- package/lib/smtp-transport-review.mjs +75 -34
- package/lib/sslbl-certificates.generated.mjs +141 -138
- package/lib/static-html-analysis.mjs +117 -146
- package/lib/structured-data-identity.mjs +2 -1
- package/lib/technology-markup.mjs +100 -0
- package/lib/threat-intelligence-manifests.mjs +11 -139
- package/lib/threat-intelligence-runtime.mjs +24 -52
- package/lib/threat-intelligence-targets.mjs +6 -34
- package/lib/threat-intelligence-types.mjs +2 -36
- package/lib/typosquat-generator.mjs +29 -16
- package/lib/urlhaus-intelligence.mjs +8 -3
- package/lib/urlscan-intelligence.mjs +1 -1
- package/lib/website-enrichment.mjs +388 -0
- package/lib/website-security-posture.mjs +30 -6
- package/lib/website-technology.mjs +111 -71
- package/lib/whois-authority.mjs +55 -5
- package/lib/whois-chain.mjs +9 -2
- package/lib/whois-common-formats.mjs +5 -5
- package/lib/whois-contracts.mjs +2 -1
- package/lib/whois-dialects.mjs +2 -2
- package/lib/whois-normalization.mjs +3 -1
- package/lib/whois-parser-support.mjs +8 -1
- package/lib/whois-parser.mjs +5 -3
- package/lib/whois-transport.mjs +23 -7
- package/package.json +14 -14
- package/packages/analysis/analyst-taxonomy.mjs +40 -0
- package/packages/analysis/bounded-json.mjs +330 -0
- package/packages/analysis/candidate-provenance-bounds.mjs +4 -0
- package/packages/analysis/cli-argument-grammar.mjs +216 -0
- package/packages/analysis/ct-query.mjs +27 -0
- package/packages/analysis/ct-response-bounds.mjs +9 -0
- package/packages/analysis/external-intelligence-risk.mjs +124 -0
- package/packages/analysis/http-evidence-bounds.mjs +8 -0
- package/packages/analysis/lookup-network-evidence-bounds.mjs +10 -0
- package/packages/analysis/opportunity-scoring.mjs +107 -0
- package/packages/analysis/perceptual-hash-comparison.mjs +34 -0
- package/packages/analysis/portable-generator.mjs +29 -0
- package/packages/analysis/project-metadata.mjs +8 -0
- package/packages/analysis/registrable-domain.mjs +56 -0
- package/packages/analysis/risk-calibration-summary.mjs +352 -0
- package/packages/analysis/risk-scoring.mjs +345 -0
- package/packages/analysis/scoring-evidence-quality.mjs +107 -0
- package/packages/analysis/semantic-version.mjs +54 -0
- package/packages/analysis/threat-intelligence-types.mjs +75 -0
- package/packages/cases/case-attachment-model.mjs +83 -0
- package/packages/cases/case-evidence-links.mjs +135 -0
- package/packages/cases/case-evidence-model.mjs +229 -69
- package/packages/cases/case-evidence-quality.mjs +5 -0
- package/packages/cases/case-export-input.mjs +55 -0
- package/packages/cases/case-incident-context.mjs +52 -0
- package/packages/cases/case-markdown.mjs +11 -0
- package/packages/cases/case-migration-model.mjs +98 -48
- package/packages/cases/case-model.mjs +1 -0
- package/packages/cases/case-recheck-model.mjs +149 -0
- package/packages/cases/case-record-contracts.mjs +1 -1
- package/packages/cases/case-record-core.mjs +4 -3
- package/packages/cases/case-record-operations.mjs +123 -133
- package/packages/cases/case-record-projection.mjs +30 -20
- package/packages/cases/case-record-version-input.mjs +19 -0
- package/packages/cases/case-report.mjs +114 -67
- package/packages/cases/case-requested-evidence.mjs +146 -0
- package/packages/cases/case-response-actions.mjs +756 -0
- package/packages/cases/case-response-model.mjs +49 -1129
- package/packages/cases/case-response-outcomes.mjs +338 -0
- package/packages/cases/case-response-packet-render.mjs +157 -0
- package/packages/cases/case-response-packet-types.mjs +1 -0
- package/packages/cases/case-response-packet-vocabulary.mjs +48 -0
- package/packages/cases/case-response-packet.mjs +58 -593
- package/packages/cases/case-response-records.mjs +93 -0
- package/packages/cases/case-response-review-inputs.mjs +382 -0
- package/packages/cases/case-response-values.mjs +85 -0
- package/packages/cases/case-review-package.mjs +45 -0
- package/packages/cases/case-selection.mjs +46 -0
- package/packages/cases/case-storage-model.mjs +26 -1
- package/packages/cases/case-workflow-metadata.mjs +225 -0
- package/packages/cases/case-workflow-migration.mjs +43 -0
- package/packages/cases/http-summary.mjs +6 -1
- package/packages/cases/response-route-freshness.mjs +16 -0
- package/packages/comparison/capture-context.mjs +61 -0
- package/packages/comparison/comparison-ledger-bulk.mjs +4 -1
- package/packages/comparison/favicon-similarity.mjs +1 -1
- package/packages/comparison/page-similarity.mjs +3 -2
- package/packages/comparison/relationship-evidence.mjs +103 -14
- package/packages/comparison/relationship-provenance.mjs +99 -0
- package/packages/contracts/analyst-interchange.mjs +234 -0
- package/packages/contracts/analyst-review-state-contract.mjs +11 -0
- package/packages/contracts/analyst-review-state.mjs +5 -4
- package/packages/contracts/case-portability.mjs +370 -114
- package/packages/contracts/case-views-contract.mjs +12 -0
- package/packages/contracts/case-views.mjs +26 -0
- package/packages/contracts/cli-command-semantics.mjs +3 -0
- package/packages/contracts/cli-grammar.mjs +6 -0
- package/packages/contracts/cli-lookup-brief.mjs +2 -0
- package/packages/contracts/cli-lookup.mjs +0 -1
- package/packages/contracts/context-review.mjs +24 -0
- package/packages/contracts/document-intake.mjs +11 -0
- package/packages/contracts/domain-control-flight-recorder.mjs +98 -44
- package/packages/contracts/domain-control-manifest.mjs +128 -74
- package/packages/contracts/domain-control-monitor.mjs +59 -45
- package/packages/contracts/domain-control-review.mjs +200 -121
- package/packages/contracts/domain-name.mjs +21 -0
- package/packages/contracts/external-observation-interchange.mjs +2 -1
- package/packages/contracts/extracted-domain-lifecycle.mjs +11 -32
- package/packages/contracts/har-review.mjs +2 -0
- package/packages/contracts/http-url.mjs +2 -0
- package/packages/contracts/identity-events.mjs +4 -0
- package/packages/contracts/investigation-package-limits.mjs +24 -0
- package/packages/contracts/investigation-portability.mjs +56 -27
- package/packages/contracts/investigation-run.mjs +18 -0
- package/packages/contracts/ip-address.mjs +44 -0
- package/packages/contracts/lookup-evidence.mjs +35 -0
- package/packages/contracts/mail-authentication.mjs +5 -0
- package/packages/contracts/message-intake.mjs +31 -0
- package/packages/contracts/offline-comparison.mjs +4 -2
- package/packages/contracts/page-fingerprints.mjs +15 -0
- package/packages/contracts/passive-mail.mjs +10 -0
- package/packages/contracts/risk-calibration.mjs +0 -1
- package/packages/contracts/schema-compatibility.mjs +1 -1
- package/packages/contracts/schema-lifecycle.mjs +83 -153
- package/packages/contracts/selected-file-limits.mjs +6 -0
- package/packages/contracts/vulnerability-identifiers.mjs +5 -0
- package/packages/contracts/web-capture.mjs +10 -0
- package/packages/contracts/workspace-portability.mjs +219 -238
- package/packages/evidence/artifact-integrity.mjs +8 -2
- package/packages/evidence/artifact-structure.mjs +105 -0
- package/packages/evidence/collection-quality.mjs +67 -0
- package/packages/evidence/decision-fact.mjs +4 -1
- package/packages/evidence/dns-query-outcome.mjs +72 -0
- package/packages/evidence/domain-control-runtime.mjs +155 -27
- package/packages/evidence/domain-posture-context.mjs +79 -0
- package/packages/evidence/email-recipient.mjs +18 -0
- package/packages/evidence/image-regions.mjs +56 -0
- package/packages/evidence/latest-observations.mjs +28 -0
- package/packages/evidence/lookup-target.mjs +108 -0
- package/packages/evidence/observation.mjs +18 -1
- package/packages/evidence/passphrase-encryption.mjs +44 -0
- package/packages/evidence/png-pixels.mjs +117 -0
- package/packages/evidence/public-address-policy.mjs +83 -0
- package/packages/evidence/record-identity.mjs +72 -0
- package/packages/evidence/registry-dates.mjs +190 -0
- package/packages/evidence/retained-file.mjs +69 -0
- package/packages/evidence/review-age.mjs +14 -0
- package/packages/interchange/bagit.mjs +342 -0
- package/packages/interchange/bounded-zip-extraction.mjs +278 -0
- package/packages/interchange/crc32.mjs +17 -0
- package/packages/interchange/defensive-indicator-export.mjs +264 -0
- package/packages/interchange/external-findings-import.mjs +93 -61
- package/packages/interchange/managed-indicator-export.mjs +35 -0
- package/packages/interchange/managed-indicator-set.mjs +210 -0
- package/packages/interchange/misp-indicator-export.mjs +93 -0
- package/packages/interchange/sharing-policy.mjs +56 -0
- package/packages/interchange/stix-indicator-export.mjs +129 -0
- package/packages/interchange/web-capture-import.mjs +389 -0
- package/packages/investigation/acquisition-decision-packet.mjs +1 -1
- package/packages/investigation/bulk-mail-exposure.mjs +9 -14
- package/packages/investigation/capture-coverage.mjs +57 -0
- package/packages/investigation/connector-provenance-review.mjs +100 -0
- package/packages/investigation/context-review.mjs +35 -0
- package/packages/investigation/document-intake.mjs +81 -0
- package/packages/investigation/docx-intake.mjs +133 -0
- package/packages/investigation/domain-history-review.mjs +100 -0
- package/packages/investigation/domain-transition-review.mjs +47 -0
- package/packages/investigation/evidence-coverage-ledger.mjs +9 -1
- package/packages/investigation/har-intake.mjs +68 -0
- package/packages/investigation/identity-event-intake.mjs +131 -0
- package/packages/investigation/identity-incident-review.mjs +39 -0
- package/packages/investigation/incident-sequence-review.mjs +58 -0
- package/packages/investigation/intake-report.mjs +19 -0
- package/packages/investigation/investigation-bagit.mjs +14 -0
- package/packages/investigation/investigation-capsule.mjs +33 -5
- package/packages/investigation/investigation-manifest.mjs +200 -0
- package/packages/investigation/investigation-package-crypto.mjs +98 -0
- package/packages/investigation/investigation-package.mjs +200 -0
- package/packages/investigation/link-intake.mjs +179 -0
- package/packages/investigation/lookup-asset-graph-inputs.mjs +38 -0
- package/packages/investigation/lookup-asset-graph.mjs +161 -141
- package/packages/investigation/lookup-claim-readiness.mjs +18 -0
- package/packages/investigation/lookup-investigation-brief.mjs +3 -3
- package/packages/investigation/lookup-task-guidance.mjs +58 -0
- package/packages/investigation/mail-authentication-review.mjs +222 -0
- package/packages/investigation/mail-header-identity.mjs +209 -0
- package/packages/investigation/message-intake.mjs +195 -0
- package/packages/investigation/page-behaviour.mjs +113 -0
- package/packages/investigation/pdf-intake.mjs +155 -0
- package/packages/investigation/platform-continuity-review.mjs +60 -0
- package/packages/investigation/qr-intake.mjs +56 -0
- package/packages/investigation/requested-action-hints.mjs +11 -0
- package/packages/investigation/selected-input-review.mjs +17 -0
- package/packages/investigation/storefront-review.mjs +70 -0
- package/packages/monitoring/analyst-review-state.mjs +38 -90
- package/packages/workspace/brand-profile-model.mjs +160 -22
- package/packages/workspace/bulk-columns.mjs +17 -0
- package/packages/workspace/bulk-review-model.mjs +39 -31
- package/packages/workspace/bulk-session-model.mjs +112 -27
- package/packages/workspace/bulk-sort.mjs +1 -0
- package/packages/workspace/campaign-model.mjs +1 -1
- package/packages/workspace/case-views.mjs +102 -0
- package/packages/workspace/detection-rule-model.mjs +1 -1
- package/packages/workspace/hostile-input.mjs +29 -8
- package/packages/workspace/investigation-guide.mjs +2 -2
- package/packages/workspace/investigation-template-model.mjs +45 -6
- package/packages/workspace/page-baseline.mjs +20 -17
- package/packages/workspace/relationship-observation-model.mjs +54 -35
- package/packages/workspace/shortlist-model.mjs +14 -5
- package/packages/workspace/watchlist-history.mjs +30 -14
- package/packages/workspace/watchlist-store.mjs +24 -6
- package/packages/workspace/website-snapshot-model.mjs +98 -33
- package/packages/workspace/workspace-archive-crypto.mjs +41 -61
- package/packages/workspace/workspace-archive.mjs +90 -25
- package/third-party-notices.txt +577 -17
- package/lib/hostname.mjs +0 -38
- package/packages/investigation/bulk-review-export.mjs +0 -63
|
@@ -1,21 +1,18 @@
|
|
|
1
|
-
// One bounded
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
|
|
5
|
-
import { Tokenizer, TokenizerMode } from 'parse5';
|
|
1
|
+
// One bounded native HTML parse over the already-captured homepage body.
|
|
2
|
+
// Consumers share element/attribute evidence instead of reparsing reconstructed
|
|
3
|
+
// markup. Only bounded transient projections leave this module, never a DOM.
|
|
4
|
+
import { htmlTreeEvents, isHtmlElement, parseBoundedHtml, MAX_STATIC_HTML_CHARS, MAX_STATIC_HTML_TAGS, MAX_STATIC_HTML_TREE_EVENTS, } from "./bounded-html-document.mjs";
|
|
6
5
|
import { MAX_CSP_META_POLICIES, MAX_RESPONSE_POLICY_HEADER_BYTES, } from "./response-policy.mjs";
|
|
6
|
+
import { MAX_FAVICON_BYTES, MAX_FAVICON_CANDIDATES } from "./outbound-request-bounds.mjs";
|
|
7
7
|
import { MAX_PAGE_PUBLICATION_DECLARATIONS, MAX_PAGE_PUBLICATION_META_ELEMENTS, MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES, } from "./homepage-metadata-contract.mjs";
|
|
8
|
-
const
|
|
9
|
-
const MAX_STATIC_HTML_TAGS = 8_192;
|
|
10
|
-
const MAX_STATIC_STRUCTURE_TOKENS = 4_096;
|
|
11
|
-
const MAX_TECHNOLOGY_TAGS = 2_048;
|
|
8
|
+
const MAX_STATIC_STRUCTURE_TOKENS = MAX_STATIC_HTML_TREE_EVENTS;
|
|
12
9
|
const MAX_TAG_LENGTH = 4_096;
|
|
13
10
|
const MAX_ATTRIBUTES_PER_TAG = 128;
|
|
14
11
|
const MAX_SCRIPT_ELEMENTS = 64;
|
|
15
12
|
const MAX_SCRIPT_REFERENCE_LENGTH = 2_048;
|
|
16
13
|
const MAX_SCRIPT_MEDIA_TYPE_LENGTH = 120;
|
|
17
|
-
const MAX_INLINE_SCRIPT_CHARS = 32_768;
|
|
18
14
|
const MAX_INLINE_SCRIPT_TOTAL_CHARS = 65_536;
|
|
15
|
+
const MAX_INLINE_SCRIPT_CHARS = MAX_INLINE_SCRIPT_TOTAL_CHARS;
|
|
19
16
|
const MAX_STATIC_VISIBLE_TEXT_CHARS = MAX_STATIC_HTML_CHARS;
|
|
20
17
|
const MAX_STATIC_FORMS = 50;
|
|
21
18
|
const MAX_STATIC_INPUTS = 500;
|
|
@@ -39,29 +36,11 @@ const PAYMENT_AUTOCOMPLETE_TOKENS = new Set([
|
|
|
39
36
|
'transaction-currency',
|
|
40
37
|
'transaction-amount',
|
|
41
38
|
]);
|
|
42
|
-
const RAW_TEXT_MODES = Object.freeze({
|
|
43
|
-
iframe: TokenizerMode.RAWTEXT,
|
|
44
|
-
noembed: TokenizerMode.RAWTEXT,
|
|
45
|
-
noframes: TokenizerMode.RAWTEXT,
|
|
46
|
-
script: TokenizerMode.SCRIPT_DATA,
|
|
47
|
-
style: TokenizerMode.RAWTEXT,
|
|
48
|
-
template: TokenizerMode.RAWTEXT,
|
|
49
|
-
textarea: TokenizerMode.RCDATA,
|
|
50
|
-
title: TokenizerMode.RCDATA,
|
|
51
|
-
xmp: TokenizerMode.RAWTEXT,
|
|
52
|
-
});
|
|
53
39
|
const VOID_TAGS = new Set([
|
|
54
40
|
'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta',
|
|
55
41
|
'param', 'source', 'track', 'wbr',
|
|
56
42
|
]);
|
|
57
|
-
const NON_VISIBLE_TEXT_TAGS = new Set(['script', 'style', 'template']);
|
|
58
|
-
const HEAD_CONTEXT_TAGS = new Set([
|
|
59
|
-
'html', 'head', 'base', 'basefont', 'bgsound', 'link', 'meta', 'title',
|
|
60
|
-
'noscript', 'noframes', 'style', 'template', 'script',
|
|
61
|
-
]);
|
|
62
|
-
const IMPLICIT_HEAD_TEXT_TAGS = new Set([
|
|
63
|
-
'title', 'noscript', 'noframes', 'style', 'template', 'script',
|
|
64
|
-
]);
|
|
43
|
+
const NON_VISIBLE_TEXT_TAGS = new Set(['head', 'script', 'style', 'template', 'noscript', 'iframe', 'noembed', 'noframes']);
|
|
65
44
|
const ROBOTS_DIRECTIVES = new Set([
|
|
66
45
|
'all', 'follow', 'index', 'max-image-preview', 'max-snippet', 'max-video-preview',
|
|
67
46
|
'noarchive', 'nocache', 'nofollow', 'noimageindex', 'noindex', 'none', 'nositelinkssearchbox',
|
|
@@ -218,31 +197,18 @@ function stylesheetCandidate(attributes) {
|
|
|
218
197
|
truncated: rel.truncated || href.truncated || disabled.truncated || media.truncated,
|
|
219
198
|
};
|
|
220
199
|
}
|
|
221
|
-
function serializedStartTag(tagName, attributes) {
|
|
222
|
-
let serialized = `<${tagName.toLowerCase()}`;
|
|
223
|
-
let limitReached = attributes.length > MAX_ATTRIBUTES_PER_TAG;
|
|
224
|
-
for (const attribute of attributes.slice(0, MAX_ATTRIBUTES_PER_TAG)) {
|
|
225
|
-
const candidate = ` ${attribute.name.toLowerCase()}="${attribute.value.toLowerCase()}"`;
|
|
226
|
-
if (serialized.length + candidate.length + 1 > MAX_TAG_LENGTH) {
|
|
227
|
-
limitReached = true;
|
|
228
|
-
break;
|
|
229
|
-
}
|
|
230
|
-
serialized += candidate;
|
|
231
|
-
}
|
|
232
|
-
return {
|
|
233
|
-
markup: serialized.length + 1 <= MAX_TAG_LENGTH ? `${serialized}>` : null,
|
|
234
|
-
limitReached,
|
|
235
|
-
};
|
|
236
|
-
}
|
|
237
200
|
function analyzeStaticHtml(value, options = {}) {
|
|
238
|
-
const
|
|
239
|
-
const inputLimitReached =
|
|
240
|
-
const html = supplied.slice(0, MAX_STATIC_HTML_CHARS);
|
|
201
|
+
const parsed = parseBoundedHtml(value);
|
|
202
|
+
const { inputLimitReached } = parsed;
|
|
241
203
|
const documentUrl = safeBaseUrl(options.baseUrl);
|
|
242
204
|
let effectiveBaseUrl = documentUrl;
|
|
243
205
|
let baseHrefState = 'absent';
|
|
244
206
|
const includeVisibleText = options.includeVisibleText === true;
|
|
245
|
-
const
|
|
207
|
+
const elements = [];
|
|
208
|
+
const iconLinks = [];
|
|
209
|
+
const iconCounts = [0, 0];
|
|
210
|
+
const tokens = [];
|
|
211
|
+
const elementStack = [];
|
|
246
212
|
const visibleTextParts = [];
|
|
247
213
|
const structureTokens = [];
|
|
248
214
|
const scripts = [];
|
|
@@ -298,11 +264,10 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
298
264
|
};
|
|
299
265
|
const robotDirectives = new Set();
|
|
300
266
|
const twitterCardTypes = new Set();
|
|
301
|
-
let tokenizer;
|
|
302
267
|
let activeInlineScript = null;
|
|
303
268
|
let tagsExamined = 0;
|
|
304
269
|
let inlineCharactersExamined = 0;
|
|
305
|
-
let tagLimitReached =
|
|
270
|
+
let tagLimitReached = parsed.constructionLimitReached;
|
|
306
271
|
let structureLimitReached = false;
|
|
307
272
|
let scriptLimitReached = false;
|
|
308
273
|
let inlineLimitReached = false;
|
|
@@ -310,11 +275,9 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
310
275
|
let cspMetaLimitReached = false;
|
|
311
276
|
let publicationMetaElements = 0;
|
|
312
277
|
let robotDirectiveTokens = 0;
|
|
313
|
-
let publicationTagLimitReached =
|
|
278
|
+
let publicationTagLimitReached = parsed.constructionLimitReached;
|
|
314
279
|
let insideExplicitHead = false;
|
|
315
|
-
let
|
|
316
|
-
let headScopeClosed = false;
|
|
317
|
-
let bodyContentStarted = false;
|
|
280
|
+
let insideHead = false;
|
|
318
281
|
let scriptElementSeen = false;
|
|
319
282
|
let nonVisibleTextDepth = 0;
|
|
320
283
|
let visibleTextCharacters = 0;
|
|
@@ -332,9 +295,8 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
332
295
|
function appendInlineScript(chars) {
|
|
333
296
|
if (!activeInlineScript || !chars)
|
|
334
297
|
return;
|
|
335
|
-
const perScriptRemaining = MAX_INLINE_SCRIPT_CHARS - activeInlineScript.inlineContent.length;
|
|
336
298
|
const totalRemaining = MAX_INLINE_SCRIPT_TOTAL_CHARS - inlineCharactersExamined;
|
|
337
|
-
const retainedLength = Math.max(0, Math.min(chars.length,
|
|
299
|
+
const retainedLength = Math.max(0, Math.min(chars.length, totalRemaining));
|
|
338
300
|
if (retainedLength > 0) {
|
|
339
301
|
activeInlineScript.inlineContent += chars.slice(0, retainedLength);
|
|
340
302
|
inlineCharactersExamined += retainedLength;
|
|
@@ -361,33 +323,96 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
361
323
|
if (retainedLength < chars.length)
|
|
362
324
|
visibleTextLimitReached = true;
|
|
363
325
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
326
|
+
function appendToken(token) {
|
|
327
|
+
if (tokens.length >= MAX_STATIC_STRUCTURE_TOKENS)
|
|
328
|
+
structureLimitReached = true;
|
|
329
|
+
else
|
|
330
|
+
tokens.push(token);
|
|
331
|
+
}
|
|
332
|
+
for (const event of htmlTreeEvents(parsed.document)) {
|
|
333
|
+
if (event.kind === 'text') {
|
|
334
|
+
appendInlineScript(event.text);
|
|
335
|
+
appendTitle(event.text);
|
|
336
|
+
appendVisibleText(event.text);
|
|
337
|
+
if (nonVisibleTextDepth === 0)
|
|
338
|
+
appendToken({ kind: 'text', value: event.text });
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
const node = event.element;
|
|
342
|
+
const tagName = node.tagName.toLowerCase();
|
|
343
|
+
const htmlElement = isHtmlElement(node);
|
|
344
|
+
if (event.kind === 'end') {
|
|
345
|
+
elementStack.pop();
|
|
346
|
+
if (tagName === 'script')
|
|
375
347
|
activeInlineScript = null;
|
|
376
|
-
|
|
348
|
+
if (htmlElement && tagName === 'title')
|
|
349
|
+
titleDepth = 0;
|
|
350
|
+
if (htmlElement && tagName === 'head') {
|
|
351
|
+
insideHead = false;
|
|
352
|
+
insideExplicitHead = false;
|
|
353
|
+
}
|
|
354
|
+
if (!htmlElement || !VOID_TAGS.has(tagName)) {
|
|
355
|
+
appendStructureToken(`/${tagName}`);
|
|
356
|
+
appendToken({ kind: 'end', name: tagName });
|
|
377
357
|
}
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
358
|
+
if (NON_VISIBLE_TEXT_TAGS.has(tagName))
|
|
359
|
+
nonVisibleTextDepth -= 1;
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
{
|
|
363
|
+
const token = { attrs: node.attrs };
|
|
364
|
+
if (NON_VISIBLE_TEXT_TAGS.has(tagName))
|
|
365
|
+
nonVisibleTextDepth += 1;
|
|
366
|
+
if (node.sourceCodeLocation?.startTag)
|
|
367
|
+
tagsExamined += 1;
|
|
368
|
+
appendStructureToken(htmlElement && VOID_TAGS.has(tagName) ? `${tagName}/` : tagName);
|
|
369
|
+
if (htmlElement && tagName === 'head') {
|
|
370
|
+
insideHead = true;
|
|
371
|
+
insideExplicitHead = Boolean(node.sourceCodeLocation?.startTag);
|
|
385
372
|
}
|
|
386
|
-
|
|
387
|
-
|
|
373
|
+
const inPublicationHead = insideHead;
|
|
374
|
+
const attributes = [];
|
|
375
|
+
let attributeLength = 0;
|
|
376
|
+
for (const attribute of node.attrs) {
|
|
377
|
+
attributeLength += attribute.name.length + attribute.value.length;
|
|
378
|
+
if (attributes.length >= MAX_ATTRIBUTES_PER_TAG || attributeLength > MAX_TAG_LENGTH)
|
|
379
|
+
break;
|
|
380
|
+
attributes.push({ name: attribute.name, value: attribute.value });
|
|
388
381
|
}
|
|
389
|
-
const
|
|
390
|
-
|
|
382
|
+
const element = {
|
|
383
|
+
name: tagName, html: htmlElement, attributes,
|
|
384
|
+
attributesTruncated: attributes.length !== node.attrs.length,
|
|
385
|
+
parent: elementStack.at(-1) ?? null, inHead: insideHead,
|
|
386
|
+
};
|
|
387
|
+
elementStack.push(elements.length);
|
|
388
|
+
elements.push(element);
|
|
389
|
+
appendToken({ kind: 'start', element });
|
|
390
|
+
// Foreign drawing attributes are not HTML technology, form or resource
|
|
391
|
+
// inputs. Their per-element omission still reaches fingerprinting, but
|
|
392
|
+
// must not make unrelated HTML evidence incomplete.
|
|
393
|
+
if (htmlElement && element.attributesTruncated)
|
|
394
|
+
tagLimitReached = true;
|
|
395
|
+
if (!htmlElement)
|
|
396
|
+
continue;
|
|
397
|
+
if (tagName === 'link') {
|
|
398
|
+
const rel = node.attrs.find((attribute) => attribute.name === 'rel')?.value;
|
|
399
|
+
const href = node.attrs.find((attribute) => attribute.name === 'href')?.value.trim();
|
|
400
|
+
if (rel && rel.length <= 128 && href) {
|
|
401
|
+
const relations = rel.toLowerCase().split(/[\t\n\f\r ]+/u);
|
|
402
|
+
const priority = relations.includes('icon') ? 0
|
|
403
|
+
: relations.some((relation) => ['apple-touch-icon', 'apple-touch-icon-precomposed'].includes(relation)) ? 1 : null;
|
|
404
|
+
// Inline image bytes have their own bound; the general 4 KiB
|
|
405
|
+
// attribute projection must not silently discard supported icons.
|
|
406
|
+
const maxHref = /^data:/iu.test(href) ? MAX_FAVICON_BYTES * 3 + 128 : 2048;
|
|
407
|
+
if (priority !== null && iconCounts[priority] < MAX_FAVICON_CANDIDATES && href.length <= maxHref) {
|
|
408
|
+
iconLinks.push({ href, priority });
|
|
409
|
+
iconCounts[priority] += 1;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
// The first document base applies even when it appears in the body;
|
|
414
|
+
// publication metadata's separate head-only policy does not govern URLs.
|
|
415
|
+
if (tagName === 'base' && baseHrefState === 'absent') {
|
|
391
416
|
const href = attributeValue(token.attrs, 'href', MAX_FORM_ATTRIBUTE_LENGTH, false);
|
|
392
417
|
if (href.truncated && !href.present) {
|
|
393
418
|
effectiveBaseUrl = documentUrl;
|
|
@@ -408,7 +433,7 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
408
433
|
}
|
|
409
434
|
}
|
|
410
435
|
}
|
|
411
|
-
if (tagName === 'title' &&
|
|
436
|
+
if (tagName === 'title' && titleParts.length === 0)
|
|
412
437
|
titleDepth = 1;
|
|
413
438
|
if (/^h[1-6]$/u.test(tagName)) {
|
|
414
439
|
const key = tagName;
|
|
@@ -450,10 +475,10 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
450
475
|
if (candidate.candidate)
|
|
451
476
|
publicationMetadata.renderBlockingCandidates.stylesheet += 1;
|
|
452
477
|
}
|
|
453
|
-
if (tagName === 'meta' &&
|
|
478
|
+
if (tagName === 'meta' && insideHead) {
|
|
454
479
|
const httpEquiv = attributeValue(token.attrs, 'http-equiv', 64);
|
|
455
480
|
if (httpEquiv.value === 'content-security-policy') {
|
|
456
|
-
const content = attributeValue(token.attrs, 'content', MAX_RESPONSE_POLICY_HEADER_BYTES);
|
|
481
|
+
const content = attributeValue(token.attrs, 'content', MAX_RESPONSE_POLICY_HEADER_BYTES, false);
|
|
457
482
|
if (content.value === null || !content.value) {
|
|
458
483
|
cspMetaLimitReached = cspMetaLimitReached || content.present || content.truncated;
|
|
459
484
|
}
|
|
@@ -584,16 +609,6 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
584
609
|
}
|
|
585
610
|
}
|
|
586
611
|
}
|
|
587
|
-
if (markup.length < MAX_TECHNOLOGY_TAGS) {
|
|
588
|
-
const serialized = serializedStartTag(tagName, token.attrs);
|
|
589
|
-
if (serialized.limitReached)
|
|
590
|
-
tagLimitReached = true;
|
|
591
|
-
if (serialized.markup)
|
|
592
|
-
markup.push(serialized.markup);
|
|
593
|
-
}
|
|
594
|
-
else {
|
|
595
|
-
tagLimitReached = true;
|
|
596
|
-
}
|
|
597
612
|
if (tagName === 'form') {
|
|
598
613
|
if (forms.formsObserved >= MAX_STATIC_FORMS) {
|
|
599
614
|
forms.truncated = true;
|
|
@@ -624,13 +639,13 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
624
639
|
}
|
|
625
640
|
if (tagName !== 'script') {
|
|
626
641
|
activeInlineScript = null;
|
|
627
|
-
|
|
642
|
+
continue;
|
|
628
643
|
}
|
|
629
644
|
scriptElementSeen = true;
|
|
630
645
|
if (scripts.length >= MAX_SCRIPT_ELEMENTS) {
|
|
631
646
|
scriptLimitReached = true;
|
|
632
647
|
activeInlineScript = null;
|
|
633
|
-
|
|
648
|
+
continue;
|
|
634
649
|
}
|
|
635
650
|
const reference = scriptReference(token.attrs);
|
|
636
651
|
const script = {
|
|
@@ -640,54 +655,8 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
640
655
|
};
|
|
641
656
|
scripts.push(script);
|
|
642
657
|
activeInlineScript = reference ? null : script;
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
const tagName = token.tagName.toLowerCase();
|
|
646
|
-
if (tagName === 'script')
|
|
647
|
-
activeInlineScript = null;
|
|
648
|
-
if (tagName === 'title' && titleDepth > 0)
|
|
649
|
-
titleDepth = 0;
|
|
650
|
-
if (tagName === 'head') {
|
|
651
|
-
insideExplicitHead = false;
|
|
652
|
-
headScopeClosed = true;
|
|
653
|
-
}
|
|
654
|
-
if (!insideExplicitHead && IMPLICIT_HEAD_TEXT_TAGS.has(tagName) && implicitHeadTextDepth > 0) {
|
|
655
|
-
implicitHeadTextDepth -= 1;
|
|
656
|
-
}
|
|
657
|
-
if (!VOID_TAGS.has(tagName))
|
|
658
|
-
appendStructureToken(`/${tagName}`);
|
|
659
|
-
if (NON_VISIBLE_TEXT_TAGS.has(tagName) && nonVisibleTextDepth > 0)
|
|
660
|
-
nonVisibleTextDepth -= 1;
|
|
661
|
-
},
|
|
662
|
-
onCharacter(token) {
|
|
663
|
-
if (!insideExplicitHead && !headScopeClosed && implicitHeadTextDepth === 0 && token.chars.trim()) {
|
|
664
|
-
bodyContentStarted = true;
|
|
665
|
-
headScopeClosed = true;
|
|
666
|
-
}
|
|
667
|
-
appendInlineScript(token.chars);
|
|
668
|
-
appendTitle(token.chars);
|
|
669
|
-
appendVisibleText(token.chars);
|
|
670
|
-
},
|
|
671
|
-
onNullCharacter(token) {
|
|
672
|
-
if (!insideExplicitHead && !headScopeClosed && implicitHeadTextDepth === 0) {
|
|
673
|
-
bodyContentStarted = true;
|
|
674
|
-
headScopeClosed = true;
|
|
675
|
-
}
|
|
676
|
-
appendInlineScript(token.chars);
|
|
677
|
-
appendTitle(token.chars);
|
|
678
|
-
appendVisibleText(token.chars);
|
|
679
|
-
},
|
|
680
|
-
onWhitespaceCharacter(token) {
|
|
681
|
-
appendInlineScript(token.chars);
|
|
682
|
-
appendTitle(token.chars);
|
|
683
|
-
appendVisibleText(token.chars);
|
|
684
|
-
},
|
|
685
|
-
onComment() { },
|
|
686
|
-
onDoctype() { },
|
|
687
|
-
onEof() { },
|
|
688
|
-
};
|
|
689
|
-
tokenizer = new Tokenizer({ sourceCodeLocationInfo: false }, handler);
|
|
690
|
-
tokenizer.write(html, true);
|
|
658
|
+
}
|
|
659
|
+
}
|
|
691
660
|
for (const attributes of pendingFormActions) {
|
|
692
661
|
const action = formAction(attributes, effectiveBaseUrl, documentUrl?.origin ?? null);
|
|
693
662
|
forms.actions[action.relationship] += 1;
|
|
@@ -723,11 +692,13 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
723
692
|
const value = titleParts.join('').replace(CONTROL_CHARACTER_RE_GLOBAL, ' ').replace(/\ufffd/gu, ' ').replace(/\s+/gu, ' ').trim();
|
|
724
693
|
if (!value)
|
|
725
694
|
return null;
|
|
726
|
-
return value.length > 200 ? `${value.slice(0,
|
|
695
|
+
return value.length > 200 ? `${value.slice(0, 199)}…` : value;
|
|
727
696
|
})(),
|
|
728
697
|
effectiveBaseUrl: effectiveBaseUrl?.toString() ?? null,
|
|
729
698
|
baseHrefState,
|
|
730
|
-
|
|
699
|
+
elements,
|
|
700
|
+
iconLinks: iconLinks.sort((left, right) => left.priority - right.priority).slice(0, MAX_FAVICON_CANDIDATES),
|
|
701
|
+
tokens,
|
|
731
702
|
visibleText: visibleTextParts.join('').replace(CONTROL_CHARACTER_RE_GLOBAL, ' ').replace(/\s+/gu, ' '),
|
|
732
703
|
structureTokens,
|
|
733
704
|
scripts,
|
|
@@ -745,4 +716,4 @@ function analyzeStaticHtml(value, options = {}) {
|
|
|
745
716
|
inlineCharactersExamined,
|
|
746
717
|
};
|
|
747
718
|
}
|
|
748
|
-
export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_PUBLICATION_META_ELEMENTS, MAX_STATIC_PUBLICATION_DECLARATIONS, MAX_STATIC_ROBOTS_DIRECTIVES, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH,
|
|
719
|
+
export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_REFERENCE_LENGTH, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_PUBLICATION_META_ELEMENTS, MAX_STATIC_PUBLICATION_DECLARATIONS, MAX_STATIC_ROBOTS_DIRECTIVES, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, analyzeStaticHtml, };
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import { domainToASCII } from 'node:url';
|
|
7
7
|
import { createObservation } from "../packages/evidence/observation.mjs";
|
|
8
8
|
import { MAX_STRUCTURED_DATA_ENTITIES, MAX_STRUCTURED_DATA_SAME_AS_HOSTS, STRUCTURED_DATA_IDENTITY_VERSION, } from "./lookup-child-profile-contract.mjs";
|
|
9
|
-
import { isUriShapedLabel } from "
|
|
9
|
+
import { isUriShapedLabel } from "../packages/analysis/portable-generator.mjs";
|
|
10
10
|
import { analyzeStaticHtml, } from "./static-html-analysis.mjs";
|
|
11
11
|
const MAX_STRUCTURED_DATA_SCRIPTS = 8;
|
|
12
12
|
const MAX_STRUCTURED_DATA_SCRIPT_CHARS = 16_384;
|
|
@@ -290,6 +290,7 @@ function analyzeStructuredDataIdentity(input = {}) {
|
|
|
290
290
|
}
|
|
291
291
|
const truncated = input.sourceTruncated === true
|
|
292
292
|
|| htmlAnalysis.inputLimitReached
|
|
293
|
+
|| htmlAnalysis.tagLimitReached
|
|
293
294
|
|| htmlAnalysis.scriptLimitReached
|
|
294
295
|
|| htmlAnalysis.inlineLimitReached
|
|
295
296
|
|| limitReached;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
const CONTROL_RE = /[\u0000-\u001f\u007f]/u;
|
|
2
|
+
export function attributeValue(element, name) {
|
|
3
|
+
return element.attributes.find((attribute) => attribute.name === name)?.value.toLowerCase() ?? '';
|
|
4
|
+
}
|
|
5
|
+
export function hasAttributes(element, ...names) {
|
|
6
|
+
return element.attributes.some((attribute) => names.includes(attribute.name));
|
|
7
|
+
}
|
|
8
|
+
function httpUrl(value, base) {
|
|
9
|
+
if (typeof value !== 'string' || value.length > 2048 || CONTROL_RE.test(value) || !value.trim())
|
|
10
|
+
return null;
|
|
11
|
+
try {
|
|
12
|
+
const url = new URL(value, base);
|
|
13
|
+
return ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password ? url : null;
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export function createTechnologyMarkupContext(analysis, effectiveBaseUrl, documentOrigin) {
|
|
20
|
+
const elements = analysis.elements.filter((element) => element.html && element.name !== 'template');
|
|
21
|
+
const documentUrl = httpUrl(documentOrigin);
|
|
22
|
+
const suppliedBase = effectiveBaseUrl ?? analysis.effectiveBaseUrl;
|
|
23
|
+
const base = httpUrl(suppliedBase) ?? documentUrl;
|
|
24
|
+
// A reserved base permits legacy minimised relative references without
|
|
25
|
+
// attributing an absolute external URL to an unknown document origin.
|
|
26
|
+
const resolutionBase = base ?? new URL('https://document.invalid/');
|
|
27
|
+
const references = (element, names) => (element.attributes.flatMap((attribute) => {
|
|
28
|
+
if (!names.includes(attribute.name))
|
|
29
|
+
return [];
|
|
30
|
+
if (attribute.value.trim().startsWith('#'))
|
|
31
|
+
return [];
|
|
32
|
+
const url = httpUrl(attribute.value, resolutionBase);
|
|
33
|
+
if (!url)
|
|
34
|
+
return [];
|
|
35
|
+
const relative = !/^(?:[a-z][a-z0-9+.-]*:|\/\/)/iu.test(attribute.value.trim());
|
|
36
|
+
const embedded = documentUrl
|
|
37
|
+
? url.origin !== documentUrl.origin
|
|
38
|
+
: Boolean(base || suppliedBase || documentOrigin) || !relative;
|
|
39
|
+
return [{ url, embedded }];
|
|
40
|
+
}));
|
|
41
|
+
const resources = [];
|
|
42
|
+
const navigation = [];
|
|
43
|
+
for (const element of elements) {
|
|
44
|
+
switch (element.name) {
|
|
45
|
+
case 'script':
|
|
46
|
+
case 'img':
|
|
47
|
+
case 'iframe':
|
|
48
|
+
case 'frame':
|
|
49
|
+
case 'source':
|
|
50
|
+
case 'audio':
|
|
51
|
+
case 'embed':
|
|
52
|
+
resources.push(...references(element, ['src']));
|
|
53
|
+
break;
|
|
54
|
+
case 'video':
|
|
55
|
+
resources.push(...references(element, ['src', 'poster']));
|
|
56
|
+
break;
|
|
57
|
+
case 'link': {
|
|
58
|
+
const relations = attributeValue(element, 'rel').trim().split(/\s+/u);
|
|
59
|
+
// Existing minimised inputs retain a link reference without its rel.
|
|
60
|
+
// Explicit document metadata or navigation relations are not assets.
|
|
61
|
+
if (!hasAttributes(element, 'rel') || relations.some((relation) => ['stylesheet', 'preload', 'modulepreload', 'icon'].includes(relation))) {
|
|
62
|
+
resources.push(...references(element, ['href']));
|
|
63
|
+
}
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
case 'object':
|
|
67
|
+
resources.push(...references(element, ['data']));
|
|
68
|
+
break;
|
|
69
|
+
case 'input':
|
|
70
|
+
if (attributeValue(element, 'type') === 'image')
|
|
71
|
+
resources.push(...references(element, ['src']));
|
|
72
|
+
break;
|
|
73
|
+
case 'a':
|
|
74
|
+
case 'area':
|
|
75
|
+
navigation.push(...references(element, ['href']));
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return { elements, resources, navigation };
|
|
80
|
+
}
|
|
81
|
+
export function elementMarker(matches, reviewMarkup) {
|
|
82
|
+
return { matches: ({ elements }) => elements.some(matches), reviewMarkup };
|
|
83
|
+
}
|
|
84
|
+
export function resourceMarker(matches, reviewPath) {
|
|
85
|
+
const sharedOrigin = reviewPath.startsWith('https://');
|
|
86
|
+
return {
|
|
87
|
+
matches: ({ resources }) => resources.some(({ url }) => matches(url)),
|
|
88
|
+
embedded: ({ resources }) => sharedOrigin || !resources.some(({ url, embedded }) => !embedded && matches(url)),
|
|
89
|
+
reviewMarkup: `<link href="${reviewPath}">`,
|
|
90
|
+
embeddedReviewMarkup: `<link href="${sharedOrigin ? reviewPath : `https://embedded.invalid${reviewPath}`}">`,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
export function navigationMarker(matches, reviewPath) {
|
|
94
|
+
return {
|
|
95
|
+
matches: ({ navigation }) => navigation.some(({ url }) => matches(url)),
|
|
96
|
+
embedded: ({ navigation }) => !navigation.some(({ url, embedded }) => !embedded && matches(url)),
|
|
97
|
+
reviewMarkup: `<a href="${reviewPath}"></a>`,
|
|
98
|
+
embeddedReviewMarkup: `<a href="https://embedded.invalid/${reviewPath}"></a>`,
|
|
99
|
+
};
|
|
100
|
+
}
|