@slicedearth/whoisleuth-cli 2.3.0 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +8 -2
- package/README.md +19 -15
- package/bin/whoisleuth.mjs +8 -1
- package/cli/archive-content-digest.mjs +14 -0
- package/cli/archive-inspect.mjs +20 -53
- package/cli/argument-values.mjs +55 -0
- package/cli/arguments.mjs +24 -408
- package/cli/artifact-validation/case-response.mjs +65 -44
- package/cli/artifact-validation/investigation-capsule.mjs +52 -31
- package/cli/artifact-validation/lookup-evidence.mjs +15 -9
- package/cli/artifact-validation/signed-review.mjs +136 -68
- package/cli/artifact-validation/structure-primitives.mjs +1 -105
- package/cli/artifact-verify.mjs +92 -24
- package/cli/assurance-arguments.mjs +56 -0
- package/cli/assurance-command-definitions.mjs +179 -0
- package/cli/assurance-command-runner.mjs +40 -5
- package/cli/bagit-folder.mjs +57 -0
- package/cli/bagit-review.mjs +23 -0
- package/cli/bounded-json.mjs +1 -1
- package/cli/bulk-checkpoint.mjs +1 -1
- package/cli/bulk-command-runner.mjs +4 -2
- package/cli/bulk-output.mjs +29 -3
- package/cli/bulk.mjs +2 -3
- package/cli/case-command.mjs +212 -0
- package/cli/case-pack.mjs +50 -32
- package/cli/ci-report.mjs +6 -3
- package/cli/collection-arguments.mjs +123 -0
- package/cli/collection-command-definitions.mjs +330 -0
- package/cli/collection-preflight.mjs +8 -1
- package/cli/command-argument-grammar.mjs +8 -206
- package/cli/command-definition.mjs +329 -0
- package/cli/command-reference.mjs +94 -1357
- package/cli/command-seeds.mjs +18 -0
- package/cli/compare.mjs +1 -1
- package/cli/completion.mjs +298 -79
- package/cli/config-profile.mjs +8 -6
- package/cli/ct-event-intake.mjs +16 -16
- package/cli/discover.mjs +5 -4
- package/cli/discovery-observation-snapshot.mjs +46 -18
- package/cli/discovery-scan-command-runner.mjs +5 -3
- package/cli/discovery-scan.mjs +7 -4
- package/cli/discovery-snapshot.mjs +2 -2
- package/cli/discovery-workflow.mjs +5 -5
- package/cli/domain-control-monitor.mjs +50 -20
- package/cli/domain-control-observations.mjs +151 -103
- package/cli/errors.mjs +37 -6
- package/cli/evidence-arguments.mjs +46 -0
- package/cli/evidence-command-definitions.mjs +69 -0
- package/cli/evidence-command-runner.mjs +18 -8
- package/cli/evidence-signing.mjs +5 -4
- package/cli/external-observation-mapping.mjs +9 -10
- package/cli/formatters/terminal-command-formats.mjs +6 -0
- package/cli/formatters/terminal-lookup.mjs +24 -46
- package/cli/formatters/terminal-shared.mjs +5 -3
- package/cli/formatters/terminal-technology.mjs +114 -0
- package/cli/history-arguments.mjs +39 -0
- package/cli/history-command-definitions.mjs +88 -0
- package/cli/indicator-set-command.mjs +33 -0
- package/cli/inline-command-families.mjs +6 -46
- package/cli/inline-command-runner.mjs +4 -31
- package/cli/input.mjs +2 -2
- package/cli/intake-command.mjs +107 -0
- package/cli/interactive-launcher.mjs +2 -160
- package/cli/interchange-report.mjs +8 -10
- package/cli/investigation-artifacts.mjs +176 -0
- package/cli/investigation-checkpoint.mjs +10 -0
- package/cli/investigation-folder.mjs +114 -0
- package/cli/investigation-manifest.mjs +1 -112
- package/cli/investigation-package-review.mjs +103 -0
- package/cli/investigation-plan.mjs +38 -147
- package/cli/investigation-recipes.mjs +177 -0
- package/cli/investigation-run.mjs +256 -35
- package/cli/local-document-checkpoint.mjs +170 -0
- package/cli/local-mmdb-review.mjs +197 -17
- package/cli/lookalike-calibration.mjs +1 -1
- package/cli/lookup-brief.mjs +2 -2
- package/cli/lookup-browser.mjs +5 -12
- package/cli/lookup-command-runner.mjs +33 -39
- package/cli/lookup-diff.mjs +47 -12
- package/cli/lookup-plan.mjs +16 -14
- package/cli/lookup-reconcile.mjs +15 -12
- package/cli/lookup-timeline.mjs +7 -6
- package/cli/mail-header-review.mjs +24 -247
- package/cli/mail-review.mjs +75 -21
- package/cli/manual.mjs +1 -0
- package/cli/network-arguments.mjs +51 -0
- package/cli/network-command-definitions.mjs +130 -0
- package/cli/network-command-runner.mjs +45 -31
- package/cli/offline-evidence-review.mjs +39 -6
- package/cli/output-file.mjs +31 -5
- package/cli/posture.mjs +1 -1
- package/cli/registry-doctor.mjs +3 -1
- package/cli/registry-fixture-scaffold.mjs +1 -1
- package/cli/registry-support.mjs +1 -1
- package/cli/retained-artifact-diff.mjs +10 -9
- package/cli/review-arguments.mjs +131 -0
- package/cli/review-command-definitions.mjs +273 -0
- package/cli/review-command-runner.mjs +104 -149
- package/cli/risk-calibration.mjs +8 -8
- package/cli/runner.mjs +74 -37
- package/cli/saved-lookup.mjs +2 -2
- package/cli/selected-input-worker.mjs +63 -0
- package/cli/sharing-review.mjs +8 -80
- package/cli/signer-trust.mjs +104 -0
- package/cli/source-reliability.mjs +75 -84
- package/cli/support-arguments.mjs +49 -0
- package/cli/support-command-definitions.mjs +209 -0
- package/cli/support-command-runner.mjs +2 -2
- package/cli/terminal-input.mjs +168 -0
- package/cli/workflow-arguments.mjs +86 -0
- package/cli/workflow-command-definitions.mjs +78 -0
- package/cli/workflow-command-runner.mjs +75 -13
- package/docs/cli-reference.md +103 -30
- package/docs/cli.md +510 -25
- package/docs/contextual-reviews.md +155 -0
- package/lib/abort.mjs +4 -1
- package/lib/analyst-taxonomy.mjs +2 -40
- package/lib/application-version.mjs +1 -1
- package/lib/availability.mjs +27 -754
- package/lib/bounded-contract-normalizers.mjs +4 -0
- package/lib/bounded-file.mjs +3 -1
- package/lib/bounded-html-document.mjs +125 -0
- package/lib/bounded-json.mjs +2 -290
- package/lib/browser-library-profile.mjs +99 -130
- package/lib/browser-library-worker.mjs +112 -0
- package/lib/candidate-provenance-bounds.mjs +2 -0
- package/lib/classify.mjs +1 -1
- package/lib/client-behavior-profile.mjs +3 -3
- package/lib/ct-query.mjs +2 -27
- package/lib/ct-response-bounds.mjs +2 -9
- package/lib/ct-search.mjs +89 -46
- package/lib/dmarc-discovery.mjs +62 -0
- package/lib/dns-convergence-review.mjs +21 -8
- package/lib/dns-delegation-health.mjs +36 -22
- package/lib/dns-inheritance-review.mjs +207 -0
- package/lib/dns-intelligence.mjs +44 -29
- package/lib/dns-mx.mjs +1 -1
- package/lib/dns-query-coverage.mjs +40 -0
- package/lib/dnssec-evidence-validation.mjs +11 -3
- package/lib/domain-assurance.mjs +1 -1
- package/lib/domain-change-packet.mjs +18 -14
- package/lib/domain-change-review.mjs +40 -16
- package/lib/domain-control-flight-recorder.mjs +301 -124
- package/lib/domain-control-manifest.mjs +126 -58
- package/lib/domain-posture-analysis.mjs +41 -11
- package/lib/domain-posture-parsers.mjs +88 -6
- package/lib/domain-posture.mjs +120 -32
- package/lib/domain-registration.mjs +392 -0
- package/lib/evidence-export-privacy.mjs +568 -0
- package/lib/evidence-export.mjs +12 -593
- package/lib/evidence-report.mjs +7 -2
- package/lib/external-intelligence-risk.mjs +2 -124
- package/lib/favicon.mjs +117 -41
- package/lib/generated/cisa-kev-catalog.mjs +50 -4
- package/lib/generated/registrar-standing-catalogue.mjs +18 -10
- package/lib/generated/retire-browser-catalog.mjs +2742 -1034
- package/lib/generated/{unicode-confusables-17.mjs → unicode-confusables.mjs} +33 -33
- package/lib/html-signals.mjs +87 -85
- package/lib/http-evidence-bounds.mjs +2 -7
- package/lib/http-intelligence.mjs +52 -27
- package/lib/idn-confusable-policy.mjs +6 -4
- package/lib/idn-confusables.mjs +1 -1
- package/lib/interchange-fidelity-registry.mjs +10 -0
- package/lib/lookup-cache.mjs +34 -11
- package/lib/lookup-child-profile-contract.mjs +24 -15
- package/lib/lookup-contract-primitives.mjs +1 -1
- package/lib/lookup-diagnostics.mjs +2 -1
- package/lib/lookup-network-evidence-bounds.mjs +1 -10
- package/lib/lookup-progress.mjs +79 -13
- package/lib/lookup-response-contract.mjs +1077 -0
- package/lib/lookup-response.mjs +11 -24
- package/lib/lookup-source-progress.mjs +24 -35
- package/lib/lookup.mjs +49 -28
- package/lib/observed-network-context.mjs +43 -27
- package/lib/opportunity-scoring.mjs +2 -107
- package/lib/outbound-identity.mjs +1 -1
- package/lib/outbound-request-bounds.mjs +4 -1
- package/lib/page-fingerprints.mjs +87 -134
- package/lib/page-role-profile.mjs +12 -8
- package/lib/perceptual-hash-comparison.mjs +2 -34
- package/lib/perceptual-hash.mjs +79 -24
- package/lib/portable-generator.mjs +2 -29
- package/lib/project-metadata.mjs +2 -8
- package/lib/rdap-bootstrap.mjs +72 -22
- package/lib/rdap-client.mjs +26 -14
- package/lib/rdap-normalization.mjs +33 -18
- package/lib/rdap-search-workbench.mjs +1 -1
- package/lib/rdap-transport.mjs +47 -20
- package/lib/rdap-types.mjs +2 -1
- package/lib/rdap.mjs +18 -3
- package/lib/registrable-domain.mjs +2 -36
- package/lib/registrar-standing-contract.mjs +2 -1
- package/lib/registrar-standing.mjs +9 -13
- package/lib/registry-capabilities.mjs +7 -3
- package/lib/registry-capability-catalogue.mjs +29 -20
- package/lib/registry-dates.mjs +2 -190
- package/lib/registry-insights.mjs +56 -18
- package/lib/response-policy.mjs +86 -48
- package/lib/risk-calibration-summary.mjs +2 -338
- package/lib/risk-scoring.mjs +2 -345
- package/lib/rpki-evidence.mjs +8 -5
- package/lib/safe-fetch.mjs +54 -47
- package/lib/scoring-evidence-quality.mjs +2 -107
- package/lib/security-txt.mjs +10 -7
- package/lib/semantic-version.mjs +2 -54
- package/lib/service-binding-dns.mjs +35 -0
- package/lib/smtp-transport-review.mjs +75 -34
- package/lib/sslbl-certificates.generated.mjs +141 -138
- package/lib/static-html-analysis.mjs +117 -146
- package/lib/structured-data-identity.mjs +2 -1
- package/lib/technology-markup.mjs +100 -0
- package/lib/threat-intelligence-manifests.mjs +11 -139
- package/lib/threat-intelligence-runtime.mjs +24 -52
- package/lib/threat-intelligence-targets.mjs +6 -34
- package/lib/threat-intelligence-types.mjs +2 -36
- package/lib/typosquat-generator.mjs +29 -16
- package/lib/urlhaus-intelligence.mjs +8 -3
- package/lib/urlscan-intelligence.mjs +1 -1
- package/lib/website-enrichment.mjs +388 -0
- package/lib/website-security-posture.mjs +30 -6
- package/lib/website-technology.mjs +111 -71
- package/lib/whois-authority.mjs +55 -5
- package/lib/whois-chain.mjs +9 -2
- package/lib/whois-common-formats.mjs +5 -5
- package/lib/whois-contracts.mjs +2 -1
- package/lib/whois-dialects.mjs +2 -2
- package/lib/whois-normalization.mjs +3 -1
- package/lib/whois-parser-support.mjs +8 -1
- package/lib/whois-parser.mjs +5 -3
- package/lib/whois-transport.mjs +23 -7
- package/package.json +14 -14
- package/packages/analysis/analyst-taxonomy.mjs +40 -0
- package/packages/analysis/bounded-json.mjs +330 -0
- package/packages/analysis/candidate-provenance-bounds.mjs +4 -0
- package/packages/analysis/cli-argument-grammar.mjs +216 -0
- package/packages/analysis/ct-query.mjs +27 -0
- package/packages/analysis/ct-response-bounds.mjs +9 -0
- package/packages/analysis/external-intelligence-risk.mjs +124 -0
- package/packages/analysis/http-evidence-bounds.mjs +8 -0
- package/packages/analysis/lookup-network-evidence-bounds.mjs +10 -0
- package/packages/analysis/opportunity-scoring.mjs +107 -0
- package/packages/analysis/perceptual-hash-comparison.mjs +34 -0
- package/packages/analysis/portable-generator.mjs +29 -0
- package/packages/analysis/project-metadata.mjs +8 -0
- package/packages/analysis/registrable-domain.mjs +56 -0
- package/packages/analysis/risk-calibration-summary.mjs +352 -0
- package/packages/analysis/risk-scoring.mjs +345 -0
- package/packages/analysis/scoring-evidence-quality.mjs +107 -0
- package/packages/analysis/semantic-version.mjs +54 -0
- package/packages/analysis/threat-intelligence-types.mjs +75 -0
- package/packages/cases/case-attachment-model.mjs +83 -0
- package/packages/cases/case-evidence-links.mjs +135 -0
- package/packages/cases/case-evidence-model.mjs +229 -69
- package/packages/cases/case-evidence-quality.mjs +5 -0
- package/packages/cases/case-export-input.mjs +55 -0
- package/packages/cases/case-incident-context.mjs +52 -0
- package/packages/cases/case-markdown.mjs +11 -0
- package/packages/cases/case-migration-model.mjs +98 -48
- package/packages/cases/case-model.mjs +1 -0
- package/packages/cases/case-recheck-model.mjs +149 -0
- package/packages/cases/case-record-contracts.mjs +1 -1
- package/packages/cases/case-record-core.mjs +4 -3
- package/packages/cases/case-record-operations.mjs +123 -133
- package/packages/cases/case-record-projection.mjs +30 -20
- package/packages/cases/case-record-version-input.mjs +19 -0
- package/packages/cases/case-report.mjs +114 -67
- package/packages/cases/case-requested-evidence.mjs +146 -0
- package/packages/cases/case-response-actions.mjs +756 -0
- package/packages/cases/case-response-model.mjs +49 -1129
- package/packages/cases/case-response-outcomes.mjs +338 -0
- package/packages/cases/case-response-packet-render.mjs +157 -0
- package/packages/cases/case-response-packet-types.mjs +1 -0
- package/packages/cases/case-response-packet-vocabulary.mjs +48 -0
- package/packages/cases/case-response-packet.mjs +58 -593
- package/packages/cases/case-response-records.mjs +93 -0
- package/packages/cases/case-response-review-inputs.mjs +382 -0
- package/packages/cases/case-response-values.mjs +85 -0
- package/packages/cases/case-review-package.mjs +45 -0
- package/packages/cases/case-selection.mjs +46 -0
- package/packages/cases/case-storage-model.mjs +26 -1
- package/packages/cases/case-workflow-metadata.mjs +225 -0
- package/packages/cases/case-workflow-migration.mjs +43 -0
- package/packages/cases/http-summary.mjs +6 -1
- package/packages/cases/response-route-freshness.mjs +16 -0
- package/packages/comparison/capture-context.mjs +61 -0
- package/packages/comparison/comparison-ledger-bulk.mjs +4 -1
- package/packages/comparison/favicon-similarity.mjs +1 -1
- package/packages/comparison/page-similarity.mjs +3 -2
- package/packages/comparison/relationship-evidence.mjs +103 -14
- package/packages/comparison/relationship-provenance.mjs +99 -0
- package/packages/contracts/analyst-interchange.mjs +234 -0
- package/packages/contracts/analyst-review-state-contract.mjs +11 -0
- package/packages/contracts/analyst-review-state.mjs +5 -4
- package/packages/contracts/case-portability.mjs +370 -114
- package/packages/contracts/case-views-contract.mjs +12 -0
- package/packages/contracts/case-views.mjs +26 -0
- package/packages/contracts/cli-command-semantics.mjs +3 -0
- package/packages/contracts/cli-grammar.mjs +6 -0
- package/packages/contracts/cli-lookup-brief.mjs +2 -0
- package/packages/contracts/cli-lookup.mjs +0 -1
- package/packages/contracts/context-review.mjs +24 -0
- package/packages/contracts/document-intake.mjs +11 -0
- package/packages/contracts/domain-control-flight-recorder.mjs +98 -44
- package/packages/contracts/domain-control-manifest.mjs +128 -74
- package/packages/contracts/domain-control-monitor.mjs +59 -45
- package/packages/contracts/domain-control-review.mjs +200 -121
- package/packages/contracts/domain-name.mjs +21 -0
- package/packages/contracts/external-observation-interchange.mjs +2 -1
- package/packages/contracts/extracted-domain-lifecycle.mjs +11 -32
- package/packages/contracts/har-review.mjs +2 -0
- package/packages/contracts/http-url.mjs +2 -0
- package/packages/contracts/identity-events.mjs +4 -0
- package/packages/contracts/investigation-package-limits.mjs +24 -0
- package/packages/contracts/investigation-portability.mjs +56 -27
- package/packages/contracts/investigation-run.mjs +18 -0
- package/packages/contracts/ip-address.mjs +44 -0
- package/packages/contracts/lookup-evidence.mjs +35 -0
- package/packages/contracts/mail-authentication.mjs +5 -0
- package/packages/contracts/message-intake.mjs +31 -0
- package/packages/contracts/offline-comparison.mjs +4 -2
- package/packages/contracts/page-fingerprints.mjs +15 -0
- package/packages/contracts/passive-mail.mjs +10 -0
- package/packages/contracts/risk-calibration.mjs +0 -1
- package/packages/contracts/schema-compatibility.mjs +1 -1
- package/packages/contracts/schema-lifecycle.mjs +83 -153
- package/packages/contracts/selected-file-limits.mjs +6 -0
- package/packages/contracts/vulnerability-identifiers.mjs +5 -0
- package/packages/contracts/web-capture.mjs +10 -0
- package/packages/contracts/workspace-portability.mjs +219 -238
- package/packages/evidence/artifact-integrity.mjs +8 -2
- package/packages/evidence/artifact-structure.mjs +105 -0
- package/packages/evidence/collection-quality.mjs +67 -0
- package/packages/evidence/decision-fact.mjs +4 -1
- package/packages/evidence/dns-query-outcome.mjs +72 -0
- package/packages/evidence/domain-control-runtime.mjs +155 -27
- package/packages/evidence/domain-posture-context.mjs +79 -0
- package/packages/evidence/email-recipient.mjs +18 -0
- package/packages/evidence/image-regions.mjs +56 -0
- package/packages/evidence/latest-observations.mjs +28 -0
- package/packages/evidence/lookup-target.mjs +108 -0
- package/packages/evidence/observation.mjs +18 -1
- package/packages/evidence/passphrase-encryption.mjs +44 -0
- package/packages/evidence/png-pixels.mjs +117 -0
- package/packages/evidence/public-address-policy.mjs +83 -0
- package/packages/evidence/record-identity.mjs +72 -0
- package/packages/evidence/registry-dates.mjs +190 -0
- package/packages/evidence/retained-file.mjs +69 -0
- package/packages/evidence/review-age.mjs +14 -0
- package/packages/interchange/bagit.mjs +342 -0
- package/packages/interchange/bounded-zip-extraction.mjs +278 -0
- package/packages/interchange/crc32.mjs +17 -0
- package/packages/interchange/defensive-indicator-export.mjs +264 -0
- package/packages/interchange/external-findings-import.mjs +93 -61
- package/packages/interchange/managed-indicator-export.mjs +35 -0
- package/packages/interchange/managed-indicator-set.mjs +210 -0
- package/packages/interchange/misp-indicator-export.mjs +93 -0
- package/packages/interchange/sharing-policy.mjs +56 -0
- package/packages/interchange/stix-indicator-export.mjs +129 -0
- package/packages/interchange/web-capture-import.mjs +389 -0
- package/packages/investigation/acquisition-decision-packet.mjs +1 -1
- package/packages/investigation/bulk-mail-exposure.mjs +9 -14
- package/packages/investigation/capture-coverage.mjs +57 -0
- package/packages/investigation/connector-provenance-review.mjs +100 -0
- package/packages/investigation/context-review.mjs +35 -0
- package/packages/investigation/document-intake.mjs +81 -0
- package/packages/investigation/docx-intake.mjs +133 -0
- package/packages/investigation/domain-history-review.mjs +100 -0
- package/packages/investigation/domain-transition-review.mjs +47 -0
- package/packages/investigation/evidence-coverage-ledger.mjs +9 -1
- package/packages/investigation/har-intake.mjs +68 -0
- package/packages/investigation/identity-event-intake.mjs +131 -0
- package/packages/investigation/identity-incident-review.mjs +39 -0
- package/packages/investigation/incident-sequence-review.mjs +58 -0
- package/packages/investigation/intake-report.mjs +19 -0
- package/packages/investigation/investigation-bagit.mjs +14 -0
- package/packages/investigation/investigation-capsule.mjs +33 -5
- package/packages/investigation/investigation-manifest.mjs +200 -0
- package/packages/investigation/investigation-package-crypto.mjs +98 -0
- package/packages/investigation/investigation-package.mjs +200 -0
- package/packages/investigation/link-intake.mjs +179 -0
- package/packages/investigation/lookup-asset-graph-inputs.mjs +38 -0
- package/packages/investigation/lookup-asset-graph.mjs +161 -141
- package/packages/investigation/lookup-claim-readiness.mjs +18 -0
- package/packages/investigation/lookup-investigation-brief.mjs +3 -3
- package/packages/investigation/lookup-task-guidance.mjs +58 -0
- package/packages/investigation/mail-authentication-review.mjs +222 -0
- package/packages/investigation/mail-header-identity.mjs +209 -0
- package/packages/investigation/message-intake.mjs +195 -0
- package/packages/investigation/page-behaviour.mjs +113 -0
- package/packages/investigation/pdf-intake.mjs +155 -0
- package/packages/investigation/platform-continuity-review.mjs +60 -0
- package/packages/investigation/qr-intake.mjs +56 -0
- package/packages/investigation/requested-action-hints.mjs +11 -0
- package/packages/investigation/selected-input-review.mjs +17 -0
- package/packages/investigation/storefront-review.mjs +70 -0
- package/packages/monitoring/analyst-review-state.mjs +38 -90
- package/packages/workspace/brand-profile-model.mjs +160 -22
- package/packages/workspace/bulk-columns.mjs +17 -0
- package/packages/workspace/bulk-review-model.mjs +39 -31
- package/packages/workspace/bulk-session-model.mjs +112 -27
- package/packages/workspace/bulk-sort.mjs +1 -0
- package/packages/workspace/campaign-model.mjs +1 -1
- package/packages/workspace/case-views.mjs +102 -0
- package/packages/workspace/detection-rule-model.mjs +1 -1
- package/packages/workspace/hostile-input.mjs +29 -8
- package/packages/workspace/investigation-guide.mjs +2 -2
- package/packages/workspace/investigation-template-model.mjs +45 -6
- package/packages/workspace/page-baseline.mjs +20 -17
- package/packages/workspace/relationship-observation-model.mjs +54 -35
- package/packages/workspace/shortlist-model.mjs +14 -5
- package/packages/workspace/watchlist-history.mjs +30 -14
- package/packages/workspace/watchlist-store.mjs +24 -6
- package/packages/workspace/website-snapshot-model.mjs +98 -33
- package/packages/workspace/workspace-archive-crypto.mjs +41 -61
- package/packages/workspace/workspace-archive.mjs +90 -25
- package/third-party-notices.txt +577 -17
- package/lib/hostname.mjs +0 -38
- package/packages/investigation/bulk-review-export.mjs +0 -63
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
// those intermediate representations exist only long enough to derive a digest.
|
|
4
4
|
import { createHash } from 'node:crypto';
|
|
5
5
|
import { analyzeStaticHtml } from "./static-html-analysis.mjs";
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
import { PAGE_FINGERPRINT_VERSION, PAGE_FINGERPRINT_PARSERS, PAGE_FINGERPRINT_TOKEN_LIMITS, } from "../packages/contracts/page-fingerprints.mjs";
|
|
7
|
+
import { MAX_HOMEPAGE_BYTES } from "./outbound-request-bounds.mjs";
|
|
8
|
+
const MAX_FINGERPRINT_SOURCE_BYTES = MAX_HOMEPAGE_BYTES;
|
|
8
9
|
const MAX_FINGERPRINT_TAG_LENGTH = 4096;
|
|
9
|
-
const MAX_FINGERPRINT_TOKENS =
|
|
10
|
+
const MAX_FINGERPRINT_TOKENS = PAGE_FINGERPRINT_TOKEN_LIMITS[PAGE_FINGERPRINT_VERSION];
|
|
10
11
|
const MAX_FINGERPRINT_ATTRIBUTES = 64;
|
|
11
12
|
const MAX_VISIBLE_TEXT_TOKENS = 8192;
|
|
12
13
|
const MAX_FORM_FINGERPRINTS = 50;
|
|
@@ -15,11 +16,6 @@ const MAX_RESOURCE_HOSTS = 30;
|
|
|
15
16
|
const MAX_IDENTIFIER_VALUES = 30;
|
|
16
17
|
const CONTROL_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/gu;
|
|
17
18
|
const HAS_CONTROL_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/u;
|
|
18
|
-
const COMMENT_RE = /<!--[\s\S]*?(?:-->|$)/g;
|
|
19
|
-
const RAW_BODY_RE = /<(script|style|noscript|textarea|template)\b([^>]*)>[\s\S]*?(?:<\/\1\s*>|$)/gi;
|
|
20
|
-
const MARKUP_TOKEN_RE = /<\/?[A-Za-z][A-Za-z0-9:-]*\b[^>]*>|[^<]+/g;
|
|
21
|
-
const TAG_NAME_RE = /^<\s*(\/?)\s*([A-Za-z][A-Za-z0-9:-]*)/;
|
|
22
|
-
const ATTRIBUTE_RE = /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
|
|
23
19
|
const WORD_RE = /[\p{L}\p{N}]+/gu;
|
|
24
20
|
const URL_ATTRIBUTE_NAMES = new Set(['action', 'cite', 'data', 'formaction', 'href', 'poster', 'src']);
|
|
25
21
|
const OMITTED_ATTRIBUTE_NAMES = new Set(['integrity', 'nonce', 'srcdoc', 'style', 'value']);
|
|
@@ -28,16 +24,12 @@ function sha256(value) {
|
|
|
28
24
|
return createHash('sha256').update(value).digest('hex');
|
|
29
25
|
}
|
|
30
26
|
function boundedSource(html) {
|
|
31
|
-
const
|
|
32
|
-
const
|
|
33
|
-
const
|
|
27
|
+
const supplied = typeof html === 'string' ? html : '';
|
|
28
|
+
const prefix = Buffer.from(supplied.slice(0, MAX_FINGERPRINT_SOURCE_BYTES), 'utf8');
|
|
29
|
+
const truncated = supplied.length > MAX_FINGERPRINT_SOURCE_BYTES || prefix.length > MAX_FINGERPRINT_SOURCE_BYTES;
|
|
30
|
+
const bytes = prefix.subarray(0, MAX_FINGERPRINT_SOURCE_BYTES);
|
|
34
31
|
return { bytes, text: bytes.toString('utf8'), truncated };
|
|
35
32
|
}
|
|
36
|
-
function staticMarkup(html) {
|
|
37
|
-
return html
|
|
38
|
-
.replace(COMMENT_RE, ' ')
|
|
39
|
-
.replace(RAW_BODY_RE, '<$1$2></$1>');
|
|
40
|
-
}
|
|
41
33
|
function normalizedExactBodyHash(value) {
|
|
42
34
|
if (!value || typeof value !== 'object')
|
|
43
35
|
return null;
|
|
@@ -56,39 +48,6 @@ function normalizedExactBodyHash(value) {
|
|
|
56
48
|
source: 'captured-response-bytes',
|
|
57
49
|
};
|
|
58
50
|
}
|
|
59
|
-
function parseAttributes(tag) {
|
|
60
|
-
const start = tag.search(/\s/);
|
|
61
|
-
if (start === -1)
|
|
62
|
-
return { values: new Map(), truncated: false };
|
|
63
|
-
const values = new Map();
|
|
64
|
-
let truncated = false;
|
|
65
|
-
let match;
|
|
66
|
-
ATTRIBUTE_RE.lastIndex = 0;
|
|
67
|
-
while ((match = ATTRIBUTE_RE.exec(tag.slice(start)))) {
|
|
68
|
-
const name = match[1]?.toLowerCase();
|
|
69
|
-
if (!name)
|
|
70
|
-
continue;
|
|
71
|
-
if (values.has(name))
|
|
72
|
-
continue;
|
|
73
|
-
if (values.size >= MAX_FINGERPRINT_ATTRIBUTES) {
|
|
74
|
-
truncated = true;
|
|
75
|
-
break;
|
|
76
|
-
}
|
|
77
|
-
values.set(name, match[2] ?? match[3] ?? match[4] ?? '');
|
|
78
|
-
}
|
|
79
|
-
return { values, truncated };
|
|
80
|
-
}
|
|
81
|
-
function decodeStableEntities(text) {
|
|
82
|
-
const named = { amp: '&', apos: "'", gt: '>', lt: '<', nbsp: ' ', quot: '"' };
|
|
83
|
-
return text.replace(/&(?:#(\d{1,7})|#x([a-f0-9]{1,6})|([a-z]{2,8}));/gi, (whole, decimal, hex, name) => {
|
|
84
|
-
if (name)
|
|
85
|
-
return named[name.toLowerCase()] ?? whole.toLowerCase();
|
|
86
|
-
const point = Number.parseInt(decimal || hex, decimal ? 10 : 16);
|
|
87
|
-
if (!Number.isInteger(point) || point <= 0 || point > 0x10ffff || (point >= 0xd800 && point <= 0xdfff))
|
|
88
|
-
return ' ';
|
|
89
|
-
return String.fromCodePoint(point);
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
51
|
function looksRandom(value) {
|
|
93
52
|
if (typeof value !== 'string')
|
|
94
53
|
return false;
|
|
@@ -105,10 +64,27 @@ function reduceDynamicValues(value) {
|
|
|
105
64
|
.replace(/\b\d{4}-\d{1,2}-\d{1,2}(?:[t\s]\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:z|[+-]\d{2}:?\d{2})?)?\b/gi, '<time>')
|
|
106
65
|
.replace(/\b\d{10,13}\b/g, '<time>')
|
|
107
66
|
.replace(/\b[a-f0-9]{16,}\b/gi, '<id>')
|
|
108
|
-
.replace(
|
|
67
|
+
.replace(/[a-z0-9_-]+/gi, (token) => {
|
|
68
|
+
// Visit each maximal token once. Repeated lookaheads at word boundaries
|
|
69
|
+
// rescan hyphenated text quadratically when a required digit is absent.
|
|
70
|
+
// Only hyphens are outside the former ASCII word-boundary match;
|
|
71
|
+
// underscores remain part of the identifier, including at either end.
|
|
72
|
+
let start = 0;
|
|
73
|
+
let end = token.length;
|
|
74
|
+
while (start < end && token[start] === '-')
|
|
75
|
+
start += 1;
|
|
76
|
+
while (end > start && token[end - 1] === '-')
|
|
77
|
+
end -= 1;
|
|
78
|
+
if (end - start < 20)
|
|
79
|
+
return token;
|
|
80
|
+
const identifier = token.slice(start, end);
|
|
81
|
+
return /[a-z]/i.test(identifier) && /\d/.test(identifier)
|
|
82
|
+
? `${token.slice(0, start)}<id>${token.slice(end)}`
|
|
83
|
+
: token;
|
|
84
|
+
});
|
|
109
85
|
}
|
|
110
86
|
function normalizeText(value, maxLength = 512) {
|
|
111
|
-
const normalized = reduceDynamicValues(
|
|
87
|
+
const normalized = reduceDynamicValues(String(value || ''))
|
|
112
88
|
.normalize('NFKC')
|
|
113
89
|
.replace(CONTROL_RE, ' ')
|
|
114
90
|
.replace(/\s+/g, ' ')
|
|
@@ -150,49 +126,33 @@ function normalizeAttribute(name, value, baseUrl) {
|
|
|
150
126
|
return name;
|
|
151
127
|
return normalized ? `${name}=${normalized}` : name;
|
|
152
128
|
}
|
|
153
|
-
function normalizedMarkupAndStructure(
|
|
154
|
-
const markup = staticMarkup(html);
|
|
129
|
+
function normalizedMarkupAndStructure(analysis, baseUrl) {
|
|
155
130
|
const normalizedTokens = [];
|
|
156
|
-
const structureTokens =
|
|
131
|
+
const structureTokens = analysis.structureTokens;
|
|
157
132
|
const visibleSegments = [];
|
|
158
|
-
let normalizedTruncated =
|
|
159
|
-
|
|
133
|
+
let normalizedTruncated = analysis.structureLimitReached;
|
|
134
|
+
const structureTruncated = analysis.structureLimitReached;
|
|
160
135
|
let attributesTruncated = false;
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
while ((match = MARKUP_TOKEN_RE.exec(markup))) {
|
|
165
|
-
const token = match[0];
|
|
166
|
-
if (token.startsWith('<')) {
|
|
167
|
-
if (token.length > MAX_FINGERPRINT_TAG_LENGTH) {
|
|
168
|
-
oversizedTag = true;
|
|
169
|
-
continue;
|
|
170
|
-
}
|
|
171
|
-
const nameMatch = token.match(TAG_NAME_RE);
|
|
172
|
-
const name = nameMatch?.[2]?.toLowerCase();
|
|
173
|
-
if (!nameMatch || !name)
|
|
174
|
-
continue;
|
|
175
|
-
const closing = Boolean(nameMatch[1]);
|
|
176
|
-
const selfClosing = !closing && (VOID_TAGS.has(name) || /\/\s*>$/.test(token));
|
|
177
|
-
const structure = closing ? `/${name}` : selfClosing ? `${name}/` : name;
|
|
178
|
-
if (structureTokens.length < MAX_FINGERPRINT_TOKENS)
|
|
179
|
-
structureTokens.push(structure);
|
|
180
|
-
else
|
|
181
|
-
structureTruncated = true;
|
|
136
|
+
const oversizedTag = analysis.elements.some((element) => element.attributesTruncated);
|
|
137
|
+
for (const token of analysis.tokens) {
|
|
138
|
+
if (token.kind !== 'text') {
|
|
182
139
|
if (normalizedTokens.length >= MAX_FINGERPRINT_TOKENS) {
|
|
183
140
|
normalizedTruncated = true;
|
|
184
141
|
continue;
|
|
185
142
|
}
|
|
186
|
-
if (
|
|
187
|
-
normalizedTokens.push(`</${name}>`);
|
|
143
|
+
if (token.kind === 'end') {
|
|
144
|
+
normalizedTokens.push(`</${token.name}>`);
|
|
188
145
|
continue;
|
|
189
146
|
}
|
|
190
|
-
const
|
|
191
|
-
|
|
147
|
+
const element = token.element;
|
|
148
|
+
const name = element.name;
|
|
149
|
+
const selfClosing = element.html && VOID_TAGS.has(name);
|
|
150
|
+
if (element.attributesTruncated || element.attributes.length > MAX_FINGERPRINT_ATTRIBUTES)
|
|
192
151
|
attributesTruncated = true;
|
|
193
|
-
const
|
|
152
|
+
const values = new Map(element.attributes.slice(0, MAX_FINGERPRINT_ATTRIBUTES).map(({ name, value }) => [name, value]));
|
|
153
|
+
const attributes = [...values.entries()]
|
|
194
154
|
.map(([attributeName, value]) => {
|
|
195
|
-
const metaKey = name === 'meta' ? String(
|
|
155
|
+
const metaKey = name === 'meta' ? String(values.get('name') || values.get('property') || '') : '';
|
|
196
156
|
if (attributeName === 'content' && /(?:csrf|nonce|session|token)/i.test(metaKey))
|
|
197
157
|
return null;
|
|
198
158
|
return normalizeAttribute(attributeName, value, baseUrl);
|
|
@@ -202,7 +162,7 @@ function normalizedMarkupAndStructure(html, baseUrl) {
|
|
|
202
162
|
normalizedTokens.push(`<${name}${attributes.length ? ` ${attributes.join(' ')}` : ''}${selfClosing ? '/' : ''}>`);
|
|
203
163
|
continue;
|
|
204
164
|
}
|
|
205
|
-
const text = normalizeText(token, MAX_FINGERPRINT_SOURCE_BYTES);
|
|
165
|
+
const text = normalizeText(token.value, MAX_FINGERPRINT_SOURCE_BYTES);
|
|
206
166
|
if (!text)
|
|
207
167
|
continue;
|
|
208
168
|
visibleSegments.push(text);
|
|
@@ -270,66 +230,55 @@ function formActionClass(value, baseUrl) {
|
|
|
270
230
|
return 'external-insecure';
|
|
271
231
|
return 'external';
|
|
272
232
|
}
|
|
273
|
-
function formStructureFingerprint(
|
|
274
|
-
const
|
|
275
|
-
|
|
233
|
+
function formStructureFingerprint(analysis, baseUrl) {
|
|
234
|
+
const formByElement = new Map();
|
|
235
|
+
const firstElementById = new Map();
|
|
276
236
|
let formLimitReached = false;
|
|
277
237
|
let controlLimitReached = false;
|
|
278
238
|
let controls = 0;
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
const name = nameMatch?.[2]?.toLowerCase();
|
|
287
|
-
if (!nameMatch || !name)
|
|
288
|
-
continue;
|
|
289
|
-
const closing = Boolean(nameMatch[1]);
|
|
290
|
-
if (name === 'form') {
|
|
291
|
-
if (closing) {
|
|
292
|
-
if (current)
|
|
293
|
-
forms.push(current);
|
|
294
|
-
current = null;
|
|
295
|
-
continue;
|
|
296
|
-
}
|
|
297
|
-
if (current)
|
|
298
|
-
forms.push(current);
|
|
299
|
-
if (forms.length >= MAX_FORM_FINGERPRINTS) {
|
|
239
|
+
for (const [index, element] of analysis.elements.entries()) {
|
|
240
|
+
const attributes = new Map(element.attributes.map(({ name, value }) => [name, value]));
|
|
241
|
+
const id = attributes.get('id');
|
|
242
|
+
if (id && !firstElementById.has(id))
|
|
243
|
+
firstElementById.set(id, index);
|
|
244
|
+
if (element.html && element.name === 'form') {
|
|
245
|
+
if (formByElement.size >= MAX_FORM_FINGERPRINTS) {
|
|
300
246
|
formLimitReached = true;
|
|
301
|
-
current = null;
|
|
302
247
|
continue;
|
|
303
248
|
}
|
|
304
|
-
const attributes = parseAttributes(token).values;
|
|
305
249
|
const rawMethod = normalizeText(attributes.get('method') || 'get', 20);
|
|
306
|
-
|
|
250
|
+
formByElement.set(index, {
|
|
307
251
|
method: /^[a-z]{1,20}$/.test(rawMethod) ? rawMethod : 'other',
|
|
308
252
|
action: formActionClass(attributes.get('action'), baseUrl),
|
|
309
253
|
controls: {},
|
|
310
|
-
};
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
for (const element of analysis.elements) {
|
|
258
|
+
const name = element.name;
|
|
259
|
+
if (!element.html || !['button', 'input', 'select', 'textarea'].includes(name))
|
|
311
260
|
continue;
|
|
261
|
+
const attributes = new Map(element.attributes.map(({ name, value }) => [name, value]));
|
|
262
|
+
let owner = attributes.has('form') ? firstElementById.get(attributes.get('form')) ?? null : element.parent;
|
|
263
|
+
if (!attributes.has('form')) {
|
|
264
|
+
while (owner !== null && !formByElement.has(owner))
|
|
265
|
+
owner = analysis.elements[owner]?.parent ?? null;
|
|
312
266
|
}
|
|
313
|
-
|
|
267
|
+
const current = owner === null ? undefined : formByElement.get(owner);
|
|
268
|
+
if (!current)
|
|
314
269
|
continue;
|
|
315
270
|
if (controls >= MAX_FORM_CONTROLS) {
|
|
316
271
|
controlLimitReached = true;
|
|
317
272
|
continue;
|
|
318
273
|
}
|
|
319
274
|
controls += 1;
|
|
320
|
-
const attributes = parseAttributes(token).values;
|
|
321
275
|
const rawType = name === 'input' || name === 'button'
|
|
322
276
|
? normalizeText(attributes.get('type') || (name === 'input' ? 'text' : 'submit'), 30)
|
|
323
277
|
: name;
|
|
324
278
|
const type = /^[a-z0-9-]{1,30}$/.test(rawType) ? `${name}:${rawType}` : `${name}:other`;
|
|
325
279
|
current.controls[type] = (current.controls[type] || 0) + 1;
|
|
326
280
|
}
|
|
327
|
-
|
|
328
|
-
forms.push(current);
|
|
329
|
-
if (forms.length > MAX_FORM_FINGERPRINTS) {
|
|
330
|
-
forms.length = MAX_FORM_FINGERPRINTS;
|
|
331
|
-
formLimitReached = true;
|
|
332
|
-
}
|
|
281
|
+
const forms = [...formByElement.values()];
|
|
333
282
|
for (const form of forms)
|
|
334
283
|
form.controls = Object.fromEntries(Object.entries(form.controls).sort(([left], [right]) => left.localeCompare(right)));
|
|
335
284
|
return forms.length ? {
|
|
@@ -337,7 +286,7 @@ function formStructureFingerprint(markup, baseUrl) {
|
|
|
337
286
|
value: sha256(JSON.stringify(forms)),
|
|
338
287
|
formCount: forms.length,
|
|
339
288
|
controlCount: controls,
|
|
340
|
-
truncated: formLimitReached || controlLimitReached,
|
|
289
|
+
truncated: formLimitReached || controlLimitReached || analysis.tagLimitReached || analysis.inputLimitReached,
|
|
341
290
|
} : null;
|
|
342
291
|
}
|
|
343
292
|
function normalizedResourceHosts(resources) {
|
|
@@ -418,21 +367,25 @@ function createPageFingerprints(html, options = {}) {
|
|
|
418
367
|
};
|
|
419
368
|
if (options.sourceTruncated === true && exact.scope === 'complete-body')
|
|
420
369
|
exact = { ...exact, scope: 'captured-prefix' };
|
|
421
|
-
const
|
|
422
|
-
|
|
370
|
+
const parsedStructure = !source.truncated && options.htmlAnalysis
|
|
371
|
+
? options.htmlAnalysis : analyzeStaticHtml(source.text, { baseUrl, includeVisibleText: true });
|
|
372
|
+
const documentIncomplete = options.sourceTruncated === true || source.truncated
|
|
373
|
+
|| parsedStructure.inputLimitReached || parsedStructure.tagLimitReached;
|
|
374
|
+
const treeIncomplete = documentIncomplete || parsedStructure.structureLimitReached;
|
|
375
|
+
const resolutionBase = parsedStructure.effectiveBaseUrl ?? baseUrl;
|
|
376
|
+
const normalized = normalizedMarkupAndStructure(parsedStructure, resolutionBase);
|
|
423
377
|
const text = visibleTextFingerprint(normalized.visibleText);
|
|
424
378
|
const structureSimilarity = simHash64(parsedStructure.structureTokens);
|
|
425
|
-
const forms = formStructureFingerprint(
|
|
379
|
+
const forms = formStructureFingerprint(parsedStructure, resolutionBase);
|
|
426
380
|
const resourceHosts = normalizedResourceHosts(options.resources);
|
|
427
381
|
const identifiers = normalizedIdentifiers(options.trackingIdentifiers, options.identifiersTruncated === true);
|
|
428
|
-
const truncated =
|
|
429
|
-
|| normalized.structureTruncated ||
|
|
430
|
-
|| parsedStructure.structureLimitReached || text?.truncated === true || forms?.truncated === true
|
|
382
|
+
const truncated = treeIncomplete || normalized.normalizedTruncated
|
|
383
|
+
|| normalized.structureTruncated || text?.truncated === true || forms?.truncated === true
|
|
431
384
|
|| resourceHosts.truncated || identifiers.truncated;
|
|
432
385
|
const limitations = [
|
|
433
386
|
'Fingerprints summarise capped static HTML and are comparison aids, not cryptographic proof of page authorship or intent.',
|
|
434
387
|
'Visible-text SimHash is a fuzzy similarity fingerprint and must not be treated as a cryptographic digest.',
|
|
435
|
-
'DOM-structure
|
|
388
|
+
'DOM-structure and form fingerprints describe parsed ancestry and explicit form references, not every browser form-owner repair, executed behaviour or proof of a shared author.',
|
|
436
389
|
];
|
|
437
390
|
if (source.truncated)
|
|
438
391
|
limitations.push(`Fingerprint input was capped at ${MAX_FINGERPRINT_SOURCE_BYTES} UTF-8 bytes.`);
|
|
@@ -453,24 +406,24 @@ function createPageFingerprints(html, options = {}) {
|
|
|
453
406
|
algorithm: 'sha256',
|
|
454
407
|
value: sha256(normalized.normalizedTokens.join('\n')),
|
|
455
408
|
tokenCount: normalized.normalizedTokens.length,
|
|
456
|
-
truncated: normalized.normalizedTruncated,
|
|
409
|
+
truncated: treeIncomplete || normalized.normalizedTruncated,
|
|
457
410
|
},
|
|
458
|
-
visibleText: text,
|
|
411
|
+
visibleText: text ? { ...text, truncated: treeIncomplete || text.truncated } : null,
|
|
459
412
|
domStructure: {
|
|
460
413
|
algorithm: 'sha256',
|
|
461
414
|
value: sha256(normalized.structureTokens.join('\n')),
|
|
462
415
|
nodeCount: normalized.structureTokens.length,
|
|
463
|
-
parser:
|
|
464
|
-
truncated: normalized.structureTruncated,
|
|
416
|
+
parser: PAGE_FINGERPRINT_PARSERS[PAGE_FINGERPRINT_VERSION],
|
|
417
|
+
truncated: treeIncomplete || normalized.structureTruncated,
|
|
465
418
|
similarity: structureSimilarity ? {
|
|
466
419
|
algorithm: 'simhash64-v1',
|
|
467
420
|
value: structureSimilarity.value,
|
|
468
421
|
tokenCount: parsedStructure.structureTokens.length,
|
|
469
422
|
featureCount: structureSimilarity.featureCount,
|
|
470
|
-
truncated:
|
|
423
|
+
truncated: treeIncomplete,
|
|
471
424
|
} : null,
|
|
472
425
|
},
|
|
473
|
-
formStructure: forms,
|
|
426
|
+
formStructure: forms ? { ...forms, truncated: documentIncomplete || forms.truncated } : null,
|
|
474
427
|
resourceHosts,
|
|
475
428
|
identifiers,
|
|
476
429
|
complete: !truncated,
|
|
@@ -24,13 +24,17 @@ function addFinding(findings, role, label, confidence, evidence) {
|
|
|
24
24
|
}
|
|
25
25
|
function analyzePageRole(input = {}) {
|
|
26
26
|
const htmlAnalysis = input.htmlAnalysis ?? analyzeStaticHtml(input.html);
|
|
27
|
-
const
|
|
27
|
+
const elements = htmlAnalysis.elements.filter((element) => element.html && element.name !== 'template');
|
|
28
|
+
// Reuse native element evidence instead of allocating and scanning a second,
|
|
29
|
+
// independently capped reconstruction of the same document.
|
|
30
|
+
const marker = (pattern) => elements.some((element) => pattern.test(element.name)
|
|
31
|
+
|| element.attributes.some(({ name, value }) => pattern.test(name) || pattern.test(value.toLowerCase())));
|
|
28
32
|
const title = boundedLowercase(input.pageTitle, MAX_ROLE_TITLE_CHARS);
|
|
29
33
|
const activityStatus = boundedLowercase(input.activityStatus, 40);
|
|
30
34
|
const forms = htmlAnalysis.forms;
|
|
31
35
|
const findings = [];
|
|
32
36
|
const challengeEvidence = [];
|
|
33
|
-
if (/(?:captcha|recaptcha|hcaptcha|turnstile|cf-chl|challenge-form|verify-human)/u
|
|
37
|
+
if (marker(/(?:captcha|recaptcha|hcaptcha|turnstile|cf-chl|challenge-form|verify-human)/u)) {
|
|
34
38
|
challengeEvidence.push('Static challenge or human-verification marker observed');
|
|
35
39
|
}
|
|
36
40
|
if (/(?:just a moment|checking your browser|verify you are human|access denied)/u.test(title)) {
|
|
@@ -43,19 +47,19 @@ function analyzePageRole(input = {}) {
|
|
|
43
47
|
if (forms.categories.username > 0 || forms.categories.email > 0) {
|
|
44
48
|
authenticationEvidence.push('Username or email-purpose input observed');
|
|
45
49
|
}
|
|
46
|
-
if (/(?:login|log-in|signin|sign-in|authentication|current-password|new-password)/u
|
|
50
|
+
if (marker(/(?:login|log-in|signin|sign-in|authentication|current-password|new-password)/u)) {
|
|
47
51
|
authenticationEvidence.push('Static authentication marker observed');
|
|
48
52
|
}
|
|
49
53
|
addFinding(findings, 'authentication', 'Authentication', forms.categories.password > 0 ? 'high' : 'medium', authenticationEvidence);
|
|
50
54
|
const commerceEvidence = [];
|
|
51
55
|
if (forms.categories.payment > 0)
|
|
52
56
|
commerceEvidence.push('Payment-purpose input observed');
|
|
53
|
-
if (/(?:checkout|shopping-cart|add-to-cart|product-price|woocommerce|shopify-payment)/u
|
|
57
|
+
if (marker(/(?:checkout|shopping-cart|add-to-cart|product-price|woocommerce|shopify-payment)/u)) {
|
|
54
58
|
commerceEvidence.push('Static commerce or checkout marker observed');
|
|
55
59
|
}
|
|
56
60
|
addFinding(findings, 'commerce', 'Commerce', forms.categories.payment > 0 ? 'high' : 'medium', commerceEvidence);
|
|
57
61
|
const supportEvidence = [];
|
|
58
|
-
if (/(?:contact|support|helpdesk|customer-service)/u
|
|
62
|
+
if (marker(/(?:contact|support|helpdesk|customer-service)/u)) {
|
|
59
63
|
supportEvidence.push('Static support or contact marker observed');
|
|
60
64
|
}
|
|
61
65
|
if (forms.formsObserved > 0 && forms.categories.password === 0 && forms.categories.payment === 0) {
|
|
@@ -65,14 +69,14 @@ function analyzePageRole(input = {}) {
|
|
|
65
69
|
const parkedEvidence = [];
|
|
66
70
|
if (activityStatus === 'parked')
|
|
67
71
|
parkedEvidence.push('Existing activity analysis classified the page as parked');
|
|
68
|
-
if (/(?:domain-for-sale|buy-this-domain|parking-page|sedoparking)/u
|
|
72
|
+
if (marker(/(?:domain-for-sale|buy-this-domain|parking-page|sedoparking)/u)) {
|
|
69
73
|
parkedEvidence.push('Static domain-sale or parking marker observed');
|
|
70
74
|
}
|
|
71
75
|
addFinding(findings, 'parked_sale', 'Parked or for sale', activityStatus === 'parked' ? 'high' : 'medium', parkedEvidence);
|
|
72
76
|
const contentEvidence = [];
|
|
73
|
-
if (
|
|
77
|
+
if (elements.some(({ name }) => name === 'article' || name === 'main'))
|
|
74
78
|
contentEvidence.push('Semantic article or main-content element observed');
|
|
75
|
-
if (
|
|
79
|
+
if (elements.some(({ name }) => name === 'h1' || name === 'h2'))
|
|
76
80
|
contentEvidence.push('Static heading structure observed');
|
|
77
81
|
addFinding(findings, 'content', 'Content or publication', contentEvidence.length > 1 ? 'medium' : 'low', contentEvidence);
|
|
78
82
|
if (!findings.length) {
|
|
@@ -1,34 +1,2 @@
|
|
|
1
|
-
//
|
|
2
|
-
|
|
3
|
-
// prevents Node and browser consumers from drifting into different matches.
|
|
4
|
-
export const PERCEPTUAL_HASH_HEX_RE = /^[0-9a-f]{16}$/u;
|
|
5
|
-
export const MIN_INFORMATIVE_HASH_BITS = 10;
|
|
6
|
-
const POPCOUNT = new Uint8Array(16);
|
|
7
|
-
for (let index = 0; index < POPCOUNT.length; index += 1) {
|
|
8
|
-
POPCOUNT[index] = (index & 1) + ((index >> 1) & 1) + ((index >> 2) & 1) + ((index >> 3) & 1);
|
|
9
|
-
}
|
|
10
|
-
export function isPerceptualHash(value) {
|
|
11
|
-
return typeof value === 'string' && PERCEPTUAL_HASH_HEX_RE.test(value);
|
|
12
|
-
}
|
|
13
|
-
function popcountHex(hex) {
|
|
14
|
-
let bits = 0;
|
|
15
|
-
for (let index = 0; index < hex.length; index += 1) {
|
|
16
|
-
bits += POPCOUNT[parseInt(hex.charAt(index), 16)] ?? 0;
|
|
17
|
-
}
|
|
18
|
-
return bits;
|
|
19
|
-
}
|
|
20
|
-
export function isInformativePerceptualHash(value) {
|
|
21
|
-
if (!isPerceptualHash(value))
|
|
22
|
-
return false;
|
|
23
|
-
const bits = popcountHex(value);
|
|
24
|
-
return bits >= MIN_INFORMATIVE_HASH_BITS && bits <= 64 - MIN_INFORMATIVE_HASH_BITS;
|
|
25
|
-
}
|
|
26
|
-
export function hammingDistanceHex(left, right) {
|
|
27
|
-
if (!isPerceptualHash(left) || !isPerceptualHash(right))
|
|
28
|
-
return null;
|
|
29
|
-
let distance = 0;
|
|
30
|
-
for (let index = 0; index < 16; index += 1) {
|
|
31
|
-
distance += POPCOUNT[parseInt(left.charAt(index), 16) ^ parseInt(right.charAt(index), 16)] ?? 0;
|
|
32
|
-
}
|
|
33
|
-
return distance;
|
|
34
|
-
}
|
|
1
|
+
// Retained public import; the runtime-neutral implementation lives in the domain layer.
|
|
2
|
+
export * from "../packages/analysis/perceptual-hash-comparison.mjs";
|