@slicedearth/whoisleuth-cli 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +1 -1
- package/NOTICE +2 -2
- package/cli/argument-values.mjs +55 -0
- package/cli/arguments.mjs +24 -470
- package/cli/artifact-validation/case-response.mjs +67 -9
- package/cli/artifact-validation/lookup-evidence.mjs +18 -4
- package/cli/artifact-validation/signed-review.mjs +61 -64
- package/cli/artifact-verify.mjs +11 -2
- package/cli/assurance-arguments.mjs +56 -0
- package/cli/assurance-command-definitions.mjs +179 -0
- package/cli/assurance-command-runner.mjs +1 -1
- package/cli/bounded-json.mjs +1 -1
- package/cli/bulk-checkpoint.mjs +1 -1
- package/cli/bulk-command-runner.mjs +6 -8
- package/cli/bulk.mjs +2 -3
- package/cli/case-command.mjs +99 -6
- package/cli/case-pack.mjs +19 -4
- package/cli/ci-report.mjs +6 -3
- package/cli/collection-arguments.mjs +123 -0
- package/cli/collection-command-definitions.mjs +330 -0
- package/cli/collection-preflight.mjs +8 -1
- package/cli/command-argument-grammar.mjs +8 -210
- package/cli/command-definition.mjs +330 -0
- package/cli/command-reference.mjs +55 -1388
- package/cli/command-seeds.mjs +18 -0
- package/cli/compare.mjs +1 -1
- package/cli/config-profile.mjs +1 -1
- package/cli/ct-event-intake.mjs +4 -4
- package/cli/discover.mjs +2 -2
- package/cli/discovery-observation-snapshot.mjs +2 -2
- package/cli/discovery-scan-command-runner.mjs +7 -9
- package/cli/discovery-scan.mjs +1 -1
- package/cli/discovery-snapshot.mjs +2 -2
- package/cli/discovery-workflow.mjs +4 -4
- package/cli/domain-control-monitor.mjs +6 -6
- package/cli/domain-control-observations.mjs +2 -1
- package/cli/domain-feed.mjs +61 -0
- package/cli/errors.mjs +5 -6
- package/cli/evidence-arguments.mjs +46 -0
- package/cli/evidence-command-definitions.mjs +69 -0
- package/cli/evidence-command-runner.mjs +2 -3
- package/cli/evidence-signing.mjs +3 -2
- package/cli/external-observation-mapping.mjs +2 -2
- package/cli/formatters/json.mjs +1 -13
- package/cli/formatters/terminal-command-formats.mjs +6 -0
- package/cli/formatters/terminal-lookup.mjs +6 -0
- package/cli/history-arguments.mjs +39 -0
- package/cli/history-command-definitions.mjs +88 -0
- package/cli/indicator-set-command.mjs +32 -0
- package/cli/inline-command-families.mjs +6 -47
- package/cli/inline-command-runner.mjs +4 -31
- package/cli/input.mjs +2 -2
- package/cli/intake-command.mjs +147 -0
- package/cli/interchange-report.mjs +4 -2
- package/cli/investigation-artifacts.mjs +3 -4
- package/cli/investigation-package-review.mjs +2 -0
- package/cli/investigation-plan.mjs +32 -154
- package/cli/investigation-recipes.mjs +177 -0
- package/cli/investigation-run.mjs +78 -12
- package/cli/local-mmdb-review.mjs +220 -17
- package/cli/lookalike-calibration.mjs +1 -1
- package/cli/lookup-brief.mjs +2 -2
- package/cli/lookup-command-runner.mjs +19 -35
- package/cli/lookup-diff.mjs +10 -9
- package/cli/lookup-plan.mjs +10 -14
- package/cli/lookup-reconcile.mjs +6 -5
- package/cli/lookup-timeline.mjs +7 -6
- package/cli/mail-header-review.mjs +24 -247
- package/cli/mail-review.mjs +9 -7
- package/cli/network-arguments.mjs +51 -0
- package/cli/network-command-definitions.mjs +130 -0
- package/cli/network-command-runner.mjs +43 -31
- package/cli/offline-evidence-review.mjs +85 -6
- package/cli/posture.mjs +1 -1
- package/cli/registry-fixture-scaffold.mjs +1 -1
- package/cli/retained-artifact-diff.mjs +8 -7
- package/cli/review-arguments.mjs +132 -0
- package/cli/review-command-definitions.mjs +279 -0
- package/cli/review-command-runner.mjs +72 -159
- package/cli/risk-calibration.mjs +14 -12
- package/cli/runner.mjs +20 -32
- package/cli/saved-lookup.mjs +2 -2
- package/cli/selected-input-worker.mjs +73 -0
- package/cli/sharing-review.mjs +5 -64
- package/cli/source-reliability.mjs +2 -2
- package/cli/support-arguments.mjs +49 -0
- package/cli/support-command-definitions.mjs +209 -0
- package/cli/support-command-runner.mjs +4 -4
- package/cli/watchlist-review.mjs +43 -0
- package/cli/workflow-arguments.mjs +100 -0
- package/cli/workflow-command-definitions.mjs +103 -0
- package/cli/workflow-command-runner.mjs +40 -1
- package/docs/cli-reference.md +27 -0
- package/docs/cli.md +322 -4
- package/docs/contextual-reviews.md +178 -0
- package/docs/infrastructure-observations.md +126 -0
- package/docs/offline-intake-context.md +213 -0
- package/lib/analyst-taxonomy.mjs +2 -40
- package/lib/application-version.mjs +1 -1
- package/lib/availability.mjs +13 -789
- package/lib/bounded-contract-normalizers.mjs +4 -0
- package/lib/bounded-json.mjs +2 -330
- package/lib/browser-library-profile.mjs +62 -84
- package/lib/candidate-provenance-bounds.mjs +2 -0
- package/lib/classify.mjs +1 -1
- package/lib/credential-form-attribution.mjs +136 -0
- package/lib/credential-surface-profile.mjs +6 -2
- package/lib/ct-query.mjs +2 -27
- package/lib/ct-response-bounds.mjs +2 -9
- package/lib/ct-search.mjs +83 -47
- package/lib/dmarc-discovery.mjs +62 -0
- package/lib/dns-convergence-review.mjs +21 -8
- package/lib/dns-delegation-health.mjs +35 -15
- package/lib/dns-inheritance-review.mjs +53 -63
- package/lib/dns-intelligence.mjs +27 -11
- package/lib/dns-query-coverage.mjs +40 -0
- package/lib/domain-assurance.mjs +1 -1
- package/lib/domain-change-packet.mjs +12 -13
- package/lib/domain-change-review.mjs +32 -14
- package/lib/domain-control-flight-recorder.mjs +103 -27
- package/lib/domain-control-manifest.mjs +25 -17
- package/lib/domain-posture-analysis.mjs +75 -7
- package/lib/domain-posture-parsers.mjs +9 -0
- package/lib/domain-posture.mjs +45 -25
- package/lib/domain-registration.mjs +392 -0
- package/lib/evidence-export-privacy.mjs +27 -17
- package/lib/evidence-export.mjs +3 -3
- package/lib/evidence-report.mjs +5 -1
- package/lib/external-intelligence-risk.mjs +2 -124
- package/lib/favicon.mjs +13 -4
- package/lib/generated/cisa-kev-catalog.mjs +41 -3
- package/lib/generated/registrar-standing-catalogue.mjs +18 -10
- package/lib/generated/retire-browser-catalog.mjs +332 -19
- package/lib/generated/{unicode-confusables-17.mjs → unicode-confusables.mjs} +33 -33
- package/lib/html-signals.mjs +13 -2
- package/lib/http-evidence-bounds.mjs +2 -8
- package/lib/http-intelligence.mjs +13 -1
- package/lib/idn-confusable-policy.mjs +6 -4
- package/lib/idn-confusables.mjs +1 -1
- package/lib/interchange-fidelity-registry.mjs +10 -0
- package/lib/lookup-child-profile-contract.mjs +28 -11
- package/lib/lookup-contract-primitives.mjs +1 -1
- package/lib/lookup-diagnostics.mjs +1 -23
- package/lib/lookup-network-evidence-bounds.mjs +1 -10
- package/lib/lookup-progress.mjs +50 -1
- package/lib/lookup-response-contract.mjs +70 -40
- package/lib/lookup-response.mjs +5 -15
- package/lib/lookup-source-progress.mjs +8 -22
- package/lib/lookup.mjs +1 -1
- package/lib/operation-budget.mjs +4 -0
- package/lib/opportunity-scoring.mjs +2 -107
- package/lib/outbound-identity.mjs +1 -1
- package/lib/page-fingerprints.mjs +18 -1
- package/lib/perceptual-hash-comparison.mjs +2 -34
- package/lib/perceptual-hash.mjs +36 -10
- package/lib/portable-generator.mjs +2 -29
- package/lib/project-metadata.mjs +2 -8
- package/lib/rdap-client.mjs +11 -7
- package/lib/registrable-domain.mjs +2 -56
- package/lib/registrar-standing-contract.mjs +1 -1
- package/lib/registrar-standing.mjs +9 -13
- package/lib/registry-capabilities.mjs +2 -2
- package/lib/registry-capability-catalogue.mjs +27 -19
- package/lib/risk-calibration-summary.mjs +2 -352
- package/lib/risk-scoring.mjs +2 -345
- package/lib/scoring-evidence-quality.mjs +2 -107
- package/lib/semantic-version.mjs +2 -54
- package/lib/service-binding-dns.mjs +46 -2
- package/lib/smtp-transport-review.mjs +48 -19
- package/lib/sslbl-certificates.generated.mjs +142 -139
- package/lib/static-html-analysis.mjs +32 -4
- package/lib/structured-data-identity.mjs +1 -1
- package/lib/technology-evidence-role.mjs +1 -51
- package/lib/threat-intelligence-manifests.mjs +1 -1
- package/lib/threat-intelligence-runtime.mjs +1 -1
- package/lib/threat-intelligence-targets.mjs +1 -1
- package/lib/threat-intelligence-types.mjs +2 -75
- package/lib/tls-intelligence.mjs +30 -5
- package/lib/typosquat-generator.mjs +11 -2
- package/lib/website-enrichment.mjs +404 -0
- package/lib/whois-authority.mjs +43 -5
- package/lib/whois-chain.mjs +12 -3
- package/lib/whois-common-formats.mjs +3 -1
- package/lib/whois-contacts.mjs +13 -2
- package/lib/whois-normalization.mjs +3 -1
- package/lib/whois-parser.mjs +5 -3
- package/package.json +15 -14
- package/packages/analysis/analyst-taxonomy.mjs +40 -0
- package/packages/analysis/bounded-json.mjs +330 -0
- package/packages/analysis/candidate-provenance-bounds.mjs +4 -0
- package/packages/analysis/cli-argument-grammar.mjs +216 -0
- package/packages/analysis/ct-query.mjs +27 -0
- package/packages/analysis/ct-response-bounds.mjs +9 -0
- package/packages/analysis/external-intelligence-risk.mjs +124 -0
- package/packages/analysis/http-evidence-bounds.mjs +8 -0
- package/packages/analysis/lookup-network-evidence-bounds.mjs +10 -0
- package/packages/analysis/opportunity-scoring.mjs +107 -0
- package/packages/analysis/perceptual-hash-comparison.mjs +34 -0
- package/packages/analysis/portable-generator.mjs +29 -0
- package/packages/analysis/project-metadata.mjs +8 -0
- package/packages/analysis/registrable-domain.mjs +56 -0
- package/packages/analysis/risk-calibration-summary.mjs +352 -0
- package/packages/analysis/risk-scoring.mjs +348 -0
- package/packages/analysis/scoring-evidence-quality.mjs +107 -0
- package/packages/analysis/semantic-version.mjs +54 -0
- package/packages/analysis/threat-intelligence-types.mjs +75 -0
- package/packages/cases/case-containment-handoff.mjs +195 -0
- package/packages/cases/case-evidence-links.mjs +151 -0
- package/packages/cases/case-evidence-model.mjs +89 -12
- package/packages/cases/case-evidence-quality.mjs +5 -0
- package/packages/cases/case-export-input.mjs +7 -5
- package/packages/cases/case-incident-context.mjs +52 -0
- package/packages/cases/case-markdown.mjs +11 -0
- package/packages/cases/case-migration-model.mjs +40 -24
- package/packages/cases/case-object-coverage.mjs +33 -0
- package/packages/cases/case-packet-correction.mjs +106 -0
- package/packages/cases/case-recheck-model.mjs +124 -36
- package/packages/cases/case-record-contracts.mjs +1 -1
- package/packages/cases/case-record-core.mjs +1 -1
- package/packages/cases/case-record-operations.mjs +111 -118
- package/packages/cases/case-record-projection.mjs +23 -4
- package/packages/cases/case-record-version-input.mjs +41 -0
- package/packages/cases/case-report.mjs +93 -27
- package/packages/cases/case-requested-evidence.mjs +146 -0
- package/packages/cases/case-response-actions.mjs +159 -1
- package/packages/cases/case-response-model.mjs +16 -1
- package/packages/cases/case-response-object.mjs +119 -0
- package/packages/cases/case-response-outcomes.mjs +108 -20
- package/packages/cases/case-response-packet-render.mjs +35 -12
- package/packages/cases/case-response-packet.mjs +55 -13
- package/packages/cases/case-response-records.mjs +0 -1
- package/packages/cases/case-response-review-inputs.mjs +60 -8
- package/packages/cases/case-storage-model.mjs +1 -1
- package/packages/cases/case-workflow-metadata.mjs +217 -0
- package/packages/cases/case-workflow-migration.mjs +43 -0
- package/packages/cases/http-summary.mjs +6 -1
- package/packages/comparison/favicon-similarity.mjs +1 -1
- package/packages/comparison/page-similarity.mjs +1 -1
- package/packages/comparison/relationship-evidence.mjs +21 -9
- package/packages/comparison/relationship-provenance.mjs +7 -1
- package/packages/contracts/analyst-interchange.mjs +35 -15
- package/packages/contracts/candidate-watch-review.mjs +54 -0
- package/packages/contracts/case-portability.mjs +360 -49
- package/packages/contracts/cli-command-semantics.mjs +5 -0
- package/packages/contracts/cli-grammar.mjs +6 -0
- package/packages/contracts/cli-lookup-brief.mjs +2 -0
- package/packages/contracts/context-review.mjs +25 -0
- package/packages/contracts/document-intake.mjs +11 -0
- package/packages/contracts/domain-control-flight-recorder.mjs +30 -14
- package/packages/contracts/domain-control-manifest.mjs +1 -1
- package/packages/contracts/domain-control-monitor.mjs +13 -2
- package/packages/contracts/domain-control-review.mjs +36 -15
- package/packages/contracts/external-observation-interchange.mjs +96 -28
- package/packages/contracts/har-review.mjs +2 -0
- package/packages/contracts/identity-events.mjs +4 -0
- package/packages/contracts/investigation-run.mjs +2 -0
- package/packages/contracts/ip-address.mjs +10 -0
- package/packages/contracts/lookup-evidence.mjs +3 -2
- package/packages/contracts/mail-authentication.mjs +5 -0
- package/packages/contracts/message-intake.mjs +37 -0
- package/packages/contracts/offline-comparison.mjs +4 -1
- package/packages/contracts/passive-mail.mjs +10 -0
- package/packages/contracts/risk-calibration.mjs +44 -29
- package/packages/contracts/schema-lifecycle.mjs +3 -1
- package/packages/contracts/web-capture.mjs +2 -2
- package/packages/contracts/workspace-portability.mjs +144 -28
- package/packages/evidence/collection-quality.mjs +67 -0
- package/packages/evidence/credential-form-attribution.mjs +60 -0
- package/packages/evidence/dns-query-outcome.mjs +72 -0
- package/packages/evidence/email-recipient.mjs +18 -0
- package/packages/evidence/image-regions.mjs +15 -0
- package/packages/evidence/lookup-target.mjs +5 -1
- package/packages/evidence/observation.mjs +2 -0
- package/packages/evidence/png-pixels.mjs +117 -0
- package/packages/evidence/public-address-policy.mjs +83 -0
- package/packages/evidence/record-identity.mjs +60 -16
- package/packages/evidence/technology-evidence-role.mjs +51 -0
- package/packages/interchange/defensive-indicator-export.mjs +264 -0
- package/packages/interchange/external-findings-import.mjs +36 -29
- package/packages/interchange/json-output.mjs +14 -0
- package/packages/interchange/managed-indicator-export.mjs +36 -0
- package/packages/interchange/managed-indicator-set.mjs +223 -0
- package/packages/interchange/misp-indicator-export.mjs +93 -0
- package/packages/interchange/sharing-policy.mjs +56 -0
- package/packages/interchange/stix-indicator-export.mjs +129 -0
- package/packages/interchange/web-capture-import.mjs +22 -15
- package/packages/investigation/acquisition-decision-packet.mjs +79 -1
- package/packages/investigation/bulk-mail-exposure.mjs +7 -13
- package/packages/investigation/capture-coverage.mjs +57 -0
- package/packages/investigation/connector-provenance-review.mjs +100 -0
- package/packages/investigation/context-review.mjs +35 -0
- package/packages/investigation/document-intake.mjs +87 -0
- package/packages/investigation/docx-intake.mjs +133 -0
- package/packages/investigation/domain-history-review.mjs +132 -0
- package/packages/investigation/domain-transition-review.mjs +49 -0
- package/packages/investigation/evidence-coverage-ledger.mjs +9 -1
- package/packages/investigation/har-intake.mjs +68 -0
- package/packages/investigation/identity-event-intake.mjs +131 -0
- package/packages/investigation/identity-incident-review.mjs +39 -0
- package/packages/investigation/incident-sequence-review.mjs +58 -0
- package/packages/investigation/infrastructure-observation.mjs +234 -0
- package/packages/investigation/intake-context.mjs +284 -0
- package/packages/investigation/intake-indicators.mjs +116 -0
- package/packages/investigation/intake-phones.mjs +155 -0
- package/packages/investigation/intake-report.mjs +20 -0
- package/packages/investigation/investigation-manifest.mjs +35 -9
- package/packages/investigation/link-intake.mjs +209 -0
- package/packages/investigation/lookup-asset-graph.mjs +1 -1
- package/packages/investigation/mail-authentication-review.mjs +222 -0
- package/packages/investigation/mail-header-identity.mjs +209 -0
- package/packages/investigation/message-intake.mjs +235 -0
- package/packages/investigation/page-behaviour.mjs +113 -0
- package/packages/investigation/pdf-intake.mjs +155 -0
- package/packages/investigation/platform-continuity-review.mjs +60 -0
- package/packages/investigation/qr-intake.mjs +56 -0
- package/packages/investigation/requested-action-hints.mjs +11 -0
- package/packages/investigation/selected-input-review.mjs +17 -0
- package/packages/investigation/storefront-review.mjs +70 -0
- package/packages/monitoring/domain-feed.mjs +321 -0
- package/packages/workspace/brand-candidate-workflow.mjs +351 -0
- package/packages/workspace/brand-keyword-campaign.mjs +231 -0
- package/packages/workspace/brand-profile-model.mjs +48 -4
- package/packages/workspace/bulk-review-model.mjs +28 -26
- package/packages/workspace/bulk-session-model.mjs +27 -13
- package/packages/workspace/campaign-model.mjs +1 -1
- package/packages/workspace/candidate-watch-handoff.mjs +195 -0
- package/packages/workspace/detection-rule-model.mjs +11 -3
- package/packages/workspace/hostile-input.mjs +1 -1
- package/packages/workspace/investigation-guide.mjs +2 -2
- package/packages/workspace/investigation-template-model.mjs +26 -2
- package/packages/workspace/page-baseline.mjs +2 -2
- package/packages/workspace/relationship-observation-model.mjs +17 -13
- package/packages/workspace/shortlist-model.mjs +24 -7
- package/packages/workspace/watchlist-history.mjs +59 -16
- package/packages/workspace/watchlist-store.mjs +152 -11
- package/packages/workspace/website-snapshot-model.mjs +28 -3
- package/packages/workspace/workspace-archive-crypto.mjs +1 -1
- package/packages/workspace/workspace-archive.mjs +5 -4
- package/third-party-notices.txt +549 -16
- package/lib/hostname.mjs +0 -2
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { normalizeDomain
|
|
2
|
-
import {
|
|
1
|
+
import { normalizeDomain } from "../evidence/domain-name.mjs";
|
|
2
|
+
import { openOrCreateCase, updateCase } from "../cases/case-record-operations.mjs";
|
|
3
|
+
import { canonicalRegistrableDomain } from "../analysis/registrable-domain.mjs";
|
|
3
4
|
import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
|
|
4
5
|
import { canonicalArtifactJsonV2 } from "../evidence/artifact-integrity.mjs";
|
|
5
6
|
import { sha256IdentityHex } from "../evidence/record-identity.mjs";
|
|
6
7
|
import { MAX_RESPONSE_LABEL_LENGTH, MAX_RESPONSE_VALUE_LENGTH, MAX_RESPONSE_LIMITATION_LENGTH, normalizeCaseEvidencePins, normalizeCaseSightings, } from "../cases/case-response-model.mjs";
|
|
7
8
|
import { EXTERNAL_FINDINGS_SCHEMA, EXTERNAL_FINDINGS_VERSION, MAX_EXTERNAL_FINDING_DOMAINS, MAX_EXTERNAL_FINDING_LIMITATIONS, MAX_EXTERNAL_FINDING_LIMITATION_LENGTH, MAX_EXTERNAL_FINDINGS, MAX_EXTERNAL_FINDINGS_IMPORT_BYTES, MAX_EXTERNAL_FINDINGS_PER_DOMAIN, } from "../contracts/external-observation-interchange.mjs";
|
|
8
9
|
import { hasUnsafeRetainedText } from "./retained-text.mjs";
|
|
10
|
+
import { readInfrastructureObservation } from "../investigation/infrastructure-observation.mjs";
|
|
9
11
|
export { EXTERNAL_FINDINGS_SCHEMA, EXTERNAL_FINDINGS_VERSION, MAX_EXTERNAL_FINDING_DOMAINS, MAX_EXTERNAL_FINDING_LIMITATIONS, MAX_EXTERNAL_FINDING_LIMITATION_LENGTH, MAX_EXTERNAL_FINDINGS, MAX_EXTERNAL_FINDINGS_IMPORT_BYTES, MAX_EXTERNAL_FINDINGS_PER_DOMAIN, };
|
|
10
12
|
export const EXTERNAL_FINDING_CATEGORIES = [
|
|
11
13
|
'certificate',
|
|
@@ -32,6 +34,7 @@ const FINDING_KEYS = new Set([
|
|
|
32
34
|
'limitations',
|
|
33
35
|
'reference',
|
|
34
36
|
'structuredObservation',
|
|
37
|
+
'infrastructureObservation',
|
|
35
38
|
]);
|
|
36
39
|
const STRUCTURED_OBSERVATION_KEYS = new Set([
|
|
37
40
|
'sourceSchema',
|
|
@@ -182,16 +185,7 @@ function structuredObservation(value, index) {
|
|
|
182
185
|
};
|
|
183
186
|
}
|
|
184
187
|
function findingKey(finding, sourceName) {
|
|
185
|
-
return
|
|
186
|
-
finding.domain,
|
|
187
|
-
finding.category,
|
|
188
|
-
finding.evidenceClass,
|
|
189
|
-
finding.summary,
|
|
190
|
-
finding.observedAt,
|
|
191
|
-
finding.completeness,
|
|
192
|
-
sourceName,
|
|
193
|
-
finding.structuredObservation?.eventId ?? '',
|
|
194
|
-
].join('\u0000');
|
|
188
|
+
return canonicalArtifactJsonV2({ sourceName, finding });
|
|
195
189
|
}
|
|
196
190
|
export function parseExternalFindingsDocument(value) {
|
|
197
191
|
const root = record(value);
|
|
@@ -199,8 +193,8 @@ export function parseExternalFindingsDocument(value) {
|
|
|
199
193
|
throw new Error('External findings must use the documented object shape without additional top-level fields.');
|
|
200
194
|
}
|
|
201
195
|
if (root.schema !== EXTERNAL_FINDINGS_SCHEMA
|
|
202
|
-
|| root.schemaVersion !==
|
|
203
|
-
throw new Error(`External findings must use ${EXTERNAL_FINDINGS_SCHEMA} schema version
|
|
196
|
+
|| (root.schemaVersion !== 4 && root.schemaVersion !== 5)) {
|
|
197
|
+
throw new Error(`External findings must use ${EXTERNAL_FINDINGS_SCHEMA} schema version 4 or 5.`);
|
|
204
198
|
}
|
|
205
199
|
const sourceValue = record(root.source);
|
|
206
200
|
if (!sourceValue || !hasOnlyKeys(sourceValue, SOURCE_KEYS)) {
|
|
@@ -215,6 +209,7 @@ export function parseExternalFindingsDocument(value) {
|
|
|
215
209
|
throw new Error(`External findings must contain between 1 and ${MAX_EXTERNAL_FINDINGS} findings.`);
|
|
216
210
|
}
|
|
217
211
|
const normalized = [];
|
|
212
|
+
let retainedBytes = new TextEncoder().encode(JSON.stringify({ schema: root.schema, schemaVersion: root.schemaVersion, source, findings: [] })).byteLength + 1;
|
|
218
213
|
const seen = new Set();
|
|
219
214
|
const domainCounts = new Map();
|
|
220
215
|
for (const [index, raw] of root.findings.entries()) {
|
|
@@ -236,7 +231,7 @@ export function parseExternalFindingsDocument(value) {
|
|
|
236
231
|
if (evidenceClass !== 'deployment_observation' && evidenceClass !== 'provider_report') {
|
|
237
232
|
throw new Error(`Finding ${index + 1} evidence class is unsupported.`);
|
|
238
233
|
}
|
|
239
|
-
|
|
234
|
+
let finding = {
|
|
240
235
|
domain,
|
|
241
236
|
category: item.category,
|
|
242
237
|
evidenceClass,
|
|
@@ -247,6 +242,21 @@ export function parseExternalFindingsDocument(value) {
|
|
|
247
242
|
reference: optionalText(item.reference, 500, `Finding ${index + 1} reference`),
|
|
248
243
|
structuredObservation: structuredObservation(item.structuredObservation, index),
|
|
249
244
|
};
|
|
245
|
+
if (Object.hasOwn(item, 'infrastructureObservation')) {
|
|
246
|
+
if (root.schemaVersion !== 5)
|
|
247
|
+
throw new TypeError('Infrastructure observations require external-findings version 5.');
|
|
248
|
+
const infrastructureObservation = readInfrastructureObservation(item.infrastructureObservation);
|
|
249
|
+
if (infrastructureObservation.target !== domain || infrastructureObservation.observedAt !== finding.observedAt)
|
|
250
|
+
throw new TypeError('Infrastructure snapshot must match its finding target and observation time.');
|
|
251
|
+
if (finding.structuredObservation)
|
|
252
|
+
throw new TypeError('Infrastructure snapshots cannot also declare a single structured observation.');
|
|
253
|
+
finding = { ...finding, infrastructureObservation };
|
|
254
|
+
}
|
|
255
|
+
if (root.schemaVersion === 5) {
|
|
256
|
+
retainedBytes += new TextEncoder().encode(JSON.stringify(finding)).byteLength + 1;
|
|
257
|
+
if (retainedBytes > MAX_EXTERNAL_FINDINGS_IMPORT_BYTES)
|
|
258
|
+
throw new TypeError('External findings exceed the retained aggregate byte bound.');
|
|
259
|
+
}
|
|
250
260
|
const key = findingKey(finding, source.name);
|
|
251
261
|
if (seen.has(key))
|
|
252
262
|
continue;
|
|
@@ -265,12 +275,14 @@ export function parseExternalFindingsDocument(value) {
|
|
|
265
275
|
throw new Error('External findings did not contain a unique valid finding.');
|
|
266
276
|
return {
|
|
267
277
|
schema: EXTERNAL_FINDINGS_SCHEMA,
|
|
268
|
-
schemaVersion:
|
|
278
|
+
schemaVersion: root.schemaVersion,
|
|
269
279
|
source,
|
|
270
280
|
findings: normalized,
|
|
271
281
|
};
|
|
272
282
|
}
|
|
273
283
|
export function serializeExternalFindingsDocument(value) {
|
|
284
|
+
if (record(value)?.schemaVersion === 5)
|
|
285
|
+
return `${JSON.stringify(parseExternalFindingsDocument(value))}\n`;
|
|
274
286
|
const serialised = JSON.stringify(value, null, 2);
|
|
275
287
|
if (typeof serialised !== 'string') {
|
|
276
288
|
throw new TypeError('External findings must be JSON-serialisable.');
|
|
@@ -302,6 +314,12 @@ function alreadyRetained(recordValue, projection) {
|
|
|
302
314
|
});
|
|
303
315
|
}
|
|
304
316
|
function structuredPinFields(finding) {
|
|
317
|
+
if (finding.infrastructureObservation)
|
|
318
|
+
return {
|
|
319
|
+
field: 'infrastructure.snapshot', category: 'infrastructure',
|
|
320
|
+
sourceSchema: { collection: 'external_observations', schema: finding.infrastructureObservation.schema, version: finding.infrastructureObservation.version },
|
|
321
|
+
infrastructureObservation: finding.infrastructureObservation,
|
|
322
|
+
};
|
|
305
323
|
const observation = finding.structuredObservation;
|
|
306
324
|
if (!observation)
|
|
307
325
|
return {};
|
|
@@ -356,6 +374,7 @@ export function externalFindingCaseProjection(finding, source) {
|
|
|
356
374
|
const omittedLimitations = finding.limitations.filter((item) => !sourceLimitations.includes(item)).length;
|
|
357
375
|
const pin = normalizeCaseEvidencePins([{
|
|
358
376
|
...structuredPinFields(finding),
|
|
377
|
+
observationHostname: finding.domain,
|
|
359
378
|
importContentSha256: identitySha256,
|
|
360
379
|
label: `External ${finding.category} finding`,
|
|
361
380
|
value,
|
|
@@ -436,17 +455,6 @@ export function mergeExternalFindingsIntoCases(current, document, now = new Date
|
|
|
436
455
|
duplicatesSkipped,
|
|
437
456
|
};
|
|
438
457
|
}
|
|
439
|
-
function targetedFinding(finding, caseDomain) {
|
|
440
|
-
if (finding.domain === caseDomain)
|
|
441
|
-
return finding;
|
|
442
|
-
const prefix = `Captured hostname ${finding.domain}. `;
|
|
443
|
-
const maximumSummaryLength = MAX_EXTERNAL_FINDING_SUMMARY_LENGTH;
|
|
444
|
-
const remaining = maximumSummaryLength - prefix.length;
|
|
445
|
-
const summary = remaining > 1 && finding.summary.length > remaining
|
|
446
|
-
? `${prefix}${finding.summary.slice(0, remaining - 1).trimEnd()}…`
|
|
447
|
-
: `${prefix}${finding.summary}`;
|
|
448
|
-
return { ...finding, domain: caseDomain, summary };
|
|
449
|
-
}
|
|
450
458
|
export function externalFindingsCaseTargets(document, caseDomain) {
|
|
451
459
|
if (canonicalRegistrableDomain(caseDomain) !== caseDomain) {
|
|
452
460
|
throw new Error('The selected Case does not have a canonical registrable-domain identity.');
|
|
@@ -476,8 +484,7 @@ export function mergeExternalFindingsIntoCase(current, caseId, document, now = n
|
|
|
476
484
|
let record = selected;
|
|
477
485
|
let findingsAdded = 0;
|
|
478
486
|
let duplicatesSkipped = 0;
|
|
479
|
-
for (const
|
|
480
|
-
const finding = targetedFinding(original, selected.domain);
|
|
487
|
+
for (const finding of document.findings) {
|
|
481
488
|
const merged = mergeExternalFindingIntoCase(cases, selected.id, finding, document.source, now);
|
|
482
489
|
cases = merged.cases;
|
|
483
490
|
record = merged.record;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
const TERMINAL_UNSAFE_JSON_RE = /[\u007f-\u009f\u2028\u2029]|\p{Default_Ignorable_Code_Point}/gu;
|
|
2
|
+
function jsonUnicodeEscape(value) {
|
|
3
|
+
const codePoint = value.codePointAt(0);
|
|
4
|
+
if (codePoint <= 0xffff)
|
|
5
|
+
return `\\u${codePoint.toString(16).padStart(4, '0')}`;
|
|
6
|
+
const offset = codePoint - 0x10000;
|
|
7
|
+
const high = 0xd800 + (offset >> 10);
|
|
8
|
+
const low = 0xdc00 + (offset & 0x3ff);
|
|
9
|
+
return `\\u${high.toString(16).padStart(4, '0')}\\u${low.toString(16).padStart(4, '0')}`;
|
|
10
|
+
}
|
|
11
|
+
/** Escape display controls without changing the parsed JSON values. */
|
|
12
|
+
export function terminalSafeJson(value, space) {
|
|
13
|
+
return JSON.stringify(value, null, space).replace(TERMINAL_UNSAFE_JSON_RE, jsonUnicodeEscape);
|
|
14
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { readManagedIndicatorSet, serializeManagedIndicatorSet } from "./managed-indicator-set.mjs";
|
|
2
|
+
import { terminalSafeJson } from "./json-output.mjs";
|
|
3
|
+
import { stixIndicatorObjects, stixIndicatorProducer } from "./stix-indicator-export.mjs";
|
|
4
|
+
import { mispIndicatorAttribute, mispIndicatorEvent } from "./misp-indicator-export.mjs";
|
|
5
|
+
/** Re-exporting a revision is deterministic; revoked object versions never advance. */
|
|
6
|
+
export async function exportManagedIndicators(input, format) {
|
|
7
|
+
const manifest = await readManagedIndicatorSet(input);
|
|
8
|
+
let document = manifest;
|
|
9
|
+
if (format === 'stix') {
|
|
10
|
+
const producer = `identity--${manifest.producerId}`;
|
|
11
|
+
document = { type: 'bundle', id: `bundle--${manifest.revisionId}`, objects: [
|
|
12
|
+
stixIndicatorProducer(producer, manifest.createdAt),
|
|
13
|
+
...manifest.entries.flatMap(entry => stixIndicatorObjects(entry.domain, entry.observation.availability, entry.observation, entry.createdAt, producer, type => `${type}--${entry.id}`)
|
|
14
|
+
.map(object => object.type !== 'indicator' ? object : { ...object,
|
|
15
|
+
modified: entry.modifiedAt, valid_until: entry.expiresAt, revoked: entry.withdrawal !== null,
|
|
16
|
+
x_whoisleuth_managed_set: manifest.id, x_whoisleuth_original_review_basis: entry.basis,
|
|
17
|
+
x_whoisleuth_review_basis: entry.reviewBasis,
|
|
18
|
+
...(entry.withdrawal ? { x_whoisleuth_withdrawal_reason: entry.withdrawal.reason } : {}),
|
|
19
|
+
})),
|
|
20
|
+
] };
|
|
21
|
+
}
|
|
22
|
+
else if (format === 'misp') {
|
|
23
|
+
const attributes = manifest.entries.map(entry => {
|
|
24
|
+
const attribute = mispIndicatorAttribute(entry.domain, entry.observation.availability, entry.observation, entry.id, entry.modifiedAt);
|
|
25
|
+
return { ...attribute, deleted: entry.withdrawal !== null,
|
|
26
|
+
comment: `${attribute.comment}; review-expires=${entry.expiresAt}; expiry requires recipient review, not automatic deletion; original-basis=${entry.basis}; latest-review=${entry.reviewBasis}${entry.withdrawal ? `; withdrawn=${entry.withdrawal.reason}` : ''}` };
|
|
27
|
+
});
|
|
28
|
+
document = { Event: { ...mispIndicatorEvent(manifest.id, manifest.createdAt, attributes),
|
|
29
|
+
info: `${manifest.name} · reviewed domain indicators`, timestamp: String(Math.floor(Date.parse(manifest.modifiedAt) / 1000)) } };
|
|
30
|
+
}
|
|
31
|
+
else if (format !== 'manifest')
|
|
32
|
+
throw new TypeError('Unsupported managed indicator export format.');
|
|
33
|
+
return { document, content: format === 'manifest' ? serializeManagedIndicatorSet(manifest) : `${terminalSafeJson(document, 2)}\n`,
|
|
34
|
+
filename: `whoisleuth-indicators-${manifest.id}-r${manifest.revision}.${format}.json`,
|
|
35
|
+
mimeType: format === 'stix' ? 'application/stix+json;charset=utf-8' : 'application/json;charset=utf-8' };
|
|
36
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { array, digest, domain, enumeration, exact, exactOptional, integer, iso, text, validateIntegrity } from "../evidence/artifact-structure.mjs";
|
|
2
|
+
import { canonicalArtifactJsonV2, sha256ArtifactDigestV2, SORTED_JSON_V2 } from "../evidence/artifact-integrity.mjs";
|
|
3
|
+
import { assertWorkspaceInputGraph } from "../workspace/hostile-input.mjs";
|
|
4
|
+
import { parseBoundedJson, boundedJsonLimitsForBytes } from "../analysis/bounded-json.mjs";
|
|
5
|
+
import { MANAGED_INDICATOR_SET_SCHEMA, MANAGED_INDICATOR_SET_VERSION, MAX_MANAGED_INDICATOR_SET_BYTES, MAX_MANAGED_INDICATOR_PLAN_BYTES } from "../contracts/analyst-interchange.mjs";
|
|
6
|
+
import { defensiveIndicatorProvenance, prepareDefensiveIndicatorExport, MAX_DEFENSIVE_INDICATOR_INPUTS } from "./defensive-indicator-export.mjs";
|
|
7
|
+
import { MAX_STIX_INDICATORS } from "./stix-indicator-export.mjs";
|
|
8
|
+
import { terminalSafeJson } from "./json-output.mjs";
|
|
9
|
+
export const MAX_MANAGED_INDICATORS = MAX_STIX_INDICATORS;
|
|
10
|
+
export const MAX_INDICATOR_REVIEW_BASIS = 1_000;
|
|
11
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u;
|
|
12
|
+
const AVAILABILITY = ['registered', 'for_sale', 'expiring'];
|
|
13
|
+
function uuid(value) {
|
|
14
|
+
if (typeof value !== 'string' || !UUID.test(value))
|
|
15
|
+
throw new TypeError('Indicator identities must be lowercase version-4 UUIDs.');
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
function timestamp(value, label) { iso(value, label); return value; }
|
|
19
|
+
function basis(value) {
|
|
20
|
+
const result = text(value, 'Indicator review basis', MAX_INDICATOR_REVIEW_BASIS).trim();
|
|
21
|
+
if (!result)
|
|
22
|
+
throw new TypeError('Explain the indicator revision.');
|
|
23
|
+
return result;
|
|
24
|
+
}
|
|
25
|
+
function bounded(value, maximumBytes) {
|
|
26
|
+
assertWorkspaceInputGraph(value, 'Indicator input', { maximumBytes });
|
|
27
|
+
if (new TextEncoder().encode(JSON.stringify(value)).byteLength > maximumBytes)
|
|
28
|
+
throw new TypeError('Indicator input exceeds its byte limit.');
|
|
29
|
+
}
|
|
30
|
+
/** Structural validation is independent of digest verification and author trust. */
|
|
31
|
+
export function validateManagedIndicatorSet(value) {
|
|
32
|
+
bounded(value, MAX_MANAGED_INDICATOR_SET_BYTES);
|
|
33
|
+
const root = exact(value, ['schema', 'version', 'id', 'producerId', 'revisionId', 'name', 'revision', 'createdAt', 'modifiedAt', 'previous', 'entries', 'integrity'], 'Managed indicator set');
|
|
34
|
+
if (root.schema !== MANAGED_INDICATOR_SET_SCHEMA || root.version !== MANAGED_INDICATOR_SET_VERSION)
|
|
35
|
+
throw new TypeError('Unsupported managed indicator set schema or version.');
|
|
36
|
+
const identities = [uuid(root.id), uuid(root.producerId), uuid(root.revisionId)];
|
|
37
|
+
if (!text(root.name, 'Indicator set name', 120).trim())
|
|
38
|
+
throw new TypeError('Name the indicator set.');
|
|
39
|
+
const revision = integer(root.revision, 'Indicator revision', 1);
|
|
40
|
+
const created = Date.parse(timestamp(root.createdAt, 'Set creation time')), modified = Date.parse(timestamp(root.modifiedAt, 'Set modification time'));
|
|
41
|
+
if (modified < created || (revision === 1 && modified !== created))
|
|
42
|
+
throw new TypeError('Indicator revision times are inconsistent.');
|
|
43
|
+
if (revision === 1) {
|
|
44
|
+
if (root.previous !== null)
|
|
45
|
+
throw new TypeError('An initial indicator revision has no predecessor.');
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
const previous = exact(root.previous, ['revisionId', 'digestSha256'], 'Preceding indicator revision');
|
|
49
|
+
identities.push(uuid(previous.revisionId));
|
|
50
|
+
digest(previous.digestSha256, 'Preceding revision digest');
|
|
51
|
+
}
|
|
52
|
+
const activeDomains = new Set();
|
|
53
|
+
for (const item of array(root.entries, 'Managed indicators', MAX_MANAGED_INDICATORS, 1)) {
|
|
54
|
+
const entry = exact(item, ['id', 'domain', 'createdAt', 'modifiedAt', 'expiresAt', 'basis', 'reviewBasis', 'observation', 'withdrawal'], 'Managed indicator');
|
|
55
|
+
identities.push(uuid(entry.id));
|
|
56
|
+
domain(entry.domain, 'Managed indicator domain');
|
|
57
|
+
const first = Date.parse(timestamp(entry.createdAt, 'Indicator creation time'));
|
|
58
|
+
const last = Date.parse(timestamp(entry.modifiedAt, 'Indicator modification time'));
|
|
59
|
+
const expires = Date.parse(timestamp(entry.expiresAt, 'Indicator review expiry'));
|
|
60
|
+
if (first < created || last < first || last > modified || expires <= first || (entry.withdrawal === null && expires <= last))
|
|
61
|
+
throw new TypeError('Indicator entry times are inconsistent.');
|
|
62
|
+
basis(entry.basis);
|
|
63
|
+
basis(entry.reviewBasis);
|
|
64
|
+
const observation = exact(entry.observation, ['availability', 'riskScore', 'riskModelVersion', 'scanDepth', 'observedAt'], 'Indicator observation');
|
|
65
|
+
enumeration(observation.availability, AVAILABILITY, 'Indicator registration observation');
|
|
66
|
+
if (observation.riskScore !== null)
|
|
67
|
+
integer(observation.riskScore, 'Retained risk score', 0, 100);
|
|
68
|
+
if (observation.riskModelVersion !== null)
|
|
69
|
+
integer(observation.riskModelVersion, 'Retained risk model', 1, 1_000);
|
|
70
|
+
enumeration(observation.scanDepth, ['fast', 'deep', 'unknown'], 'Retained scan depth');
|
|
71
|
+
iso(observation.observedAt, 'Retained observation time', true);
|
|
72
|
+
if (entry.withdrawal === null) {
|
|
73
|
+
if (activeDomains.has(entry.domain))
|
|
74
|
+
throw new TypeError('Only one active identity per domain is allowed in a managed set.');
|
|
75
|
+
activeDomains.add(entry.domain);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
const withdrawal = exact(entry.withdrawal, ['at', 'reason'], 'Indicator withdrawal');
|
|
79
|
+
if (timestamp(withdrawal.at, 'Withdrawal time') !== entry.modifiedAt)
|
|
80
|
+
throw new TypeError('Withdrawal must be the final indicator modification.');
|
|
81
|
+
basis(withdrawal.reason);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (new Set(identities).size !== identities.length)
|
|
85
|
+
throw new TypeError('Managed indicator identities must be distinct.');
|
|
86
|
+
validateIntegrity(root.integrity, 'Managed indicator integrity', root.version, MANAGED_INDICATOR_SET_VERSION);
|
|
87
|
+
return value;
|
|
88
|
+
}
|
|
89
|
+
export async function readManagedIndicatorSet(value) {
|
|
90
|
+
const checked = structuredClone(validateManagedIndicatorSet(value));
|
|
91
|
+
const { integrity, ...unsigned } = checked;
|
|
92
|
+
if (await sha256ArtifactDigestV2(unsigned) !== integrity.digestSha256)
|
|
93
|
+
throw new TypeError('Managed indicator content does not match its recorded digest.');
|
|
94
|
+
return checked;
|
|
95
|
+
}
|
|
96
|
+
export function parseManagedIndicatorJson(raw, plan = false) {
|
|
97
|
+
const maximumBytes = plan ? MAX_MANAGED_INDICATOR_PLAN_BYTES : MAX_MANAGED_INDICATOR_SET_BYTES;
|
|
98
|
+
return parseBoundedJson(raw, { maximumBytes, label: 'Managed indicator input', limits: boundedJsonLimitsForBytes(maximumBytes) });
|
|
99
|
+
}
|
|
100
|
+
/** Both writers enforce the reader's budget on final UTF-8 bytes, including escapes. */
|
|
101
|
+
export function serializeManagedIndicatorSet(manifest) {
|
|
102
|
+
const pretty = `${terminalSafeJson(manifest, 2)}\n`;
|
|
103
|
+
if (new TextEncoder().encode(pretty).byteLength <= MAX_MANAGED_INDICATOR_SET_BYTES)
|
|
104
|
+
return pretty;
|
|
105
|
+
const compact = terminalSafeJson(manifest);
|
|
106
|
+
if (new TextEncoder().encode(compact).byteLength > MAX_MANAGED_INDICATOR_SET_BYTES) {
|
|
107
|
+
throw new TypeError('The indicator revision exceeds the file byte limit after safe JSON encoding; no partial file was created.');
|
|
108
|
+
}
|
|
109
|
+
return compact;
|
|
110
|
+
}
|
|
111
|
+
export function managedIndicatorState(entry, at) {
|
|
112
|
+
iso(at, 'Review time');
|
|
113
|
+
return entry.withdrawal ? 'withdrawn' : Date.parse(entry.expiresAt) <= Date.parse(at) ? 'expired' : 'active';
|
|
114
|
+
}
|
|
115
|
+
/** A known predecessor detects removal, rollback and edits to original observations. */
|
|
116
|
+
export function compareManagedIndicatorRevisions(previous, next) {
|
|
117
|
+
validateManagedIndicatorSet(previous);
|
|
118
|
+
validateManagedIndicatorSet(next);
|
|
119
|
+
if (previous.id !== next.id || previous.producerId !== next.producerId || previous.name !== next.name || previous.createdAt !== next.createdAt
|
|
120
|
+
|| next.revision !== previous.revision + 1 || next.previous?.revisionId !== previous.revisionId || next.previous.digestSha256 !== previous.integrity.digestSha256
|
|
121
|
+
|| Math.floor(Date.parse(next.modifiedAt) / 1000) <= Math.floor(Date.parse(previous.modifiedAt) / 1000)) {
|
|
122
|
+
throw new TypeError('This is not the next revision of the selected indicator set. Keep both files and review the predecessor.');
|
|
123
|
+
}
|
|
124
|
+
const old = new Map(previous.entries.map(entry => [entry.id, entry]));
|
|
125
|
+
const changes = [];
|
|
126
|
+
for (const entry of next.entries) {
|
|
127
|
+
const retained = old.get(entry.id);
|
|
128
|
+
if (!retained) {
|
|
129
|
+
if (entry.createdAt !== next.modifiedAt || entry.modifiedAt !== next.modifiedAt || entry.withdrawal)
|
|
130
|
+
throw new TypeError('A new indicator must start in the new revision.');
|
|
131
|
+
changes.push({ id: entry.id, domain: entry.domain, kind: 'added' });
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
old.delete(entry.id);
|
|
135
|
+
if (canonicalArtifactJsonV2(retained) === canonicalArtifactJsonV2(entry))
|
|
136
|
+
continue;
|
|
137
|
+
if (retained.withdrawal || entry.modifiedAt !== next.modifiedAt || retained.domain !== entry.domain || retained.createdAt !== entry.createdAt
|
|
138
|
+
|| retained.basis !== entry.basis || canonicalArtifactJsonV2(retained.observation) !== canonicalArtifactJsonV2(entry.observation)
|
|
139
|
+
|| (entry.withdrawal && retained.expiresAt !== entry.expiresAt))
|
|
140
|
+
throw new TypeError('A revision cannot rewrite observations or modify a withdrawn identity.');
|
|
141
|
+
changes.push({ id: entry.id, domain: entry.domain, kind: entry.withdrawal ? 'withdrawn' : 'renewed' });
|
|
142
|
+
}
|
|
143
|
+
if (old.size)
|
|
144
|
+
throw new TypeError('A revision cannot remove retained indicator history.');
|
|
145
|
+
if (!changes.length)
|
|
146
|
+
throw new TypeError('No indicator change was selected.');
|
|
147
|
+
return changes;
|
|
148
|
+
}
|
|
149
|
+
export async function buildManagedIndicatorRevision(input, now = new Date().toISOString()) {
|
|
150
|
+
bounded(input, MAX_MANAGED_INDICATOR_PLAN_BYTES);
|
|
151
|
+
const plan = exactOptional(structuredClone(input), ['basis'], ['name', 'previous', 'rows', 'selectedDomains', 'officialDomains', 'allowlistedDomains', 'commonInfrastructureDomains', 'renewIds', 'withdrawIds', 'reintroduceDomains', 'expiresAt'], 'Indicator revision plan');
|
|
152
|
+
const previous = plan.previous === undefined ? null : await readManagedIndicatorSet(plan.previous);
|
|
153
|
+
const generatedAt = timestamp(now, 'Revision time');
|
|
154
|
+
const reviewBasis = basis(plan.basis);
|
|
155
|
+
const name = plan.name === undefined && previous ? previous.name : text(plan.name, 'Indicator set name', 120).trim();
|
|
156
|
+
if (!name || (previous && name !== previous.name))
|
|
157
|
+
throw new TypeError('A revision retains the original set name.');
|
|
158
|
+
function values(key, maximum, read) {
|
|
159
|
+
const result = array(plan[key] ?? [], key, maximum).map(read);
|
|
160
|
+
if (new Set(result).size !== result.length)
|
|
161
|
+
throw new TypeError('Revision selections must not contain duplicates.');
|
|
162
|
+
return result;
|
|
163
|
+
}
|
|
164
|
+
const domains = (key) => values(key, MAX_DEFENSIVE_INDICATOR_INPUTS, item => { domain(item, 'Selected domain'); return item; });
|
|
165
|
+
const selectedDomains = domains('selectedDomains');
|
|
166
|
+
const preflight = prepareDefensiveIndicatorExport(array(plan.rows ?? [], 'Reviewed rows', MAX_DEFENSIVE_INDICATOR_INPUTS), {
|
|
167
|
+
selectedDomains, officialDomains: domains('officialDomains'), allowlistedDomains: domains('allowlistedDomains'), commonInfrastructureDomains: domains('commonInfrastructureDomains'),
|
|
168
|
+
}, MAX_MANAGED_INDICATORS);
|
|
169
|
+
if (preflight.truncated)
|
|
170
|
+
throw new TypeError('The selected revision exceeds the managed-set entry limit; no partial revision was created.');
|
|
171
|
+
const renew = new Set(values('renewIds', MAX_MANAGED_INDICATORS, uuid));
|
|
172
|
+
const withdraw = new Set(values('withdrawIds', MAX_MANAGED_INDICATORS, uuid));
|
|
173
|
+
const reintroduce = new Set(domains('reintroduceDomains'));
|
|
174
|
+
if ([...renew].some(id => withdraw.has(id)))
|
|
175
|
+
throw new TypeError('An indicator cannot be renewed and withdrawn in one revision.');
|
|
176
|
+
const entries = structuredClone(previous?.entries ?? []);
|
|
177
|
+
const known = new Map(entries.map(entry => [entry.id, entry]));
|
|
178
|
+
for (const id of [...renew, ...withdraw])
|
|
179
|
+
if (!known.has(id) || known.get(id).withdrawal)
|
|
180
|
+
throw new TypeError('Select an existing, non-withdrawn indicator identity.');
|
|
181
|
+
const active = new Set(entries.filter(entry => !entry.withdrawal).map(entry => entry.domain));
|
|
182
|
+
const withdrawn = new Set(entries.filter(entry => entry.withdrawal).map(entry => entry.domain));
|
|
183
|
+
const additions = preflight.entries.filter(entry => !active.has(entry.domain));
|
|
184
|
+
if ([...reintroduce].some(value => !withdrawn.has(value) || !additions.some(entry => entry.domain === value)))
|
|
185
|
+
throw new TypeError('Reintroduction must select a withdrawn domain being added with a new identity.');
|
|
186
|
+
for (const entry of additions)
|
|
187
|
+
if (withdrawn.has(entry.domain) && !reintroduce.has(entry.domain))
|
|
188
|
+
throw new TypeError('Explicitly approve a new identity before reintroducing a withdrawn domain.');
|
|
189
|
+
let expiresAt = null;
|
|
190
|
+
if (additions.length || renew.size) {
|
|
191
|
+
expiresAt = timestamp(plan.expiresAt, 'New review expiry');
|
|
192
|
+
if (Date.parse(expiresAt) <= Date.parse(generatedAt))
|
|
193
|
+
throw new TypeError('The new review expiry must be in the future.');
|
|
194
|
+
}
|
|
195
|
+
else if (plan.expiresAt !== undefined)
|
|
196
|
+
timestamp(plan.expiresAt, 'New review expiry');
|
|
197
|
+
for (const entry of entries) {
|
|
198
|
+
if (renew.has(entry.id)) {
|
|
199
|
+
entry.expiresAt = expiresAt;
|
|
200
|
+
entry.modifiedAt = generatedAt;
|
|
201
|
+
entry.reviewBasis = reviewBasis;
|
|
202
|
+
}
|
|
203
|
+
if (withdraw.has(entry.id)) {
|
|
204
|
+
entry.withdrawal = { at: generatedAt, reason: reviewBasis };
|
|
205
|
+
entry.modifiedAt = generatedAt;
|
|
206
|
+
entry.reviewBasis = reviewBasis;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
const randomId = () => globalThis.crypto.randomUUID();
|
|
210
|
+
for (const entry of additions)
|
|
211
|
+
entries.push({ id: randomId(), domain: entry.domain, createdAt: generatedAt, modifiedAt: generatedAt, expiresAt: expiresAt, basis: reviewBasis, reviewBasis,
|
|
212
|
+
observation: { availability: entry.source.availability, ...defensiveIndicatorProvenance(entry.source) }, withdrawal: null });
|
|
213
|
+
if (!additions.length && !renew.size && !withdraw.size)
|
|
214
|
+
throw new TypeError('No eligible addition, renewal or withdrawal was selected.');
|
|
215
|
+
const unsigned = { schema: MANAGED_INDICATOR_SET_SCHEMA, version: MANAGED_INDICATOR_SET_VERSION,
|
|
216
|
+
id: previous?.id ?? randomId(), producerId: previous?.producerId ?? randomId(), revisionId: randomId(), name,
|
|
217
|
+
revision: (previous?.revision ?? 0) + 1, createdAt: previous?.createdAt ?? generatedAt, modifiedAt: generatedAt,
|
|
218
|
+
previous: previous ? { revisionId: previous.revisionId, digestSha256: previous.integrity.digestSha256 } : null, entries };
|
|
219
|
+
const manifest = validateManagedIndicatorSet({ ...unsigned, integrity: { algorithm: 'SHA-256', canonicalization: SORTED_JSON_V2, digestSha256: await sha256ArtifactDigestV2(unsigned) } });
|
|
220
|
+
serializeManagedIndicatorSet(manifest);
|
|
221
|
+
const changes = previous ? compareManagedIndicatorRevisions(previous, manifest) : manifest.entries.map(entry => ({ id: entry.id, domain: entry.domain, kind: 'added' }));
|
|
222
|
+
return { manifest, changes, unchanged: manifest.entries.length - changes.length, exclusions: preflight.exclusions };
|
|
223
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// Pure MISP event JSON for local analyst import. The event stays unpublished,
|
|
2
|
+
// organization-only, non-IDS, and non-correlating until a MISP user reviews it.
|
|
3
|
+
import { collectDefensiveIndicatorCandidates, defensiveIndicatorProvenance, MAX_DEFENSIVE_INDICATORS, } from "./defensive-indicator-export.mjs";
|
|
4
|
+
import { MISP_INDICATOR_EXPORT_VERSION } from "../contracts/analyst-interchange.mjs";
|
|
5
|
+
export { MISP_INDICATOR_EXPORT_VERSION } from "../contracts/analyst-interchange.mjs";
|
|
6
|
+
export const MAX_MISP_ATTRIBUTES = MAX_DEFENSIVE_INDICATORS;
|
|
7
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
8
|
+
const CONTROL_RE = /[\x00-\x1f\x7f]/;
|
|
9
|
+
const WARNING = 'Review before operational use; false positives are possible.';
|
|
10
|
+
function record(value) {
|
|
11
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
12
|
+
? value
|
|
13
|
+
: {};
|
|
14
|
+
}
|
|
15
|
+
function isoTimestamp(value) {
|
|
16
|
+
if (typeof value !== 'string' || value.length > 64 || CONTROL_RE.test(value))
|
|
17
|
+
return null;
|
|
18
|
+
const parsed = Date.parse(value);
|
|
19
|
+
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
|
|
20
|
+
}
|
|
21
|
+
function defaultUuidFactory() {
|
|
22
|
+
const uuid = globalThis.crypto?.randomUUID?.();
|
|
23
|
+
if (!uuid)
|
|
24
|
+
throw new Error('Secure random identifiers are unavailable for the MISP export.');
|
|
25
|
+
return uuid;
|
|
26
|
+
}
|
|
27
|
+
function nextUuid(uuidFactory, used) {
|
|
28
|
+
const value = uuidFactory();
|
|
29
|
+
if (typeof value !== 'string' || !UUID_RE.test(value)) {
|
|
30
|
+
throw new Error('The MISP identifier factory returned an invalid UUID.');
|
|
31
|
+
}
|
|
32
|
+
const uuid = value.toLowerCase();
|
|
33
|
+
if (used.has(uuid))
|
|
34
|
+
throw new Error('The MISP identifier factory returned a duplicate UUID.');
|
|
35
|
+
used.add(uuid);
|
|
36
|
+
return uuid;
|
|
37
|
+
}
|
|
38
|
+
function attributeComment(availability, provenance) {
|
|
39
|
+
const { riskScore: score, riskModelVersion: modelVersion, scanDepth, observedAt } = provenance;
|
|
40
|
+
return [
|
|
41
|
+
'Heuristic Bulk finding',
|
|
42
|
+
`availability=${availability}`,
|
|
43
|
+
`risk=${score ?? 'unknown'}`,
|
|
44
|
+
...(modelVersion === null ? [] : [`risk-model=v${modelVersion}`]),
|
|
45
|
+
`scan-depth=${scanDepth}`,
|
|
46
|
+
`observed-at=${observedAt ?? 'unknown'}`,
|
|
47
|
+
`timestamp-basis=${observedAt ? 'scan' : 'unknown'}`,
|
|
48
|
+
WARNING,
|
|
49
|
+
].join('; ');
|
|
50
|
+
}
|
|
51
|
+
export function mispIndicatorAttribute(domain, availability, provenance, uuid, modifiedAt) {
|
|
52
|
+
return {
|
|
53
|
+
uuid, type: 'domain', category: 'Network activity', value: domain,
|
|
54
|
+
to_ids: false, distribution: '5', timestamp: String(Math.floor(Date.parse(modifiedAt) / 1000)),
|
|
55
|
+
...(provenance.observedAt ? { first_seen: provenance.observedAt, last_seen: provenance.observedAt } : {}),
|
|
56
|
+
comment: attributeComment(availability, provenance), disable_correlation: true, deleted: false,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
export function mispIndicatorEvent(uuid, generatedAt, attributes) {
|
|
60
|
+
return { uuid, date: generatedAt.slice(0, 10),
|
|
61
|
+
info: `WHOISleuth heuristic defensive-domain candidates for analyst review (export v${MISP_INDICATOR_EXPORT_VERSION})`,
|
|
62
|
+
threat_level_id: '4', analysis: '0', distribution: '0', published: false,
|
|
63
|
+
timestamp: String(Math.floor(Date.parse(generatedAt) / 1000)), publish_timestamp: '0', disable_correlation: true, Attribute: attributes };
|
|
64
|
+
}
|
|
65
|
+
export function buildMispIndicatorExport(records, options = {}) {
|
|
66
|
+
const generatedAt = isoTimestamp(options.generatedAt) || new Date().toISOString();
|
|
67
|
+
const uuidFactory = typeof options.uuidFactory === 'function'
|
|
68
|
+
? options.uuidFactory
|
|
69
|
+
: defaultUuidFactory;
|
|
70
|
+
let collected;
|
|
71
|
+
try {
|
|
72
|
+
collected = collectDefensiveIndicatorCandidates(records, MAX_MISP_ATTRIBUTES);
|
|
73
|
+
}
|
|
74
|
+
catch (cause) {
|
|
75
|
+
if (cause instanceof TypeError)
|
|
76
|
+
throw new TypeError('MISP indicator export requires an array of Bulk results.');
|
|
77
|
+
throw cause;
|
|
78
|
+
}
|
|
79
|
+
const used = new Set();
|
|
80
|
+
const eventUuid = nextUuid(uuidFactory, used);
|
|
81
|
+
const attributes = collected.entries.map(({ domain, source }) => mispIndicatorAttribute(domain, record(source).availability, defensiveIndicatorProvenance(source), nextUuid(uuidFactory, used), generatedAt));
|
|
82
|
+
const payload = { Event: mispIndicatorEvent(eventUuid, generatedAt, attributes) };
|
|
83
|
+
return {
|
|
84
|
+
version: MISP_INDICATOR_EXPORT_VERSION,
|
|
85
|
+
format: 'misp',
|
|
86
|
+
generatedAt,
|
|
87
|
+
domains: collected.domains,
|
|
88
|
+
truncated: collected.truncated,
|
|
89
|
+
filename: `whoisleuth-defensive-domains-${generatedAt.slice(0, 10)}.misp.json`,
|
|
90
|
+
mimeType: 'application/json;charset=utf-8',
|
|
91
|
+
content: `${JSON.stringify(payload, null, 2)}\n`,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/** Sharing declarations are not encryption, redaction or an authority to disclose. */
|
|
2
|
+
export const TLP_MARKINGS = ['clear', 'green', 'amber', 'amber-strict', 'red'];
|
|
3
|
+
export const RECIPIENT_SCOPES = ['public', 'community', 'organization', 'named-recipients'];
|
|
4
|
+
const RISKY_KEYS = new Set([
|
|
5
|
+
'authorization', 'cookie', 'cookies', 'credential', 'credentials', 'email', 'emails',
|
|
6
|
+
'entities', 'password', 'phone', 'raw', 'rawrdap', 'rawwhois', 'registrant', 'session', 'token',
|
|
7
|
+
]);
|
|
8
|
+
const MARKING_KEYS = new Set([
|
|
9
|
+
'informationmarking', 'marking', 'markings', 'sharingmarking',
|
|
10
|
+
'tlp', 'tlplabel', 'tlpmarking', 'trafficlightprotocol',
|
|
11
|
+
]);
|
|
12
|
+
export function markingFromText(value) {
|
|
13
|
+
const normalized = value.toUpperCase().replace(/\s+/gu, '').replace('TLP:', '').replace('+STRICT', '-STRICT').toLowerCase();
|
|
14
|
+
return TLP_MARKINGS.includes(normalized) ? normalized : null;
|
|
15
|
+
}
|
|
16
|
+
export function expectedScope(marking) {
|
|
17
|
+
if (marking === 'clear')
|
|
18
|
+
return 'public';
|
|
19
|
+
if (marking === 'green')
|
|
20
|
+
return 'community';
|
|
21
|
+
if (marking === 'red')
|
|
22
|
+
return 'named-recipients';
|
|
23
|
+
return 'organization';
|
|
24
|
+
}
|
|
25
|
+
export function tlpLabel(marking) {
|
|
26
|
+
return `TLP:${marking === 'amber-strict' ? 'AMBER+STRICT' : marking.toUpperCase()}`;
|
|
27
|
+
}
|
|
28
|
+
/** Scan an already admitted, byte/node/depth-bounded JSON tree, not arbitrary objects. */
|
|
29
|
+
export function scanSharingMetadata(root) {
|
|
30
|
+
const stack = [{ value: root, marking: false }];
|
|
31
|
+
const markings = new Set();
|
|
32
|
+
let riskyKeyCount = 0;
|
|
33
|
+
while (stack.length) {
|
|
34
|
+
const next = stack.pop();
|
|
35
|
+
if (typeof next.value === 'string') {
|
|
36
|
+
if (next.marking) {
|
|
37
|
+
const marking = markingFromText(next.value);
|
|
38
|
+
if (marking)
|
|
39
|
+
markings.add(marking);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
else if (Array.isArray(next.value)) {
|
|
43
|
+
for (const value of next.value)
|
|
44
|
+
stack.push({ value, marking: next.marking });
|
|
45
|
+
}
|
|
46
|
+
else if (next.value && typeof next.value === 'object') {
|
|
47
|
+
for (const [key, value] of Object.entries(next.value)) {
|
|
48
|
+
const normalized = key.toLowerCase().replace(/[^a-z0-9]/gu, '');
|
|
49
|
+
if (RISKY_KEYS.has(normalized))
|
|
50
|
+
riskyKeyCount++;
|
|
51
|
+
stack.push({ value, marking: MARKING_KEYS.has(normalized) });
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return { importedMarkings: [...markings], riskyKeyCount };
|
|
56
|
+
}
|