@slicedearth/whoisleuth-cli 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +6 -0
- package/README.md +19 -15
- package/bin/whoisleuth.mjs +8 -1
- package/cli/archive-content-digest.mjs +14 -0
- package/cli/archive-inspect.mjs +20 -53
- package/cli/arguments.mjs +68 -6
- package/cli/artifact-validation/case-response.mjs +47 -41
- package/cli/artifact-validation/investigation-capsule.mjs +52 -31
- package/cli/artifact-validation/lookup-evidence.mjs +12 -6
- package/cli/artifact-validation/signed-review.mjs +83 -61
- package/cli/artifact-validation/structure-primitives.mjs +1 -105
- package/cli/artifact-verify.mjs +90 -24
- package/cli/assurance-command-runner.mjs +39 -4
- package/cli/bagit-folder.mjs +57 -0
- package/cli/bagit-review.mjs +23 -0
- package/cli/bounded-json.mjs +1 -1
- package/cli/bulk-command-runner.mjs +2 -0
- package/cli/bulk-output.mjs +29 -3
- package/cli/case-command.mjs +195 -0
- package/cli/case-pack.mjs +39 -31
- package/cli/command-argument-grammar.mjs +4 -0
- package/cli/command-reference.mjs +145 -75
- package/cli/completion.mjs +298 -79
- package/cli/config-profile.mjs +7 -5
- package/cli/ct-event-intake.mjs +12 -12
- package/cli/discover.mjs +4 -3
- package/cli/discovery-observation-snapshot.mjs +44 -16
- package/cli/discovery-scan-command-runner.mjs +2 -0
- package/cli/discovery-scan.mjs +7 -4
- package/cli/discovery-workflow.mjs +1 -1
- package/cli/domain-control-monitor.mjs +50 -20
- package/cli/domain-control-observations.mjs +149 -102
- package/cli/errors.mjs +33 -1
- package/cli/evidence-command-runner.mjs +16 -5
- package/cli/evidence-signing.mjs +2 -2
- package/cli/external-observation-mapping.mjs +7 -8
- package/cli/formatters/terminal-lookup.mjs +18 -46
- package/cli/formatters/terminal-shared.mjs +5 -3
- package/cli/formatters/terminal-technology.mjs +114 -0
- package/cli/inline-command-families.mjs +1 -0
- package/cli/interactive-launcher.mjs +2 -160
- package/cli/interchange-report.mjs +4 -8
- package/cli/investigation-artifacts.mjs +176 -0
- package/cli/investigation-checkpoint.mjs +10 -0
- package/cli/investigation-folder.mjs +114 -0
- package/cli/investigation-manifest.mjs +1 -112
- package/cli/investigation-package-review.mjs +103 -0
- package/cli/investigation-plan.mjs +32 -19
- package/cli/investigation-run.mjs +188 -33
- package/cli/local-document-checkpoint.mjs +170 -0
- package/cli/lookup-browser.mjs +5 -12
- package/cli/lookup-command-runner.mjs +17 -6
- package/cli/lookup-diff.mjs +37 -3
- package/cli/lookup-plan.mjs +11 -5
- package/cli/lookup-reconcile.mjs +10 -8
- package/cli/mail-review.mjs +66 -14
- package/cli/manual.mjs +1 -0
- package/cli/network-command-runner.mjs +2 -0
- package/cli/output-file.mjs +31 -5
- package/cli/registry-doctor.mjs +3 -1
- package/cli/registry-support.mjs +1 -1
- package/cli/retained-artifact-diff.mjs +2 -2
- package/cli/review-command-runner.mjs +45 -0
- package/cli/runner.mjs +55 -6
- package/cli/sharing-review.mjs +21 -34
- package/cli/signer-trust.mjs +104 -0
- package/cli/source-reliability.mjs +74 -83
- package/cli/terminal-input.mjs +168 -0
- package/cli/workflow-command-runner.mjs +68 -13
- package/docs/cli-reference.md +76 -30
- package/docs/cli.md +289 -25
- package/lib/abort.mjs +4 -1
- package/lib/availability.mjs +161 -112
- package/lib/bounded-file.mjs +3 -1
- package/lib/bounded-html-document.mjs +125 -0
- package/lib/bounded-json.mjs +43 -3
- package/lib/browser-library-profile.mjs +42 -51
- package/lib/browser-library-worker.mjs +112 -0
- package/lib/client-behavior-profile.mjs +3 -3
- package/lib/ct-search.mjs +11 -4
- package/lib/dns-delegation-health.mjs +24 -14
- package/lib/dns-inheritance-review.mjs +217 -0
- package/lib/dns-intelligence.mjs +36 -23
- package/lib/dns-mx.mjs +1 -1
- package/lib/dnssec-evidence-validation.mjs +11 -3
- package/lib/domain-change-packet.mjs +7 -2
- package/lib/domain-change-review.mjs +14 -8
- package/lib/domain-control-flight-recorder.mjs +219 -118
- package/lib/domain-control-manifest.mjs +117 -57
- package/lib/domain-posture-analysis.mjs +20 -7
- package/lib/domain-posture-parsers.mjs +88 -6
- package/lib/domain-posture.mjs +113 -26
- package/lib/evidence-export-privacy.mjs +561 -0
- package/lib/evidence-export.mjs +11 -592
- package/lib/evidence-report.mjs +3 -1
- package/lib/favicon.mjs +117 -41
- package/lib/generated/cisa-kev-catalog.mjs +12 -4
- package/lib/generated/registrar-standing-catalogue.mjs +7 -7
- package/lib/generated/retire-browser-catalog.mjs +2335 -940
- package/lib/hostname.mjs +2 -38
- package/lib/html-signals.mjs +82 -85
- package/lib/http-evidence-bounds.mjs +2 -1
- package/lib/http-intelligence.mjs +39 -26
- package/lib/lookup-cache.mjs +34 -11
- package/lib/lookup-child-profile-contract.mjs +24 -15
- package/lib/lookup-diagnostics.mjs +24 -1
- package/lib/lookup-progress.mjs +30 -13
- package/lib/lookup-response-contract.mjs +1048 -0
- package/lib/lookup-response.mjs +7 -10
- package/lib/lookup-source-progress.mjs +17 -14
- package/lib/lookup.mjs +48 -27
- package/lib/observed-network-context.mjs +43 -27
- package/lib/outbound-request-bounds.mjs +4 -1
- package/lib/page-fingerprints.mjs +69 -133
- package/lib/page-role-profile.mjs +12 -8
- package/lib/perceptual-hash.mjs +43 -14
- package/lib/rdap-bootstrap.mjs +72 -22
- package/lib/rdap-client.mjs +15 -7
- package/lib/rdap-normalization.mjs +33 -18
- package/lib/rdap-search-workbench.mjs +1 -1
- package/lib/rdap-transport.mjs +47 -20
- package/lib/rdap-types.mjs +2 -1
- package/lib/rdap.mjs +18 -3
- package/lib/registrable-domain.mjs +21 -1
- package/lib/registrar-standing-contract.mjs +1 -0
- package/lib/registry-capabilities.mjs +5 -1
- package/lib/registry-capability-catalogue.mjs +2 -1
- package/lib/registry-dates.mjs +2 -190
- package/lib/registry-insights.mjs +56 -18
- package/lib/response-policy.mjs +86 -48
- package/lib/risk-calibration-summary.mjs +17 -3
- package/lib/rpki-evidence.mjs +8 -5
- package/lib/safe-fetch.mjs +54 -47
- package/lib/security-txt.mjs +10 -7
- package/lib/service-binding-dns.mjs +8 -0
- package/lib/smtp-transport-review.mjs +27 -15
- package/lib/sslbl-certificates.generated.mjs +136 -135
- package/lib/static-html-analysis.mjs +117 -146
- package/lib/structured-data-identity.mjs +1 -0
- package/lib/technology-markup.mjs +100 -0
- package/lib/threat-intelligence-manifests.mjs +11 -139
- package/lib/threat-intelligence-runtime.mjs +24 -52
- package/lib/threat-intelligence-targets.mjs +6 -34
- package/lib/threat-intelligence-types.mjs +39 -0
- package/lib/typosquat-generator.mjs +19 -15
- package/lib/urlhaus-intelligence.mjs +8 -3
- package/lib/urlscan-intelligence.mjs +1 -1
- package/lib/website-security-posture.mjs +30 -6
- package/lib/website-technology.mjs +111 -71
- package/lib/whois-authority.mjs +14 -2
- package/lib/whois-chain.mjs +9 -2
- package/lib/whois-common-formats.mjs +5 -5
- package/lib/whois-contracts.mjs +2 -1
- package/lib/whois-dialects.mjs +2 -2
- package/lib/whois-parser-support.mjs +8 -1
- package/lib/whois-transport.mjs +23 -7
- package/package.json +2 -1
- package/packages/cases/case-attachment-model.mjs +83 -0
- package/packages/cases/case-evidence-model.mjs +156 -62
- package/packages/cases/case-export-input.mjs +53 -0
- package/packages/cases/case-migration-model.mjs +59 -25
- package/packages/cases/case-model.mjs +1 -0
- package/packages/cases/case-recheck-model.mjs +91 -0
- package/packages/cases/case-record-core.mjs +3 -2
- package/packages/cases/case-record-operations.mjs +89 -27
- package/packages/cases/case-record-projection.mjs +23 -20
- package/packages/cases/case-report.mjs +63 -52
- package/packages/cases/case-response-actions.mjs +654 -0
- package/packages/cases/case-response-model.mjs +41 -1129
- package/packages/cases/case-response-outcomes.mjs +317 -0
- package/packages/cases/case-response-packet-render.mjs +153 -0
- package/packages/cases/case-response-packet-types.mjs +1 -0
- package/packages/cases/case-response-packet-vocabulary.mjs +48 -0
- package/packages/cases/case-response-packet.mjs +38 -587
- package/packages/cases/case-response-records.mjs +94 -0
- package/packages/cases/case-response-review-inputs.mjs +371 -0
- package/packages/cases/case-response-values.mjs +85 -0
- package/packages/cases/case-review-package.mjs +45 -0
- package/packages/cases/case-selection.mjs +46 -0
- package/packages/cases/case-storage-model.mjs +25 -0
- package/packages/cases/response-route-freshness.mjs +16 -0
- package/packages/comparison/capture-context.mjs +61 -0
- package/packages/comparison/comparison-ledger-bulk.mjs +4 -1
- package/packages/comparison/page-similarity.mjs +2 -1
- package/packages/comparison/relationship-evidence.mjs +85 -8
- package/packages/comparison/relationship-provenance.mjs +93 -0
- package/packages/contracts/analyst-interchange.mjs +216 -0
- package/packages/contracts/analyst-review-state-contract.mjs +11 -0
- package/packages/contracts/analyst-review-state.mjs +5 -4
- package/packages/contracts/case-portability.mjs +279 -116
- package/packages/contracts/case-views-contract.mjs +12 -0
- package/packages/contracts/case-views.mjs +26 -0
- package/packages/contracts/cli-command-semantics.mjs +1 -0
- package/packages/contracts/cli-lookup.mjs +0 -1
- package/packages/contracts/domain-control-flight-recorder.mjs +80 -42
- package/packages/contracts/domain-control-manifest.mjs +127 -73
- package/packages/contracts/domain-control-monitor.mjs +48 -45
- package/packages/contracts/domain-control-review.mjs +179 -121
- package/packages/contracts/domain-name.mjs +21 -0
- package/packages/contracts/external-observation-interchange.mjs +2 -1
- package/packages/contracts/extracted-domain-lifecycle.mjs +11 -32
- package/packages/contracts/http-url.mjs +2 -0
- package/packages/contracts/investigation-package-limits.mjs +24 -0
- package/packages/contracts/investigation-portability.mjs +56 -27
- package/packages/contracts/investigation-run.mjs +16 -0
- package/packages/contracts/ip-address.mjs +34 -0
- package/packages/contracts/lookup-evidence.mjs +35 -0
- package/packages/contracts/offline-comparison.mjs +1 -2
- package/packages/contracts/page-fingerprints.mjs +15 -0
- package/packages/contracts/risk-calibration.mjs +0 -1
- package/packages/contracts/schema-compatibility.mjs +1 -1
- package/packages/contracts/schema-lifecycle.mjs +83 -153
- package/packages/contracts/selected-file-limits.mjs +6 -0
- package/packages/contracts/vulnerability-identifiers.mjs +5 -0
- package/packages/contracts/web-capture.mjs +10 -0
- package/packages/contracts/workspace-portability.mjs +160 -225
- package/packages/evidence/artifact-integrity.mjs +8 -2
- package/packages/evidence/artifact-structure.mjs +105 -0
- package/packages/evidence/decision-fact.mjs +4 -1
- package/packages/evidence/domain-control-runtime.mjs +155 -27
- package/packages/evidence/domain-posture-context.mjs +79 -0
- package/packages/evidence/image-regions.mjs +56 -0
- package/packages/evidence/latest-observations.mjs +28 -0
- package/packages/evidence/lookup-target.mjs +104 -0
- package/packages/evidence/observation.mjs +16 -1
- package/packages/evidence/passphrase-encryption.mjs +44 -0
- package/packages/evidence/record-identity.mjs +72 -0
- package/packages/evidence/registry-dates.mjs +190 -0
- package/packages/evidence/retained-file.mjs +69 -0
- package/packages/evidence/review-age.mjs +14 -0
- package/packages/interchange/bagit.mjs +342 -0
- package/packages/interchange/bounded-zip-extraction.mjs +278 -0
- package/packages/interchange/crc32.mjs +17 -0
- package/packages/interchange/external-findings-import.mjs +89 -49
- package/packages/interchange/web-capture-import.mjs +382 -0
- package/packages/investigation/bulk-mail-exposure.mjs +2 -1
- package/packages/investigation/investigation-bagit.mjs +14 -0
- package/packages/investigation/investigation-capsule.mjs +33 -5
- package/packages/investigation/investigation-manifest.mjs +200 -0
- package/packages/investigation/investigation-package-crypto.mjs +98 -0
- package/packages/investigation/investigation-package.mjs +200 -0
- package/packages/investigation/lookup-asset-graph-inputs.mjs +38 -0
- package/packages/investigation/lookup-asset-graph.mjs +161 -141
- package/packages/investigation/lookup-claim-readiness.mjs +18 -0
- package/packages/investigation/lookup-investigation-brief.mjs +3 -3
- package/packages/investigation/lookup-task-guidance.mjs +58 -0
- package/packages/monitoring/analyst-review-state.mjs +38 -90
- package/packages/workspace/brand-profile-model.mjs +158 -20
- package/packages/workspace/bulk-columns.mjs +17 -0
- package/packages/workspace/bulk-review-model.mjs +11 -5
- package/packages/workspace/bulk-session-model.mjs +95 -20
- package/packages/workspace/bulk-sort.mjs +1 -0
- package/packages/workspace/case-views.mjs +102 -0
- package/packages/workspace/hostile-input.mjs +29 -8
- package/packages/workspace/investigation-template-model.mjs +19 -4
- package/packages/workspace/page-baseline.mjs +18 -15
- package/packages/workspace/relationship-observation-model.mjs +41 -26
- package/packages/workspace/watchlist-history.mjs +2 -2
- package/packages/workspace/watchlist-store.mjs +8 -2
- package/packages/workspace/website-snapshot-model.mjs +70 -30
- package/packages/workspace/workspace-archive-crypto.mjs +41 -61
- package/packages/workspace/workspace-archive.mjs +85 -21
- package/third-party-notices.txt +29 -2
- package/packages/investigation/bulk-review-export.mjs +0 -63
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { normalizeDomain } from "../cases/case-model.mjs";
|
|
2
2
|
import { CASE_DISPOSITIONS as CASE_DISPOSITION_OPTIONS } from "../cases/case-record-contracts.mjs";
|
|
3
|
-
import { normalizeBulkPresentationSortKey } from "./bulk-sort.mjs";
|
|
3
|
+
import { BULK_SORT_KEYS, normalizeBulkPresentationSortKey } from "./bulk-sort.mjs";
|
|
4
|
+
import { normalizeBulkResultColumns } from "./bulk-columns.mjs";
|
|
4
5
|
import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
|
|
5
6
|
import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
|
|
6
|
-
import { BULK_REVIEW_SCHEMA, BULK_REVIEW_SCHEMA_VERSION, MAX_BULK_REVIEW_NAME_LENGTH, MAX_BULK_REVIEW_PRESETS, MAX_BULK_REVIEW_ROWS, MAX_BULK_REVIEW_STORE_BYTES, } from "../contracts/workspace-portability.mjs";
|
|
7
|
+
import { BULK_REVIEW_SCHEMA, BULK_REVIEW_SCHEMA_VERSION, BULK_REVIEW_EXPORT_SUPPORTED_VERSIONS, MAX_BULK_REVIEW_NAME_LENGTH, MAX_BULK_REVIEW_PRESETS, MAX_BULK_REVIEW_ROWS, MAX_BULK_REVIEW_STORE_BYTES, } from "../contracts/workspace-portability.mjs";
|
|
7
8
|
export { BULK_REVIEW_SCHEMA, BULK_REVIEW_SCHEMA_VERSION, MAX_BULK_REVIEW_NAME_LENGTH, MAX_BULK_REVIEW_PRESETS, MAX_BULK_REVIEW_ROWS, MAX_BULK_REVIEW_STORE_BYTES, } from "../contracts/workspace-portability.mjs";
|
|
8
9
|
export const BULK_REVIEW_STATES = ['unreviewed', 'reviewing', 'reviewed', 'deferred'];
|
|
9
10
|
export const BULK_SOURCE_FILTERS = ['', 'complete', 'limited', 'unrecorded'];
|
|
@@ -19,7 +20,7 @@ const LIFECYCLE_FILTERS = new Set(BULK_LIFECYCLE_FILTERS);
|
|
|
19
20
|
const AGE_FILTERS = new Set(BULK_AGE_FILTERS);
|
|
20
21
|
const MAIL_FILTERS = new Set(BULK_MAIL_FILTERS);
|
|
21
22
|
const GROUPS = new Set(BULK_GROUP_OPTIONS);
|
|
22
|
-
const SORT_KEYS = new Set(
|
|
23
|
+
const SORT_KEYS = new Set(BULK_SORT_KEYS);
|
|
23
24
|
const REVIEW_STATES = new Set(BULK_REVIEW_STATES);
|
|
24
25
|
const CASE_DISPOSITIONS = new Set(['', 'untracked', ...CASE_DISPOSITION_OPTIONS.map((item) => item.value)]);
|
|
25
26
|
const SIGNAL_FILTERS = new Set(['favicon', 'password', 'phishing', 'asset_reuse', 'idn']);
|
|
@@ -67,6 +68,7 @@ function normalizeView(raw) {
|
|
|
67
68
|
groupBy: setValue(value.groupBy, GROUPS),
|
|
68
69
|
sortKey,
|
|
69
70
|
sortDirection: value.sortDirection === 1 ? 1 : -1,
|
|
71
|
+
columns: normalizeBulkResultColumns(value.columns),
|
|
70
72
|
};
|
|
71
73
|
}
|
|
72
74
|
function normalizePreset(raw, fallback) {
|
|
@@ -113,6 +115,10 @@ function sourceLists(raw) {
|
|
|
113
115
|
export function normalizeBulkReviewStore(raw) {
|
|
114
116
|
assertWorkspaceInputGraph(raw, 'Bulk-review store');
|
|
115
117
|
assertWorkspaceDeclaredVersion(raw, 'Bulk-review store');
|
|
118
|
+
const declaredVersion = record(raw).version;
|
|
119
|
+
if (typeof declaredVersion === 'number' && declaredVersion > BULK_REVIEW_SCHEMA_VERSION) {
|
|
120
|
+
throw new Error(`This Bulk review store uses newer schema ${declaredVersion}. Update the app before changing it.`);
|
|
121
|
+
}
|
|
116
122
|
const fallback = new Date(0).toISOString();
|
|
117
123
|
const source = sourceLists(raw);
|
|
118
124
|
const presets = new Map();
|
|
@@ -154,7 +160,7 @@ export function enforceBulkReviewBudget(raw) {
|
|
|
154
160
|
while (byteLength(JSON.stringify(store)) > MAX_BULK_REVIEW_STORE_BYTES && store.presets.length > 1)
|
|
155
161
|
store.presets.pop();
|
|
156
162
|
if (byteLength(JSON.stringify(store)) > MAX_BULK_REVIEW_STORE_BYTES) {
|
|
157
|
-
throw new Error('Bulk review preferences exceed the
|
|
163
|
+
throw new Error('Bulk review preferences exceed the workspace storage limit.');
|
|
158
164
|
}
|
|
159
165
|
return store;
|
|
160
166
|
}
|
|
@@ -217,7 +223,7 @@ export function mergeBulkReviewStores(localRaw, importedRaw) {
|
|
|
217
223
|
if (importedRecord.schema !== BULK_REVIEW_SCHEMA) {
|
|
218
224
|
throw new Error('This file is not a WHOISleuth Bulk review export.');
|
|
219
225
|
}
|
|
220
|
-
if (
|
|
226
|
+
if (!BULK_REVIEW_EXPORT_SUPPORTED_VERSIONS.some((version) => version === importedRecord.version)) {
|
|
221
227
|
if (typeof importedRecord.version === 'number' && importedRecord.version > BULK_REVIEW_SCHEMA_VERSION) {
|
|
222
228
|
throw new Error(`This Bulk review export uses newer schema ${importedRecord.version}. Update the app before importing it.`);
|
|
223
229
|
}
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
import { normalizeDomain } from "../cases/case-model.mjs";
|
|
5
5
|
import { normalizeCaaCritical } from "./dns-record-normalization.mjs";
|
|
6
6
|
import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
|
|
7
|
+
import { RELATIONSHIP_EVIDENCE_VERSION } from "../contracts/offline-comparison.mjs";
|
|
8
|
+
import { normalizeRelationshipSourceProjection } from "../comparison/relationship-provenance.mjs";
|
|
7
9
|
import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
|
|
8
10
|
import { BULK_SESSION_SCHEMA, BULK_SESSION_SCHEMA_VERSION, MAX_BULK_PROFILE_CONTEXT_LIMITATION_LENGTH, MAX_BULK_SESSIONS, MAX_BULK_SESSION_ARRAY_VALUES, MAX_BULK_SESSION_NAME_LENGTH, MAX_BULK_SESSION_ROWS, MAX_BULK_SESSION_SOURCES, MAX_BULK_SESSION_STORE_BYTES, MAX_BULK_SESSION_TEXT_LENGTH, SUPPORTED_BULK_SESSION_SCHEMA_VERSIONS, } from "../contracts/workspace-portability.mjs";
|
|
9
11
|
export { BULK_SESSION_SCHEMA, BULK_SESSION_SCHEMA_VERSION, MAX_BULK_PROFILE_CONTEXT_LIMITATION_LENGTH, MAX_BULK_SESSIONS, MAX_BULK_SESSION_ARRAY_VALUES, MAX_BULK_SESSION_NAME_LENGTH, MAX_BULK_SESSION_ROWS, MAX_BULK_SESSION_SOURCES, MAX_BULK_SESSION_STORE_BYTES, MAX_BULK_SESSION_TEXT_LENGTH, SUPPORTED_BULK_SESSION_SCHEMA_VERSIONS, } from "../contracts/workspace-portability.mjs";
|
|
@@ -175,7 +177,7 @@ function normalizeSourceCoverage(value) {
|
|
|
175
177
|
if (!SOURCE_RE.test(source) || !SOURCE_STATES.has(state) || seen.has(source))
|
|
176
178
|
continue;
|
|
177
179
|
seen.add(source);
|
|
178
|
-
output.push({ source, state: state });
|
|
180
|
+
output.push({ source, state: state, observedAt: normalizeExplicitIsoTimestamp(item?.observedAt) });
|
|
179
181
|
if (output.length >= MAX_BULK_SESSION_SOURCES)
|
|
180
182
|
break;
|
|
181
183
|
}
|
|
@@ -198,8 +200,11 @@ function nullableHash(value, expression) {
|
|
|
198
200
|
}
|
|
199
201
|
function normalizeRelationship(value) {
|
|
200
202
|
const item = record(value);
|
|
203
|
+
if (item?.version !== undefined && item.version !== 2 && item.version !== RELATIONSHIP_EVIDENCE_VERSION) {
|
|
204
|
+
throw new TypeError('This Bulk relationship projection uses an unsupported version. No source evidence was interpreted.');
|
|
205
|
+
}
|
|
201
206
|
return {
|
|
202
|
-
version:
|
|
207
|
+
version: RELATIONSHIP_EVIDENCE_VERSION,
|
|
203
208
|
nameservers: boundedStrings(item?.nameservers, 20),
|
|
204
209
|
ipAddresses: boundedStrings(item?.ipAddresses, 50),
|
|
205
210
|
trackingIdentifiers: boundedTrackingIdentifiers(item?.trackingIdentifiers),
|
|
@@ -207,6 +212,7 @@ function normalizeRelationship(value) {
|
|
|
207
212
|
faviconHash: nullableHash(item?.faviconHash, HASH_64_RE),
|
|
208
213
|
faviconPHash: nullableHash(item?.faviconPHash, PHASH_RE),
|
|
209
214
|
certificateFingerprint: nullableHash(item?.certificateFingerprint, HASH_64_RE),
|
|
215
|
+
sourceEvidence: item?.version === RELATIONSHIP_EVIDENCE_VERSION ? normalizeRelationshipSourceProjection(item.sourceEvidence) : {},
|
|
210
216
|
truncated: item?.truncated === true,
|
|
211
217
|
};
|
|
212
218
|
}
|
|
@@ -270,14 +276,14 @@ export function normalizeBulkComparisonEvidence(value) {
|
|
|
270
276
|
},
|
|
271
277
|
};
|
|
272
278
|
}
|
|
273
|
-
export function normalizeBulkSessionResult(value) {
|
|
279
|
+
export function normalizeBulkSessionResult(value, inheritedProfileContext) {
|
|
274
280
|
const item = record(value);
|
|
275
281
|
const domain = normalizeDomain(item?.domain);
|
|
276
282
|
const status = boundedText(item?.status, 20);
|
|
277
283
|
const scanDepth = boundedText(item?.scanDepth, 10);
|
|
278
284
|
if (!item || !domain || !RESULT_STATES.has(status) || !MODES.has(scanDepth))
|
|
279
285
|
return null;
|
|
280
|
-
const rawProfileContext = record(item.profileContext);
|
|
286
|
+
const rawProfileContext = record(Object.hasOwn(item, 'profileContext') ? item.profileContext : inheritedProfileContext);
|
|
281
287
|
if (!rawProfileContext || !['ready', 'unavailable'].includes(String(rawProfileContext.sourceState)))
|
|
282
288
|
return null;
|
|
283
289
|
const profileContext = normalizeBulkProfileContext(rawProfileContext);
|
|
@@ -311,6 +317,7 @@ export function normalizeBulkSessionResult(value) {
|
|
|
311
317
|
trusted: profileClaimsUsable && !readyWithoutActiveProfile && TRUST_STATES.has(trusted) ? trusted : null,
|
|
312
318
|
error: boundedText(item.error),
|
|
313
319
|
scanDepth: scanDepth,
|
|
320
|
+
observedAt: normalizeExplicitIsoTimestamp(item.observedAt),
|
|
314
321
|
createdDate: boundedText(item.createdDate, 64) || null,
|
|
315
322
|
expiryDate: boundedText(item.expiryDate, 64) || null,
|
|
316
323
|
privacyProtected: nullableBoolean(item.privacyProtected),
|
|
@@ -343,7 +350,7 @@ export function normalizeBulkSessionResult(value) {
|
|
|
343
350
|
profileContext,
|
|
344
351
|
};
|
|
345
352
|
}
|
|
346
|
-
export function normalizeBulkSession(value) {
|
|
353
|
+
export function normalizeBulkSession(value, sourceStoreVersion) {
|
|
347
354
|
const item = record(value);
|
|
348
355
|
const id = boundedText(item?.id, 128);
|
|
349
356
|
const name = boundedText(item?.name, MAX_BULK_SESSION_NAME_LENGTH);
|
|
@@ -367,23 +374,43 @@ export function normalizeBulkSession(value) {
|
|
|
367
374
|
return null;
|
|
368
375
|
if (!Array.isArray(item.results))
|
|
369
376
|
return null;
|
|
377
|
+
const rawProfileContext = record(item.profileContext);
|
|
378
|
+
if (!rawProfileContext)
|
|
379
|
+
return null;
|
|
380
|
+
const declaredProfileContext = normalizeBulkProfileContext(rawProfileContext);
|
|
381
|
+
// Only an explicitly versioned current document may omit identical row
|
|
382
|
+
// context. Bare runtime rows and public schema 4 must carry their own context;
|
|
383
|
+
// an explicit invalid/null row value never falls back to the session.
|
|
384
|
+
const inheritedProfileContext = sourceStoreVersion === 5
|
|
385
|
+
&& declaredProfileContext.sourceState !== 'mixed'
|
|
386
|
+
&& rawProfileContext.sourceState === declaredProfileContext.sourceState
|
|
387
|
+
? declaredProfileContext
|
|
388
|
+
: undefined;
|
|
370
389
|
const allowed = new Set(domains);
|
|
371
390
|
const results = [];
|
|
372
391
|
const seen = new Set();
|
|
373
392
|
for (const candidate of Array.isArray(item.results) ? item.results.slice(0, MAX_BULK_SESSION_ROWS * 2) : []) {
|
|
374
|
-
const
|
|
393
|
+
const publicLegacyRow = sourceStoreVersion === 4;
|
|
394
|
+
if (publicLegacyRow && record(record(candidate)?.relationship)?.version === RELATIONSHIP_EVIDENCE_VERSION) {
|
|
395
|
+
throw new TypeError('Bulk schema 4 cannot contain newer relationship source evidence; no evidence was interpreted.');
|
|
396
|
+
}
|
|
397
|
+
const result = normalizeBulkSessionResult(candidate, inheritedProfileContext);
|
|
375
398
|
if (!result || !allowed.has(result.domain) || seen.has(result.domain)) {
|
|
376
399
|
return null;
|
|
377
400
|
}
|
|
401
|
+
// Public schema 4 never recorded a row observation time or qualified
|
|
402
|
+
// relationship sources. Ignore added fields rather than backdating them.
|
|
403
|
+
if (publicLegacyRow) {
|
|
404
|
+
result.observedAt = null;
|
|
405
|
+
result.relationship.sourceEvidence = {};
|
|
406
|
+
result.sourceCoverage = result.sourceCoverage.map((source) => ({ ...source, observedAt: null }));
|
|
407
|
+
}
|
|
378
408
|
seen.add(result.domain);
|
|
379
409
|
results.push(result);
|
|
380
410
|
}
|
|
381
411
|
if ((state === 'complete') !== (results.length === domains.length))
|
|
382
412
|
return null;
|
|
383
413
|
const profileContext = summarizeBulkProfileContexts(results);
|
|
384
|
-
if (!record(item.profileContext))
|
|
385
|
-
return null;
|
|
386
|
-
const declaredProfileContext = normalizeBulkProfileContext(item.profileContext);
|
|
387
414
|
if (!sameProfileContext(declaredProfileContext, profileContext))
|
|
388
415
|
return null;
|
|
389
416
|
return {
|
|
@@ -407,7 +434,9 @@ export function bulkSessionStoreVersion(raw) {
|
|
|
407
434
|
: null;
|
|
408
435
|
}
|
|
409
436
|
export function normalizeBulkSessionStore(raw) {
|
|
410
|
-
|
|
437
|
+
// Preparation may combine an existing store and one incoming store before
|
|
438
|
+
// whole-session eviction. The persisted document still has its 4 MiB cap.
|
|
439
|
+
assertWorkspaceInputGraph(raw, 'Bulk-session store', { maximumBytes: MAX_BULK_SESSION_STORE_BYTES * 2 });
|
|
411
440
|
assertWorkspaceDeclaredVersion(raw, 'Bulk-session store');
|
|
412
441
|
const value = record(raw);
|
|
413
442
|
if (value?.schema === BULK_SESSION_SCHEMA
|
|
@@ -423,7 +452,7 @@ export function normalizeBulkSessionStore(raw) {
|
|
|
423
452
|
: [];
|
|
424
453
|
const byId = new Map();
|
|
425
454
|
for (const candidate of candidates.slice(0, MAX_BULK_SESSIONS * 4)) {
|
|
426
|
-
const session = normalizeBulkSession(candidate);
|
|
455
|
+
const session = normalizeBulkSession(candidate, bulkSessionStoreVersion(raw) ?? undefined);
|
|
427
456
|
if (!session)
|
|
428
457
|
continue;
|
|
429
458
|
const existing = byId.get(session.id);
|
|
@@ -438,8 +467,21 @@ export function normalizeBulkSessionStore(raw) {
|
|
|
438
467
|
.slice(0, MAX_BULK_SESSIONS),
|
|
439
468
|
};
|
|
440
469
|
}
|
|
470
|
+
/** Current wire representation of an already-normalized session. */
|
|
471
|
+
export function bulkSessionStorageValue(session) {
|
|
472
|
+
return {
|
|
473
|
+
...session,
|
|
474
|
+
results: session.results.map(({ profileContext, ...row }) => sameProfileContext(profileContext, session.profileContext)
|
|
475
|
+
? row
|
|
476
|
+
: { ...row, profileContext }),
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
/** Current wire bytes for sessions already admitted by the collection owner. */
|
|
480
|
+
export function serializeNormalizedBulkSessions(sessions) {
|
|
481
|
+
return JSON.stringify({ schema: BULK_SESSION_SCHEMA, version: BULK_SESSION_SCHEMA_VERSION, sessions: sessions.map(bulkSessionStorageValue) });
|
|
482
|
+
}
|
|
441
483
|
export function serializeBulkSessionStore(raw) {
|
|
442
|
-
return
|
|
484
|
+
return serializeNormalizedBulkSessions(normalizeBulkSessionStore(raw).sessions);
|
|
443
485
|
}
|
|
444
486
|
function byteLength(value) {
|
|
445
487
|
return new TextEncoder().encode(value).byteLength;
|
|
@@ -447,11 +489,16 @@ function byteLength(value) {
|
|
|
447
489
|
export function enforceBulkSessionStoreBudget(raw) {
|
|
448
490
|
const store = normalizeBulkSessionStore(raw);
|
|
449
491
|
let pruned = 0;
|
|
450
|
-
|
|
492
|
+
const sessionBytes = store.sessions.map((session) => byteLength(JSON.stringify(bulkSessionStorageValue(session))));
|
|
493
|
+
let bytes = byteLength(JSON.stringify({ ...store, sessions: [] }))
|
|
494
|
+
+ sessionBytes.reduce((total, size) => total + size, 0)
|
|
495
|
+
+ Math.max(0, sessionBytes.length - 1);
|
|
496
|
+
while (bytes > MAX_BULK_SESSION_STORE_BYTES && store.sessions.length > 1) {
|
|
497
|
+
bytes -= sessionBytes.pop() + 1;
|
|
451
498
|
store.sessions.pop();
|
|
452
499
|
pruned += 1;
|
|
453
500
|
}
|
|
454
|
-
if (
|
|
501
|
+
if (bytes > MAX_BULK_SESSION_STORE_BYTES) {
|
|
455
502
|
throw new Error('This Bulk session exceeds the 4 MiB browser-local session limit.');
|
|
456
503
|
}
|
|
457
504
|
return { store, pruned };
|
|
@@ -474,9 +521,29 @@ export function upsertBulkSession(raw, sessionRaw) {
|
|
|
474
521
|
sessions: bounded.store.sessions,
|
|
475
522
|
session,
|
|
476
523
|
added: index < 0,
|
|
477
|
-
pruned: bounded.
|
|
524
|
+
pruned: Math.max(0, next.length - bounded.store.sessions.length),
|
|
478
525
|
};
|
|
479
526
|
}
|
|
527
|
+
/** Ephemeral retention preview, never part of the stored or exported format. */
|
|
528
|
+
export function prepareBulkSessionSave(raw, sessionRaw) {
|
|
529
|
+
const current = normalizeBulkSessionStore(raw).sessions;
|
|
530
|
+
const result = upsertBulkSession(current, sessionRaw);
|
|
531
|
+
if (!result.sessions.some((session) => session.id === result.session.id)) {
|
|
532
|
+
throw new Error('This Bulk session cannot fit alongside the saved sessions. Export or remove saved sessions before trying again. Nothing was changed.');
|
|
533
|
+
}
|
|
534
|
+
const retained = new Set(result.sessions.map((session) => session.id));
|
|
535
|
+
const removed = current.filter((session) => !retained.has(session.id));
|
|
536
|
+
return {
|
|
537
|
+
...result,
|
|
538
|
+
current,
|
|
539
|
+
removed,
|
|
540
|
+
removedBytes: removed.reduce((total, session) => total + byteLength(JSON.stringify(bulkSessionStorageValue(session))), 0),
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
/** Confirmation applies to the exact records displayed, not just their IDs. */
|
|
544
|
+
export function bulkSessionSavePreviewIsCurrent(raw, preview) {
|
|
545
|
+
return serializeBulkSessionStore(raw) === serializeNormalizedBulkSessions(preview.current);
|
|
546
|
+
}
|
|
480
547
|
export function deleteBulkSession(raw, idRaw) {
|
|
481
548
|
const id = boundedText(idRaw, 128);
|
|
482
549
|
return normalizeBulkSessionStore({
|
|
@@ -588,7 +655,7 @@ export function buildBulkSessionExport(sessions, generatedAt = new Date().toISOS
|
|
|
588
655
|
schema: BULK_SESSION_SCHEMA,
|
|
589
656
|
version: BULK_SESSION_SCHEMA_VERSION,
|
|
590
657
|
generatedAt: timestamp(generatedAt, new Date().toISOString()),
|
|
591
|
-
sessions: store.sessions,
|
|
658
|
+
sessions: store.sessions.map(bulkSessionStorageValue),
|
|
592
659
|
limitations: [
|
|
593
660
|
'Compact Bulk results and source states only; raw source payloads and contact records are excluded.',
|
|
594
661
|
'Import is non-destructive and does not resume network collection automatically.',
|
|
@@ -596,8 +663,8 @@ export function buildBulkSessionExport(sessions, generatedAt = new Date().toISOS
|
|
|
596
663
|
};
|
|
597
664
|
}
|
|
598
665
|
export function mergeBulkSessions(localRaw, importedRaw) {
|
|
599
|
-
assertWorkspaceInputGraph(localRaw, 'Local Bulk-session store');
|
|
600
|
-
assertWorkspaceInputGraph(importedRaw, 'Imported Bulk-session document');
|
|
666
|
+
assertWorkspaceInputGraph(localRaw, 'Local Bulk-session store', { maximumBytes: MAX_BULK_SESSION_STORE_BYTES * 2 });
|
|
667
|
+
assertWorkspaceInputGraph(importedRaw, 'Imported Bulk-session document', { maximumBytes: MAX_BULK_SESSION_STORE_BYTES * 2 });
|
|
601
668
|
assertWorkspacePortableVersion(importedRaw, BULK_SESSION_SCHEMA_VERSION, 'Imported Bulk-session document');
|
|
602
669
|
const imported = record(importedRaw);
|
|
603
670
|
if (!imported || imported.schema !== BULK_SESSION_SCHEMA || !Array.isArray(imported.sessions)) {
|
|
@@ -614,7 +681,7 @@ export function mergeBulkSessions(localRaw, importedRaw) {
|
|
|
614
681
|
let updated = 0;
|
|
615
682
|
let skipped = 0;
|
|
616
683
|
for (const candidate of imported.sessions.slice(0, MAX_BULK_SESSIONS * 4)) {
|
|
617
|
-
const normalized = normalizeBulkSession(candidate);
|
|
684
|
+
const normalized = normalizeBulkSession(candidate, Number(imported.version));
|
|
618
685
|
if (!normalized) {
|
|
619
686
|
skipped += 1;
|
|
620
687
|
continue;
|
|
@@ -649,8 +716,16 @@ export function mergeBulkSessions(localRaw, importedRaw) {
|
|
|
649
716
|
skipped += 1;
|
|
650
717
|
continue;
|
|
651
718
|
}
|
|
719
|
+
// An import is additive: count and byte pressure must not silently remove
|
|
720
|
+
// a local session (or an earlier admitted import). The preview reports
|
|
721
|
+
// the skipped candidate and the analyst can export/remove data explicitly.
|
|
722
|
+
const merged = upsertBulkSession(sessions, session);
|
|
723
|
+
if (merged.pruned > 0 || !merged.sessions.some((item) => item.id === session.id)) {
|
|
724
|
+
skipped += 1;
|
|
725
|
+
continue;
|
|
726
|
+
}
|
|
652
727
|
added += 1;
|
|
653
|
-
sessions =
|
|
728
|
+
sessions = merged.sessions;
|
|
654
729
|
}
|
|
655
730
|
const bounded = enforceBulkSessionStoreBudget({
|
|
656
731
|
schema: BULK_SESSION_SCHEMA,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export const BULK_SORT_KEYS = ['domain', 'availability', 'confidence', 'risk', 'opportunity', 'activity', 'registrar', 'mutation'];
|
|
1
2
|
/** Opportunity remains a retained data/export field, but is no longer a Bulk presentation sort. */
|
|
2
3
|
export function normalizeBulkPresentationSortKey(key) {
|
|
3
4
|
return key === 'opportunity' ? 'risk' : key;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { CASE_VIEWS_SCHEMA, CASE_VIEWS_VERSION, CASE_VIEW_SORTS, MAX_CASE_VIEWS, MAX_CASE_VIEWS_BYTES, MAX_CASE_VIEW_NAME_LENGTH, MAX_CASE_VIEW_SEARCH_LENGTH, } from "../contracts/case-views-contract.mjs";
|
|
2
|
+
import { isValidDisposition, isValidStatus } from "../cases/case-record-decisions.mjs";
|
|
3
|
+
import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
|
|
4
|
+
import { assertWorkspaceInputGraph, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
|
|
5
|
+
const encoder = new TextEncoder();
|
|
6
|
+
const control = /[\u0000-\u001f\u007f]/u;
|
|
7
|
+
function object(raw, keys, label) {
|
|
8
|
+
const value = ordinaryWorkspaceRecord(raw, label);
|
|
9
|
+
if (!value || Object.keys(value).length !== keys.length || Object.keys(value).some(key => !keys.includes(key)))
|
|
10
|
+
throw new Error(`${label} has missing or unsupported fields.`);
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
function text(raw, maximum, label) {
|
|
14
|
+
if (typeof raw !== 'string' || raw.length > maximum || control.test(raw))
|
|
15
|
+
throw new Error(`${label} is invalid or exceeds ${maximum} characters.`);
|
|
16
|
+
return raw;
|
|
17
|
+
}
|
|
18
|
+
function time(raw) {
|
|
19
|
+
const value = normalizeExplicitIsoTimestamp(raw);
|
|
20
|
+
if (!value)
|
|
21
|
+
throw new Error('A saved Case view has an invalid timestamp.');
|
|
22
|
+
return value;
|
|
23
|
+
}
|
|
24
|
+
export function normalizeCaseViewFilters(raw) {
|
|
25
|
+
const value = object(raw, ['status', 'disposition', 'search', 'sort'], 'Case filters');
|
|
26
|
+
if (value.status !== '' && !isValidStatus(value.status))
|
|
27
|
+
throw new Error('Unsupported Case status filter.');
|
|
28
|
+
if (value.disposition !== '' && !isValidDisposition(value.disposition))
|
|
29
|
+
throw new Error('Unsupported Case disposition filter.');
|
|
30
|
+
const sort = CASE_VIEW_SORTS.find(option => option.value === value.sort)?.value;
|
|
31
|
+
if (!sort)
|
|
32
|
+
throw new Error('Unsupported Case sort order.');
|
|
33
|
+
return { status: value.status, disposition: value.disposition, search: text(value.search, MAX_CASE_VIEW_SEARCH_LENGTH, 'Case search'), sort };
|
|
34
|
+
}
|
|
35
|
+
export function emptyCaseViewsStore() {
|
|
36
|
+
return { schema: CASE_VIEWS_SCHEMA, version: CASE_VIEWS_VERSION, views: [] };
|
|
37
|
+
}
|
|
38
|
+
export function caseViewsStoreVersion(raw) {
|
|
39
|
+
const value = ordinaryWorkspaceRecord(raw, 'Case views');
|
|
40
|
+
return value?.schema === CASE_VIEWS_SCHEMA && Number.isSafeInteger(value.version) ? value.version : null;
|
|
41
|
+
}
|
|
42
|
+
/** Strict, lossless admission: invalid views and capacity excess never prune saved work. */
|
|
43
|
+
export function normalizeCaseViewsStore(raw) {
|
|
44
|
+
if (raw === null || raw === undefined)
|
|
45
|
+
return emptyCaseViewsStore();
|
|
46
|
+
assertWorkspaceInputGraph(raw, 'Saved Case views', { maximumBytes: MAX_CASE_VIEWS_BYTES });
|
|
47
|
+
const store = object(raw, ['schema', 'version', 'views'], 'Case views');
|
|
48
|
+
if (store.schema !== CASE_VIEWS_SCHEMA || store.version !== CASE_VIEWS_VERSION)
|
|
49
|
+
throw new Error('Unsupported saved Case view format. No views were changed.');
|
|
50
|
+
if (!Array.isArray(store.views) || store.views.length > MAX_CASE_VIEWS)
|
|
51
|
+
throw new Error(`Keep at most ${MAX_CASE_VIEWS} saved Case views. Delete unwanted views explicitly before adding more.`);
|
|
52
|
+
const ids = new Set();
|
|
53
|
+
const views = [];
|
|
54
|
+
let bytes = encoder.encode(JSON.stringify(emptyCaseViewsStore())).byteLength;
|
|
55
|
+
for (const candidate of store.views) {
|
|
56
|
+
const value = object(candidate, ['id', 'name', 'filters', 'createdAt', 'updatedAt'], 'Saved Case view');
|
|
57
|
+
if (typeof value.id !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/u.test(value.id) || ids.has(value.id))
|
|
58
|
+
throw new Error('Invalid or duplicate saved Case view identity.');
|
|
59
|
+
ids.add(value.id);
|
|
60
|
+
const name = text(value.name, MAX_CASE_VIEW_NAME_LENGTH, 'Case view name').trim();
|
|
61
|
+
if (!name)
|
|
62
|
+
throw new Error('Name this Case view.');
|
|
63
|
+
const createdAt = time(value.createdAt);
|
|
64
|
+
const updatedAt = time(value.updatedAt);
|
|
65
|
+
if (Date.parse(updatedAt) < Date.parse(createdAt))
|
|
66
|
+
throw new Error('A saved Case view update precedes its creation.');
|
|
67
|
+
const view = { id: value.id, name, filters: normalizeCaseViewFilters(value.filters), createdAt, updatedAt };
|
|
68
|
+
bytes += encoder.encode(JSON.stringify(view)).byteLength + (views.length ? 1 : 0);
|
|
69
|
+
if (bytes > MAX_CASE_VIEWS_BYTES)
|
|
70
|
+
throw new Error('Saved Case views exceed their storage limit. Existing views were not removed.');
|
|
71
|
+
views.push(view);
|
|
72
|
+
}
|
|
73
|
+
return { ...emptyCaseViewsStore(), views };
|
|
74
|
+
}
|
|
75
|
+
export function serializeCaseViewsStore(raw) {
|
|
76
|
+
return JSON.stringify(normalizeCaseViewsStore(raw));
|
|
77
|
+
}
|
|
78
|
+
export function serialiseCaseViewsJson(raw) {
|
|
79
|
+
return `${JSON.stringify(normalizeCaseViewsStore(raw), null, 2)}\n`;
|
|
80
|
+
}
|
|
81
|
+
export function mergeCaseViews(local, incoming) {
|
|
82
|
+
const current = normalizeCaseViewsStore(local);
|
|
83
|
+
const imported = normalizeCaseViewsStore(incoming);
|
|
84
|
+
const views = new Map(current.views.map(view => [view.id, view]));
|
|
85
|
+
let added = 0, updated = 0, skipped = 0;
|
|
86
|
+
for (const view of imported.views) {
|
|
87
|
+
const existing = views.get(view.id);
|
|
88
|
+
if (!existing) {
|
|
89
|
+
views.set(view.id, view);
|
|
90
|
+
added += 1;
|
|
91
|
+
}
|
|
92
|
+
else if (JSON.stringify(existing) === JSON.stringify(view))
|
|
93
|
+
continue;
|
|
94
|
+
else if (Date.parse(view.updatedAt) > Date.parse(existing.updatedAt) && view.createdAt === existing.createdAt) {
|
|
95
|
+
views.set(view.id, view);
|
|
96
|
+
updated += 1;
|
|
97
|
+
}
|
|
98
|
+
else
|
|
99
|
+
skipped += 1;
|
|
100
|
+
}
|
|
101
|
+
return { store: normalizeCaseViewsStore({ ...current, views: [...views.values()] }), added, updated, skipped };
|
|
102
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { MAX_WORKSPACE_INPUT_ARRAY_LENGTH, MAX_WORKSPACE_INPUT_GRAPH_DEPTH, MAX_WORKSPACE_INPUT_GRAPH_NODES, MAX_WORKSPACE_INPUT_OBJECT_KEYS, MAX_WORKSPACE_INPUT_STRING_CODE_UNITS, } from "../contracts/workspace-portability.mjs";
|
|
2
|
+
import { boundedJsonLimitsForBytes } from "../../lib/bounded-json.mjs";
|
|
2
3
|
function inputError(label, detail) {
|
|
3
4
|
return new TypeError(`${label} must be bounded ordinary JSON data; ${detail}.`);
|
|
4
5
|
}
|
|
@@ -27,9 +28,7 @@ function ownDataDescriptors(value, label) {
|
|
|
27
28
|
return [key, descriptor];
|
|
28
29
|
});
|
|
29
30
|
}
|
|
30
|
-
|
|
31
|
-
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
32
|
-
return null;
|
|
31
|
+
function ordinaryRecordDescriptors(value, label) {
|
|
33
32
|
let prototype;
|
|
34
33
|
try {
|
|
35
34
|
prototype = Object.getPrototypeOf(value);
|
|
@@ -47,19 +46,33 @@ export function ordinaryWorkspaceRecord(value, label = 'Workspace input') {
|
|
|
47
46
|
if (descriptors.some(([, descriptor]) => !descriptor.enumerable)) {
|
|
48
47
|
throw inputError(label, 'non-enumerable object fields are not supported');
|
|
49
48
|
}
|
|
49
|
+
return descriptors;
|
|
50
|
+
}
|
|
51
|
+
export function ordinaryWorkspaceRecord(value, label = 'Workspace input') {
|
|
52
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
53
|
+
return null;
|
|
54
|
+
ordinaryRecordDescriptors(value, label);
|
|
50
55
|
return value;
|
|
51
56
|
}
|
|
52
|
-
export function assertWorkspaceInputGraph(value, label = 'Workspace input') {
|
|
57
|
+
export function assertWorkspaceInputGraph(value, label = 'Workspace input', options = {}) {
|
|
58
|
+
const limits = options.maximumBytes === undefined ? null : boundedJsonLimitsForBytes(options.maximumBytes);
|
|
59
|
+
const maximumStringCodeUnits = limits?.maximumStringCodeUnits ?? MAX_WORKSPACE_INPUT_STRING_CODE_UNITS;
|
|
53
60
|
const pending = [{ value, depth: 0, label }];
|
|
54
61
|
const seen = new WeakSet();
|
|
55
62
|
let nodes = 0;
|
|
63
|
+
let values = 0;
|
|
64
|
+
let keys = 0;
|
|
56
65
|
let stringCodeUnits = 0;
|
|
57
66
|
while (pending.length) {
|
|
58
67
|
const current = pending.pop();
|
|
59
68
|
const candidate = current.value;
|
|
69
|
+
values += 1;
|
|
70
|
+
if (limits && values > limits.maximumValues) {
|
|
71
|
+
throw inputError(label, 'the graph exceeds the byte-budgeted value ceiling');
|
|
72
|
+
}
|
|
60
73
|
if (typeof candidate === 'string') {
|
|
61
74
|
stringCodeUnits += candidate.length;
|
|
62
|
-
if (stringCodeUnits >
|
|
75
|
+
if (stringCodeUnits > maximumStringCodeUnits) {
|
|
63
76
|
throw inputError(label, 'aggregate text exceeds the string ceiling');
|
|
64
77
|
}
|
|
65
78
|
continue;
|
|
@@ -76,7 +89,7 @@ export function assertWorkspaceInputGraph(value, label = 'Workspace input') {
|
|
|
76
89
|
throw inputError(label, 'the nesting depth exceeds the graph ceiling');
|
|
77
90
|
}
|
|
78
91
|
nodes += 1;
|
|
79
|
-
if (nodes > MAX_WORKSPACE_INPUT_GRAPH_NODES) {
|
|
92
|
+
if (!limits && nodes > MAX_WORKSPACE_INPUT_GRAPH_NODES) {
|
|
80
93
|
throw inputError(label, 'the graph exceeds the node ceiling');
|
|
81
94
|
}
|
|
82
95
|
if (seen.has(candidate))
|
|
@@ -118,8 +131,16 @@ export function assertWorkspaceInputGraph(value, label = 'Workspace input') {
|
|
|
118
131
|
}
|
|
119
132
|
continue;
|
|
120
133
|
}
|
|
121
|
-
const
|
|
122
|
-
|
|
134
|
+
const descriptors = ordinaryRecordDescriptors(candidate, current.label);
|
|
135
|
+
if (limits) {
|
|
136
|
+
keys += descriptors.length;
|
|
137
|
+
if (keys > limits.maximumKeys)
|
|
138
|
+
throw inputError(label, 'the graph exceeds the byte-budgeted key ceiling');
|
|
139
|
+
stringCodeUnits += descriptors.reduce((total, [key]) => total + key.length, 0);
|
|
140
|
+
if (stringCodeUnits > maximumStringCodeUnits) {
|
|
141
|
+
throw inputError(label, 'aggregate text exceeds the string ceiling');
|
|
142
|
+
}
|
|
143
|
+
}
|
|
123
144
|
for (const [key, descriptor] of descriptors) {
|
|
124
145
|
pending.push({
|
|
125
146
|
value: descriptor.value,
|
|
@@ -85,6 +85,9 @@ export function saveInvestigationTemplate(localRaw, candidateRaw) {
|
|
|
85
85
|
if (!candidate)
|
|
86
86
|
throw new Error('The investigation template is incomplete or invalid.');
|
|
87
87
|
const local = normalizeInvestigationTemplateStore(localRaw).templates;
|
|
88
|
+
if (!local.some((item) => item.id === candidate.id) && local.length >= MAX_INVESTIGATION_TEMPLATES) {
|
|
89
|
+
throw new Error(`Investigation-template storage is full (${MAX_INVESTIGATION_TEMPLATES} templates). Export and remove a template before saving another; no templates were changed.`);
|
|
90
|
+
}
|
|
88
91
|
return normalizeInvestigationTemplateStore([
|
|
89
92
|
candidate,
|
|
90
93
|
...local.filter((item) => item.id !== candidate.id),
|
|
@@ -126,11 +129,23 @@ export function mergeInvestigationTemplates(localRaw, incomingRaw) {
|
|
|
126
129
|
const byId = new Map(local.map((item) => [item.id, item]));
|
|
127
130
|
let added = 0;
|
|
128
131
|
let updated = 0;
|
|
132
|
+
let skipped = 0;
|
|
129
133
|
for (const item of incoming) {
|
|
130
|
-
|
|
134
|
+
const existing = byId.get(item.id);
|
|
135
|
+
if (existing) {
|
|
136
|
+
if (item.updatedAt <= existing.updatedAt) {
|
|
137
|
+
skipped += 1;
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
131
140
|
updated += 1;
|
|
132
|
-
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
if (byId.size >= MAX_INVESTIGATION_TEMPLATES) {
|
|
144
|
+
skipped += 1;
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
133
147
|
added += 1;
|
|
148
|
+
}
|
|
134
149
|
byId.set(item.id, item);
|
|
135
150
|
}
|
|
136
151
|
const templates = normalizeInvestigationTemplateStore([...byId.values()]).templates;
|
|
@@ -138,7 +153,7 @@ export function mergeInvestigationTemplates(localRaw, incomingRaw) {
|
|
|
138
153
|
templates,
|
|
139
154
|
added,
|
|
140
155
|
updated,
|
|
141
|
-
skipped
|
|
142
|
-
pruned:
|
|
156
|
+
skipped,
|
|
157
|
+
pruned: 0,
|
|
143
158
|
};
|
|
144
159
|
}
|
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
import { normalizeDomain } from "../cases/case-model.mjs";
|
|
6
6
|
import { isInformativePerceptualHash as isInformativeFaviconHash } from "../../lib/perceptual-hash-comparison.mjs";
|
|
7
7
|
import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
|
|
8
|
-
import {
|
|
9
|
-
|
|
8
|
+
import { PAGE_FINGERPRINT_TOKEN_LIMITS } from "../contracts/page-fingerprints.mjs";
|
|
9
|
+
import { MAX_BASELINE_IDENTIFIERS, MAX_BASELINE_RESOURCE_HOSTS, MAX_BASELINE_TITLE_LENGTH, PAGE_BASELINE_VERSION, PAGE_FINGERPRINT_PARSERS, PAGE_IDENTITY_VERSION, } from "../contracts/workspace-portability.mjs";
|
|
10
|
+
export { MAX_BASELINE_IDENTIFIERS, MAX_BASELINE_RESOURCE_HOSTS, MAX_BASELINE_TITLE_LENGTH, PAGE_BASELINE_VERSION, PAGE_FINGERPRINT_VERSION, PAGE_FINGERPRINT_PARSERS, PAGE_IDENTITY_VERSION, } from "../contracts/workspace-portability.mjs";
|
|
10
11
|
const SHA256_RE = /^[a-f0-9]{64}$/i;
|
|
11
12
|
const SIMHASH_RE = /^[a-f0-9]{16}$/i;
|
|
12
13
|
const CONTROL_RE = /[\x00-\x1f\x7f]/;
|
|
13
|
-
const MAX_HTML_TOKENS = 4096;
|
|
14
14
|
const MAX_TEXT_TOKENS = 8192;
|
|
15
15
|
const MAX_FORMS = 50;
|
|
16
16
|
const MAX_FORM_CONTROLS = 500;
|
|
@@ -50,12 +50,12 @@ function shaComponent(raw, countKey, maximum) {
|
|
|
50
50
|
truncated: value.truncated === true,
|
|
51
51
|
};
|
|
52
52
|
}
|
|
53
|
-
function visibleTextComponent(raw) {
|
|
53
|
+
function visibleTextComponent(raw, maximum = MAX_TEXT_TOKENS) {
|
|
54
54
|
const value = record(raw);
|
|
55
55
|
if (!value || value.algorithm !== 'simhash64-v1' || typeof value.value !== 'string' || !SIMHASH_RE.test(value.value))
|
|
56
56
|
return null;
|
|
57
|
-
const tokenCount = count(value.tokenCount,
|
|
58
|
-
const featureCount = count(value.featureCount,
|
|
57
|
+
const tokenCount = count(value.tokenCount, maximum);
|
|
58
|
+
const featureCount = count(value.featureCount, maximum);
|
|
59
59
|
if (tokenCount === null || featureCount === null)
|
|
60
60
|
return null;
|
|
61
61
|
return {
|
|
@@ -66,20 +66,21 @@ function visibleTextComponent(raw) {
|
|
|
66
66
|
truncated: value.truncated === true,
|
|
67
67
|
};
|
|
68
68
|
}
|
|
69
|
-
function domComponent(raw) {
|
|
69
|
+
function domComponent(raw, fingerprintVersion) {
|
|
70
70
|
const value = record(raw);
|
|
71
|
-
if (!value || value.algorithm !== 'sha256' || value.parser !==
|
|
71
|
+
if (!value || value.algorithm !== 'sha256' || value.parser !== PAGE_FINGERPRINT_PARSERS[fingerprintVersion])
|
|
72
72
|
return null;
|
|
73
73
|
const digest = sha256(value.value);
|
|
74
|
-
const
|
|
74
|
+
const maximum = PAGE_FINGERPRINT_TOKEN_LIMITS[fingerprintVersion];
|
|
75
|
+
const nodeCount = count(value.nodeCount, maximum);
|
|
75
76
|
if (!digest || nodeCount === null)
|
|
76
77
|
return null;
|
|
77
|
-
const similarity = value.similarity == null ? null : visibleTextComponent(value.similarity);
|
|
78
|
+
const similarity = value.similarity == null ? null : visibleTextComponent(value.similarity, maximum);
|
|
78
79
|
return {
|
|
79
80
|
algorithm: 'sha256',
|
|
80
81
|
value: digest,
|
|
81
82
|
nodeCount,
|
|
82
|
-
parser:
|
|
83
|
+
parser: PAGE_FINGERPRINT_PARSERS[fingerprintVersion],
|
|
83
84
|
truncated: value.truncated === true,
|
|
84
85
|
...(value.similarity == null ? {} : { similarity }),
|
|
85
86
|
};
|
|
@@ -175,11 +176,13 @@ export function normalizePageBaseline(raw) {
|
|
|
175
176
|
const lookupDomain = normalizeDomain(value.lookupDomain);
|
|
176
177
|
const observedAt = timestamp(value.observedAt);
|
|
177
178
|
const pageIdentityVersion = count(value.pageIdentityVersion, PAGE_IDENTITY_VERSION);
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
const
|
|
179
|
+
if (typeof value.fingerprintVersion !== 'number' || !Object.hasOwn(PAGE_FINGERPRINT_PARSERS, value.fingerprintVersion))
|
|
180
|
+
return null;
|
|
181
|
+
const fingerprintVersion = value.fingerprintVersion;
|
|
182
|
+
const normalizedHtml = shaComponent(value.normalizedHtml, 'tokenCount', PAGE_FINGERPRINT_TOKEN_LIMITS[fingerprintVersion]);
|
|
183
|
+
const domStructure = domComponent(value.domStructure, fingerprintVersion);
|
|
181
184
|
if (!domain || !lookupDomain || !observedAt || pageIdentityVersion !== PAGE_IDENTITY_VERSION
|
|
182
|
-
||
|
|
185
|
+
|| !normalizedHtml || !domStructure)
|
|
183
186
|
return null;
|
|
184
187
|
const visibleText = value.visibleText == null ? null : visibleTextComponent(value.visibleText);
|
|
185
188
|
const formStructure = value.formStructure == null ? null : formComponent(value.formStructure);
|