@slicedearth/whoisleuth-cli 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +6 -0
- package/README.md +19 -15
- package/bin/whoisleuth.mjs +8 -1
- package/cli/archive-content-digest.mjs +14 -0
- package/cli/archive-inspect.mjs +20 -53
- package/cli/arguments.mjs +68 -6
- package/cli/artifact-validation/case-response.mjs +47 -41
- package/cli/artifact-validation/investigation-capsule.mjs +52 -31
- package/cli/artifact-validation/lookup-evidence.mjs +12 -6
- package/cli/artifact-validation/signed-review.mjs +83 -61
- package/cli/artifact-validation/structure-primitives.mjs +1 -105
- package/cli/artifact-verify.mjs +90 -24
- package/cli/assurance-command-runner.mjs +39 -4
- package/cli/bagit-folder.mjs +57 -0
- package/cli/bagit-review.mjs +23 -0
- package/cli/bounded-json.mjs +1 -1
- package/cli/bulk-command-runner.mjs +2 -0
- package/cli/bulk-output.mjs +29 -3
- package/cli/case-command.mjs +195 -0
- package/cli/case-pack.mjs +39 -31
- package/cli/command-argument-grammar.mjs +4 -0
- package/cli/command-reference.mjs +145 -75
- package/cli/completion.mjs +298 -79
- package/cli/config-profile.mjs +7 -5
- package/cli/ct-event-intake.mjs +12 -12
- package/cli/discover.mjs +4 -3
- package/cli/discovery-observation-snapshot.mjs +44 -16
- package/cli/discovery-scan-command-runner.mjs +2 -0
- package/cli/discovery-scan.mjs +7 -4
- package/cli/discovery-workflow.mjs +1 -1
- package/cli/domain-control-monitor.mjs +50 -20
- package/cli/domain-control-observations.mjs +149 -102
- package/cli/errors.mjs +33 -1
- package/cli/evidence-command-runner.mjs +16 -5
- package/cli/evidence-signing.mjs +2 -2
- package/cli/external-observation-mapping.mjs +7 -8
- package/cli/formatters/terminal-lookup.mjs +18 -46
- package/cli/formatters/terminal-shared.mjs +5 -3
- package/cli/formatters/terminal-technology.mjs +114 -0
- package/cli/inline-command-families.mjs +1 -0
- package/cli/interactive-launcher.mjs +2 -160
- package/cli/interchange-report.mjs +4 -8
- package/cli/investigation-artifacts.mjs +176 -0
- package/cli/investigation-checkpoint.mjs +10 -0
- package/cli/investigation-folder.mjs +114 -0
- package/cli/investigation-manifest.mjs +1 -112
- package/cli/investigation-package-review.mjs +103 -0
- package/cli/investigation-plan.mjs +32 -19
- package/cli/investigation-run.mjs +188 -33
- package/cli/local-document-checkpoint.mjs +170 -0
- package/cli/lookup-browser.mjs +5 -12
- package/cli/lookup-command-runner.mjs +17 -6
- package/cli/lookup-diff.mjs +37 -3
- package/cli/lookup-plan.mjs +11 -5
- package/cli/lookup-reconcile.mjs +10 -8
- package/cli/mail-review.mjs +66 -14
- package/cli/manual.mjs +1 -0
- package/cli/network-command-runner.mjs +2 -0
- package/cli/output-file.mjs +31 -5
- package/cli/registry-doctor.mjs +3 -1
- package/cli/registry-support.mjs +1 -1
- package/cli/retained-artifact-diff.mjs +2 -2
- package/cli/review-command-runner.mjs +45 -0
- package/cli/runner.mjs +55 -6
- package/cli/sharing-review.mjs +21 -34
- package/cli/signer-trust.mjs +104 -0
- package/cli/source-reliability.mjs +74 -83
- package/cli/terminal-input.mjs +168 -0
- package/cli/workflow-command-runner.mjs +68 -13
- package/docs/cli-reference.md +76 -30
- package/docs/cli.md +289 -25
- package/lib/abort.mjs +4 -1
- package/lib/availability.mjs +161 -112
- package/lib/bounded-file.mjs +3 -1
- package/lib/bounded-html-document.mjs +125 -0
- package/lib/bounded-json.mjs +43 -3
- package/lib/browser-library-profile.mjs +42 -51
- package/lib/browser-library-worker.mjs +112 -0
- package/lib/client-behavior-profile.mjs +3 -3
- package/lib/ct-search.mjs +11 -4
- package/lib/dns-delegation-health.mjs +24 -14
- package/lib/dns-inheritance-review.mjs +217 -0
- package/lib/dns-intelligence.mjs +36 -23
- package/lib/dns-mx.mjs +1 -1
- package/lib/dnssec-evidence-validation.mjs +11 -3
- package/lib/domain-change-packet.mjs +7 -2
- package/lib/domain-change-review.mjs +14 -8
- package/lib/domain-control-flight-recorder.mjs +219 -118
- package/lib/domain-control-manifest.mjs +117 -57
- package/lib/domain-posture-analysis.mjs +20 -7
- package/lib/domain-posture-parsers.mjs +88 -6
- package/lib/domain-posture.mjs +113 -26
- package/lib/evidence-export-privacy.mjs +561 -0
- package/lib/evidence-export.mjs +11 -592
- package/lib/evidence-report.mjs +3 -1
- package/lib/favicon.mjs +117 -41
- package/lib/generated/cisa-kev-catalog.mjs +12 -4
- package/lib/generated/registrar-standing-catalogue.mjs +7 -7
- package/lib/generated/retire-browser-catalog.mjs +2335 -940
- package/lib/hostname.mjs +2 -38
- package/lib/html-signals.mjs +82 -85
- package/lib/http-evidence-bounds.mjs +2 -1
- package/lib/http-intelligence.mjs +39 -26
- package/lib/lookup-cache.mjs +34 -11
- package/lib/lookup-child-profile-contract.mjs +24 -15
- package/lib/lookup-diagnostics.mjs +24 -1
- package/lib/lookup-progress.mjs +30 -13
- package/lib/lookup-response-contract.mjs +1048 -0
- package/lib/lookup-response.mjs +7 -10
- package/lib/lookup-source-progress.mjs +17 -14
- package/lib/lookup.mjs +48 -27
- package/lib/observed-network-context.mjs +43 -27
- package/lib/outbound-request-bounds.mjs +4 -1
- package/lib/page-fingerprints.mjs +69 -133
- package/lib/page-role-profile.mjs +12 -8
- package/lib/perceptual-hash.mjs +43 -14
- package/lib/rdap-bootstrap.mjs +72 -22
- package/lib/rdap-client.mjs +15 -7
- package/lib/rdap-normalization.mjs +33 -18
- package/lib/rdap-search-workbench.mjs +1 -1
- package/lib/rdap-transport.mjs +47 -20
- package/lib/rdap-types.mjs +2 -1
- package/lib/rdap.mjs +18 -3
- package/lib/registrable-domain.mjs +21 -1
- package/lib/registrar-standing-contract.mjs +1 -0
- package/lib/registry-capabilities.mjs +5 -1
- package/lib/registry-capability-catalogue.mjs +2 -1
- package/lib/registry-dates.mjs +2 -190
- package/lib/registry-insights.mjs +56 -18
- package/lib/response-policy.mjs +86 -48
- package/lib/risk-calibration-summary.mjs +17 -3
- package/lib/rpki-evidence.mjs +8 -5
- package/lib/safe-fetch.mjs +54 -47
- package/lib/security-txt.mjs +10 -7
- package/lib/service-binding-dns.mjs +8 -0
- package/lib/smtp-transport-review.mjs +27 -15
- package/lib/sslbl-certificates.generated.mjs +136 -135
- package/lib/static-html-analysis.mjs +117 -146
- package/lib/structured-data-identity.mjs +1 -0
- package/lib/technology-markup.mjs +100 -0
- package/lib/threat-intelligence-manifests.mjs +11 -139
- package/lib/threat-intelligence-runtime.mjs +24 -52
- package/lib/threat-intelligence-targets.mjs +6 -34
- package/lib/threat-intelligence-types.mjs +39 -0
- package/lib/typosquat-generator.mjs +19 -15
- package/lib/urlhaus-intelligence.mjs +8 -3
- package/lib/urlscan-intelligence.mjs +1 -1
- package/lib/website-security-posture.mjs +30 -6
- package/lib/website-technology.mjs +111 -71
- package/lib/whois-authority.mjs +14 -2
- package/lib/whois-chain.mjs +9 -2
- package/lib/whois-common-formats.mjs +5 -5
- package/lib/whois-contracts.mjs +2 -1
- package/lib/whois-dialects.mjs +2 -2
- package/lib/whois-parser-support.mjs +8 -1
- package/lib/whois-transport.mjs +23 -7
- package/package.json +2 -1
- package/packages/cases/case-attachment-model.mjs +83 -0
- package/packages/cases/case-evidence-model.mjs +156 -62
- package/packages/cases/case-export-input.mjs +53 -0
- package/packages/cases/case-migration-model.mjs +59 -25
- package/packages/cases/case-model.mjs +1 -0
- package/packages/cases/case-recheck-model.mjs +91 -0
- package/packages/cases/case-record-core.mjs +3 -2
- package/packages/cases/case-record-operations.mjs +89 -27
- package/packages/cases/case-record-projection.mjs +23 -20
- package/packages/cases/case-report.mjs +63 -52
- package/packages/cases/case-response-actions.mjs +654 -0
- package/packages/cases/case-response-model.mjs +41 -1129
- package/packages/cases/case-response-outcomes.mjs +317 -0
- package/packages/cases/case-response-packet-render.mjs +153 -0
- package/packages/cases/case-response-packet-types.mjs +1 -0
- package/packages/cases/case-response-packet-vocabulary.mjs +48 -0
- package/packages/cases/case-response-packet.mjs +38 -587
- package/packages/cases/case-response-records.mjs +94 -0
- package/packages/cases/case-response-review-inputs.mjs +371 -0
- package/packages/cases/case-response-values.mjs +85 -0
- package/packages/cases/case-review-package.mjs +45 -0
- package/packages/cases/case-selection.mjs +46 -0
- package/packages/cases/case-storage-model.mjs +25 -0
- package/packages/cases/response-route-freshness.mjs +16 -0
- package/packages/comparison/capture-context.mjs +61 -0
- package/packages/comparison/comparison-ledger-bulk.mjs +4 -1
- package/packages/comparison/page-similarity.mjs +2 -1
- package/packages/comparison/relationship-evidence.mjs +85 -8
- package/packages/comparison/relationship-provenance.mjs +93 -0
- package/packages/contracts/analyst-interchange.mjs +216 -0
- package/packages/contracts/analyst-review-state-contract.mjs +11 -0
- package/packages/contracts/analyst-review-state.mjs +5 -4
- package/packages/contracts/case-portability.mjs +279 -116
- package/packages/contracts/case-views-contract.mjs +12 -0
- package/packages/contracts/case-views.mjs +26 -0
- package/packages/contracts/cli-command-semantics.mjs +1 -0
- package/packages/contracts/cli-lookup.mjs +0 -1
- package/packages/contracts/domain-control-flight-recorder.mjs +80 -42
- package/packages/contracts/domain-control-manifest.mjs +127 -73
- package/packages/contracts/domain-control-monitor.mjs +48 -45
- package/packages/contracts/domain-control-review.mjs +179 -121
- package/packages/contracts/domain-name.mjs +21 -0
- package/packages/contracts/external-observation-interchange.mjs +2 -1
- package/packages/contracts/extracted-domain-lifecycle.mjs +11 -32
- package/packages/contracts/http-url.mjs +2 -0
- package/packages/contracts/investigation-package-limits.mjs +24 -0
- package/packages/contracts/investigation-portability.mjs +56 -27
- package/packages/contracts/investigation-run.mjs +16 -0
- package/packages/contracts/ip-address.mjs +34 -0
- package/packages/contracts/lookup-evidence.mjs +35 -0
- package/packages/contracts/offline-comparison.mjs +1 -2
- package/packages/contracts/page-fingerprints.mjs +15 -0
- package/packages/contracts/risk-calibration.mjs +0 -1
- package/packages/contracts/schema-compatibility.mjs +1 -1
- package/packages/contracts/schema-lifecycle.mjs +83 -153
- package/packages/contracts/selected-file-limits.mjs +6 -0
- package/packages/contracts/vulnerability-identifiers.mjs +5 -0
- package/packages/contracts/web-capture.mjs +10 -0
- package/packages/contracts/workspace-portability.mjs +160 -225
- package/packages/evidence/artifact-integrity.mjs +8 -2
- package/packages/evidence/artifact-structure.mjs +105 -0
- package/packages/evidence/decision-fact.mjs +4 -1
- package/packages/evidence/domain-control-runtime.mjs +155 -27
- package/packages/evidence/domain-posture-context.mjs +79 -0
- package/packages/evidence/image-regions.mjs +56 -0
- package/packages/evidence/latest-observations.mjs +28 -0
- package/packages/evidence/lookup-target.mjs +104 -0
- package/packages/evidence/observation.mjs +16 -1
- package/packages/evidence/passphrase-encryption.mjs +44 -0
- package/packages/evidence/record-identity.mjs +72 -0
- package/packages/evidence/registry-dates.mjs +190 -0
- package/packages/evidence/retained-file.mjs +69 -0
- package/packages/evidence/review-age.mjs +14 -0
- package/packages/interchange/bagit.mjs +342 -0
- package/packages/interchange/bounded-zip-extraction.mjs +278 -0
- package/packages/interchange/crc32.mjs +17 -0
- package/packages/interchange/external-findings-import.mjs +89 -49
- package/packages/interchange/web-capture-import.mjs +382 -0
- package/packages/investigation/bulk-mail-exposure.mjs +2 -1
- package/packages/investigation/investigation-bagit.mjs +14 -0
- package/packages/investigation/investigation-capsule.mjs +33 -5
- package/packages/investigation/investigation-manifest.mjs +200 -0
- package/packages/investigation/investigation-package-crypto.mjs +98 -0
- package/packages/investigation/investigation-package.mjs +200 -0
- package/packages/investigation/lookup-asset-graph-inputs.mjs +38 -0
- package/packages/investigation/lookup-asset-graph.mjs +161 -141
- package/packages/investigation/lookup-claim-readiness.mjs +18 -0
- package/packages/investigation/lookup-investigation-brief.mjs +3 -3
- package/packages/investigation/lookup-task-guidance.mjs +58 -0
- package/packages/monitoring/analyst-review-state.mjs +38 -90
- package/packages/workspace/brand-profile-model.mjs +158 -20
- package/packages/workspace/bulk-columns.mjs +17 -0
- package/packages/workspace/bulk-review-model.mjs +11 -5
- package/packages/workspace/bulk-session-model.mjs +95 -20
- package/packages/workspace/bulk-sort.mjs +1 -0
- package/packages/workspace/case-views.mjs +102 -0
- package/packages/workspace/hostile-input.mjs +29 -8
- package/packages/workspace/investigation-template-model.mjs +19 -4
- package/packages/workspace/page-baseline.mjs +18 -15
- package/packages/workspace/relationship-observation-model.mjs +41 -26
- package/packages/workspace/watchlist-history.mjs +2 -2
- package/packages/workspace/watchlist-store.mjs +8 -2
- package/packages/workspace/website-snapshot-model.mjs +70 -30
- package/packages/workspace/workspace-archive-crypto.mjs +41 -61
- package/packages/workspace/workspace-archive.mjs +85 -21
- package/third-party-notices.txt +29 -2
- package/packages/investigation/bulk-review-export.mjs +0 -63
package/cli/ct-event-intake.mjs
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { domainToASCII } from 'node:url';
|
|
3
3
|
import { exactKeys, requireBoundedString, requireIsoTimestamp, requireRecord, } from "../lib/bounded-contract-normalizers.mjs";
|
|
4
|
+
import { EXTERNAL_FINDINGS_SCHEMA, EXTERNAL_FINDINGS_VERSION, MAX_EXTERNAL_FINDINGS, MAX_EXTERNAL_FINDING_DOMAINS, MAX_EXTERNAL_FINDINGS_PER_DOMAIN, MAX_EXTERNAL_FINDING_LIMITATIONS, MAX_EXTERNAL_FINDING_LIMITATION_LENGTH, retainExternalFindingLimitations, } from "../packages/interchange/external-findings-import.mjs";
|
|
4
5
|
export const CT_EVENT_BATCH_SCHEMA = 'whoisleuth.ct-event-batch';
|
|
5
6
|
export const CT_EVENT_BATCH_VERSION = 1;
|
|
6
7
|
export const MAX_CT_EVENT_INPUT_BYTES = 4 * 1024 * 1024;
|
|
7
8
|
export const MAX_CT_EVENTS = 500;
|
|
8
9
|
export const MAX_CT_EVENT_NAMES = 100;
|
|
9
|
-
export const MAX_CT_EXTERNAL_FINDINGS =
|
|
10
|
-
export const MAX_CT_EXTERNAL_DOMAINS =
|
|
11
|
-
export const MAX_CT_EXTERNAL_FINDINGS_PER_DOMAIN =
|
|
10
|
+
export const MAX_CT_EXTERNAL_FINDINGS = MAX_EXTERNAL_FINDINGS;
|
|
11
|
+
export const MAX_CT_EXTERNAL_DOMAINS = MAX_EXTERNAL_FINDING_DOMAINS;
|
|
12
|
+
export const MAX_CT_EXTERNAL_FINDINGS_PER_DOMAIN = MAX_EXTERNAL_FINDINGS_PER_DOMAIN;
|
|
12
13
|
const ROOT_KEYS = new Set(['schema', 'version', 'source', 'events']);
|
|
13
14
|
const SOURCE_KEYS = new Set(['name', 'reference', 'collectedAt']);
|
|
14
15
|
const EVENT_KEYS = new Set([
|
|
@@ -47,9 +48,9 @@ function domain(value, label) {
|
|
|
47
48
|
function boundedLimitations(value, label) {
|
|
48
49
|
if (value === undefined)
|
|
49
50
|
return Object.freeze([]);
|
|
50
|
-
if (!Array.isArray(value) || value.length >
|
|
51
|
-
throw new TypeError(`${label} must contain no more than
|
|
52
|
-
return Object.freeze([...new Set(value.map((item, index) => requireBoundedString(item, `${label}[${index}]`,
|
|
51
|
+
if (!Array.isArray(value) || value.length > MAX_EXTERNAL_FINDING_LIMITATIONS)
|
|
52
|
+
throw new TypeError(`${label} must contain no more than ${MAX_EXTERNAL_FINDING_LIMITATIONS} entries.`);
|
|
53
|
+
return Object.freeze([...new Set(value.map((item, index) => requireBoundedString(item, `${label}[${index}]`, MAX_EXTERNAL_FINDING_LIMITATION_LENGTH)))]);
|
|
53
54
|
}
|
|
54
55
|
export function buildCtEventFindings(inputRaw) {
|
|
55
56
|
const input = requireRecord(inputRaw, 'Certificate event batch');
|
|
@@ -155,19 +156,18 @@ export function buildCtEventFindings(inputRaw) {
|
|
|
155
156
|
namesComplete: finding.structuredObservation.namesComplete
|
|
156
157
|
&& selectedNamesByEvent.get(finding.structuredObservation.eventId) === finding.structuredObservation.dnsNameCount,
|
|
157
158
|
}),
|
|
158
|
-
limitations:
|
|
159
|
-
...finding.limitations,
|
|
159
|
+
limitations: retainExternalFindingLimitations(finding.limitations, [
|
|
160
160
|
'The supplied event is an observation, not proof that the certificate was served by the named domain or that the domain operator requested it.',
|
|
161
161
|
...(finding.structuredObservation.namesComplete
|
|
162
162
|
&& selectedNamesByEvent.get(finding.structuredObservation.eventId) !== finding.structuredObservation.dnsNameCount
|
|
163
163
|
? ['The bounded import did not retain every DNS name from this certificate event.']
|
|
164
164
|
: []),
|
|
165
|
-
...(truncated ? [`The batch contained ${unique.length} unique domain observations; this import preserves the first ${
|
|
166
|
-
]
|
|
165
|
+
...(truncated ? [`The batch contained ${unique.length} unique domain observations; this import preserves the first ${selected.length} in deterministic order.`] : []),
|
|
166
|
+
]),
|
|
167
167
|
}));
|
|
168
168
|
return Object.freeze({
|
|
169
|
-
schema:
|
|
170
|
-
schemaVersion:
|
|
169
|
+
schema: EXTERNAL_FINDINGS_SCHEMA,
|
|
170
|
+
schemaVersion: EXTERNAL_FINDINGS_VERSION,
|
|
171
171
|
source,
|
|
172
172
|
findings: Object.freeze(findings),
|
|
173
173
|
});
|
package/cli/discover.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
2
|
import { decodeBoundedUtf8 } from "../lib/bounded-file.mjs";
|
|
3
3
|
import { CliUsageError } from "./arguments.mjs";
|
|
4
|
+
import { normalizeDiscoverySuffix } from "../lib/registrable-domain.mjs";
|
|
4
5
|
const DEFAULT_DISCOVERY_TLDS = Object.freeze(['com', 'net', 'org']);
|
|
5
6
|
const MAX_DISCOVERY_TLD_TEXT_LENGTH = 1024;
|
|
6
7
|
const MAX_DISCOVERY_TLD_TOKENS_INSPECTED = 80;
|
|
@@ -40,9 +41,9 @@ function normalizeDiscoveryTlds(raw, maximum) {
|
|
|
40
41
|
const values = [];
|
|
41
42
|
const seen = new Set();
|
|
42
43
|
for (const token of tokens) {
|
|
43
|
-
const value = token
|
|
44
|
-
if (
|
|
45
|
-
throw new CliUsageError(`Invalid TLD "${token}".`);
|
|
44
|
+
const value = normalizeDiscoverySuffix(token);
|
|
45
|
+
if (!value)
|
|
46
|
+
throw new CliUsageError(`Invalid TLD or public suffix "${token}".`);
|
|
46
47
|
if (seen.has(value))
|
|
47
48
|
continue;
|
|
48
49
|
seen.add(value);
|
|
@@ -171,6 +171,20 @@ async function readExisting(path) {
|
|
|
171
171
|
throw boundedCliInputError(error, 'Discovery observation snapshot');
|
|
172
172
|
}
|
|
173
173
|
}
|
|
174
|
+
const DNS_MATERIAL_FIELDS = ['a', 'aaaa', 'ns', 'mx', 'hasNullMx', 'hasSpf', 'hasDmarc', 'status'];
|
|
175
|
+
function comparableValue(value) {
|
|
176
|
+
return Array.isArray(value) ? [...new Set(value.map((item) => String(item).toLowerCase()))].sort() : value;
|
|
177
|
+
}
|
|
178
|
+
function dnsMaterial(dns) {
|
|
179
|
+
return JSON.stringify(DNS_MATERIAL_FIELDS.map((field) => comparableValue(dns[field])));
|
|
180
|
+
}
|
|
181
|
+
function componentRecency(previousAt, previousMaterial, currentAt, currentMaterial) {
|
|
182
|
+
if (!previousAt || Date.parse(currentAt) > Date.parse(previousAt))
|
|
183
|
+
return 'newer';
|
|
184
|
+
if (Date.parse(currentAt) < Date.parse(previousAt))
|
|
185
|
+
return 'older';
|
|
186
|
+
return previousMaterial === currentMaterial ? 'equal' : 'conflicting';
|
|
187
|
+
}
|
|
174
188
|
function currentAttempt(domain, item, previous, generatedAt, deep) {
|
|
175
189
|
const suppliedObservedAt = item.observedAt === null || item.observedAt === undefined
|
|
176
190
|
? null
|
|
@@ -182,35 +196,46 @@ function currentAttempt(domain, item, previous, generatedAt, deep) {
|
|
|
182
196
|
const availability = item.ok ? record(record(item.result).availability) : {};
|
|
183
197
|
const currentAvailabilityState = availabilityState(item) || 'unknown';
|
|
184
198
|
const registrationObserved = item.ok && currentAvailabilityState !== 'unknown';
|
|
199
|
+
const currentConfidence = registrationObserved && typeof availability.confidence === 'string'
|
|
200
|
+
? availability.confidence.replace(/[\u0000-\u001f\u007f]+/gu, ' ').trim().slice(0, 40) || 'unknown'
|
|
201
|
+
: 'unknown';
|
|
185
202
|
const currentDns = bulkDnsSummary(item);
|
|
186
203
|
const dnsComplete = deep && currentDns.status === 'success';
|
|
187
204
|
const dnsRetainable = deep && currentDns.status === 'partial';
|
|
188
205
|
const previousDnsRetainable = previous && ['partial', 'success'].includes(previous.dns.status);
|
|
189
|
-
const
|
|
190
|
-
|
|
206
|
+
const registrationRecency = componentRecency(previous?.registrationObservedAt, JSON.stringify([previous?.availabilityState, previous?.confidence]), attemptAt, JSON.stringify([currentAvailabilityState, currentConfidence]));
|
|
207
|
+
const dnsRecency = componentRecency(previous?.dnsObservedAt, previous ? dnsMaterial(previous.dns) : '', attemptAt, dnsMaterial(currentDns));
|
|
208
|
+
const registrationAdmitted = registrationObserved && ['newer', 'equal'].includes(registrationRecency);
|
|
209
|
+
const dnsAdmitted = (dnsComplete || (dnsRetainable && !previousDnsRetainable)) && ['newer', 'equal'].includes(dnsRecency);
|
|
210
|
+
const dns = dnsAdmitted
|
|
211
|
+
? dnsRecency === 'equal' && previous ? previous.dns : currentDns
|
|
191
212
|
: previousDnsRetainable ? previous.dns : currentDns;
|
|
192
|
-
const registrationObservedAt =
|
|
193
|
-
const dnsObservedAt =
|
|
213
|
+
const registrationObservedAt = registrationAdmitted ? attemptAt : previous?.registrationObservedAt ?? null;
|
|
214
|
+
const dnsObservedAt = dnsAdmitted
|
|
194
215
|
? attemptAt
|
|
195
216
|
: previous?.dnsObservedAt ?? null;
|
|
196
217
|
const unavailableComponents = [];
|
|
197
|
-
if (!
|
|
218
|
+
if (!registrationAdmitted)
|
|
198
219
|
unavailableComponents.push('registration');
|
|
199
|
-
if (deep && !dnsComplete)
|
|
220
|
+
if (deep && !(dnsComplete && dnsAdmitted))
|
|
200
221
|
unavailableComponents.push('dns');
|
|
201
222
|
const attemptState = !item.ok
|
|
202
223
|
? 'error'
|
|
203
224
|
: unavailableComponents.length ? 'partial' : 'success';
|
|
204
|
-
const confidence =
|
|
205
|
-
?
|
|
225
|
+
const confidence = registrationAdmitted
|
|
226
|
+
? currentConfidence
|
|
206
227
|
: previous?.confidence ?? 'unknown';
|
|
207
|
-
const retainedAvailabilityState =
|
|
228
|
+
const retainedAvailabilityState = registrationAdmitted
|
|
208
229
|
? currentAvailabilityState
|
|
209
230
|
: previous?.availabilityState ?? 'unknown';
|
|
210
231
|
return {
|
|
211
|
-
registrationObserved,
|
|
212
|
-
dnsComplete,
|
|
232
|
+
registrationObserved: registrationAdmitted,
|
|
233
|
+
dnsComplete: dnsComplete && dnsAdmitted,
|
|
213
234
|
unavailableComponents,
|
|
235
|
+
withheld: [
|
|
236
|
+
...(registrationObserved && !registrationAdmitted ? [`${registrationRecency === 'older' ? 'Older' : 'Conflicting equal-time'} registration observations`] : []),
|
|
237
|
+
...((dnsComplete || dnsRetainable) && ['older', 'conflicting'].includes(dnsRecency) ? [`${dnsRecency === 'older' ? 'Older' : 'Conflicting equal-time'} DNS observations`] : []),
|
|
238
|
+
],
|
|
214
239
|
observation: {
|
|
215
240
|
domain,
|
|
216
241
|
observedAt: latestObservedAt(registrationObservedAt, dnsObservedAt),
|
|
@@ -219,7 +244,7 @@ function currentAttempt(domain, item, previous, generatedAt, deep) {
|
|
|
219
244
|
latestAttemptAt: attemptAt,
|
|
220
245
|
collectionOrigin: item.collectionOrigin ?? 'current_run',
|
|
221
246
|
latestAttemptState: attemptState,
|
|
222
|
-
latestRegistrationState:
|
|
247
|
+
latestRegistrationState: registrationAdmitted ? 'observed' : 'unavailable',
|
|
223
248
|
latestDnsState: deep ? currentDns.status : 'not_requested',
|
|
224
249
|
availabilityState: retainedAvailabilityState,
|
|
225
250
|
confidence,
|
|
@@ -233,7 +258,7 @@ function materialChanges(previous, current, attempt = {
|
|
|
233
258
|
}) {
|
|
234
259
|
const changes = [];
|
|
235
260
|
const compare = (field, before, after) => {
|
|
236
|
-
if (JSON.stringify(before) !== JSON.stringify(after))
|
|
261
|
+
if (JSON.stringify(comparableValue(before)) !== JSON.stringify(comparableValue(after)))
|
|
237
262
|
changes.push({ field, before, after });
|
|
238
263
|
};
|
|
239
264
|
if (attempt.registrationObserved) {
|
|
@@ -241,10 +266,9 @@ function materialChanges(previous, current, attempt = {
|
|
|
241
266
|
compare('confidence', previous.confidence, current.confidence);
|
|
242
267
|
}
|
|
243
268
|
if (attempt.dnsComplete) {
|
|
244
|
-
for (const field of
|
|
269
|
+
for (const field of DNS_MATERIAL_FIELDS) {
|
|
245
270
|
compare(`dns.${field}`, previous.dns[field], current.dns[field]);
|
|
246
271
|
}
|
|
247
|
-
compare('dns.status', previous.dns.status, current.dns.status);
|
|
248
272
|
}
|
|
249
273
|
return changes;
|
|
250
274
|
}
|
|
@@ -265,6 +289,7 @@ async function updateDiscoveryObservationSnapshot(path, candidates, items, confi
|
|
|
265
289
|
const changed = [];
|
|
266
290
|
const unavailable = [];
|
|
267
291
|
const unavailableComponents = [];
|
|
292
|
+
const withheld = new Map();
|
|
268
293
|
for (let index = 0; index < domains.length; index++) {
|
|
269
294
|
const domain = domains[index];
|
|
270
295
|
const item = items[index];
|
|
@@ -273,6 +298,8 @@ async function updateDiscoveryObservationSnapshot(path, candidates, items, confi
|
|
|
273
298
|
const previous = priorByDomain.get(domain);
|
|
274
299
|
const attempt = currentAttempt(domain, item, previous, currentGeneratedAt, configuration.deep);
|
|
275
300
|
const current = attempt.observation;
|
|
301
|
+
for (const reason of attempt.withheld)
|
|
302
|
+
withheld.set(reason, (withheld.get(reason) ?? 0) + 1);
|
|
276
303
|
if (attempt.unavailableComponents.length) {
|
|
277
304
|
unavailable.push(domain);
|
|
278
305
|
unavailableComponents.push({ domain, components: attempt.unavailableComponents });
|
|
@@ -310,7 +337,8 @@ async function updateDiscoveryObservationSnapshot(path, candidates, items, confi
|
|
|
310
337
|
unchanged: prior === null ? 0 : domains.filter((domain) => !changed.some((item) => item.domain === domain) && !unavailable.includes(domain)).length,
|
|
311
338
|
limitations: [
|
|
312
339
|
'Only bounded registration-state and DNS summaries are retained; raw registry records, contacts, page contents, and request details are excluded.',
|
|
313
|
-
'Registration and DNS retain separate observation times.
|
|
340
|
+
'Registration and DNS retain separate observation times. Failed, incomplete, older or conflicting equal-time components preserve the prior usable baseline and are unavailable for a new comparison, not evidence of removal.',
|
|
341
|
+
...[...withheld].map(([reason, count]) => `${reason}: ${count}. The retained component baseline was not replaced.`),
|
|
314
342
|
'Material differences are observation changes for analyst review and do not establish ownership, control, intent, safety, or maliciousness.',
|
|
315
343
|
],
|
|
316
344
|
};
|
|
@@ -163,6 +163,8 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
|
|
|
163
163
|
context.writeStdout(formatDiscoveryScanJsonLines(document));
|
|
164
164
|
else if (args.output === 'csv')
|
|
165
165
|
context.writeStdout(formatDiscoveryScanCsv(document));
|
|
166
|
+
else if (args.output === 'csv_metadata')
|
|
167
|
+
context.writeStdout(formatDiscoveryScanCsv(document, true));
|
|
166
168
|
else if (args.output === 'domains')
|
|
167
169
|
context.writeStdout(formatDiscoveryScanDomains(document));
|
|
168
170
|
else
|
package/cli/discovery-scan.mjs
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { Buffer } from 'node:buffer';
|
|
3
3
|
import { decodeBoundedUtf8 } from "../lib/bounded-file.mjs";
|
|
4
|
+
import { MAX_DISCOVERY_SUFFIX_LENGTH } from "../lib/registrable-domain.mjs";
|
|
4
5
|
import { runBulkLookups } from "./bulk.mjs";
|
|
5
|
-
import { REGISTERED_STATES, availabilityState, bulkDnsSummary } from "./bulk-output.mjs";
|
|
6
|
+
import { REGISTERED_STATES, BULK_CSV_METADATA_COLUMNS, availabilityState, bulkCsvMetadataValues, bulkDnsSummary } from "./bulk-output.mjs";
|
|
6
7
|
import { cliCsvCell } from "./csv.mjs";
|
|
7
8
|
import { CliUsageError } from "./errors.mjs";
|
|
8
9
|
import { terminalSafeJson } from "./formatters/json.mjs";
|
|
@@ -70,7 +71,7 @@ function parseDiscoveryScanAllowlist(text, classifyQuery) {
|
|
|
70
71
|
function normalizedCandidate(candidate) {
|
|
71
72
|
const domain = boundedText(candidate.domain, 253).toLowerCase();
|
|
72
73
|
const source = boundedText(candidate.source, 253).toLowerCase();
|
|
73
|
-
const tld = boundedText(candidate.tld,
|
|
74
|
+
const tld = boundedText(candidate.tld, MAX_DISCOVERY_SUFFIX_LENGTH).toLowerCase();
|
|
74
75
|
const mutationTypes = Array.isArray(candidate.mutationTypes)
|
|
75
76
|
? [...new Set(candidate.mutationTypes.flatMap((value) => {
|
|
76
77
|
const normalized = boundedText(value, 80);
|
|
@@ -308,12 +309,14 @@ function formatDiscoveryScanJsonLines(document) {
|
|
|
308
309
|
? `${document.results.map((item) => terminalSafeJson({ ...item, generatedAt: document.generatedAt })).join('\n')}\n`
|
|
309
310
|
: '';
|
|
310
311
|
}
|
|
311
|
-
function formatDiscoveryScanCsv(document) {
|
|
312
|
-
const header = ['domain', 'availability', 'confidence', 'review_lane', 'mutation_types', 'a', 'aaaa', 'ns', 'mx', 'relationship_ids', 'error'
|
|
312
|
+
function formatDiscoveryScanCsv(document, includeMetadata = false) {
|
|
313
|
+
const header = ['domain', 'availability', 'confidence', 'review_lane', 'mutation_types', 'a', 'aaaa', 'ns', 'mx', 'relationship_ids', 'error',
|
|
314
|
+
...(includeMetadata ? BULK_CSV_METADATA_COLUMNS : [])];
|
|
313
315
|
const rows = document.results.map((item) => [
|
|
314
316
|
item.domain, item.availabilityState, item.confidence, item.review.lane, item.mutationTypes,
|
|
315
317
|
item.dnsSummary.a, item.dnsSummary.aaaa, item.dnsSummary.ns, item.dnsSummary.mx,
|
|
316
318
|
item.relationshipIds, 'error' in item ? item.error : '',
|
|
319
|
+
...(includeMetadata ? bulkCsvMetadataValues(item, document) : []),
|
|
317
320
|
].map(cliCsvCell).join(','));
|
|
318
321
|
return `${[header.join(','), ...rows].join('\n')}\n`;
|
|
319
322
|
}
|
|
@@ -47,7 +47,7 @@ async function generateDiscoveryCandidates(args, dependencies, context) {
|
|
|
47
47
|
...(args.preset === 'custom' ? { mutationTypes: mutationFamilies } : {}),
|
|
48
48
|
});
|
|
49
49
|
if (!result.inputValid) {
|
|
50
|
-
throw new CliUsageError('discover requires a valid brand label or domain
|
|
50
|
+
throw new CliUsageError('discover requires a valid brand label or registrable domain, without a subdomain or URL.');
|
|
51
51
|
}
|
|
52
52
|
const normalizedDictionary = generator.normalizeCustomDictionaryTerms(dictionaryText);
|
|
53
53
|
const metadata = {
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { classifyQuery } from "../lib/classify.mjs";
|
|
2
2
|
import { scanBoundedJson } from "../lib/bounded-json.mjs";
|
|
3
|
-
import { buildDomainControlFlightRecorder, validateDomainControlFlightRecorderDocument, } from "../lib/domain-control-flight-recorder.mjs";
|
|
4
|
-
import { DOMAIN_CONTROL_FLIGHT_RECORDER_SCHEMA, DOMAIN_CONTROL_FLIGHT_RECORDER_INPUT_SCHEMA, DOMAIN_CONTROL_FLIGHT_RECORDER_VERSION, } from "../packages/contracts/domain-control-flight-recorder.mjs";
|
|
5
|
-
import { DOMAIN_CONTROL_REVIEW_SCHEMA, DOMAIN_CONTROL_REVIEW_VERSION, CLI_DOMAIN_CONTROL_REVIEW_INPUT_SCHEMA, CLI_DOMAIN_CONTROL_REVIEW_VERSION, } from "../packages/contracts/domain-control-review.mjs";
|
|
6
|
-
import { CLI_DOMAIN_CONTROL_MONITOR_SCHEMA, CLI_DOMAIN_CONTROL_MONITOR_VERSION, DOMAIN_CONTROL_MONITOR_COLLECTION_KEYS, DOMAIN_CONTROL_MONITOR_FAILURE_CATEGORIES, DOMAIN_CONTROL_MONITOR_FAILURE_KEYS, DOMAIN_CONTROL_MONITOR_LIMITATIONS, DOMAIN_CONTROL_MONITOR_MANIFEST_KEYS, DOMAIN_CONTROL_MONITOR_ROOT_KEYS, MAX_DOMAIN_CONTROL_MONITOR_CONCURRENCY, MAX_DOMAIN_CONTROL_MONITOR_DOMAINS as CONTRACT_MAX_DOMAIN_CONTROL_MONITOR_DOMAINS, MAX_DOMAIN_CONTROL_MONITOR_ERROR_LENGTH, MAX_DOMAIN_CONTROL_MONITOR_FAILURES, MAX_DOMAIN_CONTROL_MONITOR_INPUT_BYTES, MAX_DOMAIN_CONTROL_MONITOR_JSON_CONTAINER_ITEMS, MAX_DOMAIN_CONTROL_MONITOR_JSON_DEPTH, MAX_DOMAIN_CONTROL_MONITOR_JSON_KEYS, MAX_DOMAIN_CONTROL_MONITOR_JSON_VALUES, MIN_DOMAIN_CONTROL_MONITOR_CONCURRENCY, MIN_DOMAIN_CONTROL_MONITOR_DOMAINS, } from "../packages/contracts/domain-control-monitor.mjs";
|
|
3
|
+
import { buildDomainControlFlightRecorder, validateDomainControlFlightRecorderDocument, normalizeDomainControlFlightRecorderObservations, } from "../lib/domain-control-flight-recorder.mjs";
|
|
4
|
+
import { DOMAIN_CONTROL_FLIGHT_RECORDER_SCHEMA, DOMAIN_CONTROL_FLIGHT_RECORDER_INPUT_SCHEMA, DOMAIN_CONTROL_FLIGHT_RECORDER_VERSION, PUBLIC_DOMAIN_CONTROL_FLIGHT_RECORDER_VERSION, } from "../packages/contracts/domain-control-flight-recorder.mjs";
|
|
5
|
+
import { DOMAIN_CONTROL_REVIEW_SCHEMA, DOMAIN_CONTROL_REVIEW_VERSION, PUBLIC_DOMAIN_CONTROL_REVIEW_VERSION, CLI_DOMAIN_CONTROL_REVIEW_INPUT_SCHEMA, CLI_DOMAIN_CONTROL_REVIEW_VERSION, } from "../packages/contracts/domain-control-review.mjs";
|
|
6
|
+
import { CLI_DOMAIN_CONTROL_MONITOR_SCHEMA, CLI_DOMAIN_CONTROL_MONITOR_VERSION, PUBLIC_CLI_DOMAIN_CONTROL_MONITOR_VERSION, SUPPORTED_CLI_DOMAIN_CONTROL_MONITOR_VERSIONS, DOMAIN_CONTROL_MONITOR_COLLECTION_KEYS, DOMAIN_CONTROL_MONITOR_FAILURE_CATEGORIES, DOMAIN_CONTROL_MONITOR_FAILURE_KEYS, DOMAIN_CONTROL_MONITOR_LIMITATIONS, DOMAIN_CONTROL_MONITOR_MANIFEST_KEYS, DOMAIN_CONTROL_MONITOR_ROOT_KEYS, MAX_DOMAIN_CONTROL_MONITOR_CONCURRENCY, MAX_DOMAIN_CONTROL_MONITOR_DOMAINS as CONTRACT_MAX_DOMAIN_CONTROL_MONITOR_DOMAINS, MAX_DOMAIN_CONTROL_MONITOR_ERROR_LENGTH, MAX_DOMAIN_CONTROL_MONITOR_FAILURES, MAX_DOMAIN_CONTROL_MONITOR_INPUT_BYTES, MAX_DOMAIN_CONTROL_MONITOR_JSON_CONTAINER_ITEMS, MAX_DOMAIN_CONTROL_MONITOR_JSON_DEPTH, MAX_DOMAIN_CONTROL_MONITOR_JSON_KEYS, MAX_DOMAIN_CONTROL_MONITOR_JSON_VALUES, MIN_DOMAIN_CONTROL_MONITOR_CONCURRENCY, MIN_DOMAIN_CONTROL_MONITOR_DOMAINS, } from "../packages/contracts/domain-control-monitor.mjs";
|
|
7
7
|
import { validateDomainControlReviewDocument, verifyDomainControlManifest, } from "../lib/domain-control-manifest.mjs";
|
|
8
8
|
import { requireDomainName, requireIsoTimestamp } from "../lib/bounded-contract-normalizers.mjs";
|
|
9
9
|
import { buildCliLookupDocument } from "./formatters/json.mjs";
|
|
10
|
-
import { buildCliDomainControlReview, } from "./domain-control-observations.mjs";
|
|
10
|
+
import { buildCliDomainControlReview, DOMAIN_CONTROL_REVIEW_SOURCE_FIELDS, } from "./domain-control-observations.mjs";
|
|
11
11
|
import { CliUsageError } from "./errors.mjs";
|
|
12
12
|
export { CLI_DOMAIN_CONTROL_MONITOR_SCHEMA, CLI_DOMAIN_CONTROL_MONITOR_VERSION };
|
|
13
13
|
export const MAX_DOMAIN_CONTROL_MONITOR_DOMAINS = CONTRACT_MAX_DOMAIN_CONTROL_MONITOR_DOMAINS;
|
|
@@ -58,9 +58,12 @@ function previousSnapshot(input, currentGeneratedAt) {
|
|
|
58
58
|
return { observations: [] };
|
|
59
59
|
const parsed = parseMonitorJson(input, 'Previous monitor snapshot');
|
|
60
60
|
const root = exactMonitorRecord(parsed, DOMAIN_CONTROL_MONITOR_ROOT_KEYS, 'Previous monitor snapshot');
|
|
61
|
-
if (root.schema !== CLI_DOMAIN_CONTROL_MONITOR_SCHEMA ||
|
|
62
|
-
throw new CliUsageError(`Previous monitor snapshot must use ${CLI_DOMAIN_CONTROL_MONITOR_SCHEMA} version ${
|
|
61
|
+
if (root.schema !== CLI_DOMAIN_CONTROL_MONITOR_SCHEMA || !SUPPORTED_CLI_DOMAIN_CONTROL_MONITOR_VERSIONS.some((version) => version === root.version)) {
|
|
62
|
+
throw new CliUsageError(`Previous monitor snapshot must use ${CLI_DOMAIN_CONTROL_MONITOR_SCHEMA} supported version ${SUPPORTED_CLI_DOMAIN_CONTROL_MONITOR_VERSIONS.join(' or ')}.`);
|
|
63
63
|
}
|
|
64
|
+
const legacy = root.version === PUBLIC_CLI_DOMAIN_CONTROL_MONITOR_VERSION;
|
|
65
|
+
const reviewVersion = legacy ? PUBLIC_DOMAIN_CONTROL_REVIEW_VERSION : DOMAIN_CONTROL_REVIEW_VERSION;
|
|
66
|
+
const flightVersion = legacy ? PUBLIC_DOMAIN_CONTROL_FLIGHT_RECORDER_VERSION : DOMAIN_CONTROL_FLIGHT_RECORDER_VERSION;
|
|
64
67
|
let previousGeneratedAt;
|
|
65
68
|
try {
|
|
66
69
|
previousGeneratedAt = requireIsoTimestamp(root.generatedAt, 'Previous monitor snapshot generatedAt');
|
|
@@ -83,7 +86,7 @@ function previousSnapshot(input, currentGeneratedAt) {
|
|
|
83
86
|
catch {
|
|
84
87
|
throw new CliUsageError('Previous monitor snapshot manifest expiresAt must be a valid ISO 8601 timestamp.');
|
|
85
88
|
}
|
|
86
|
-
if (Date.parse(manifest.expiresAt) <= Date.parse(previousGeneratedAt)) {
|
|
89
|
+
if (legacy && Date.parse(manifest.expiresAt) <= Date.parse(previousGeneratedAt)) {
|
|
87
90
|
throw new CliUsageError('Previous monitor snapshot manifest must have been unexpired when the checkpoint was generated.');
|
|
88
91
|
}
|
|
89
92
|
const collection = exactMonitorRecord(root.collection, DOMAIN_CONTROL_MONITOR_COLLECTION_KEYS, 'Previous monitor snapshot collection');
|
|
@@ -124,25 +127,31 @@ function previousSnapshot(input, currentGeneratedAt) {
|
|
|
124
127
|
let flightRecorder;
|
|
125
128
|
try {
|
|
126
129
|
review = validateDomainControlReviewDocument(root.review);
|
|
130
|
+
if (review.version !== reviewVersion)
|
|
131
|
+
throw new TypeError('Mismatched review version.');
|
|
127
132
|
}
|
|
128
133
|
catch {
|
|
129
|
-
throw new CliUsageError(`Previous monitor snapshot review must use the exact ${DOMAIN_CONTROL_REVIEW_SCHEMA} version ${
|
|
134
|
+
throw new CliUsageError(`Previous monitor snapshot review must use the exact ${DOMAIN_CONTROL_REVIEW_SCHEMA} version ${reviewVersion} contract.`);
|
|
130
135
|
}
|
|
131
136
|
try {
|
|
132
137
|
flightRecorder = validateDomainControlFlightRecorderDocument(root.flightRecorder);
|
|
138
|
+
if (flightRecorder.version !== flightVersion)
|
|
139
|
+
throw new TypeError('Mismatched flight-recorder version.');
|
|
133
140
|
}
|
|
134
141
|
catch {
|
|
135
|
-
throw new CliUsageError(`Previous monitor snapshot flight recorder must use the exact ${DOMAIN_CONTROL_FLIGHT_RECORDER_SCHEMA} version ${
|
|
142
|
+
throw new CliUsageError(`Previous monitor snapshot flight recorder must use the exact ${DOMAIN_CONTROL_FLIGHT_RECORDER_SCHEMA} version ${flightVersion} contract.`);
|
|
136
143
|
}
|
|
137
144
|
if (!Array.isArray(root.limitations)
|
|
138
145
|
|| root.limitations.length !== DOMAIN_CONTROL_MONITOR_LIMITATIONS.length
|
|
139
146
|
|| root.limitations.some((value, index) => value !== DOMAIN_CONTROL_MONITOR_LIMITATIONS[index])) {
|
|
140
147
|
throw new CliUsageError('Previous monitor snapshot limitations are invalid.');
|
|
141
148
|
}
|
|
149
|
+
let observations;
|
|
142
150
|
try {
|
|
151
|
+
observations = normalizeDomainControlFlightRecorderObservations(root.observations, flightVersion);
|
|
143
152
|
buildDomainControlFlightRecorder({
|
|
144
153
|
schema: DOMAIN_CONTROL_FLIGHT_RECORDER_INPUT_SCHEMA,
|
|
145
|
-
version:
|
|
154
|
+
version: flightVersion,
|
|
146
155
|
observations: root.observations,
|
|
147
156
|
approvedWindows: [],
|
|
148
157
|
}, previousGeneratedAt);
|
|
@@ -150,20 +159,30 @@ function previousSnapshot(input, currentGeneratedAt) {
|
|
|
150
159
|
catch {
|
|
151
160
|
throw new CliUsageError('Previous monitor snapshot observations must use the exact bounded flight-recorder input contract.');
|
|
152
161
|
}
|
|
153
|
-
const observations = root.observations;
|
|
154
162
|
const observationDomainOrder = observations.map((observation) => observation.domain);
|
|
155
163
|
const observationDomains = new Set(observationDomainOrder);
|
|
156
164
|
const requestedDomains = review.domains.slice(0, requested).map((item) => item.domain);
|
|
157
165
|
const expectedObservationDomains = requestedDomains.filter((domain) => !failureDomains.has(domain));
|
|
158
166
|
const expectedFailureDomains = requestedDomains.filter((domain) => !observationDomains.has(domain));
|
|
167
|
+
const mismatchedSource = !legacy && review.domains.some((item) => {
|
|
168
|
+
const observation = observations.find((candidate) => candidate.domain === item.domain);
|
|
169
|
+
return item.comparisons.some((comparison) => {
|
|
170
|
+
if (comparison.source === null)
|
|
171
|
+
return false;
|
|
172
|
+
const field = observation?.fields.find((candidate) => DOMAIN_CONTROL_REVIEW_SOURCE_FIELDS[candidate.id] === comparison.field);
|
|
173
|
+
return !field || field.source !== comparison.source || field.observedAt !== comparison.observedAt;
|
|
174
|
+
});
|
|
175
|
+
});
|
|
159
176
|
if (review.domains.length < requested
|
|
177
|
+
|| mismatchedSource
|
|
160
178
|
|| observationDomains.size !== observations.length
|
|
161
179
|
|| observationDomainOrder.some((domain, index) => domain !== expectedObservationDomains[index])
|
|
162
180
|
|| observationDomainOrder.length !== expectedObservationDomains.length
|
|
163
181
|
|| failureDomainOrder.some((domain, index) => domain !== expectedFailureDomains[index])
|
|
164
182
|
|| failureDomainOrder.length !== expectedFailureDomains.length
|
|
165
|
-
|| observations.some((observation) => observation.
|
|
166
|
-
|
|
183
|
+
|| observations.some((observation) => legacy ? observation.capturedAt !== previousGeneratedAt
|
|
184
|
+
: Date.parse(observation.capturedAt) > Date.parse(previousGeneratedAt))
|
|
185
|
+
|| review.domains.some((item) => item.comparisons.some((comparison) => legacy && comparison.observedAt !== null && comparison.observedAt !== previousGeneratedAt))
|
|
167
186
|
|| review.generatedAt !== previousGeneratedAt
|
|
168
187
|
|| review.manifest.digestSha256 !== manifest.digestSha256
|
|
169
188
|
|| review.manifest.expiresAt !== manifest.expiresAt
|
|
@@ -173,7 +192,11 @@ function previousSnapshot(input, currentGeneratedAt) {
|
|
|
173
192
|
|| [...observationDomains].some((domain) => !flightRecorder.domains.includes(domain))) {
|
|
174
193
|
throw new CliUsageError('Previous monitor snapshot embedded documents are inconsistent with its observation projection.');
|
|
175
194
|
}
|
|
176
|
-
|
|
195
|
+
// Published checkpoints recorded run time, not independently retained source
|
|
196
|
+
// clocks. Their values remain available without upgrading that timestamp.
|
|
197
|
+
return { observations: legacy ? observations.map((item) => Object.freeze({
|
|
198
|
+
...item, fields: Object.freeze(item.fields.map((field) => Object.freeze({ ...field, observedAt: null }))),
|
|
199
|
+
})) : observations };
|
|
177
200
|
}
|
|
178
201
|
function monitorFailureMessage() {
|
|
179
202
|
return DOMAIN_CONTROL_MONITOR_FAILURE_CATEGORIES[0];
|
|
@@ -189,9 +212,9 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
|
|
|
189
212
|
|| options.concurrency > MAX_DOMAIN_CONTROL_MONITOR_CONCURRENCY) {
|
|
190
213
|
throw new CliUsageError(`Domain-control monitor concurrency must be from ${MIN_DOMAIN_CONTROL_MONITOR_CONCURRENCY} to ${MAX_DOMAIN_CONTROL_MONITOR_CONCURRENCY}.`);
|
|
191
214
|
}
|
|
192
|
-
let
|
|
215
|
+
let startedAt;
|
|
193
216
|
try {
|
|
194
|
-
|
|
217
|
+
startedAt = requireIsoTimestamp(options.now(), 'Domain-control monitor time');
|
|
195
218
|
}
|
|
196
219
|
catch {
|
|
197
220
|
throw new CliUsageError('Domain-control monitor time must be a valid ISO 8601 timestamp.');
|
|
@@ -204,11 +227,11 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
|
|
|
204
227
|
catch {
|
|
205
228
|
throw new CliUsageError('Domain-control manifest must satisfy its supported bounded integrity contract.');
|
|
206
229
|
}
|
|
207
|
-
const checkedAt = Date.parse(
|
|
230
|
+
const checkedAt = Date.parse(startedAt);
|
|
208
231
|
if (Date.parse(manifest.expiresAt) <= checkedAt) {
|
|
209
232
|
throw new CliUsageError('Domain-control monitor requires an unexpired manifest.');
|
|
210
233
|
}
|
|
211
|
-
const previous = previousSnapshot(previousInput,
|
|
234
|
+
const previous = previousSnapshot(previousInput, startedAt);
|
|
212
235
|
const entries = manifest.entries.slice(0, Math.min(options.limit, MAX_DOMAIN_CONTROL_MONITOR_DOMAINS));
|
|
213
236
|
const lookups = new Array(entries.length);
|
|
214
237
|
const failuresByIndex = new Array(entries.length);
|
|
@@ -225,7 +248,10 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
|
|
|
225
248
|
const classified = classifyQuery(entry.domain);
|
|
226
249
|
const result = await options.executeLookup(classified, { fast: false, compact: false, ...(options.signal ? { signal: options.signal } : {}) });
|
|
227
250
|
options.signal?.throwIfAborted();
|
|
228
|
-
|
|
251
|
+
const capturedAt = requireIsoTimestamp(options.now(), 'Domain-control capture time');
|
|
252
|
+
if (Date.parse(capturedAt) < checkedAt)
|
|
253
|
+
throw new CliUsageError('Domain-control capture time must not precede the run.');
|
|
254
|
+
lookups[index] = buildCliLookupDocument(entry.domain, classified, result, capturedAt, 'deep');
|
|
229
255
|
}
|
|
230
256
|
catch {
|
|
231
257
|
if (options.signal?.aborted) {
|
|
@@ -245,6 +271,10 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
|
|
|
245
271
|
const successful = lookups.filter(Boolean);
|
|
246
272
|
if (!successful.length)
|
|
247
273
|
throw new Error('No domain-control monitor lookup completed successfully.');
|
|
274
|
+
const generatedAt = requireIsoTimestamp(options.now(), 'Domain-control completion time');
|
|
275
|
+
if (Date.parse(generatedAt) < checkedAt || successful.some((lookup) => Date.parse(String(lookup.generatedAt)) > Date.parse(generatedAt))) {
|
|
276
|
+
throw new CliUsageError('Domain-control completion time must not precede its captures.');
|
|
277
|
+
}
|
|
248
278
|
const review = buildCliDomainControlReview(JSON.stringify({
|
|
249
279
|
schema: CLI_DOMAIN_CONTROL_REVIEW_INPUT_SCHEMA,
|
|
250
280
|
version: CLI_DOMAIN_CONTROL_REVIEW_VERSION,
|