@slicedearth/whoisleuth-cli 1.47.4 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -177
- package/SECURITY.md +6 -3
- package/bin/whoisleuth.mjs +2 -1
- package/cli/archive-inspect.mjs +28 -10
- package/cli/arguments.mjs +204 -174
- package/cli/artifact-structure.mjs +837 -163
- package/cli/artifact-verify.mjs +90 -61
- package/cli/bounded-json.mjs +1 -1
- package/cli/bulk-checkpoint.mjs +33 -20
- package/cli/bulk.mjs +9 -2
- package/cli/case-pack.mjs +180 -576
- package/cli/ci-report.mjs +6 -1
- package/cli/collection-preflight.mjs +2 -2
- package/cli/command-catalogue.mjs +12 -3
- package/cli/command-reference.mjs +910 -92
- package/cli/completion.mjs +504 -226
- package/cli/config-profile.mjs +13 -3
- package/cli/discover.mjs +7 -1
- package/cli/discovery-observation-snapshot.mjs +24 -28
- package/cli/discovery-scan-command-runner.mjs +9 -9
- package/cli/discovery-scan.mjs +17 -3
- package/cli/discovery-snapshot.mjs +15 -7
- package/cli/domain-control-monitor.mjs +203 -30
- package/cli/domain-control-observations.mjs +15 -21
- package/cli/errors.mjs +8 -3
- package/cli/evidence-signing.mjs +10 -20
- package/cli/external-observation-mapping.mjs +3 -3
- package/cli/formatters/json.mjs +20 -32
- package/cli/formatters/terminal.mjs +583 -22
- package/cli/input.mjs +7 -2
- package/cli/interactive-launcher.mjs +226 -0
- package/cli/interchange-report.mjs +19 -19
- package/cli/investigation-manifest.mjs +1 -1
- package/cli/investigation-plan.mjs +167 -16
- package/cli/investigation-run.mjs +9 -1
- package/cli/lookalike-calibration.mjs +3 -1
- package/cli/lookup-brief.mjs +13 -8
- package/cli/lookup-browser.mjs +844 -0
- package/cli/lookup-command-runner.mjs +94 -31
- package/cli/lookup-diff.mjs +2 -2
- package/cli/lookup-plan.mjs +1 -0
- package/cli/mail-review.mjs +206 -94
- package/cli/manual.mjs +5 -5
- package/cli/offline-evidence-review.mjs +18 -7
- package/cli/open-asset-model-bridge.mjs +1 -1
- package/cli/output-file.mjs +70 -14
- package/cli/page-compare.mjs +11 -6
- package/cli/progress-events.mjs +2 -1
- package/cli/progress.mjs +3 -3
- package/cli/registry-cohort.mjs +9 -51
- package/cli/registry-doctor.mjs +67 -28
- package/cli/retained-artifact-diff.mjs +7 -8
- package/cli/risk-calibration.mjs +85 -38
- package/cli/runner.mjs +189 -75
- package/cli/saved-lookup.mjs +424 -57
- package/cli/sharing-review.mjs +6 -1
- package/cli/source-reliability.mjs +75 -36
- package/cli/terminal-presentation.mjs +43 -18
- package/docs/cli-reference.md +125 -1708
- package/docs/cli.md +86 -254
- package/frontend/src/lib/analysis/domain-control-manifest-core.js +3 -145
- package/frontend/src/lib/analysis/domain-control-records.js +3 -88
- package/lib/availability.mjs +58 -40
- package/lib/bounded-contract-normalizers.mjs +2 -4
- package/lib/bounded-file.mjs +78 -4
- package/lib/bounded-json.mjs +51 -13
- package/lib/browser-library-profile.mjs +1 -1
- package/lib/classify.mjs +56 -5
- package/lib/client-behavior-profile.mjs +1 -1
- package/lib/credential-surface-profile.mjs +1 -1
- package/lib/cryptographic-assurance.mjs +35 -17
- package/lib/ct-query.mjs +3 -3
- package/lib/ct-search.mjs +37 -21
- package/lib/distributed-operation-budget.mjs +4 -2
- package/lib/dns-delegation-health.mjs +9 -6
- package/lib/dns-intelligence.mjs +1 -1
- package/lib/dnssec-chain-validation.mjs +2 -3
- package/lib/dnssec-evidence-validation.mjs +13 -7
- package/lib/domain-assurance.mjs +12 -12
- package/lib/domain-change-packet.mjs +1 -1
- package/lib/domain-control-flight-recorder.mjs +442 -78
- package/lib/domain-control-manifest.mjs +314 -59
- package/lib/domain-posture-analysis.mjs +5 -2
- package/lib/encrypted-dns-contract.mjs +2 -3
- package/lib/evidence-export.mjs +518 -36
- package/lib/external-intelligence-risk.mjs +26 -10
- package/lib/feature-policy.mjs +1 -1
- package/lib/homepage-metadata-contract.mjs +290 -0
- package/lib/hostname.mjs +1 -1
- package/lib/html-signals.mjs +89 -5
- package/lib/http-intelligence.mjs +331 -2
- package/lib/interchange-fidelity-registry.mjs +28 -26
- package/lib/lookup-cache.mjs +6 -4
- package/lib/lookup.mjs +25 -2
- package/lib/observed-network-context.mjs +1 -1
- package/lib/page-fingerprints.mjs +2 -2
- package/lib/page-role-profile.mjs +1 -1
- package/lib/portable-generator.mjs +8 -0
- package/lib/project-metadata.mjs +1 -1
- package/lib/provider-policy-admission.mjs +3 -1
- package/lib/rdap-bootstrap.mjs +76 -16
- package/lib/rdap-client.mjs +20 -12
- package/lib/rdap-normalization.mjs +306 -75
- package/lib/rdap-transport.mjs +2 -2
- package/lib/rdap-validation.mjs +65 -14
- package/lib/rdap.mjs +2 -2
- package/lib/registrable-domain.mjs +36 -0
- package/lib/registry-insights.mjs +7 -0
- package/lib/response-policy.mjs +27 -4
- package/lib/risk-calibration-summary.mjs +35 -33
- package/lib/risk-scoring.mjs +4 -3
- package/lib/safe-fetch.mjs +62 -40
- package/lib/security-txt.mjs +1 -1
- package/lib/smtp-transport-review.mjs +8 -5
- package/lib/sslbl-intelligence.mjs +6 -8
- package/lib/static-html-analysis.mjs +301 -4
- package/lib/structured-data-identity.mjs +51 -19
- package/lib/threat-intelligence-fixtures.mjs +2 -4
- package/lib/threat-intelligence-manifests.mjs +2 -7
- package/lib/threat-intelligence-runtime.mjs +3 -5
- package/lib/threat-intelligence-types.mjs +3 -0
- package/lib/threatfox-intelligence.mjs +7 -3
- package/lib/tls-intelligence.mjs +4 -4
- package/lib/urlhaus-intelligence.mjs +4 -2
- package/lib/urlscan-intelligence.mjs +6 -5
- package/lib/website-security-posture.mjs +1 -1
- package/lib/website-technology.mjs +1 -1
- package/lib/whois-authority.mjs +43 -8
- package/lib/whois-chain.mjs +25 -3
- package/lib/whois-normalization.mjs +3 -1
- package/lib/whois-transport.mjs +12 -3
- package/lib/zone-intent-review.mjs +1 -1
- package/package.json +13 -5
- package/{frontend/src/lib/analysis/case-brand-profile-references.js → packages/cases/case-brand-profile-references.mjs} +2 -2
- package/{frontend/src/lib/analysis/case-evidence-model.js → packages/cases/case-evidence-model.mjs} +35 -14
- package/{frontend/src/lib/analysis/case-investigation-branch-model.js → packages/cases/case-investigation-branch-model.mjs} +10 -12
- package/packages/cases/case-migration-model.mjs +476 -0
- package/{frontend/src/lib/analysis/case-model.js → packages/cases/case-model.mjs} +3 -3
- package/{frontend/src/lib/analysis/case-record-contracts.js → packages/cases/case-record-contracts.mjs} +3 -21
- package/{frontend/src/lib/analysis/case-record-core.js → packages/cases/case-record-core.mjs} +58 -31
- package/packages/cases/case-record-model.mjs +9 -0
- package/{frontend/src/lib/analysis/case-record-operations.js → packages/cases/case-record-operations.mjs} +72 -29
- package/{frontend/src/lib/analysis/case-report.js → packages/cases/case-report.mjs} +108 -14
- package/packages/cases/case-response-model.mjs +1604 -0
- package/packages/cases/case-response-packet.mjs +1312 -0
- package/{frontend/src/lib/analysis/case-storage-model.js → packages/cases/case-storage-model.mjs} +2 -2
- package/{frontend/src/lib/analysis/http-summary.js → packages/cases/http-summary.mjs} +1 -1
- package/{frontend/src/lib/analysis/comparison-ledger-bulk.js → packages/comparison/comparison-ledger-bulk.mjs} +2 -2
- package/{frontend/src/lib/analysis/comparison-ledger-contract.js → packages/comparison/comparison-ledger-contract.mjs} +4 -4
- package/{frontend/src/lib/analysis/comparison-ledger-json.js → packages/comparison/comparison-ledger-json.mjs} +1 -0
- package/{frontend/src/lib/analysis/comparison-ledger-serialization.js → packages/comparison/comparison-ledger-serialization.mjs} +1 -0
- package/packages/comparison/favicon-similarity.mjs +78 -0
- package/{frontend/src/lib/analysis/page-similarity.js → packages/comparison/page-similarity.mjs} +4 -3
- package/{frontend/src/lib/analysis/relationship-evidence.js → packages/comparison/relationship-evidence.mjs} +6 -7
- package/packages/contracts/analyst-review-state-contract.mjs +61 -0
- package/packages/contracts/analyst-review-state.mjs +50 -0
- package/packages/contracts/case-portability.mjs +1310 -0
- package/packages/contracts/cli-command-catalogue.mjs +2 -0
- package/packages/contracts/cli-lookup.mjs +394 -0
- package/packages/contracts/domain-control-flight-recorder.mjs +461 -0
- package/packages/contracts/domain-control-manifest.mjs +1027 -0
- package/packages/contracts/domain-control-monitor.mjs +310 -0
- package/packages/contracts/domain-control-review.mjs +828 -0
- package/packages/contracts/domain-name.mjs +4 -0
- package/packages/contracts/external-observation-interchange.mjs +224 -0
- package/packages/contracts/extracted-domain-lifecycle.mjs +218 -0
- package/packages/contracts/investigation-portability.mjs +331 -0
- package/packages/contracts/offline-comparison.mjs +160 -0
- package/packages/contracts/risk-calibration.mjs +1288 -0
- package/packages/contracts/schema-compatibility.mjs +71 -0
- package/packages/contracts/schema-lifecycle.mjs +1702 -0
- package/packages/contracts/sslbl-snapshot.mjs +17 -0
- package/packages/contracts/workspace-portability.mjs +1347 -0
- package/{frontend/src/lib/analysis/artifact-integrity.js → packages/evidence/artifact-integrity.mjs} +13 -11
- package/packages/evidence/decision-fact.mjs +735 -0
- package/packages/evidence/domain-control-runtime.mjs +547 -0
- package/packages/evidence/domain-name.mjs +41 -0
- package/packages/evidence/observation.mjs +384 -0
- package/{frontend/src/lib/analysis/external-findings-import.js → packages/interchange/external-findings-import.mjs} +24 -22
- package/packages/interchange/retained-text.mjs +8 -0
- package/{frontend/src/lib/analysis/acquisition-decision-packet.js → packages/investigation/acquisition-decision-packet.mjs} +9 -6
- package/{frontend/src/lib/analysis/bulk-domain-comparison.js → packages/investigation/bulk-domain-comparison.mjs} +5 -6
- package/{frontend/src/lib/analysis/bulk-mail-exposure.js → packages/investigation/bulk-mail-exposure.mjs} +4 -6
- package/{frontend/src/lib/analysis/bulk-review-export.js → packages/investigation/bulk-review-export.mjs} +5 -5
- package/packages/investigation/bulk-review-policy.mjs +1 -0
- package/{frontend/src/lib/analysis/investigation-capsule.js → packages/investigation/investigation-capsule.mjs} +30 -32
- package/packages/investigation/lookup-artefact-inputs.mjs +1 -0
- package/{frontend/src/lib/analysis/lookup-asset-graph.js → packages/investigation/lookup-asset-graph.mjs} +4 -2
- package/{frontend/src/lib/analysis/lookup-claim-passport.js → packages/investigation/lookup-claim-passport.mjs} +6 -10
- package/{frontend/src/lib/analysis/lookup-claim-readiness.js → packages/investigation/lookup-claim-readiness.mjs} +9 -2
- package/packages/investigation/lookup-investigation-brief.mjs +172 -0
- package/packages/monitoring/analyst-review-state.mjs +680 -0
- package/{frontend/src/lib/analysis/brand-profile-model.js → packages/workspace/brand-profile-model.mjs} +71 -48
- package/{frontend/src/lib/analysis/bulk-review-model.js → packages/workspace/bulk-review-model.mjs} +13 -14
- package/{frontend/src/lib/analysis/bulk-session-model.js → packages/workspace/bulk-session-model.mjs} +33 -59
- package/{frontend/src/lib/analysis/bulk-sort.js → packages/workspace/bulk-sort.mjs} +8 -5
- package/{frontend/src/lib/analysis/campaign-model.js → packages/workspace/campaign-model.mjs} +20 -22
- package/{frontend/src/lib/analysis/detection-rule-model.js → packages/workspace/detection-rule-model.mjs} +15 -17
- package/packages/workspace/hostile-input.mjs +172 -0
- package/{frontend/src/lib/analysis/investigation-guide.js → packages/workspace/investigation-guide.mjs} +8 -45
- package/{frontend/src/lib/analysis/investigation-template-model.js → packages/workspace/investigation-template-model.mjs} +18 -21
- package/{frontend/src/lib/analysis/page-baseline.js → packages/workspace/page-baseline.mjs} +6 -13
- package/{frontend/src/lib/analysis/relationship-observation-model.js → packages/workspace/relationship-observation-model.mjs} +13 -17
- package/{frontend/src/lib/analysis/shortlist-model.js → packages/workspace/shortlist-model.mjs} +20 -18
- package/{frontend/src/lib/analysis/watchlist-history.js → packages/workspace/watchlist-history.mjs} +9 -16
- package/{frontend/src/lib/analysis/watchlist-store.js → packages/workspace/watchlist-store.mjs} +24 -17
- package/{frontend/src/lib/analysis/website-snapshot-model.js → packages/workspace/website-snapshot-model.mjs} +13 -13
- package/{frontend/src/lib/analysis/workspace-archive-crypto.js → packages/workspace/workspace-archive-crypto.mjs} +49 -19
- package/{frontend/src/lib/analysis/workspace-archive.js → packages/workspace/workspace-archive.mjs} +110 -89
- package/third-party-notices.txt +14 -14
- package/frontend/src/lib/analysis/bulk-retry-plan.js +0 -79
- package/frontend/src/lib/analysis/bulk-source-coverage.js +0 -76
- package/frontend/src/lib/analysis/case-migration-model.js +0 -305
- package/frontend/src/lib/analysis/case-record-model.js +0 -9
- package/frontend/src/lib/analysis/case-response-model.js +0 -728
- package/frontend/src/lib/analysis/case-response-packet.js +0 -583
- package/frontend/src/lib/analysis/lookup-decision-support.js +0 -542
- package/frontend/src/lib/analysis/lookup-investigation-brief.js +0 -153
- package/frontend/src/lib/analysis/lookup-presentation.js +0 -120
- package/frontend/src/lib/analysis/lookup-response.js +0 -2
- package/frontend/src/lib/analysis/lookup-source-refresh.js +0 -277
- package/frontend/src/lib/analysis/lookup-summary-model.js +0 -398
- package/frontend/src/lib/analysis/scoring.js +0 -55
- package/frontend/src/lib/analysis/utils.js +0 -255
- package/frontend/src/lib/analysis/visualization-bounds.js +0 -26
- package/frontend/src/lib/analysis/visualization-force-graph.js +0 -352
- package/frontend/src/lib/analysis/visualization-matrix.js +0 -131
- package/frontend/src/lib/analysis/visualization-models.js +0 -338
- package/frontend/src/lib/analysis/visualization-time-series.js +0 -185
- package/lib/bounded-json-response.mjs +0 -170
- package/lib/lookup-response-contract.mjs +0 -1034
- package/lib/observation.mjs +0 -115
- /package/{frontend/src/lib/analysis/opaque-reference-id.js → packages/cases/opaque-reference-id.mjs} +0 -0
- /package/{frontend/src/lib/analysis/acquisition-due-diligence.js → packages/investigation/acquisition-due-diligence.mjs} +0 -0
- /package/{frontend/src/lib/analysis/evidence-coverage-ledger.js → packages/investigation/evidence-coverage-ledger.mjs} +0 -0
- /package/{frontend/src/lib/analysis/dns-record-normalization.js → packages/workspace/dns-record-normalization.mjs} +0 -0
|
@@ -1,153 +0,0 @@
|
|
|
1
|
-
export const LOOKUP_INVESTIGATION_BRIEF_SCHEMA = 'whoisleuth.investigation-brief';
|
|
2
|
-
export const LOOKUP_INVESTIGATION_BRIEF_VERSION = 1;
|
|
3
|
-
const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f]/gu;
|
|
4
|
-
const MAX_FACTS = 12;
|
|
5
|
-
const MAX_LIMITATIONS = 20;
|
|
6
|
-
function text(value, maximum = 320) {
|
|
7
|
-
return String(value ?? '')
|
|
8
|
-
.replace(CONTROL_CHARACTERS, ' ')
|
|
9
|
-
.replace(/\s+/gu, ' ')
|
|
10
|
-
.trim()
|
|
11
|
-
.slice(0, maximum);
|
|
12
|
-
}
|
|
13
|
-
function timestamp(value) {
|
|
14
|
-
if (typeof value !== 'string')
|
|
15
|
-
return null;
|
|
16
|
-
const parsed = Date.parse(value);
|
|
17
|
-
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
|
|
18
|
-
}
|
|
19
|
-
function uniqueText(values, maximum = MAX_LIMITATIONS) {
|
|
20
|
-
const output = [];
|
|
21
|
-
const seen = new Set();
|
|
22
|
-
for (const value of values) {
|
|
23
|
-
const normalized = text(value);
|
|
24
|
-
if (!normalized || seen.has(normalized))
|
|
25
|
-
continue;
|
|
26
|
-
output.push(normalized);
|
|
27
|
-
seen.add(normalized);
|
|
28
|
-
if (output.length >= maximum)
|
|
29
|
-
break;
|
|
30
|
-
}
|
|
31
|
-
return output;
|
|
32
|
-
}
|
|
33
|
-
function taskQuestion(support) {
|
|
34
|
-
return support.guidance.questions[0]
|
|
35
|
-
?? 'What does the collected evidence establish, and what remains uncertain?';
|
|
36
|
-
}
|
|
37
|
-
export function buildLookupInvestigationBrief(input) {
|
|
38
|
-
const generatedAt = timestamp(input.generatedAt) ?? new Date().toISOString();
|
|
39
|
-
const verifiedFacts = input.summary.facts
|
|
40
|
-
.filter((fact) => fact.value !== '—')
|
|
41
|
-
.slice(0, MAX_FACTS);
|
|
42
|
-
const contradictions = input.decisionSupport.entries
|
|
43
|
-
.filter((entry) => entry.state === 'conflict');
|
|
44
|
-
const unknowns = input.decisionSupport.entries
|
|
45
|
-
.filter((entry) => entry.state === 'uncertain');
|
|
46
|
-
const limitedSources = input.quality.entries
|
|
47
|
-
.filter((entry) => entry.state !== 'complete')
|
|
48
|
-
.map((entry) => `${entry.label}: ${entry.statusLabel}`);
|
|
49
|
-
const graphKinds = uniqueText(input.graph.edges.map((edge) => edge.label), 12);
|
|
50
|
-
return {
|
|
51
|
-
schema: LOOKUP_INVESTIGATION_BRIEF_SCHEMA,
|
|
52
|
-
schemaVersion: LOOKUP_INVESTIGATION_BRIEF_VERSION,
|
|
53
|
-
generatedAt,
|
|
54
|
-
target: text(input.target, 253) || 'Unknown target',
|
|
55
|
-
targetType: text(input.targetType, 40) || 'unknown',
|
|
56
|
-
task: input.task,
|
|
57
|
-
taskLabel: input.decisionSupport.guidance.label,
|
|
58
|
-
question: taskQuestion(input.decisionSupport),
|
|
59
|
-
summary: input.decisionSupport.guidance.summary,
|
|
60
|
-
observation: {
|
|
61
|
-
observedAt: input.quality.observedAt,
|
|
62
|
-
evidenceAgeDays: input.quality.ageDays,
|
|
63
|
-
completeSources: input.quality.completeCount,
|
|
64
|
-
limitedSources: input.quality.limitedCount,
|
|
65
|
-
freshnessPolicy: input.quality.freshnessPolicy,
|
|
66
|
-
},
|
|
67
|
-
verifiedFacts,
|
|
68
|
-
contradictions,
|
|
69
|
-
unknowns,
|
|
70
|
-
nextActions: input.decisionSupport.actions,
|
|
71
|
-
relationships: {
|
|
72
|
-
nodes: input.graph.nodes.length,
|
|
73
|
-
edges: input.graph.edges.length,
|
|
74
|
-
truncated: input.graph.truncated,
|
|
75
|
-
kinds: graphKinds,
|
|
76
|
-
},
|
|
77
|
-
limitations: uniqueText([
|
|
78
|
-
...limitedSources,
|
|
79
|
-
...input.graph.limitations,
|
|
80
|
-
'This brief is a deterministic organisation of collected and derived evidence, not an attribution, ownership, safety, availability, or maliciousness conclusion.',
|
|
81
|
-
'Analyst assertions, hypotheses, and decisions must remain separate from observed evidence.',
|
|
82
|
-
]),
|
|
83
|
-
};
|
|
84
|
-
}
|
|
85
|
-
function markdown(value) {
|
|
86
|
-
return text(value, 2_000)
|
|
87
|
-
.replace(/&/gu, '&')
|
|
88
|
-
.replace(/</gu, '<')
|
|
89
|
-
.replace(/>/gu, '>')
|
|
90
|
-
.replace(/([\\`*_{}[\]()#+\-.!|=~])/gu, '\\$1');
|
|
91
|
-
}
|
|
92
|
-
function factLine(fact) {
|
|
93
|
-
const sources = fact.provenance.sources.length
|
|
94
|
-
? ` Sources: ${fact.provenance.sources.map(markdown).join(', ')}.`
|
|
95
|
-
: '';
|
|
96
|
-
return `- **${markdown(fact.label)}:** ${markdown(fact.value)}.${sources}`;
|
|
97
|
-
}
|
|
98
|
-
function decisionLine(entry) {
|
|
99
|
-
return `- **${markdown(entry.title)}:** ${markdown(entry.detail)} Sources: ${entry.sources.map(markdown).join(', ') || 'not reported'}.`;
|
|
100
|
-
}
|
|
101
|
-
export function formatLookupInvestigationBriefMarkdown(brief) {
|
|
102
|
-
const lines = [
|
|
103
|
-
`# Investigation brief: ${markdown(brief.target)}`,
|
|
104
|
-
'',
|
|
105
|
-
`Generated ${markdown(brief.generatedAt)} from a ${markdown(brief.targetType)} Lookup.`,
|
|
106
|
-
'',
|
|
107
|
-
'## Decision context',
|
|
108
|
-
'',
|
|
109
|
-
`- **Task:** ${markdown(brief.taskLabel)}`,
|
|
110
|
-
`- **Question:** ${markdown(brief.question)}`,
|
|
111
|
-
`- **Summary:** ${markdown(brief.summary)}`,
|
|
112
|
-
`- **Evidence observed:** ${markdown(brief.observation.observedAt ?? 'not reported')}`,
|
|
113
|
-
`- **Source quality:** ${brief.observation.completeSources} complete; ${brief.observation.limitedSources} limited`,
|
|
114
|
-
`- **Freshness policy:** v${brief.observation.freshnessPolicy.version} ${markdown(brief.observation.freshnessPolicy.id)} for ${markdown(brief.observation.freshnessPolicy.task)}; registration ${brief.observation.freshnessPolicy.thresholdsDays.registration}d, network ${brief.observation.freshnessPolicy.thresholdsDays.network}d, web ${brief.observation.freshnessPolicy.thresholdsDays.web}d`,
|
|
115
|
-
'',
|
|
116
|
-
'## Verified normalised facts',
|
|
117
|
-
'',
|
|
118
|
-
...(brief.verifiedFacts.length ? brief.verifiedFacts.map(factLine) : ['- No normalised fact was available for this brief.']),
|
|
119
|
-
'',
|
|
120
|
-
'## Contradictory evidence',
|
|
121
|
-
'',
|
|
122
|
-
...(brief.contradictions.length ? brief.contradictions.map(decisionLine) : ['- No explicit contradiction was derived from the supported comparisons.']),
|
|
123
|
-
'',
|
|
124
|
-
'## Unknowns and incomplete comparisons',
|
|
125
|
-
'',
|
|
126
|
-
...(brief.unknowns.length ? brief.unknowns.map(decisionLine) : ['- No explicit uncertainty entry was derived. Review source quality before treating this as complete.']),
|
|
127
|
-
'',
|
|
128
|
-
'## Recommended next manual steps',
|
|
129
|
-
'',
|
|
130
|
-
...(brief.nextActions.length
|
|
131
|
-
? brief.nextActions.map((action) => `- **${markdown(action.label)}:** ${markdown(action.reason)}`)
|
|
132
|
-
: ['- Review the underlying source evidence and its limitations.']),
|
|
133
|
-
'',
|
|
134
|
-
'## Observed relationship map',
|
|
135
|
-
'',
|
|
136
|
-
`- ${brief.relationships.nodes} nodes and ${brief.relationships.edges} edges${brief.relationships.truncated ? '; bounded output was truncated' : ''}.`,
|
|
137
|
-
`- Relationship types: ${brief.relationships.kinds.map(markdown).join(', ') || 'none observed'}.`,
|
|
138
|
-
'',
|
|
139
|
-
'## Limitations',
|
|
140
|
-
'',
|
|
141
|
-
...brief.limitations.map((limitation) => `- ${markdown(limitation)}`),
|
|
142
|
-
'',
|
|
143
|
-
];
|
|
144
|
-
return lines.join('\n');
|
|
145
|
-
}
|
|
146
|
-
export function lookupInvestigationBriefFilename(brief) {
|
|
147
|
-
const target = brief.target
|
|
148
|
-
.toLowerCase()
|
|
149
|
-
.replace(/[^a-z0-9.-]+/gu, '-')
|
|
150
|
-
.replace(/^-+|-+$/gu, '')
|
|
151
|
-
.slice(0, 80) || 'lookup';
|
|
152
|
-
return `whoisleuth-investigation-brief-${target}-${brief.generatedAt.slice(0, 10)}.md`;
|
|
153
|
-
}
|
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
export const LOOKUP_PRESENTATION_STORAGE_KEY = 'whoisleuth:lookup-presentation:v1';
|
|
2
|
-
export const MAX_LOOKUP_PRESENTATION_SERIALIZED_BYTES = 1_024;
|
|
3
|
-
export const MAX_LOOKUP_URL_QUERY_LENGTH = 4_096;
|
|
4
|
-
export const LOOKUP_TASK_VIEWS = Object.freeze([
|
|
5
|
-
Object.freeze({ id: 'general', label: 'General investigation' }),
|
|
6
|
-
Object.freeze({ id: 'acquisition', label: 'Acquisition review' }),
|
|
7
|
-
Object.freeze({ id: 'brand', label: 'Brand review' }),
|
|
8
|
-
Object.freeze({ id: 'incident', label: 'Incident response' }),
|
|
9
|
-
Object.freeze({ id: 'owned', label: 'Owned-domain posture' }),
|
|
10
|
-
]);
|
|
11
|
-
const TASKS = new Set(LOOKUP_TASK_VIEWS.map((option) => option.id));
|
|
12
|
-
const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f]/u;
|
|
13
|
-
export function parseLookupTaskContext(value) {
|
|
14
|
-
if (typeof value !== 'string' || value.length > 16)
|
|
15
|
-
return null;
|
|
16
|
-
return TASKS.has(value) ? value : null;
|
|
17
|
-
}
|
|
18
|
-
export function normalizeLookupTaskView(value) {
|
|
19
|
-
return typeof value === 'string' && TASKS.has(value)
|
|
20
|
-
? value
|
|
21
|
-
: 'general';
|
|
22
|
-
}
|
|
23
|
-
function lookupUrlQuery(value) {
|
|
24
|
-
if (value === null || value.length > MAX_LOOKUP_URL_QUERY_LENGTH || CONTROL_CHARACTERS.test(value))
|
|
25
|
-
return null;
|
|
26
|
-
const normalized = value.trim();
|
|
27
|
-
return normalized ? normalized : null;
|
|
28
|
-
}
|
|
29
|
-
function lookupUrlDepth(value) {
|
|
30
|
-
return value === 'fast' || value === 'deep' ? value : null;
|
|
31
|
-
}
|
|
32
|
-
function record(value) {
|
|
33
|
-
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
34
|
-
? value
|
|
35
|
-
: null;
|
|
36
|
-
}
|
|
37
|
-
export function lookupResultDepth(value, fallback = null) {
|
|
38
|
-
const result = record(value);
|
|
39
|
-
if (!result)
|
|
40
|
-
return fallback;
|
|
41
|
-
const availability = record(result.availability);
|
|
42
|
-
if (availability?.deepScanComplete === false)
|
|
43
|
-
return 'fast';
|
|
44
|
-
if (availability?.deepScanComplete === true)
|
|
45
|
-
return 'deep';
|
|
46
|
-
const whois = record(result.whois);
|
|
47
|
-
if (whois?.skipped === true
|
|
48
|
-
&& typeof whois.detail === 'string'
|
|
49
|
-
&& /fast rdap-only mode/iu.test(whois.detail))
|
|
50
|
-
return 'fast';
|
|
51
|
-
const scanModes = [
|
|
52
|
-
record(result.reverseDns)?.scanMode,
|
|
53
|
-
record(availability?.dns)?.scanMode,
|
|
54
|
-
record(availability?.http)?.scanMode,
|
|
55
|
-
];
|
|
56
|
-
if (scanModes.includes('deep'))
|
|
57
|
-
return 'deep';
|
|
58
|
-
if (scanModes.includes('fast'))
|
|
59
|
-
return 'fast';
|
|
60
|
-
return fallback;
|
|
61
|
-
}
|
|
62
|
-
export function reconcileLookupUrlState(current, searchParams, preferredTask) {
|
|
63
|
-
const explicitQuery = lookupUrlQuery(searchParams.get('q'));
|
|
64
|
-
const explicitDepth = lookupUrlDepth(searchParams.get('depth'));
|
|
65
|
-
const queryChanged = explicitQuery !== null && explicitQuery !== current.query;
|
|
66
|
-
const depthChanged = explicitDepth !== null && explicitDepth !== current.depth;
|
|
67
|
-
const conflictsWithResult = current.result !== null && ((explicitQuery !== null && explicitQuery !== current.completedTarget)
|
|
68
|
-
|| (explicitDepth !== null && explicitDepth !== current.retainedResultDepth));
|
|
69
|
-
return {
|
|
70
|
-
query: explicitQuery ?? current.query,
|
|
71
|
-
depth: explicitDepth ?? current.depth,
|
|
72
|
-
task: parseLookupTaskContext(searchParams.get('task')) ?? normalizeLookupTaskView(preferredTask),
|
|
73
|
-
result: conflictsWithResult ? null : current.result,
|
|
74
|
-
completedTarget: conflictsWithResult ? '' : current.completedTarget,
|
|
75
|
-
error: conflictsWithResult || queryChanged || depthChanged ? '' : current.error,
|
|
76
|
-
};
|
|
77
|
-
}
|
|
78
|
-
export function readLookupPresentation(storage) {
|
|
79
|
-
try {
|
|
80
|
-
const serialized = storage.getItem(LOOKUP_PRESENTATION_STORAGE_KEY);
|
|
81
|
-
if (serialized === null
|
|
82
|
-
|| serialized.length > MAX_LOOKUP_PRESENTATION_SERIALIZED_BYTES
|
|
83
|
-
|| new TextEncoder().encode(serialized).byteLength > MAX_LOOKUP_PRESENTATION_SERIALIZED_BYTES) {
|
|
84
|
-
return { task: 'general' };
|
|
85
|
-
}
|
|
86
|
-
const stored = JSON.parse(serialized);
|
|
87
|
-
if (stored?.version === 1) {
|
|
88
|
-
return {
|
|
89
|
-
task: normalizeLookupTaskView(stored.task),
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
catch {
|
|
94
|
-
// Invalid or unavailable browser storage falls back to the stable defaults.
|
|
95
|
-
}
|
|
96
|
-
return { task: 'general' };
|
|
97
|
-
}
|
|
98
|
-
export function writeLookupPresentation(storage, state) {
|
|
99
|
-
try {
|
|
100
|
-
storage.setItem(LOOKUP_PRESENTATION_STORAGE_KEY, JSON.stringify({
|
|
101
|
-
version: 1,
|
|
102
|
-
task: normalizeLookupTaskView(state.task),
|
|
103
|
-
}));
|
|
104
|
-
}
|
|
105
|
-
catch {
|
|
106
|
-
// The caller's in-memory selection remains valid when storage is unavailable.
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
const PRIORITY = Object.freeze({
|
|
110
|
-
general: Object.freeze(['overview', 'registry', 'web-evidence', 'relationships-history', 'source-quality', 'case-response', 'advanced-evidence']),
|
|
111
|
-
acquisition: Object.freeze(['overview', 'registry', 'relationships-history', 'web-evidence', 'source-quality', 'case-response', 'advanced-evidence']),
|
|
112
|
-
brand: Object.freeze(['overview', 'web-evidence', 'relationships-history', 'registry', 'source-quality', 'case-response', 'advanced-evidence']),
|
|
113
|
-
incident: Object.freeze(['overview', 'web-evidence', 'relationships-history', 'advanced-evidence', 'registry', 'source-quality', 'case-response']),
|
|
114
|
-
owned: Object.freeze(['overview', 'registry', 'web-evidence', 'relationships-history', 'source-quality', 'case-response', 'advanced-evidence']),
|
|
115
|
-
});
|
|
116
|
-
export function prioritizeLookupSectionLinks(links, task) {
|
|
117
|
-
const order = PRIORITY[normalizeLookupTaskView(task)];
|
|
118
|
-
const rank = new Map(order.map((id, index) => [`#${id}`, index]));
|
|
119
|
-
return [...links].sort((left, right) => ((rank.get(left.href) ?? order.length) - (rank.get(right.href) ?? order.length)));
|
|
120
|
-
}
|
|
@@ -1,277 +0,0 @@
|
|
|
1
|
-
import { BoundedJsonResponseError, requestJsonCapped, } from "../bounded-json-response.js";
|
|
2
|
-
export const LOOKUP_SOURCE_REFRESH_VERSION = 1;
|
|
3
|
-
export const LOOKUP_SOURCE_STALE_AFTER_DAYS = 7;
|
|
4
|
-
export const LOOKUP_FRESHNESS_POLICY_VERSION = 1;
|
|
5
|
-
export const LOOKUP_SOURCE_REFRESH_TIMEOUT_MS = 40_000;
|
|
6
|
-
export const MAX_LOOKUP_SOURCE_REFRESH_KEYS = 512;
|
|
7
|
-
export const MAX_LOOKUP_SOURCE_REFRESH_BYTES = 2 * 1024 * 1024;
|
|
8
|
-
export const MAX_LOOKUP_SOURCE_REFRESH_HISTORY = 12;
|
|
9
|
-
const DOMAIN_EVIDENCE_IDS = new Set([
|
|
10
|
-
'availability',
|
|
11
|
-
'client-behavior',
|
|
12
|
-
'dns',
|
|
13
|
-
'http',
|
|
14
|
-
'page-identity',
|
|
15
|
-
'page-role',
|
|
16
|
-
'security-posture',
|
|
17
|
-
'technology',
|
|
18
|
-
'tls',
|
|
19
|
-
]);
|
|
20
|
-
const NETWORK_EVIDENCE_IDS = new Set(['availability', 'dns', 'reverse-dns', 'network-context']);
|
|
21
|
-
const WEB_EVIDENCE_IDS = new Set(['http', 'tls', 'page-identity', 'page-role', 'client-behavior', 'security-posture', 'technology']);
|
|
22
|
-
const TASK_FRESHNESS_THRESHOLDS = Object.freeze({
|
|
23
|
-
general: Object.freeze({ registration: 30, network: 7, web: 3 }),
|
|
24
|
-
acquisition: Object.freeze({ registration: 7, network: 3, web: 3 }),
|
|
25
|
-
brand: Object.freeze({ registration: 30, network: 3, web: 1 }),
|
|
26
|
-
incident: Object.freeze({ registration: 14, network: 1, web: 1 }),
|
|
27
|
-
owned: Object.freeze({ registration: 30, network: 7, web: 3 }),
|
|
28
|
-
});
|
|
29
|
-
function freshnessDays(value, fallback) {
|
|
30
|
-
const parsed = Number(value);
|
|
31
|
-
return Number.isFinite(parsed) ? Math.max(1, Math.min(365, Math.round(parsed))) : fallback;
|
|
32
|
-
}
|
|
33
|
-
export function buildLookupFreshnessPolicy(task, input) {
|
|
34
|
-
const defaults = TASK_FRESHNESS_THRESHOLDS[task];
|
|
35
|
-
const custom = input?.id === 'analyst-custom';
|
|
36
|
-
return {
|
|
37
|
-
version: LOOKUP_FRESHNESS_POLICY_VERSION,
|
|
38
|
-
id: custom ? 'analyst-custom' : 'task-default',
|
|
39
|
-
task,
|
|
40
|
-
thresholdsDays: custom ? {
|
|
41
|
-
registration: freshnessDays(input?.thresholdsDays?.registration, defaults.registration),
|
|
42
|
-
network: freshnessDays(input?.thresholdsDays?.network, defaults.network),
|
|
43
|
-
web: freshnessDays(input?.thresholdsDays?.web, defaults.web),
|
|
44
|
-
} : { ...defaults },
|
|
45
|
-
};
|
|
46
|
-
}
|
|
47
|
-
function observedAgeDays(observedAt, now) {
|
|
48
|
-
if (typeof observedAt !== 'string' || typeof now !== 'string')
|
|
49
|
-
return null;
|
|
50
|
-
const observedMs = Date.parse(observedAt);
|
|
51
|
-
const nowMs = Date.parse(now);
|
|
52
|
-
if (!Number.isFinite(observedMs) || !Number.isFinite(nowMs))
|
|
53
|
-
return null;
|
|
54
|
-
return Math.max(0, Math.floor((nowMs - observedMs) / 86_400_000));
|
|
55
|
-
}
|
|
56
|
-
function limited(entries, ids) {
|
|
57
|
-
return entries.some((entry) => ids.has(entry.id) && entry.manualReviewSuggested);
|
|
58
|
-
}
|
|
59
|
-
function availableIds(entries, ids) {
|
|
60
|
-
return entries.filter((entry) => ids.has(entry.id)).map((entry) => entry.id).slice(0, 12);
|
|
61
|
-
}
|
|
62
|
-
export function buildLookupSourceRefreshPlan(ledger, observedAt, now = new Date().toISOString(), options = {}) {
|
|
63
|
-
const ageDays = observedAgeDays(observedAt, now);
|
|
64
|
-
const freshnessPolicy = buildLookupFreshnessPolicy(options.task ?? 'general', options.freshnessPolicy);
|
|
65
|
-
const entries = ledger.entries.slice(0, 24);
|
|
66
|
-
const plans = [];
|
|
67
|
-
const groups = [
|
|
68
|
-
{
|
|
69
|
-
id: 'rdap',
|
|
70
|
-
label: 'Registry RDAP',
|
|
71
|
-
endpoint: '/api/rdap',
|
|
72
|
-
ids: new Set(['rdap']),
|
|
73
|
-
disclosure: 'Starts one bounded registry RDAP operation for this target.',
|
|
74
|
-
threshold: 'registration',
|
|
75
|
-
},
|
|
76
|
-
{
|
|
77
|
-
id: 'whois',
|
|
78
|
-
label: 'WHOIS',
|
|
79
|
-
endpoint: '/api/whois',
|
|
80
|
-
ids: new Set(['whois']),
|
|
81
|
-
disclosure: 'Starts one bounded referral-aware WHOIS operation for this target.',
|
|
82
|
-
threshold: 'registration',
|
|
83
|
-
},
|
|
84
|
-
{
|
|
85
|
-
id: 'availability',
|
|
86
|
-
label: 'Domain evidence',
|
|
87
|
-
endpoint: '/api/availability',
|
|
88
|
-
ids: DOMAIN_EVIDENCE_IDS,
|
|
89
|
-
disclosure: 'Repeats the bounded domain-evidence branch, including eligible DNS, HTTP, page, and TLS work for the selected depth.',
|
|
90
|
-
threshold: 'network',
|
|
91
|
-
},
|
|
92
|
-
];
|
|
93
|
-
for (const group of groups) {
|
|
94
|
-
const evidenceIds = availableIds(entries, group.ids);
|
|
95
|
-
if (!evidenceIds.length)
|
|
96
|
-
continue;
|
|
97
|
-
const isLimited = limited(entries, group.ids);
|
|
98
|
-
const thresholds = evidenceIds.map((id) => WEB_EVIDENCE_IDS.has(id)
|
|
99
|
-
? freshnessPolicy.thresholdsDays.web
|
|
100
|
-
: NETWORK_EVIDENCE_IDS.has(id)
|
|
101
|
-
? freshnessPolicy.thresholdsDays.network
|
|
102
|
-
: freshnessPolicy.thresholdsDays[group.threshold]);
|
|
103
|
-
const staleAfterDays = Math.min(...thresholds);
|
|
104
|
-
const evidenceAges = evidenceIds
|
|
105
|
-
.map((id) => observedAgeDays(options.observedAtByEvidence?.[id] ?? observedAt, now))
|
|
106
|
-
.filter((value) => value !== null);
|
|
107
|
-
const groupAgeDays = evidenceAges.length ? Math.max(...evidenceAges) : ageDays;
|
|
108
|
-
const groupStale = groupAgeDays !== null && groupAgeDays >= staleAfterDays;
|
|
109
|
-
if (!isLimited && !groupStale)
|
|
110
|
-
continue;
|
|
111
|
-
plans.push({
|
|
112
|
-
id: group.id,
|
|
113
|
-
label: group.label,
|
|
114
|
-
endpoint: group.endpoint,
|
|
115
|
-
evidenceIds,
|
|
116
|
-
reason: isLimited ? 'limited' : 'stale',
|
|
117
|
-
requestDisclosure: group.disclosure,
|
|
118
|
-
supersedesObservedAt: typeof observedAt === 'string' && Number.isFinite(Date.parse(observedAt))
|
|
119
|
-
? new Date(observedAt).toISOString()
|
|
120
|
-
: null,
|
|
121
|
-
ageDays: groupAgeDays,
|
|
122
|
-
staleAfterDays,
|
|
123
|
-
});
|
|
124
|
-
}
|
|
125
|
-
const stale = plans.some((item) => item.reason === 'stale');
|
|
126
|
-
return {
|
|
127
|
-
version: LOOKUP_SOURCE_REFRESH_VERSION,
|
|
128
|
-
stale,
|
|
129
|
-
ageDays,
|
|
130
|
-
freshnessPolicy,
|
|
131
|
-
items: plans,
|
|
132
|
-
limitations: [
|
|
133
|
-
'A source refresh is displayed separately and never merged into the original unified Lookup envelope.',
|
|
134
|
-
'Run a complete Lookup before saving, comparing, or exporting replacement evidence collected at one review time.',
|
|
135
|
-
'A retry can remain partial or unavailable and never proves that missing evidence is absent.',
|
|
136
|
-
'Freshness thresholds organise review only. They do not make an older observation false or a newer observation complete.',
|
|
137
|
-
],
|
|
138
|
-
};
|
|
139
|
-
}
|
|
140
|
-
function record(value) {
|
|
141
|
-
return value && typeof value === 'object' && !Array.isArray(value)
|
|
142
|
-
? value
|
|
143
|
-
: {};
|
|
144
|
-
}
|
|
145
|
-
function text(value, maximum = 180) {
|
|
146
|
-
return String(value ?? '')
|
|
147
|
-
.replace(/[\u0000-\u001f\u007f]+/gu, ' ')
|
|
148
|
-
.replace(/\s+/gu, ' ')
|
|
149
|
-
.trim()
|
|
150
|
-
.slice(0, maximum);
|
|
151
|
-
}
|
|
152
|
-
function summarizeSource(plan, body, observedAt, depth, supersedesObservedAt) {
|
|
153
|
-
const base = {
|
|
154
|
-
version: LOOKUP_SOURCE_REFRESH_VERSION,
|
|
155
|
-
id: plan.id,
|
|
156
|
-
observedAt,
|
|
157
|
-
attemptedAt: observedAt,
|
|
158
|
-
reason: plan.reason,
|
|
159
|
-
evidenceIds: plan.evidenceIds.slice(0, 12),
|
|
160
|
-
supersedesObservedAt,
|
|
161
|
-
};
|
|
162
|
-
if (plan.id === 'rdap') {
|
|
163
|
-
const upstreamStatus = Number(body.upstreamStatus);
|
|
164
|
-
const parsed = record(body.parsed);
|
|
165
|
-
const complete = upstreamStatus === 200 && Object.keys(parsed).length > 0;
|
|
166
|
-
return {
|
|
167
|
-
...base,
|
|
168
|
-
state: complete ? 'complete' : 'limited',
|
|
169
|
-
detail: complete
|
|
170
|
-
? `Registry RDAP returned a validated ${upstreamStatus} response.`
|
|
171
|
-
: `Registry RDAP returned ${Number.isFinite(upstreamStatus) ? `HTTP ${upstreamStatus}` : 'no complete structured record'}.`,
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
if (plan.id === 'whois') {
|
|
175
|
-
const chain = Array.isArray(body.chain) ? body.chain : [];
|
|
176
|
-
const chainStatus = text(record(body.parsed).chainStatus, 40);
|
|
177
|
-
const complete = chain.length > 0 && chainStatus === 'complete';
|
|
178
|
-
return {
|
|
179
|
-
...base,
|
|
180
|
-
state: complete ? 'complete' : 'limited',
|
|
181
|
-
detail: complete
|
|
182
|
-
? `WHOIS returned a complete ${chain.length}-hop referral chain.`
|
|
183
|
-
: `WHOIS returned ${chain.length} hop${chain.length === 1 ? '' : 's'} with ${chainStatus || 'unknown'} chain status.`,
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
const state = text(body.state, 40) || 'unknown';
|
|
187
|
-
const complete = body.deepScanComplete === true
|
|
188
|
-
|| (depth === 'fast' && state !== 'unknown');
|
|
189
|
-
const sourceStates = ['dns', 'http', 'tls']
|
|
190
|
-
.map((key) => text(record(body[key]).status, 40))
|
|
191
|
-
.filter(Boolean);
|
|
192
|
-
return {
|
|
193
|
-
...base,
|
|
194
|
-
state: complete ? 'complete' : 'limited',
|
|
195
|
-
detail: `Domain evidence returned ${state}${sourceStates.length ? `; DNS, HTTP, and TLS states: ${sourceStates.join(', ')}` : ''}.`,
|
|
196
|
-
};
|
|
197
|
-
}
|
|
198
|
-
function unavailableRefreshResult(plan, attemptedAt, supersedesObservedAt, detail) {
|
|
199
|
-
return {
|
|
200
|
-
version: LOOKUP_SOURCE_REFRESH_VERSION,
|
|
201
|
-
id: plan.id,
|
|
202
|
-
state: 'unavailable',
|
|
203
|
-
detail,
|
|
204
|
-
observedAt: null,
|
|
205
|
-
attemptedAt,
|
|
206
|
-
reason: plan.reason,
|
|
207
|
-
evidenceIds: plan.evidenceIds.slice(0, 12),
|
|
208
|
-
supersedesObservedAt,
|
|
209
|
-
};
|
|
210
|
-
}
|
|
211
|
-
export async function requestLookupSourceRefresh(plan, query, depth, options = {}) {
|
|
212
|
-
const normalizedQuery = text(query, 4096);
|
|
213
|
-
if (!normalizedQuery)
|
|
214
|
-
return { ok: false, message: 'A valid Lookup target is required.' };
|
|
215
|
-
const timeoutMs = Number.isFinite(options.timeoutMs)
|
|
216
|
-
? Math.max(1, Math.min(LOOKUP_SOURCE_REFRESH_TIMEOUT_MS, Math.round(Number(options.timeoutMs))))
|
|
217
|
-
: LOOKUP_SOURCE_REFRESH_TIMEOUT_MS;
|
|
218
|
-
const attemptedAt = options.now?.() ?? new Date().toISOString();
|
|
219
|
-
const supersedesObservedAt = options.supersedesObservedAt === undefined
|
|
220
|
-
? plan.supersedesObservedAt
|
|
221
|
-
: options.supersedesObservedAt;
|
|
222
|
-
try {
|
|
223
|
-
const url = `${plan.endpoint}?q=${encodeURIComponent(normalizedQuery)}${plan.id === 'availability' && depth === 'fast' ? '&fast=true' : ''}`;
|
|
224
|
-
const { response, body: rawBody } = await requestJsonCapped(url, {
|
|
225
|
-
credentials: 'same-origin',
|
|
226
|
-
}, {
|
|
227
|
-
...(options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}),
|
|
228
|
-
maximumBytes: MAX_LOOKUP_SOURCE_REFRESH_BYTES,
|
|
229
|
-
timeoutMs,
|
|
230
|
-
});
|
|
231
|
-
const body = record(rawBody);
|
|
232
|
-
if (!response.ok) {
|
|
233
|
-
return {
|
|
234
|
-
ok: true,
|
|
235
|
-
value: unavailableRefreshResult(plan, attemptedAt, supersedesObservedAt, text(body.error, 240) || `The source returned HTTP ${response.status}; no new observation was recorded.`),
|
|
236
|
-
};
|
|
237
|
-
}
|
|
238
|
-
if (Object.keys(body).length > MAX_LOOKUP_SOURCE_REFRESH_KEYS) {
|
|
239
|
-
return { ok: false, message: 'Source refresh returned an oversized record.' };
|
|
240
|
-
}
|
|
241
|
-
return {
|
|
242
|
-
ok: true,
|
|
243
|
-
value: summarizeSource(plan, body, attemptedAt, depth, supersedesObservedAt),
|
|
244
|
-
};
|
|
245
|
-
}
|
|
246
|
-
catch (cause) {
|
|
247
|
-
if (cause instanceof BoundedJsonResponseError && cause.code === 'response_too_large') {
|
|
248
|
-
return { ok: true, value: unavailableRefreshResult(plan, attemptedAt, supersedesObservedAt, 'The source response exceeded the local limit; no new observation was recorded.') };
|
|
249
|
-
}
|
|
250
|
-
return {
|
|
251
|
-
ok: true,
|
|
252
|
-
value: unavailableRefreshResult(plan, attemptedAt, supersedesObservedAt, cause instanceof BoundedJsonResponseError && cause.code === 'timeout'
|
|
253
|
-
? 'The source refresh timed out; no new observation was recorded.'
|
|
254
|
-
: 'The source refresh could not be completed; no new observation was recorded.'),
|
|
255
|
-
};
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
export function mergeLookupSourceRefreshLedger(current, result) {
|
|
259
|
-
const candidates = [
|
|
260
|
-
...(current?.version === LOOKUP_SOURCE_REFRESH_VERSION ? current.entries : []),
|
|
261
|
-
result,
|
|
262
|
-
];
|
|
263
|
-
const byObservation = new Map();
|
|
264
|
-
for (const entry of candidates.slice(-MAX_LOOKUP_SOURCE_REFRESH_HISTORY * 4)) {
|
|
265
|
-
byObservation.set(`${entry.id}:${entry.attemptedAt}`, entry);
|
|
266
|
-
}
|
|
267
|
-
const ordered = [...byObservation.values()]
|
|
268
|
-
.sort((left, right) => Date.parse(left.attemptedAt) - Date.parse(right.attemptedAt));
|
|
269
|
-
const truncated = Boolean(current?.truncated)
|
|
270
|
-
|| ordered.length > MAX_LOOKUP_SOURCE_REFRESH_HISTORY
|
|
271
|
-
|| candidates.length > MAX_LOOKUP_SOURCE_REFRESH_HISTORY;
|
|
272
|
-
return {
|
|
273
|
-
version: LOOKUP_SOURCE_REFRESH_VERSION,
|
|
274
|
-
entries: ordered.slice(-MAX_LOOKUP_SOURCE_REFRESH_HISTORY),
|
|
275
|
-
truncated,
|
|
276
|
-
};
|
|
277
|
-
}
|