@slicedearth/whoisleuth-cli 1.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/LICENSE +661 -0
  2. package/LICENSES/Retire.js-Apache-2.0.txt +199 -0
  3. package/NOTICE +76 -0
  4. package/README.md +84 -0
  5. package/TRADEMARKS.md +41 -0
  6. package/bin/whoisleuth.mjs +35 -0
  7. package/cli/archive-inspect.mjs +198 -0
  8. package/cli/arguments.mjs +1022 -0
  9. package/cli/artifact-verify.mjs +210 -0
  10. package/cli/bulk-checkpoint.mjs +218 -0
  11. package/cli/bulk-command-runner.mjs +109 -0
  12. package/cli/bulk-output.mjs +112 -0
  13. package/cli/bulk.mjs +133 -0
  14. package/cli/compare.mjs +238 -0
  15. package/cli/completion.mjs +175 -0
  16. package/cli/csv.mjs +1 -0
  17. package/cli/discover.mjs +50 -0
  18. package/cli/discovery-command-runner.mjs +36 -0
  19. package/cli/discovery-observation-snapshot.mjs +315 -0
  20. package/cli/discovery-scan-command-runner.mjs +141 -0
  21. package/cli/discovery-scan.mjs +325 -0
  22. package/cli/discovery-snapshot.mjs +133 -0
  23. package/cli/discovery-workflow.mjs +73 -0
  24. package/cli/doctor.mjs +137 -0
  25. package/cli/errors.mjs +21 -0
  26. package/cli/evidence-command-arguments.mjs +145 -0
  27. package/cli/evidence-command-runner.mjs +106 -0
  28. package/cli/evidence-signing.mjs +215 -0
  29. package/cli/exit-codes.mjs +9 -0
  30. package/cli/export-evidence.mjs +28 -0
  31. package/cli/formatters/evidence-report.mjs +3 -0
  32. package/cli/formatters/html.mjs +69 -0
  33. package/cli/formatters/json.mjs +135 -0
  34. package/cli/formatters/markdown.mjs +3 -0
  35. package/cli/formatters/terminal.mjs +658 -0
  36. package/cli/http.mjs +27 -0
  37. package/cli/lookup-command-runner.mjs +87 -0
  38. package/cli/lookup-diff.mjs +171 -0
  39. package/cli/mail-review.mjs +204 -0
  40. package/cli/manual.mjs +34 -0
  41. package/cli/network-command-runner.mjs +88 -0
  42. package/cli/output-file.mjs +97 -0
  43. package/cli/page-compare.mjs +123 -0
  44. package/cli/posture.mjs +20 -0
  45. package/cli/progress-events.mjs +53 -0
  46. package/cli/progress.mjs +56 -0
  47. package/cli/registry-access.mjs +3 -0
  48. package/cli/registry-support.mjs +120 -0
  49. package/cli/risk-calibration.mjs +294 -0
  50. package/cli/runner-types.mjs +1 -0
  51. package/cli/runner.mjs +726 -0
  52. package/cli/saved-lookup.mjs +87 -0
  53. package/cli/source-reliability.mjs +727 -0
  54. package/cli/strict-exit.mjs +55 -0
  55. package/cli/terminal-presentation.mjs +147 -0
  56. package/docs/cli.md +827 -0
  57. package/frontend/src/lib/analysis/acquisition-decision-packet.js +93 -0
  58. package/frontend/src/lib/analysis/acquisition-due-diligence.js +391 -0
  59. package/frontend/src/lib/analysis/artifact-integrity.js +21 -0
  60. package/frontend/src/lib/analysis/brand-profile-model.js +421 -0
  61. package/frontend/src/lib/analysis/bulk-domain-comparison.js +246 -0
  62. package/frontend/src/lib/analysis/bulk-mail-exposure.js +192 -0
  63. package/frontend/src/lib/analysis/bulk-retry-plan.js +86 -0
  64. package/frontend/src/lib/analysis/bulk-review-export.js +62 -0
  65. package/frontend/src/lib/analysis/bulk-review-model.js +266 -0
  66. package/frontend/src/lib/analysis/bulk-session-model.js +515 -0
  67. package/frontend/src/lib/analysis/bulk-sort.js +49 -0
  68. package/frontend/src/lib/analysis/campaign-model.js +265 -0
  69. package/frontend/src/lib/analysis/case-evidence-model.js +642 -0
  70. package/frontend/src/lib/analysis/case-migration-model.js +248 -0
  71. package/frontend/src/lib/analysis/case-model.js +5 -0
  72. package/frontend/src/lib/analysis/case-record-contracts.js +44 -0
  73. package/frontend/src/lib/analysis/case-record-core.js +232 -0
  74. package/frontend/src/lib/analysis/case-record-model.js +7 -0
  75. package/frontend/src/lib/analysis/case-record-operations.js +224 -0
  76. package/frontend/src/lib/analysis/case-response-model.js +640 -0
  77. package/frontend/src/lib/analysis/case-response-packet.js +573 -0
  78. package/frontend/src/lib/analysis/case-storage-model.js +93 -0
  79. package/frontend/src/lib/analysis/detection-rule-model.js +306 -0
  80. package/frontend/src/lib/analysis/dns-record-normalization.js +11 -0
  81. package/frontend/src/lib/analysis/http-summary.js +165 -0
  82. package/frontend/src/lib/analysis/investigation-guide.js +500 -0
  83. package/frontend/src/lib/analysis/investigation-template-model.js +144 -0
  84. package/frontend/src/lib/analysis/page-baseline.js +272 -0
  85. package/frontend/src/lib/analysis/page-similarity.js +179 -0
  86. package/frontend/src/lib/analysis/relationship-evidence.js +265 -0
  87. package/frontend/src/lib/analysis/relationship-observation-model.js +423 -0
  88. package/frontend/src/lib/analysis/scoring.js +135 -0
  89. package/frontend/src/lib/analysis/shortlist-model.js +209 -0
  90. package/frontend/src/lib/analysis/utils.js +265 -0
  91. package/frontend/src/lib/analysis/watchlist-history.js +643 -0
  92. package/frontend/src/lib/analysis/watchlist-store.js +127 -0
  93. package/frontend/src/lib/analysis/website-snapshot-model.js +302 -0
  94. package/frontend/src/lib/analysis/workspace-archive-crypto.js +282 -0
  95. package/frontend/src/lib/analysis/workspace-archive.js +472 -0
  96. package/lib/abort.mjs +26 -0
  97. package/lib/abusech-auth.mjs +19 -0
  98. package/lib/availability.mjs +716 -0
  99. package/lib/bounded-contract-normalizers.mjs +61 -0
  100. package/lib/browser-library-profile.mjs +310 -0
  101. package/lib/bulk-comparison-evidence.mjs +83 -0
  102. package/lib/classify.mjs +93 -0
  103. package/lib/client-behavior-profile.mjs +83 -0
  104. package/lib/credential-surface-profile.mjs +66 -0
  105. package/lib/csv.mjs +23 -0
  106. package/lib/ct-query.mjs +27 -0
  107. package/lib/ct-search.mjs +393 -0
  108. package/lib/distributed-operation-budget.mjs +461 -0
  109. package/lib/dns-delegation-health.mjs +344 -0
  110. package/lib/dns-intelligence.mjs +739 -0
  111. package/lib/dns-mx.mjs +24 -0
  112. package/lib/dns-resolver-selection.mjs +33 -0
  113. package/lib/domain-posture-analysis.mjs +254 -0
  114. package/lib/domain-posture-parsers.mjs +358 -0
  115. package/lib/domain-posture.mjs +745 -0
  116. package/lib/error-detail.mjs +10 -0
  117. package/lib/evidence-export.mjs +431 -0
  118. package/lib/evidence-report-markdown.mjs +71 -0
  119. package/lib/evidence-report.mjs +391 -0
  120. package/lib/external-intelligence-risk.mjs +108 -0
  121. package/lib/favicon.mjs +146 -0
  122. package/lib/feature-policy.mjs +70 -0
  123. package/lib/generated/retire-browser-catalog.mjs +8724 -0
  124. package/lib/generated/unicode-confusables-17.mjs +93 -0
  125. package/lib/hostname.mjs +38 -0
  126. package/lib/html-signals.mjs +680 -0
  127. package/lib/http-intelligence.mjs +232 -0
  128. package/lib/idn-confusable-policy.mjs +86 -0
  129. package/lib/idn-confusables.mjs +465 -0
  130. package/lib/lookup-cache.mjs +130 -0
  131. package/lib/lookup-progress.mjs +291 -0
  132. package/lib/lookup-source-progress.mjs +109 -0
  133. package/lib/lookup-threat-provider-inventory.mjs +13 -0
  134. package/lib/lookup.mjs +592 -0
  135. package/lib/observation.mjs +114 -0
  136. package/lib/observed-network-context.mjs +355 -0
  137. package/lib/operation-budget.mjs +489 -0
  138. package/lib/page-fingerprints.mjs +481 -0
  139. package/lib/page-role-profile.mjs +117 -0
  140. package/lib/perceptual-hash.mjs +451 -0
  141. package/lib/rdap-attempts.mjs +34 -0
  142. package/lib/rdap-bootstrap.mjs +173 -0
  143. package/lib/rdap-capabilities.mjs +292 -0
  144. package/lib/rdap-client.mjs +125 -0
  145. package/lib/rdap-normalization.mjs +691 -0
  146. package/lib/rdap-registrar.mjs +196 -0
  147. package/lib/rdap-transport.mjs +39 -0
  148. package/lib/rdap-types.mjs +1 -0
  149. package/lib/rdap-validation.mjs +94 -0
  150. package/lib/rdap.mjs +16 -0
  151. package/lib/registry-access.mjs +32 -0
  152. package/lib/registry-capabilities.mjs +117 -0
  153. package/lib/registry-capability-catalogue.mjs +1206 -0
  154. package/lib/registry-comparison.mjs +318 -0
  155. package/lib/registry-dates.mjs +190 -0
  156. package/lib/registry-insights.mjs +325 -0
  157. package/lib/response-policy.mjs +256 -0
  158. package/lib/risk-scoring.mjs +164 -0
  159. package/lib/safe-fetch.mjs +446 -0
  160. package/lib/security-txt.mjs +359 -0
  161. package/lib/service-binding-dns.mjs +608 -0
  162. package/lib/sslbl-certificates.generated.mjs +143 -0
  163. package/lib/sslbl-intelligence.mjs +241 -0
  164. package/lib/static-html-analysis.mjs +340 -0
  165. package/lib/structured-data-identity.mjs +323 -0
  166. package/lib/threat-intelligence-contract.mjs +5 -0
  167. package/lib/threat-intelligence-definition-registry.mjs +34 -0
  168. package/lib/threat-intelligence-fixtures.mjs +82 -0
  169. package/lib/threat-intelligence-manifests.mjs +512 -0
  170. package/lib/threat-intelligence-runtime.mjs +662 -0
  171. package/lib/threat-intelligence-targets.mjs +168 -0
  172. package/lib/threat-intelligence-types.mjs +3 -0
  173. package/lib/threatfox-intelligence.mjs +307 -0
  174. package/lib/tls-intelligence.mjs +814 -0
  175. package/lib/typosquat-generator.mjs +884 -0
  176. package/lib/urlhaus-intelligence.mjs +361 -0
  177. package/lib/urlscan-intelligence.mjs +349 -0
  178. package/lib/website-security-posture.mjs +325 -0
  179. package/lib/website-technology.mjs +437 -0
  180. package/lib/whois-authority.mjs +98 -0
  181. package/lib/whois-chain.mjs +261 -0
  182. package/lib/whois-contacts.mjs +163 -0
  183. package/lib/whois-contracts.mjs +3 -0
  184. package/lib/whois-normalization.mjs +27 -0
  185. package/lib/whois-transport.mjs +100 -0
  186. package/lib/whois-values.mjs +92 -0
  187. package/lib/whois.mjs +1187 -0
  188. package/package.json +55 -0
  189. package/third-party-notices.txt +5571 -0
@@ -0,0 +1,192 @@
1
+ import { sha256ArtifactDigest } from "./artifact-integrity.js";
2
+ import { normalizeBulkSessionResult, } from "./bulk-session-model.js";
3
+ export const BULK_MAIL_EXPOSURE_SCHEMA = 'whoisleuth.bulk-mail-exposure';
4
+ export const BULK_MAIL_EXPOSURE_VERSION = 1;
5
+ export const MAX_BULK_MAIL_EXPOSURE_ROWS = 2_000;
6
+ const STATE_LABELS = {
7
+ authenticated_mail: 'Receiving mail with SPF and DMARC',
8
+ evidence_incomplete: 'Mail evidence incomplete',
9
+ mail_auth_gap: 'Receiving mail with an authentication gap',
10
+ mail_auth_incomplete: 'Receiving mail with incomplete authentication evidence',
11
+ no_explicit_mx: 'No explicit MX observed',
12
+ null_mx: 'Null MX observed',
13
+ };
14
+ function timestamp(value) {
15
+ if (typeof value !== 'string' || !Number.isFinite(Date.parse(value)))
16
+ return null;
17
+ return new Date(value).toISOString();
18
+ }
19
+ function baselineLabel(profile) {
20
+ if (profile === 'defensive_no_mail')
21
+ return 'Configured no-mail baseline';
22
+ if (profile === 'parked')
23
+ return 'Configured parked-domain baseline';
24
+ if (profile === 'standard')
25
+ return 'Configured standard-mail baseline';
26
+ return 'No active Brand Profile baseline';
27
+ }
28
+ function dnsCoverage(row) {
29
+ return row.sourceCoverage.find((source) => source.source === 'dns') ?? null;
30
+ }
31
+ function exposureState(row) {
32
+ const coverage = dnsCoverage(row);
33
+ if (row.scanDepth !== 'deep' || !coverage || !['complete', 'partial'].includes(coverage.state)) {
34
+ return 'evidence_incomplete';
35
+ }
36
+ if (row.hasNullMx === true)
37
+ return 'null_mx';
38
+ if (row.hasMx === true) {
39
+ if (row.hasSpf === true && row.hasDmarc === true)
40
+ return 'authenticated_mail';
41
+ if (row.hasSpf === false || row.hasDmarc === false)
42
+ return 'mail_auth_gap';
43
+ return 'mail_auth_incomplete';
44
+ }
45
+ if (row.hasMx === false && row.hasNullMx === false)
46
+ return 'no_explicit_mx';
47
+ return 'evidence_incomplete';
48
+ }
49
+ function stateDetail(state) {
50
+ if (state === 'authenticated_mail') {
51
+ return 'An explicit receiving MX plus SPF and DMARC publications were observed.';
52
+ }
53
+ if (state === 'null_mx') {
54
+ return 'A null MX explicitly declaring no inbound mail was observed.';
55
+ }
56
+ if (state === 'mail_auth_gap') {
57
+ return 'A receiving MX was observed while SPF or DMARC was observed absent.';
58
+ }
59
+ if (state === 'mail_auth_incomplete') {
60
+ return 'A receiving MX was observed, but SPF or DMARC collection was incomplete.';
61
+ }
62
+ if (state === 'no_explicit_mx') {
63
+ return 'DNS collection completed without an explicit MX. This is not proof that SMTP delivery is impossible.';
64
+ }
65
+ return 'The compact scan did not retain enough settled DNS evidence to classify mail exposure.';
66
+ }
67
+ function relation(state, profile) {
68
+ if (!profile || state === 'evidence_incomplete' || state === 'mail_auth_incomplete') {
69
+ return {
70
+ baselineRelation: 'inconclusive',
71
+ baselineDetail: profile
72
+ ? 'Incomplete evidence prevents a reliable comparison with the configured baseline.'
73
+ : 'Set an active Brand Profile to compare this observation with a configured official-domain mail posture.',
74
+ };
75
+ }
76
+ if (profile === 'defensive_no_mail') {
77
+ return state === 'null_mx'
78
+ ? {
79
+ baselineRelation: 'aligned',
80
+ baselineDetail: 'The observed null MX matches the configured no-mail posture.',
81
+ }
82
+ : {
83
+ baselineRelation: 'review',
84
+ baselineDetail: 'The observed candidate posture differs from the configured no-mail baseline.',
85
+ };
86
+ }
87
+ if (profile === 'standard') {
88
+ return ['authenticated_mail', 'mail_auth_gap'].includes(state)
89
+ ? {
90
+ baselineRelation: state === 'authenticated_mail' ? 'aligned' : 'review',
91
+ baselineDetail: state === 'authenticated_mail'
92
+ ? 'Receiving mail plus SPF and DMARC matches the configured standard-mail baseline.'
93
+ : 'Receiving mail was observed with an authentication gap relative to the configured baseline.',
94
+ }
95
+ : {
96
+ baselineRelation: 'review',
97
+ baselineDetail: 'The observed candidate posture differs from the configured standard-mail baseline.',
98
+ };
99
+ }
100
+ return {
101
+ baselineRelation: 'review',
102
+ baselineDetail: 'The parked-domain profile is context rather than a claim about the expected candidate mail posture.',
103
+ };
104
+ }
105
+ function rowLimitations(row, state) {
106
+ const output = [];
107
+ const coverage = dnsCoverage(row);
108
+ if (!coverage)
109
+ output.push('DNS source coverage was not recorded for this compact result.');
110
+ else if (coverage.state !== 'complete')
111
+ output.push(`DNS source coverage was ${coverage.state}.`);
112
+ if (row.scanDepth !== 'deep')
113
+ output.push('Fast mode does not collect the complete mail evidence used by this review.');
114
+ if (state === 'no_explicit_mx') {
115
+ output.push('No explicit MX is not equivalent to a null MX or proof that the domain cannot receive mail.');
116
+ }
117
+ output.push('Mail configuration alone does not establish use, intent, control, safety, or maliciousness.');
118
+ return output.slice(0, 5);
119
+ }
120
+ export function buildBulkMailExposureReport(rowsRaw, options = {}) {
121
+ const generatedAt = timestamp(options.generatedAt) ?? new Date().toISOString();
122
+ const officialDomains = [...new Set((options.officialDomains ?? [])
123
+ .map((domain) => domain.trim().toLowerCase().replace(/\.$/u, ''))
124
+ .filter(Boolean))].slice(0, 20);
125
+ const rows = rowsRaw
126
+ .map(normalizeBulkSessionResult)
127
+ .filter((row) => Boolean(row))
128
+ .filter((row) => row.trusted === null)
129
+ .slice(0, MAX_BULK_MAIL_EXPOSURE_ROWS)
130
+ .map((item) => {
131
+ const state = exposureState(item);
132
+ return {
133
+ domain: item.domain,
134
+ state,
135
+ label: STATE_LABELS[state],
136
+ detail: stateDetail(state),
137
+ ...relation(state, options.profile ?? null),
138
+ mutationTypes: item.mutationTypes.slice(0, 40),
139
+ registration: item.availability,
140
+ sourceCoverage: item.sourceCoverage.slice(0, 12),
141
+ limitations: rowLimitations(item, state),
142
+ };
143
+ });
144
+ const counts = {
145
+ authenticated_mail: 0,
146
+ evidence_incomplete: 0,
147
+ mail_auth_gap: 0,
148
+ mail_auth_incomplete: 0,
149
+ no_explicit_mx: 0,
150
+ null_mx: 0,
151
+ };
152
+ for (const item of rows)
153
+ counts[item.state] += 1;
154
+ return {
155
+ version: BULK_MAIL_EXPOSURE_VERSION,
156
+ generatedAt,
157
+ observedAt: timestamp(options.observedAt),
158
+ baseline: {
159
+ profile: options.profile ?? null,
160
+ officialDomains,
161
+ label: baselineLabel(options.profile ?? null),
162
+ limitations: [
163
+ 'The Brand Profile describes an analyst-configured official-domain posture; it is not a live observation.',
164
+ 'Candidate similarity and mail configuration do not establish use, control, intent, safety, or maliciousness.',
165
+ ],
166
+ },
167
+ rows,
168
+ counts,
169
+ limitations: [
170
+ 'This review uses compact Bulk evidence already collected and makes no additional request.',
171
+ 'Null MX, no explicit MX, receiving mail, authentication gaps, and incomplete evidence remain separate states.',
172
+ 'SMTP delivery, mailbox existence, catch-all behavior, and message acceptance were not tested.',
173
+ ],
174
+ };
175
+ }
176
+ export async function buildBulkMailExposureExport(report) {
177
+ const unsigned = {
178
+ schema: BULK_MAIL_EXPOSURE_SCHEMA,
179
+ version: BULK_MAIL_EXPOSURE_VERSION,
180
+ report,
181
+ };
182
+ const digestSha256 = await sha256ArtifactDigest(unsigned);
183
+ const document = {
184
+ ...unsigned,
185
+ integrity: { algorithm: 'SHA-256', digestSha256 },
186
+ };
187
+ return {
188
+ document,
189
+ content: `${JSON.stringify(document, null, 2)}\n`,
190
+ filename: `whoisleuth-mail-exposure-${report.generatedAt.slice(0, 10)}.json`,
191
+ };
192
+ }
@@ -0,0 +1,86 @@
1
+ import { normalizeBulkSessionResult, } from "./bulk-session-model.js";
2
+ export const MAX_BULK_RETRY_ROWS = 200;
3
+ export const BULK_REVIEW_STALE_AFTER_DAYS = 7;
4
+ const LIMITED_STATES = new Set([
5
+ 'error',
6
+ 'partial',
7
+ 'skipped',
8
+ 'unavailable',
9
+ 'unsupported',
10
+ ]);
11
+ function freshness(value, now = Date.now()) {
12
+ if (typeof value !== 'string' || !Number.isFinite(Date.parse(value))) {
13
+ return { observedAt: null, ageDays: null, state: 'unknown' };
14
+ }
15
+ const observedAt = new Date(value).toISOString();
16
+ const ageDays = Math.max(0, Math.floor((now - Date.parse(observedAt)) / 86_400_000));
17
+ return {
18
+ observedAt,
19
+ ageDays,
20
+ state: ageDays >= BULK_REVIEW_STALE_AFTER_DAYS ? 'stale' : 'current',
21
+ };
22
+ }
23
+ function retryRow(value, isStale) {
24
+ const limitedSources = value.sourceCoverage
25
+ .filter((item) => LIMITED_STATES.has(item.state))
26
+ .map((item) => item.source)
27
+ .sort()
28
+ .slice(0, 12);
29
+ const reasons = [];
30
+ if (value.status === 'error')
31
+ reasons.push('error');
32
+ if (limitedSources.length)
33
+ reasons.push('limited_source');
34
+ if (isStale)
35
+ reasons.push('stale');
36
+ return reasons.length ? { domain: value.domain, reasons, limitedSources } : null;
37
+ }
38
+ export function buildBulkRetryPlan(values, mode, observedAtRaw = null, now = Date.now()) {
39
+ const observationFreshness = freshness(observedAtRaw, now);
40
+ const candidates = values
41
+ .map(normalizeBulkSessionResult)
42
+ .filter((value) => Boolean(value))
43
+ .map((value) => retryRow(value, observationFreshness.state === 'stale'))
44
+ .filter((value) => Boolean(value));
45
+ const rows = candidates.slice(0, MAX_BULK_RETRY_ROWS);
46
+ return {
47
+ mode,
48
+ lookupRequests: rows.length,
49
+ rows,
50
+ destinations: mode === 'deep'
51
+ ? ['registry RDAP', 'registrar RDAP when advertised', 'WHOIS', 'DNS resolvers', 'target HTTP(S)', 'target TLS']
52
+ : ['registry RDAP', 'authoritative DNS delegation when registration sources are inconclusive'],
53
+ capped: candidates.length > rows.length,
54
+ freshness: observationFreshness,
55
+ limitations: [
56
+ 'This repeats the selected lookup profile; the current API does not request only one upstream source.',
57
+ `Evidence is marked stale after ${BULK_REVIEW_STALE_AFTER_DAYS} days for review purposes; this is a local observation age, not an upstream record age.`,
58
+ 'Optional third-party intelligence providers are not contacted by Bulk.',
59
+ 'A retry can remain partial or fail and does not convert unavailable evidence into absence.',
60
+ ],
61
+ };
62
+ }
63
+ export function preservePriorBulkResult(previousRaw, currentRaw) {
64
+ const previous = normalizeBulkSessionResult(previousRaw);
65
+ const current = normalizeBulkSessionResult(currentRaw);
66
+ if (!previous || !current || previous.domain !== current.domain || previous.status !== 'complete') {
67
+ return { preserve: false, reason: '' };
68
+ }
69
+ if (current.status === 'error') {
70
+ return { preserve: true, reason: 'The retry failed, so the prior settled result was retained.' };
71
+ }
72
+ if (previous.scanDepth === 'deep' && current.scanDepth === 'fast') {
73
+ return { preserve: true, reason: 'The retry returned only Fast evidence, so the prior Deep result was retained.' };
74
+ }
75
+ const currentSources = new Map(current.sourceCoverage.map((item) => [item.source, item.state]));
76
+ const regressed = previous.sourceCoverage
77
+ .filter((item) => item.state === 'complete')
78
+ .map((item) => item.source)
79
+ .filter((source) => currentSources.get(source) !== 'complete');
80
+ return regressed.length
81
+ ? {
82
+ preserve: true,
83
+ reason: `The retry reduced complete coverage for ${regressed.slice(0, 6).join(', ')}, so the prior settled result was retained.`,
84
+ }
85
+ : { preserve: false, reason: '' };
86
+ }
@@ -0,0 +1,62 @@
1
+ import { normalizeBulkSessionResult, } from "./bulk-session-model.js";
2
+ import { BULK_REVIEW_STATES, } from "./bulk-review-model.js";
3
+ import { sha256ArtifactDigest } from "./artifact-integrity.js";
4
+ export const BULK_REVIEW_MANIFEST_SCHEMA = 'whoisleuth.bulk-review-manifest';
5
+ export const BULK_REVIEW_MANIFEST_VERSION = 1;
6
+ const MAX_MANIFEST_ROWS = 2_000;
7
+ const REVIEW_STATE_SET = new Set(BULK_REVIEW_STATES);
8
+ function timestamp(value, fallback) {
9
+ if (typeof value !== 'string' || !Number.isFinite(Date.parse(value)))
10
+ return fallback;
11
+ return new Date(value).toISOString();
12
+ }
13
+ function reviewStateMap(values) {
14
+ return new Map(values.slice(0, MAX_MANIFEST_ROWS * 2).flatMap((item) => (REVIEW_STATE_SET.has(item.state)
15
+ ? [[item.domain, item.state]]
16
+ : [])));
17
+ }
18
+ export async function buildBulkReviewManifest(input) {
19
+ const now = new Date().toISOString();
20
+ const generatedAt = timestamp(input.generatedAt, now);
21
+ const observedAt = timestamp(input.observedAt, generatedAt);
22
+ const states = reviewStateMap(input.reviewStates);
23
+ const rows = input.rows
24
+ .map(normalizeBulkSessionResult)
25
+ .filter((item) => Boolean(item))
26
+ .slice(0, MAX_MANIFEST_ROWS)
27
+ .map((item) => ({
28
+ domain: item.domain,
29
+ reviewState: states.get(item.domain) || 'unreviewed',
30
+ resultState: item.status,
31
+ scanDepth: item.scanDepth,
32
+ sourceCoverage: item.sourceCoverage,
33
+ }));
34
+ const unsigned = {
35
+ schema: BULK_REVIEW_MANIFEST_SCHEMA,
36
+ version: BULK_REVIEW_MANIFEST_VERSION,
37
+ generatedAt,
38
+ observedAt,
39
+ lookupProfile: input.lookupProfile,
40
+ selection: {
41
+ count: rows.length,
42
+ domains: rows.map((item) => item.domain),
43
+ },
44
+ view: input.view,
45
+ rows,
46
+ limitations: [
47
+ 'This manifest records the explicit review selection and view context for a separate CSV export.',
48
+ 'It contains compact source states only and excludes raw payloads, contact records, notes, and transient request state.',
49
+ 'Reproducing the filters does not reproduce upstream responses or imply that evidence remains current.',
50
+ ],
51
+ };
52
+ const digestSha256 = await sha256ArtifactDigest(unsigned);
53
+ const document = {
54
+ ...unsigned,
55
+ integrity: { algorithm: 'SHA-256', digestSha256 },
56
+ };
57
+ return {
58
+ document,
59
+ content: `${JSON.stringify(document, null, 2)}\n`,
60
+ filename: `whoisleuth-bulk-review-${generatedAt.slice(0, 10)}.manifest.json`,
61
+ };
62
+ }
@@ -0,0 +1,266 @@
1
+ import { normalizeDomain } from "./case-model.js";
2
+ export const BULK_REVIEW_SCHEMA = 'whoisleuth.bulk-review';
3
+ export const BULK_REVIEW_SCHEMA_VERSION = 1;
4
+ export const MAX_BULK_REVIEW_PRESETS = 24;
5
+ export const MAX_BULK_REVIEW_ROWS = 1_900;
6
+ export const MAX_BULK_REVIEW_STORE_BYTES = 512 * 1024;
7
+ export const MAX_BULK_REVIEW_NAME_LENGTH = 80;
8
+ export const BULK_REVIEW_STATES = ['unreviewed', 'reviewing', 'reviewed', 'deferred'];
9
+ const CONTROL_RE = /[\u0000-\u001f\u007f]/u;
10
+ const SAFE_ID_RE = /^[A-Za-z0-9_-]{1,128}$/u;
11
+ const PRIMARY_FILTERS = new Set(['all', 'available', 'registered', 'high_risk', 'trusted', 'errors']);
12
+ const SOURCE_FILTERS = new Set(['', 'complete', 'limited', 'failed']);
13
+ const LIFECYCLE_FILTERS = new Set(['', 'new', 'expiring', 'aged', 'unknown']);
14
+ const AGE_FILTERS = new Set(['', '7d', '30d', '90d', '365d', 'older', 'unknown']);
15
+ const MAIL_FILTERS = new Set(['', 'mx', 'no_mx', 'spf_missing', 'dmarc_missing', 'unknown']);
16
+ const GROUPS = new Set(['', 'registrar', 'nameserver', 'source_state', 'lifecycle', 'mail']);
17
+ const SORT_KEYS = new Set(['domain', 'availability', 'risk', 'opportunity', 'activity', 'registrar', 'mutation']);
18
+ const REVIEW_STATES = new Set(BULK_REVIEW_STATES);
19
+ const CASE_DISPOSITIONS = new Set(['', 'untracked', 'unreviewed', 'suspicious', 'confirmed_abuse', 'false_positive', 'expected', 'closed_no_action']);
20
+ const SIGNAL_FILTERS = new Set(['favicon', 'password', 'phishing', 'asset_reuse', 'idn']);
21
+ function record(value) {
22
+ return value && typeof value === 'object' && !Array.isArray(value)
23
+ ? value
24
+ : {};
25
+ }
26
+ function text(value, maximum) {
27
+ if (typeof value !== 'string' || CONTROL_RE.test(value))
28
+ return '';
29
+ return value.trim().slice(0, maximum);
30
+ }
31
+ function timestamp(value, fallback) {
32
+ const normalized = text(value, 64);
33
+ const parsed = Date.parse(normalized);
34
+ return Number.isFinite(parsed) ? new Date(parsed).toISOString() : fallback;
35
+ }
36
+ function safeId(value) {
37
+ return typeof value === 'string' && SAFE_ID_RE.test(value) ? value : '';
38
+ }
39
+ function id(prefix) {
40
+ if (typeof globalThis.crypto?.randomUUID === 'function')
41
+ return globalThis.crypto.randomUUID();
42
+ return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
43
+ }
44
+ function setValue(value, values, fallback = '') {
45
+ return typeof value === 'string' && values.has(value) ? value : fallback;
46
+ }
47
+ function normalizeView(raw) {
48
+ const value = record(raw);
49
+ const signalFilters = Array.isArray(value.signalFilters)
50
+ ? [...new Set(value.signalFilters.filter((item) => typeof item === 'string' && SIGNAL_FILTERS.has(item)))].slice(0, SIGNAL_FILTERS.size)
51
+ : [];
52
+ const sortKey = typeof value.sortKey === 'string' && SORT_KEYS.has(value.sortKey)
53
+ ? value.sortKey
54
+ : 'risk';
55
+ return {
56
+ primaryFilter: setValue(value.primaryFilter, PRIMARY_FILTERS, 'all'),
57
+ mutationFilter: text(value.mutationFilter, 60),
58
+ signalFilters,
59
+ sourceFilter: setValue(value.sourceFilter, SOURCE_FILTERS),
60
+ lifecycleFilter: setValue(value.lifecycleFilter, LIFECYCLE_FILTERS),
61
+ ageFilter: setValue(value.ageFilter, AGE_FILTERS),
62
+ mailFilter: setValue(value.mailFilter, MAIL_FILTERS),
63
+ registrarFilter: text(value.registrarFilter, 200),
64
+ caseDispositionFilter: setValue(value.caseDispositionFilter, CASE_DISPOSITIONS),
65
+ reviewStateFilter: setValue(value.reviewStateFilter, new Set(['', ...BULK_REVIEW_STATES])),
66
+ groupBy: setValue(value.groupBy, GROUPS),
67
+ sortKey,
68
+ sortDirection: value.sortDirection === 1 ? 1 : -1,
69
+ };
70
+ }
71
+ function normalizePreset(raw, fallback) {
72
+ const value = record(raw);
73
+ const name = text(value.name, MAX_BULK_REVIEW_NAME_LENGTH);
74
+ if (!name)
75
+ return null;
76
+ const createdAt = timestamp(value.createdAt, fallback);
77
+ return {
78
+ kind: 'preset',
79
+ id: safeId(value.id) || id('bulk-view'),
80
+ name,
81
+ view: normalizeView(value.view),
82
+ createdAt,
83
+ updatedAt: timestamp(value.updatedAt, createdAt),
84
+ };
85
+ }
86
+ function normalizeRow(raw, fallback) {
87
+ const value = record(raw);
88
+ const domain = normalizeDomain(value.domain);
89
+ if (!domain)
90
+ return null;
91
+ return {
92
+ kind: 'row',
93
+ id: `d-${domain}`,
94
+ domain,
95
+ state: setValue(value.state, REVIEW_STATES, 'unreviewed'),
96
+ updatedAt: timestamp(value.updatedAt, fallback),
97
+ };
98
+ }
99
+ function sourceLists(raw) {
100
+ if (Array.isArray(raw)) {
101
+ return {
102
+ presets: raw.filter((item) => record(item).kind === 'preset'),
103
+ rows: raw.filter((item) => record(item).kind === 'row'),
104
+ };
105
+ }
106
+ const value = record(raw);
107
+ return {
108
+ presets: Array.isArray(value.presets) ? value.presets : [],
109
+ rows: Array.isArray(value.rows) ? value.rows : [],
110
+ };
111
+ }
112
+ export function normalizeBulkReviewStore(raw) {
113
+ const fallback = new Date(0).toISOString();
114
+ const source = sourceLists(raw);
115
+ const presets = new Map();
116
+ for (const candidate of source.presets.slice(0, MAX_BULK_REVIEW_PRESETS * 4)) {
117
+ const normalized = normalizePreset(candidate, fallback);
118
+ if (!normalized)
119
+ continue;
120
+ const existing = presets.get(normalized.id);
121
+ if (!existing || normalized.updatedAt >= existing.updatedAt)
122
+ presets.set(normalized.id, normalized);
123
+ }
124
+ const rows = new Map();
125
+ for (const candidate of source.rows.slice(0, MAX_BULK_REVIEW_ROWS * 2)) {
126
+ const normalized = normalizeRow(candidate, fallback);
127
+ if (!normalized)
128
+ continue;
129
+ const existing = rows.get(normalized.domain);
130
+ if (!existing || normalized.updatedAt >= existing.updatedAt)
131
+ rows.set(normalized.domain, normalized);
132
+ }
133
+ return {
134
+ schema: BULK_REVIEW_SCHEMA,
135
+ version: BULK_REVIEW_SCHEMA_VERSION,
136
+ presets: [...presets.values()]
137
+ .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt) || left.id.localeCompare(right.id))
138
+ .slice(0, MAX_BULK_REVIEW_PRESETS),
139
+ rows: [...rows.values()]
140
+ .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt) || left.domain.localeCompare(right.domain))
141
+ .slice(0, MAX_BULK_REVIEW_ROWS),
142
+ };
143
+ }
144
+ function byteLength(value) {
145
+ return new TextEncoder().encode(value).byteLength;
146
+ }
147
+ export function enforceBulkReviewBudget(raw) {
148
+ const store = normalizeBulkReviewStore(raw);
149
+ while (byteLength(JSON.stringify(store)) > MAX_BULK_REVIEW_STORE_BYTES && store.rows.length)
150
+ store.rows.pop();
151
+ while (byteLength(JSON.stringify(store)) > MAX_BULK_REVIEW_STORE_BYTES && store.presets.length > 1)
152
+ store.presets.pop();
153
+ if (byteLength(JSON.stringify(store)) > MAX_BULK_REVIEW_STORE_BYTES) {
154
+ throw new Error('Bulk review preferences exceed the browser-local storage limit.');
155
+ }
156
+ return store;
157
+ }
158
+ export function bulkReviewStoreVersion(raw) {
159
+ const value = record(raw);
160
+ return typeof value.version === 'number' && Number.isSafeInteger(value.version) && value.version > 0
161
+ ? value.version
162
+ : Array.isArray(raw) ? 1 : null;
163
+ }
164
+ export function serializeBulkReviewStore(raw) {
165
+ return JSON.stringify(enforceBulkReviewBudget(raw));
166
+ }
167
+ export function upsertBulkReviewPreset(raw, input, now = new Date().toISOString()) {
168
+ const store = normalizeBulkReviewStore(raw);
169
+ const candidate = normalizePreset({ ...record(input), updatedAt: now }, now);
170
+ if (!candidate)
171
+ throw new Error('Enter a name before saving the Bulk review view.');
172
+ const existing = store.presets.find((item) => item.id === candidate.id);
173
+ candidate.createdAt = existing?.createdAt ?? candidate.createdAt;
174
+ return enforceBulkReviewBudget({
175
+ ...store,
176
+ presets: [candidate, ...store.presets.filter((item) => item.id !== candidate.id)],
177
+ });
178
+ }
179
+ export function removeBulkReviewPreset(raw, presetId) {
180
+ const store = normalizeBulkReviewStore(raw);
181
+ return { ...store, presets: store.presets.filter((item) => item.id !== presetId) };
182
+ }
183
+ export function setBulkReviewRowState(raw, domain, state, now = new Date().toISOString()) {
184
+ const store = normalizeBulkReviewStore(raw);
185
+ const normalizedDomain = normalizeDomain(domain);
186
+ if (!normalizedDomain || typeof state !== 'string' || !REVIEW_STATES.has(state)) {
187
+ throw new Error('Choose a valid domain review state.');
188
+ }
189
+ const row = normalizeRow({ domain: normalizedDomain, state, updatedAt: now }, now);
190
+ if (!row)
191
+ throw new Error('Choose a valid domain review state.');
192
+ return enforceBulkReviewBudget({
193
+ ...store,
194
+ rows: state === 'unreviewed'
195
+ ? store.rows.filter((item) => item.domain !== normalizedDomain)
196
+ : [row, ...store.rows.filter((item) => item.domain !== normalizedDomain)],
197
+ });
198
+ }
199
+ export function bulkReviewRecords(raw) {
200
+ const store = normalizeBulkReviewStore(raw);
201
+ return [...store.presets, ...store.rows];
202
+ }
203
+ export function bulkReviewStoreFromRecords(records) {
204
+ return normalizeBulkReviewStore(records);
205
+ }
206
+ export function buildBulkReviewExport(raw) {
207
+ return enforceBulkReviewBudget(raw);
208
+ }
209
+ export function mergeBulkReviewStores(localRaw, importedRaw) {
210
+ const importedRecord = record(importedRaw);
211
+ if (importedRecord.schema !== BULK_REVIEW_SCHEMA) {
212
+ throw new Error('This file is not a WHOISleuth Bulk review export.');
213
+ }
214
+ if (importedRecord.version !== BULK_REVIEW_SCHEMA_VERSION) {
215
+ if (typeof importedRecord.version === 'number' && importedRecord.version > BULK_REVIEW_SCHEMA_VERSION) {
216
+ throw new Error(`This Bulk review export uses newer schema ${importedRecord.version}. Update the app before importing it.`);
217
+ }
218
+ throw new Error(`Expected Bulk review schema ${BULK_REVIEW_SCHEMA_VERSION}.`);
219
+ }
220
+ const local = normalizeBulkReviewStore(localRaw);
221
+ const imported = normalizeBulkReviewStore(importedRaw);
222
+ const presets = new Map(local.presets.map((item) => [item.id, item]));
223
+ const rows = new Map(local.rows.map((item) => [item.domain, item]));
224
+ let added = 0;
225
+ let updated = 0;
226
+ let skipped = 0;
227
+ for (const candidate of imported.presets) {
228
+ const existing = presets.get(candidate.id);
229
+ if (!existing) {
230
+ presets.set(candidate.id, candidate);
231
+ added += 1;
232
+ }
233
+ else if (candidate.updatedAt > existing.updatedAt) {
234
+ presets.set(candidate.id, candidate);
235
+ updated += 1;
236
+ }
237
+ else {
238
+ skipped += 1;
239
+ }
240
+ }
241
+ for (const candidate of imported.rows) {
242
+ const existing = rows.get(candidate.domain);
243
+ if (!existing) {
244
+ rows.set(candidate.domain, candidate);
245
+ added += 1;
246
+ }
247
+ else if (candidate.updatedAt > existing.updatedAt) {
248
+ rows.set(candidate.domain, candidate);
249
+ updated += 1;
250
+ }
251
+ else {
252
+ skipped += 1;
253
+ }
254
+ }
255
+ return {
256
+ store: enforceBulkReviewBudget({
257
+ schema: BULK_REVIEW_SCHEMA,
258
+ version: BULK_REVIEW_SCHEMA_VERSION,
259
+ presets: [...presets.values()],
260
+ rows: [...rows.values()],
261
+ }),
262
+ added,
263
+ updated,
264
+ skipped,
265
+ };
266
+ }