@slicedearth/whoisleuth-cli 2.3.0 → 2.4.0

package/lib/safe-fetch.mjs

@@ -23,8 +23,10 @@ import * as net from 'node:net';
 import * as crypto from 'node:crypto';
 import { Agent, fetch as undiciFetch } from 'undici';
 import { MAX_OUTBOUND_REDIRECTS } from "./outbound-request-bounds.mjs";
+import { MAX_OUTBOUND_HTTP_URL_CHARACTERS } from "../packages/contracts/http-url.mjs";
+import { ipv6Groups } from "../packages/contracts/ip-address.mjs";
 const MAX_REDIRECTS = MAX_OUTBOUND_REDIRECTS;
-const MAX_SAFE_FETCH_URL_LENGTH = 4096;
+const MAX_SAFE_FETCH_URL_LENGTH = MAX_OUTBOUND_HTTP_URL_CHARACTERS;
 const MAX_SAFE_FETCH_ADDRESS_CANDIDATES = 64;
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 async function defaultSafeFetchRequest(url, options) {
@@ -70,47 +72,6 @@ function isPrivateIpv4(ip) {
         return true; // multicast/reserved
     return false;
 }
-// Expands a valid IPv6 address string to its full 8-group, zero-padded hex
-// form - e.g. "::1" -> all-zero groups ending in "0001". Needed because the
-// same address can be written multiple ways (a "::" shorthand for a run of
-// zero groups, an embedded IPv4 tail as either dotted-decimal or two hex
-// groups), and a hostile authoritative DNS server - the exact threat this
-// module exists to guard against - is free to choose whichever string form
-// it wants. Checking string prefixes on the as-written form (the previous
-// approach) only catches whichever form happened to be anticipated; e.g.
-// "::ffff:127.0.0.1" and "::ffff:7f00:1" are the same address, but only the
-// first was recognized as IPv4-mapped loopback.
-function expandIpv6Groups(ip) {
-    const hasDoubleColon = ip.includes('::');
-    const [headStr, tailStr] = hasDoubleColon ? ip.split('::') : [ip, undefined];
-    // A dotted-decimal segment (the tail of "::ffff:127.0.0.1") is two
-    // 16-bit groups' worth of bits - convert it to hex groups first so the
-    // rest of this function only ever deals with plain hex.
-    const toHexGroups = (part) => {
-        if (!part)
-            return [];
-        return part.split(':').flatMap((group) => {
-            if (!group.includes('.'))
-                return [group];
-            const bytes = group.split('.').map(Number);
-            const [first = -1, second = -1, third = -1, fourth = -1] = bytes;
-            if (bytes.length !== 4
-                || [first, second, third, fourth].some((byte) => !Number.isInteger(byte)))
-                return [];
-            return [
-                (((first << 8) | second) >>> 0).toString(16),
-                (((third << 8) | fourth) >>> 0).toString(16),
-            ];
-        });
-    };
-    const headGroups = toHexGroups(headStr);
-    const tailGroups = hasDoubleColon ? toHexGroups(tailStr) : [];
-    if (!hasDoubleColon)
-        return headGroups.map((g) => g.padStart(4, '0').toLowerCase());
-    const missing = 8 - headGroups.length - tailGroups.length;
-    const middleGroups = new Array(Math.max(missing, 0)).fill('0');
-    return [...headGroups, ...middleGroups, ...tailGroups].map((g) => g.padStart(4, '0').toLowerCase());
-}
 function groupsToIpv4(hiGroup, loGroup) {
     const hi = parseInt(hiGroup, 16);
     const lo = parseInt(loGroup, 16);
@@ -119,7 +80,7 @@ function groupsToIpv4(hiGroup, loGroup) {
     return [(hi >> 8) & 0xff, hi & 0xff, (lo >> 8) & 0xff, lo & 0xff].join('.');
 }
 function isPrivateIpv6(ip) {
-    const groups = expandIpv6Groups(ip.toLowerCase());
+    const groups = ipv6Groups(ip);
     if (groups.length !== 8)
         return true; // couldn't parse cleanly - fail closed
     const [g0 = '', g1 = '', g2 = '', g3 = '', g4 = '', g5 = '', g6 = '', g7 = '',] = groups;
@@ -319,6 +280,47 @@ async function raceWithSignal(promise, signal) {
         void promise.then((value) => { signal.removeEventListener('abort', abort); resolve(value); }, (error) => { signal.removeEventListener('abort', abort); reject(error); });
     });
 }
+// Request bodies that can be replayed must be owned before the first await.
+// Streams and multipart encoders remain valid for one request, never a replay.
+function ownedRequestBody(body) {
+    if (body instanceof URLSearchParams)
+        return new URLSearchParams(body);
+    if (body instanceof ArrayBuffer)
+        return body.slice(0);
+    if (ArrayBuffer.isView(body))
+        return Uint8Array.from(new Uint8Array(body.buffer, body.byteOffset, body.byteLength));
+    return body;
+}
+function redirectedRequest(options, status, source, destination) {
+    const next = { ...options, headers: new Headers(options.headers) };
+    const method = (options.method ?? 'GET').toUpperCase();
+    if ((status === 301 || status === 302) && method === 'POST'
+        || status === 303 && method !== 'GET' && method !== 'HEAD') {
+        next.method = 'GET';
+        next.body = null;
+        for (const name of ['content-encoding', 'content-language', 'content-location', 'content-type', 'content-length', 'transfer-encoding'])
+            next.headers.delete(name);
+    }
+    else if (next.body != null && !(typeof next.body === 'string' || next.body instanceof Blob
+        || next.body instanceof URLSearchParams || next.body instanceof ArrayBuffer || ArrayBuffer.isView(next.body))) {
+        throw new Error('Refusing to replay a streaming or multipart request body after a redirect');
+    }
+    if (source.origin !== destination.origin) {
+        // This collector is not a general authenticated browser. Do not forward a
+        // caller's body or custom credentials to another authority or a downgrade.
+        if (next.body != null || !['GET', 'HEAD'].includes((next.method ?? 'GET').toUpperCase())) {
+            throw new Error('Refusing to forward a request body or mutation across origins');
+        }
+        const publicHeaders = new Set(['accept', 'accept-encoding', 'accept-language', 'user-agent', 'range']);
+        for (const name of [...next.headers.keys()])
+            if (!publicHeaders.has(name))
+                next.headers.delete(name);
+        next.credentials = 'omit';
+        next.referrer = '';
+        next.referrerPolicy = 'no-referrer';
+    }
+    return next;
+}
 // Detailed form of the shared safe request engine. It follows redirects
 // manually under the same DNS validation and connection-pinning policy as
 // safeFetch(), while retaining a bounded hop trace for consumers that need
@@ -343,10 +345,14 @@ async function safeFetchDetailed(url, options = {}, dependencies = {}) {
     const hops = [];
     let currentUrl = requestedUrl;
     let redirectCount = 0;
+    let requestOptions = { ...options, headers: new Headers(options.headers),
+        ...(options.body !== undefined ? { body: ownedRequestBody(options.body) } : {}) };
     while (true) {
         const parsed = new URL(currentUrl);
         const hostname = resolutionHostname(parsed);
-        const signal = options.signal ?? null;
+        const signal = requestOptions.signal ?? null;
+        if (signal?.aborted)
+            throw abortError(signal);
         const records = normalizePublicAddressCandidates(await raceWithSignal(resolveAddresses(hostname), signal), hostname);
         if (signal?.aborted)
             throw abortError(signal);
@@ -354,7 +360,7 @@ async function safeFetchDetailed(url, options = {}, dependencies = {}) {
         const hopStartedAt = now();
         // `dispatcher` is a supported undici extension used to pin the connection.
         // Keep the non-standard property local to this request boundary.
-        const fetchOptions = { ...options, redirect: 'manual', dispatcher };
+        const fetchOptions = { ...requestOptions, redirect: 'manual', dispatcher };
         let response;
         try {
             response = await request(currentUrl, fetchOptions);
@@ -384,6 +390,7 @@ async function safeFetchDetailed(url, options = {}, dependencies = {}) {
         if (nextUrl && redirectCount < maxRedirects) {
             await response.body?.cancel().catch(() => { });
             await closeDispatcher(dispatcher);
+            requestOptions = redirectedRequest(requestOptions, response.status, parsed, new URL(nextUrl));
             redirectCount += 1;
             currentUrl = nextUrl;
             continue;
@@ -418,8 +425,8 @@ async function safeFetch(url, options = {}, redirectsLeft = MAX_REDIRECTS) {
 // them can be made to hold an arbitrarily large body in memory. `truncated`
 // tells the caller whether the real body was actually cut short - callers
 // that need well-formed content (e.g. JSON) should treat that as a hard
-// failure rather than parsing a partial blob; callers that only need to
-// scan for a substring (e.g. plain-text/HTML matching) can safely ignore it.
+// failure rather than parsing a partial blob. Partial-evidence consumers may
+// use the prefix, but must preserve its incomplete scope.
 async function readTextCapped(res, maxBytes, options = {}) {
     const hasher = options.includeSha256 === true ? crypto.createHash('sha256') : null;
     const result = (text, truncated, bytesRead) => ({

package/lib/security-txt.mjs

@@ -4,7 +4,7 @@
 // Risk. Published contact and policy values are normalized into a small,
 // bounded record; the response body itself is discarded after parsing.
 import { safeFetchDetailed, readTextCapped } from "./safe-fetch.mjs";
-import { createObservation } from "../packages/evidence/observation.mjs";
+import { createObservation, normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
 const SECURITY_TXT_VERSION = 1;
 const SECURITY_TXT_PATH = '/.well-known/security.txt';
 const SECURITY_TXT_TIMEOUT_MS = 6_000;
@@ -18,7 +18,6 @@ const CONTROL_CHARACTER_RE = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u;
 const URI_CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f]/u;
 const FIELD_RE = /^([A-Za-z][A-Za-z-]*):[ \t]*(.*)$/u;
 const LANGUAGE_RE = /^[a-z]{2,8}(?:-[a-z0-9]{1,8})*$/iu;
-const RFC3339_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/u;
 const CONTACT_PROTOCOLS = new Set(['https:', 'mailto:', 'tel:']);
 const HTTPS_PROTOCOLS = new Set(['https:']);
 const ENCRYPTION_PROTOCOLS = new Set(['https:', 'dns:', 'openpgp4fpr:']);
@@ -222,8 +221,9 @@ function parseSecurityTxt(text, options = {}) {
             malformedCount += languages.length - valid.length;
         }
         else if (field === 'expires') {
-            if (RFC3339_RE.test(value) && Number.isFinite(Date.parse(value)))
-                expires.push(new Date(value).toISOString());
+            const expiry = normalizeExplicitIsoTimestamp(value);
+            if (expiry)
+                expires.push(expiry);
             else
                 malformedCount += 1;
         }
@@ -235,15 +235,18 @@ function parseSecurityTxt(text, options = {}) {
     const boundedPolicies = uniqueBounded(policies);
     const boundedEncryption = uniqueBounded(encryption);
     const boundedCanonical = uniqueBounded(canonical);
+    const boundedCanonicalComparison = uniqueBounded(canonicalComparison);
+    const canonicalComparisonTruncated = valuesTruncated || boundedCanonicalComparison.truncated;
     const boundedLanguages = uniqueBounded(preferredLanguages);
-    valuesTruncated ||= [boundedContacts, boundedPolicies, boundedEncryption, boundedCanonical, boundedLanguages]
+    valuesTruncated ||= [boundedContacts, boundedPolicies, boundedEncryption, boundedCanonical, boundedCanonicalComparison, boundedLanguages]
         .some((entry) => entry.truncated);
     const expiresAt = expires.length === 1 ? expires[0] : null;
     const finalUrl = options.finalUrl ? normalizedHttpsUrl(options.finalUrl) : null;
     const finalComparisonUrl = options.finalUrl ? normalizedCanonicalComparisonUrl(options.finalUrl) : null;
     const canonicalMatches = boundedCanonical.values.length && finalUrl && finalComparisonUrl
-        ? canonicalComparison.slice(0, MAX_SECURITY_TXT_VALUES)
-            .some((value) => value === finalComparisonUrl)
+        ? boundedCanonicalComparison.values.includes(finalComparisonUrl)
+            ? true
+            : canonicalComparisonTruncated ? null : false
         : null;
     const stale = Boolean(expiresAt && Date.parse(expiresAt) <= (options.now ?? Date.now()));
     const requiredMalformed = boundedContacts.values.length === 0 || expires.length !== 1;

package/lib/service-binding-dns.mjs

@@ -442,6 +442,7 @@ function parseResolverEndpoint(value) {
 }
 function defaultTcpExchange(query, resolver, options) {
     return new Promise((resolve, reject) => {
+        options.signal?.throwIfAborted();
         const socket = createConnection({
             host: resolver.address,
             port: resolver.port,
@@ -457,12 +458,19 @@ function defaultTcpExchange(query, resolver, options) {
             settled = true;
             if (hardDeadline)
                 clearTimeout(hardDeadline);
+            options.signal?.removeEventListener('abort', abort);
             socket.destroy();
             if (error)
                 reject(error);
             else
                 resolve(response);
         };
+        const abort = () => finish(options.signal?.reason ?? new Error('DNS TCP query cancelled'));
+        options.signal?.addEventListener('abort', abort, { once: true });
+        if (options.signal?.aborted) {
+            abort();
+            return;
+        }
         hardDeadline = setTimeout(() => finish(new ServiceBindingDnsError('DNS TCP query timed out', 'ETIMEOUT')), options.timeoutMs);
         hardDeadline.unref();
         socket.setTimeout(options.timeoutMs, () => finish(new ServiceBindingDnsError('DNS TCP query timed out', 'ETIMEOUT')));

package/lib/smtp-transport-review.mjs

@@ -25,6 +25,7 @@ const MAX_SMTP_RESPONSE_BYTES = 16 * 1024;
 const MAX_SMTP_REPLY_LINES = 64;
 const MAX_SMTP_LINE_BYTES = 1_000;
 const MAX_SMTP_CAPABILITIES = 32;
+const SMTP_CAPABILITY_OMISSION = 'The retained EHLO capability inventory is incomplete; additional names were omitted. STARTTLS detection used the full bounded reply.';
 const MAX_SMTP_CERTIFICATE_BYTES = 256 * 1024;
 const SMTP_ENDPOINT_TIMEOUT_MS = 8_000;
 const MAIL_TRANSPORT_TOTAL_TIMEOUT_MS = 30_000;
@@ -138,19 +139,27 @@ function normalizeSmtpReply(linesValue) {
     const sha256 = crypto.createHash('sha256').update(`${lines.join('\r\n')}\r\n`, 'ascii').digest('hex');
     return Object.freeze({ code, lines: Object.freeze(lines), bytes, sha256 });
 }
-function smtpCapabilities(reply) {
-    if (!reply || reply.code !== 250)
-        return Object.freeze([]);
+function smtpCapabilityInventory(reply) {
+    // The reply has already passed line and byte admission. The display cap must
+    // not decide which protocol extensions the endpoint advertised.
+    const seen = new Set();
     const output = [];
-    for (const line of reply.lines.slice(1)) {
+    for (const line of reply?.code === 250 ? reply.lines.slice(1) : []) {
         const capability = line.slice(4).trim().split(/\s+/u)[0]?.toUpperCase() || '';
-        if (!/^[A-Z0-9][A-Z0-9-]{0,31}$/u.test(capability) || output.includes(capability))
+        if (!/^[A-Z0-9][A-Z0-9-]{0,31}$/u.test(capability) || seen.has(capability))
             continue;
-        if (output.length >= MAX_SMTP_CAPABILITIES)
-            break;
-        output.push(capability);
+        seen.add(capability);
+        if (output.length < MAX_SMTP_CAPABILITIES)
+            output.push(capability);
     }
-    return Object.freeze(output.sort());
+    return Object.freeze({
+        capabilities: Object.freeze(output.sort()),
+        starttlsAdvertised: seen.has('STARTTLS'),
+        truncated: seen.size > output.length,
+    });
+}
+function smtpCapabilities(reply) {
+    return smtpCapabilityInventory(reply).capabilities;
 }
 async function runSmtpConversation(hostname, connection) {
     try {
@@ -162,8 +171,7 @@ async function runSmtpConversation(hostname, connection) {
         connection.assertIdle();
         await connection.write(EHLO_COMMAND);
         const ehlo = await connection.readReply();
-        const capabilities = smtpCapabilities(ehlo);
-        const starttlsAdvertised = capabilities.includes('STARTTLS');
+        const { capabilities, starttlsAdvertised } = smtpCapabilityInventory(ehlo);
         if (!starttlsAdvertised) {
             const diagnostics = connection.diagnostics();
             return Object.freeze({ connectedAddress: connection.remoteAddress, greeting, ehlo, capabilities, starttlsAdvertised, starttlsReply: null, starttlsState: 'not_advertised', tls: null, ...diagnostics });
@@ -486,11 +494,12 @@ function addressAuthentication(selected) {
         })
         : Object.freeze({ state: 'unavailable', detail: 'No address candidate was retained for cryptographic authentication review.' });
 }
-function endpointLimitations(addressState) {
+function endpointLimitations(addressState, capabilitiesTruncated = false) {
     const addressLimitation = addressState === 'connected'
         ? 'The endpoint observation is one point-in-time connection to one revalidated and pinned public address. Other addresses or later observations may differ.'
         : 'The endpoint did not produce confirmed connection evidence. Its address field retains only the highest completed public selection or revalidation stage, if any.';
     return Object.freeze([
+        ...(capabilitiesTruncated ? [SMTP_CAPABILITY_OMISSION] : []),
         addressLimitation,
         'Public-address validation, fresh revalidation, and connection pinning do not cryptographically authenticate the selected A or AAAA RRset or any CNAME chain.',
         'Only the SMTP greeting, EHLO capability names, and optional STARTTLS negotiation were observed. No authentication, recipient, mailbox, catch-all, relay, or message command was attempted.',
@@ -587,10 +596,12 @@ async function collectEndpoint(options) {
         const tlsaComplete = !['indeterminate', 'timed_out', 'unavailable'].includes(tlsa.state)
             && (tlsa.state !== 'validated' || !tlsa.dane || !['partial', 'unavailable', 'invalid'].includes(tlsa.dane.state));
         const certificateComplete = smtp.starttlsState !== 'negotiated' || certificate.observation.state === 'observed';
+        const capabilitiesTruncated = smtpCapabilityInventory(smtp.ehlo).truncated;
         const complete = smtpState === 'observed'
             && dnssec.report.completeness === 'complete'
             && tlsaComplete
-            && certificateComplete;
+            && certificateComplete
+            && !capabilitiesTruncated;
         return Object.freeze({
             host: options.host,
             observedAt: options.observedAt,
@@ -604,7 +615,7 @@ async function collectEndpoint(options) {
             dnssec: dnssec.report,
             tlsa,
             failure: null,
-            limitations: endpointLimitations(addressState),
+            limitations: endpointLimitations(addressState, capabilitiesTruncated),
         });
     }
     catch (error) {
@@ -627,7 +638,7 @@ async function collectEndpoint(options) {
             dnssec: dnssec?.report ?? null,
             tlsa: unavailableTlsa('unavailable', 'TLSA and DANE comparison was unavailable because endpoint collection did not complete.'),
             failure: Object.freeze({ stage, detail }),
-            limitations: endpointLimitations(addressState),
+            limitations: endpointLimitations(addressState, smtpCapabilityInventory(connectedSmtp?.ehlo ?? null).truncated),
         });
     }
 }
@@ -748,6 +759,7 @@ async function collectMailTransportReview(inputValue, options, dependencies = {}
             retries: 0,
         }),
         limitations: Object.freeze([
+            ...(endpoints.some((endpoint) => endpoint.limitations.includes(SMTP_CAPABILITY_OMISSION)) ? [SMTP_CAPABILITY_OMISSION] : []),
             'This command performs active network collection only for the selected MX hosts in this explicitly authorised run. It is not invoked by Lookup, Bulk, monitoring, or automatic recipes.',
             'DNSSEC, TLSA or DANE, PKIX, certificate identity, STARTTLS, SMTP transport, MTA-STS context, and TLS-RPT context retain separate states and provenance. One family never supplies or upgrades another family.',
             ACTIVE_DANE_TA_LIMITATION,