@slicedearth/whoisleuth-cli 2.2.0 → 2.4.0

package/lib/lookup-cache.mjs

@@ -10,6 +10,7 @@
 // In-memory, so (like the bootstrap cache and the rate limiter) this
 // applies globally on server.mts's one long-lived process, but only within
 // a single warm container on Netlify Functions.
+import { abortable } from "./abort.mjs";
 const TTL_MS = 3 * 60 * 1000; // 3 minutes
 const SWEEP_INTERVAL_MS = 5 * 60 * 1000;
 // Bounds worst-case memory regardless of the TTL/sweep - without this, a
@@ -72,7 +73,7 @@ function getCached(key) {
         deleteEntry(key);
         return undefined;
     }
-    return entry.value;
+    return structuredClone(entry.value);
 }
 function setCached(key, value) {
     sweepExpiredEntries(Date.now());
@@ -80,9 +81,16 @@ function setCached(key, value) {
     const size = approxByteSize(value);
     // A value whose serialized size cannot be measured cannot participate in a
     // byte-bounded cache. Return it to the current caller without retaining it.
-    if (size === null)
-        return;
-    store.set(key, { value, expiresAt: Date.now() + TTL_MS, size });
+    if (size === null || size > MAX_TOTAL_BYTES)
+        return { value, owned: false };
+    let snapshot;
+    try {
+        snapshot = structuredClone(value);
+    }
+    catch {
+        return { value, owned: false };
+    }
+    store.set(key, { value: snapshot, expiresAt: Date.now() + TTL_MS, size });
     totalBytes += size;
     while (store.size > 0 && (store.size > MAX_ENTRIES || totalBytes > MAX_TOTAL_BYTES)) {
         const oldestKey = store.keys().next().value;
@@ -90,6 +98,12 @@ function setCached(key, value) {
             break;
         deleteEntry(oldestKey);
     }
+    return { value: snapshot, owned: true };
+}
+// Stored data and coalesced results belong to the cache, not to a caller. Each
+// consumer receives its own copy, including the first and concurrent callers.
+function callerResult(result) {
+    return result.owned ? structuredClone(result.value) : result.value;
 }
 // Concurrent callers for the same key (e.g. a bulk scan's concurrent
 // workers, or a fast scan immediately followed by a deep-check, hitting the
@@ -103,25 +117,34 @@ const inFlight = new Map();
 // The cache intentionally holds heterogeneous public lookup results. Callers
 // supply distinct result contracts, so this internal compatibility boundary
 // stays dynamically typed rather than asserting one shared payload shape.
-async function cached(key, factory) {
+async function cached(key, factory, signal) {
+    signal?.throwIfAborted();
     const hit = getCached(key);
     if (hit !== undefined)
         return hit;
+    // A deadline-scoped caller may reuse completed public data, but must own
+    // its cancellable work rather than aborting another caller's shared request.
+    if (signal) {
+        const value = await abortable(factory, signal);
+        signal.throwIfAborted();
+        return callerResult(setCached(key, value));
+    }
     const pending = inFlight.get(key);
     if (pending)
-        return pending;
-    const promise = (async () => {
+        return callerResult(await pending);
+    // Register pending work before invoking the factory, including factories
+    // that throw synchronously. A rejected promise must never poison the key.
+    const promise = Promise.resolve().then(async () => {
         try {
             const value = await factory();
-            setCached(key, value);
-            return value;
+            return setCached(key, value);
         }
         finally {
             inFlight.delete(key);
         }
-    })();
+    });
     inFlight.set(key, promise);
-    return promise;
+    return callerResult(await promise);
 }
 // Expired entries are swept lazily during cache activity. This preserves the
 // long-running Express cleanup behavior without creating an import-time timer,

package/lib/lookup-child-profile-contract.mjs

@@ -1,19 +1,21 @@
 // Browser-safe version and collection boundaries shared by Lookup child
 // profile producers and the client response trust boundary.
 import { MAX_OBSERVATION_LIMITATIONS, MAX_OBSERVATION_LIMITATION_LENGTH, OBSERVATION_VERSION, normalizeExplicitIsoTimestamp, } from "../packages/evidence/observation.mjs";
-import { PAGE_FINGERPRINT_VERSION, PAGE_IDENTITY_VERSION, } from "../packages/contracts/workspace-portability.mjs";
+import { PAGE_FINGERPRINT_PARSERS, PAGE_IDENTITY_VERSION, } from "../packages/contracts/workspace-portability.mjs";
 import { validPagePublicationMetadata } from "./homepage-metadata-contract.mjs";
+import { PAGE_FINGERPRINT_TOKEN_LIMITS } from "../packages/contracts/page-fingerprints.mjs";
 import { MAX_LOOKUP_TLS_ALT_NAMES, MAX_LOOKUP_TLS_CERTIFICATE_POLICIES, MAX_LOOKUP_TLS_CHAIN_CERTIFICATES, MAX_LOOKUP_TLS_FINDINGS, MAX_LOOKUP_TLS_NAME_VALUES, TLS_PROFILE_VERSION, } from "./lookup-network-evidence-bounds.mjs";
 import { isJsonObject, normalizedDomain, validBoundedString, validHttpProvenanceUrl, validObservationDiagnostics, validOptionalNullableText, validStringArray, validUint, } from "./lookup-contract-primitives.mjs";
 import { TECHNOLOGY_EVIDENCE_ROLE_ORDER } from "./technology-evidence-role.mjs";
-export const TECHNOLOGY_PROFILE_VERSION = 11;
-export const SUPPORTED_TECHNOLOGY_PROFILE_VERSIONS = Object.freeze([10, TECHNOLOGY_PROFILE_VERSION]);
+import { MAX_HOMEPAGE_BYTES } from "./outbound-request-bounds.mjs";
+export const TECHNOLOGY_PROFILE_VERSION = 12;
+export const SUPPORTED_TECHNOLOGY_PROFILE_VERSIONS = Object.freeze([10, 11, TECHNOLOGY_PROFILE_VERSION]);
 export const MAX_TECHNOLOGY_FINDINGS = 24;
 export const MAX_EVIDENCE_PER_TECHNOLOGY = 4;
 export const MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH = 180;
 export const BROWSER_LIBRARY_PROFILE_VERSION = 2;
 export const MAX_LIBRARY_FINDINGS = 16;
-export const WEBSITE_SECURITY_POSTURE_VERSION = 2;
+export const WEBSITE_SECURITY_POSTURE_VERSION = 3;
 export const MAX_SECURITY_POSTURE_FINDINGS = 32;
 export const CREDENTIAL_SURFACE_PROFILE_VERSION = 1;
 export const STRUCTURED_DATA_IDENTITY_VERSION = 1;
@@ -169,6 +171,7 @@ function validFingerprintIdentifier(value) {
         && /^[A-Z0-9-]{1,64}$/u.test(value.value);
 }
 function validPageFingerprintProfile(value) {
+    const tokenLimit = PAGE_FINGERPRINT_TOKEN_LIMITS[value.fingerprintVersion];
     const normalizedHtml = value.normalizedHtml;
     const visibleText = value.visibleText;
     const domStructure = value.domStructure;
@@ -181,7 +184,7 @@ function validPageFingerprintProfile(value) {
         || normalizedHtml.algorithm !== 'sha256'
         || typeof normalizedHtml.value !== 'string'
         || !SHA256_RE.test(normalizedHtml.value)
-        || !validUint(normalizedHtml.tokenCount, 4_096)
+        || !validUint(normalizedHtml.tokenCount, tokenLimit)
         || typeof normalizedHtml.truncated !== 'boolean'
         || !(visibleText === null || isJsonObject(visibleText)
             && hasExactKeys(visibleText, ['algorithm', 'value', 'tokenCount', 'featureCount', 'truncated'])
@@ -197,16 +200,16 @@ function validPageFingerprintProfile(value) {
         || domStructure.algorithm !== 'sha256'
         || typeof domStructure.value !== 'string'
         || !SHA256_RE.test(domStructure.value)
-        || !validUint(domStructure.nodeCount, 4_096)
-        || domStructure.parser !== 'static-tag-sequence-v1'
+        || !validUint(domStructure.nodeCount, tokenLimit)
+        || domStructure.parser !== PAGE_FINGERPRINT_PARSERS[value.fingerprintVersion]
         || typeof domStructure.truncated !== 'boolean'
         || !(domStructure.similarity === undefined || domStructure.similarity === null || isJsonObject(domStructure.similarity)
             && hasExactKeys(domStructure.similarity, ['algorithm', 'value', 'tokenCount', 'featureCount', 'truncated'])
             && domStructure.similarity.algorithm === 'simhash64-v1'
             && typeof domStructure.similarity.value === 'string'
             && SIMHASH64_RE.test(domStructure.similarity.value)
-            && validUint(domStructure.similarity.tokenCount, 4_096)
-            && validUint(domStructure.similarity.featureCount, 4_096)
+            && validUint(domStructure.similarity.tokenCount, tokenLimit)
+            && validUint(domStructure.similarity.featureCount, tokenLimit)
             && typeof domStructure.similarity.truncated === 'boolean')
         || !(formStructure === null || isJsonObject(formStructure)
             && hasExactKeys(formStructure, ['algorithm', 'value', 'formCount', 'controlCount', 'truncated'])
@@ -461,7 +464,7 @@ function browserLibraryProfileContractState(value) {
         || !isJsonObject(knownExploitedCatalog)
         || !hasOnlyKeys(knownExploitedCatalog, ['name', 'version', 'releasedAt'])
         || !['name', 'version', 'releasedAt'].every((field) => validBoundedString(knownExploitedCatalog[field], 160))
-        || normalizeExplicitIsoTimestamp(knownExploitedCatalog.releasedAt) !== knownExploitedCatalog.releasedAt
+        || normalizeExplicitIsoTimestamp(knownExploitedCatalog.releasedAt) === null
         || !Array.isArray(profile.findings)
         || profile.findings.length > MAX_LIBRARY_FINDINGS)
         return 'invalid';
@@ -488,6 +491,8 @@ function browserLibraryProfileContractState(value) {
             || !validExactStringArray(candidate.knownExploitedIdentifiers, 16, 80)
             || !validExactStringArray(candidate.weaknessClasses, 12, 80))
             return 'invalid';
+        // Published profiles could retain legacy upstream identifier spellings.
+        // Keep that reader boundary; current catalogue writers validate syntax.
         if (findingIds.has(candidate.id)
             || !candidate.advisoryIdentifiers.every((identifier) => typeof identifier === 'string'
                 && /^(?:CVE-[0-9X-]+|GHSA-[A-Z0-9-]+)$/u.test(identifier))
@@ -495,7 +500,9 @@ function browserLibraryProfileContractState(value) {
                 && /^CVE-[0-9X-]+$/u.test(identifier))
             || !candidate.weaknessClasses.every((identifier) => typeof identifier === 'string'
                 && /^CWE-[0-9]+$/u.test(identifier))
-            || Number(candidate.advisoryCount) < candidate.advisoryIdentifiers.length
+            // One advisory can have several identifiers, including CVE and GHSA
+            // aliases. Their count is not the number of matching advisory records.
+            || (Number(candidate.advisoryCount) === 0 && candidate.advisoryIdentifiers.length > 0)
             || Number(candidate.knownExploitedCount) < candidate.knownExploitedIdentifiers.length)
             return 'invalid';
         findingIds.add(candidate.id);
@@ -503,7 +510,7 @@ function browserLibraryProfileContractState(value) {
     return 'supported';
 }
 function pageFingerprintContractState(value) {
-    const versionState = childVersionState(value, 'fingerprintVersion', [PAGE_FINGERPRINT_VERSION]);
+    const versionState = childVersionState(value, 'fingerprintVersion', Object.keys(PAGE_FINGERPRINT_PARSERS).map(Number));
     if (versionState !== 'supported')
         return versionState;
     const profile = value;
@@ -520,7 +527,9 @@ function pageFingerprintContractState(value) {
         || typeof profile.exact.value !== 'string'
         || !SHA256_RE.test(profile.exact.value)
         || !['complete-body', 'captured-prefix'].includes(String(profile.exact.scope))
-        || !validUint(profile.exact.bytes, 300_000)
+        // The published first-generation writer had a fixed 300,000-byte ceiling.
+        // Current native fingerprints share the collector's source-size policy.
+        || !validUint(profile.exact.bytes, profile.fingerprintVersion === 1 ? 300_000 : MAX_HOMEPAGE_BYTES)
         || !['captured-response-bytes', 'decoded-markup'].includes(String(profile.exact.source))
         || !validPageFingerprintProfile(profile)
         || profile.complete !== (profile.truncated !== true))
@@ -647,7 +656,7 @@ function tlsProfileContractState(value) {
     return 'supported';
 }
 function securityPostureContractState(value) {
-    const versionState = childVersionState(value, 'postureVersion', [WEBSITE_SECURITY_POSTURE_VERSION]);
+    const versionState = childVersionState(value, 'postureVersion', [2, WEBSITE_SECURITY_POSTURE_VERSION]);
     if (versionState !== 'supported')
         return versionState;
     const profile = value;
@@ -855,4 +864,4 @@ function sanitizeLookupChildProfiles(value) {
     }
     return output;
 }
-export { sanitizeLookupChildProfiles, validSecurityPostureFinding, validTlsChainCertificate, validTlsFinding, };
+export { sanitizeLookupChildProfiles, technologyProfileContractState, validSecurityPostureFinding, validTlsChainCertificate, validTlsFinding, };

package/lib/lookup-diagnostics.mjs

@@ -3,6 +3,29 @@ import { OPERATION_BUDGET_ERROR_CODE } from "./operation-budget.mjs";
 import { FEATURE_DISABLED_ERROR_CODE } from "./feature-policy.mjs";
 const LOOKUP_DIAGNOSTICS_VERSION = 8;
 const LOOKUP_LEGACY_DIAGNOSTICS_VERSION = 7;
+const LOOKUP_SOURCE_STATES = Object.freeze([
+    'success', 'partial', 'not_found', 'skipped', 'error', 'unsupported',
+    'not_applicable', 'unavailable', 'rate_limited', 'complete', 'disabled', 'stale',
+]);
+const SOURCE_STATES = new Set(LOOKUP_SOURCE_STATES);
+function lookupDiagnosticStates(value) {
+    const record = (input) => input && typeof input === 'object' && !Array.isArray(input)
+        ? input : {};
+    const diagnostics = record(value);
+    const candidates = {
+        rdap: record(diagnostics.rdap).status,
+        registrar_rdap: record(record(diagnostics.rdap).registrar).status,
+        whois: record(diagnostics.whois).status,
+        availability: record(diagnostics.availability).status,
+        reverse_dns: record(diagnostics.reverseDns).status,
+        network_context: record(diagnostics.network).status,
+        security_txt: record(diagnostics.securityTxt).status,
+        sslbl: record(diagnostics.sslbl).status,
+    };
+    return Object.freeze(Object.fromEntries(Object.entries(candidates).map(([source, state]) => [
+        source, typeof state === 'string' && SOURCE_STATES.has(state) ? state : null,
+    ])));
+}
 const LOOKUP_TIMING_VERSION = 1;
 const MAX_LOOKUP_TIMING_MS = 120_000;
 const LOOKUP_TIMING_SOURCE_ORDER = Object.freeze([
@@ -93,4 +116,4 @@ function createLookupTimingTracker(enabled, now) {
         },
     };
 }
-export { LOOKUP_DIAGNOSTICS_VERSION, LOOKUP_LEGACY_DIAGNOSTICS_VERSION, LOOKUP_TIMING_VERSION, MAX_LOOKUP_TIMING_MS, LOOKUP_ERROR_CODES, boundedSourceDetail, createLookupTimingTracker, errorMessage, };
+export { LOOKUP_DIAGNOSTICS_VERSION, LOOKUP_LEGACY_DIAGNOSTICS_VERSION, LOOKUP_SOURCE_STATES, LOOKUP_TIMING_VERSION, MAX_LOOKUP_TIMING_MS, LOOKUP_ERROR_CODES, boundedSourceDetail, createLookupTimingTracker, errorMessage, lookupDiagnosticStates, };

package/lib/lookup-progress.mjs

@@ -1,6 +1,7 @@
 // Portable bounded NDJSON Lookup progress contract. Partial events are
 // presentation-only; only a validated final ordinary Lookup result may cross
 // the persistence boundary.
+import { scanBoundedJson, MAX_BOUNDED_JSON_DEPTH, MAX_BOUNDED_JSON_KEYS, MAX_BOUNDED_JSON_VALUES } from "./bounded-json.mjs";
 export const LOOKUP_PROGRESS_SCHEMA = 'whoisleuth.lookup-progress';
 export const LOOKUP_PROGRESS_VERSION = 1;
 export const MAX_LOOKUP_PROGRESS_SOURCES = 16;
@@ -241,20 +242,25 @@ export function createLookupProgressNdjsonDecoder(onEvent) {
         throw new TypeError('NDJSON decoder requires an event consumer.');
     const decoder = new TextDecoder('utf-8', { fatal: true });
     let pending = '';
+    let pendingBytes = 0;
     let totalBytes = 0;
     let eventCount = 0;
     function consumeLine(line) {
         if (!line)
             return;
-        if (byteLength(line) > MAX_LOOKUP_PROGRESS_FINAL_BYTES + MAX_LOOKUP_PROGRESS_FRAGMENT_BYTES) {
-            throw new TypeError('Lookup progress line exceeds its byte bound.');
-        }
         eventCount += 1;
         if (eventCount > MAX_LOOKUP_PROGRESS_EVENTS) {
             throw new TypeError('Lookup progress stream contains too many events.');
         }
         let parsed;
         try {
+            // The ordinary result keeps its original limits after decoding. Allow
+            // only the fixed progress envelope overhead around that same document.
+            scanBoundedJson(line, {
+                maximumDepth: MAX_BOUNDED_JSON_DEPTH + 1,
+                maximumKeys: MAX_BOUNDED_JSON_KEYS + 8,
+                maximumValues: MAX_BOUNDED_JSON_VALUES + MAX_LOOKUP_PROGRESS_SOURCES + 16,
+            });
             parsed = JSON.parse(line);
         }
         catch {
@@ -269,16 +275,26 @@ export function createLookupProgressNdjsonDecoder(onEvent) {
         if (totalBytes > MAX_LOOKUP_PROGRESS_STREAM_BYTES) {
             throw new TypeError('Lookup progress stream exceeds its total byte bound.');
         }
-        pending += decoder.decode(chunk, { stream: true });
-        let newline = pending.indexOf('\n');
-        while (newline >= 0) {
-            const line = pending.slice(0, newline).replace(/\r$/u, '');
-            pending = pending.slice(newline + 1);
-            consumeLine(line);
-            newline = pending.indexOf('\n');
-        }
-        if (byteLength(pending) > MAX_LOOKUP_PROGRESS_FINAL_BYTES + MAX_LOOKUP_PROGRESS_FRAGMENT_BYTES) {
-            throw new TypeError('Lookup progress line exceeds its byte bound.');
+        // Count incoming bytes once, before decoding or accumulation. LF cannot
+        // occur inside a valid UTF-8 sequence; the fatal decoder still receives
+        // every byte, including framing, so chunk boundaries cannot hide errors.
+        for (let start = 0; start < chunk.length;) {
+            const newline = chunk.indexOf(10, start);
+            const end = newline < 0 ? chunk.length : newline;
+            pendingBytes += end - start;
+            const maximum = MAX_LOOKUP_PROGRESS_FINAL_BYTES + MAX_LOOKUP_PROGRESS_FRAGMENT_BYTES;
+            const trailingCr = end > start ? chunk[end - 1] === 13 : pending.endsWith('\r');
+            if (pendingBytes > maximum + (trailingCr ? 1 : 0)) {
+                throw new TypeError('Lookup progress line exceeds its byte bound.');
+            }
+            const decoded = decoder.decode(chunk.subarray(start, end + (newline < 0 ? 0 : 1)), { stream: true });
+            pending += newline < 0 ? decoded : decoded.slice(0, -1);
+            if (newline >= 0) {
+                consumeLine(pending.replace(/\r$/u, ''));
+                pending = '';
+                pendingBytes = 0;
+            }
+            start = end + 1;
         }
     }
     function finish() {
@@ -286,6 +302,7 @@ export function createLookupProgressNdjsonDecoder(onEvent) {
         if (pending)
             consumeLine(pending.replace(/\r$/u, ''));
         pending = '';
+        pendingBytes = 0;
     }
     return Object.freeze({ push, finish });
 }