@aglyn/tenant-data-admin 1.0.0-beta.221 → 1.0.0-beta.223

package/src/lib/server/web-risk.js

@@ -19,7 +19,7 @@ import { _ as _extends } from "@swc/helpers/_/_extends";
  *
  * `@aglyn/shared-util-email/link-reputation` decides which hosts are foreign
  * and turns a listing into a strong `web-risk-link` signal. This is the store
- * half: the `uris.search` client, the cache, the kill switch.
+ * half: the `uris.search` client, the cache, the switches.
  *
  * ## The client
  *
@@ -29,52 +29,83 @@ import { _ as _extends } from "@swc/helpers/_/_extends";
  * SOCIAL_ENGINEERING, MALWARE and UNWANTED_SOFTWARE as the platform's own
  * service account — the firebase-admin credential every other server-side
  * Google call here already uses (`client-error-report.ts`), so there is no
- * key of its own. Only `https://<host>/` is ever sent: see the module header
- * of `link-reputation.ts`.
+ * key of its own. What is sent depends on the lookup mode, below.
+ *
+ * ## The lookup mode (AGL-3459)
+ *
+ * `platformSettings/webRisk.lookupMode`:
+ *
+ * - `'host'` — the default, and anything that is not `'url'`. Only
+ *   `https://<host>/` is ever sent.
+ * - `'url'` — the host first, from the cache or a call; then, for a host
+ *   that is not itself listed, each of its links' addresses as
+ *   `normalizeReputationLink` reduced them: scheme, host and path, never a
+ *   query string, a fragment or a `user:password@`. A listed address is a
+ *   hit on a clean host, so a kit at a path on a compromised site holds the
+ *   page. At most `MAX_REPUTATION_URLS_PER_LOOKUP` addresses per lookup,
+ *   inside the same deadline as the hosts.
+ *
+ * `'url'` sends more than a host to Google, so it stays off until the
+ * Subprocessors page names that purpose — see the `webrisk.googleapis.com`
+ * entry in `apps/console/constants/subprocessor-inventory.ts`.
  *
  * ## Not evidence when it fails
  *
  * A call is given {@link WEB_RISK_TIMEOUT_MS}, and a whole lookup
- * {@link WEB_RISK_DEADLINE_MS}. A host whose answer did not arrive, or whose
- * call failed, is UNKNOWN: it holds nothing and logs a warning. A credential
- * the API refuses (the API is not enabled on a self-hosted project, say) backs
- * the lookup off for a quarter of an hour, so a misconfiguration is one log
- * line rather than one per page.
+ * {@link WEB_RISK_DEADLINE_MS} — hosts and addresses together. A host or an
+ * address whose answer did not arrive, or whose call failed, is UNKNOWN: it
+ * holds nothing and logs a warning. A credential the API refuses (the API is
+ * not enabled on a self-hosted project, say) backs the lookup off for a
+ * quarter of an hour, so a misconfiguration is one log line rather than one
+ * per page.
  *
  * ## The cache
  *
- * Per host, in two layers:
+ * Per host and per address, in two layers, with the same rules for both:
  *
- * - this process's memory — a listed host until the answer's `expireTime`,
- *   which Web Risk sets and which may not be exceeded; a clean one for
+ * - this process's memory — a listing until the answer's `expireTime`,
+ *   which Web Risk sets and which may not be exceeded; a clean answer for
  *   {@link WEB_RISK_MEMORY_CLEAN_TTL_MS}, short so a listing the daily
- *   re-check writes reaches every warm render within minutes;
- * - `webRiskVerdicts/{host}` — server-only, one document per host, a clean
- *   answer kept {@link WEB_RISK_CLEAN_TTL_MS}. Bounded by the number of
- *   distinct foreign hosts and rewritten in place, so it needs no TTL policy.
+ *   re-check writes reaches every warm render within minutes. Capped at
+ *   {@link WEB_RISK_MEMORY_MAX_ENTRIES}, then started again;
+ * - the store, server-only, a clean answer kept {@link WEB_RISK_CLEAN_TTL_MS}:
+ *   `webRiskVerdicts/{host}`, bounded by the number of distinct foreign
+ *   hosts and rewritten in place; and `webRiskUrlVerdicts/{sha256(address)}`,
+ *   whose answers the daily re-check deletes a day after they expire
+ *   ({@link reapExpiredWebRiskUrlVerdicts}), because addresses are many more
+ *   than hosts and a path is worth keeping no longer than it is used.
  *
- * Read in one `getAll` per lookup, so a page's twenty hosts cost one round
- * trip, and nothing at all once they are in memory.
+ * Read in one `getAll` per lookup, so a page's hosts and addresses cost one
+ * round trip, and nothing at all once they are in memory.
  *
  * ## The kill switch
  *
  * `platformSettings/webRisk` `{ enabled: false }` stops every lookup within a
  * minute — every host reads unknown, so nothing holds on it — without a
  * deploy. Absent or anything else means on.
- *=========================================*/ import { TENANT_APEX } from "@aglyn/aglyn/app-utils/tenant-apex";
-import { LINK_THREAT_TYPES, MAX_REPUTATION_HOSTS_PER_LOOKUP, normalizeReputationHost, setLinkReputationLookup } from "@aglyn/shared-util-email/link-reputation";
+ *=========================================*/ import { createHash } from "crypto";
+import { TENANT_APEX } from "@aglyn/aglyn/app-utils/tenant-apex";
+import { LINK_THREAT_TYPES, MAX_REPUTATION_HOSTS_PER_LOOKUP, MAX_REPUTATION_URLS_PER_LOOKUP, normalizeReputationHost, normalizeReputationLink, pickReputationUrls, setLinkReputationLookup } from "@aglyn/shared-util-email/link-reputation";
+import { boundedAwait } from "@aglyn/shared-util-http/bounded-await";
 import { getApp } from "firebase-admin/app";
 import firebaseAdmin from "./firebase-admin.js";
 export const WEB_RISK_SEARCH_ENDPOINT = 'https://webrisk.googleapis.com/v1/uris:search';
-export const WEB_RISK_CACHE_COLLECTION = 'webRiskVerdicts';
-/** The kill switch: `{ enabled: false }` turns every lookup off. */ export const WEB_RISK_SETTINGS_COLLECTION = 'platformSettings';
+/** One answer per host. */ export const WEB_RISK_CACHE_COLLECTION = 'webRiskVerdicts';
+/** One answer per address, in `'url'` mode (AGL-3459), keyed by the address's SHA-256. */ export const WEB_RISK_URL_CACHE_COLLECTION = 'webRiskUrlVerdicts';
+/**
+ * The switches: `{ enabled: false }` turns every lookup off;
+ * `lookupMode: 'url'` looks up links' addresses as well as their hosts.
+ */ export const WEB_RISK_SETTINGS_COLLECTION = 'platformSettings';
 export const WEB_RISK_SETTINGS_DOC = 'webRisk';
 /** One call's budget. */ export const WEB_RISK_TIMEOUT_MS = 1000;
-/** One lookup's budget, however many hosts it asks about. */ export const WEB_RISK_DEADLINE_MS = 1200;
+/** One lookup's budget, however many hosts and addresses it asks about. */ export const WEB_RISK_DEADLINE_MS = 1200;
+/** The service account's token exchange; a cached token answers at once. */ export const WEB_RISK_TOKEN_DEADLINE_MS = 3000;
 /** How long a clean answer is kept in the store. */ export const WEB_RISK_CLEAN_TTL_MS = 12 * 60 * 60000;
 /** How long a clean answer is trusted from this process's memory. */ export const WEB_RISK_MEMORY_CLEAN_TTL_MS = 10 * 60000;
 /** A listing whose answer named no `expireTime` is kept this long. */ export const WEB_RISK_LISTED_FALLBACK_TTL_MS = 5 * 60000;
-/** How long the switch's answer is trusted. */ const SWITCH_TTL_MS = 60000;
+/** The most answers this process remembers before it starts again. */ export const WEB_RISK_MEMORY_MAX_ENTRIES = 20000;
+/** How long past its expiry an address's stored answer is deleted. */ export const WEB_RISK_URL_VERDICT_GRACE_MS = 24 * 60 * 60000;
+/** How long the switches' answer is trusted. */ const SWITCH_TTL_MS = 60000;
 /** How long a refused credential stops the lookup. */ const REFUSED_BACKOFF_MS = 15 * 60000;
 /** How long a rate limit stops the lookup. */ const THROTTLED_BACKOFF_MS = 60000;
 /** At most one warning per this long, so an outage is not a log flood. */ const WARN_EVERY_MS = 60000;
@@ -108,8 +139,7 @@ export const WEB_RISK_SETTINGS_DOC = 'webRisk';
     return `https://${host}/`;
 }
 /** The default client: `uris.search` over HTTPS with a bearer token. */ export function createWebRiskHttpClient(options) {
-    var _options_fetch, _options_timeoutMs;
-    const request = (_options_fetch = options.fetch) != null ? _options_fetch : fetch;
+    var _options_timeoutMs;
     const timeoutMs = (_options_timeoutMs = options.timeoutMs) != null ? _options_timeoutMs : WEB_RISK_TIMEOUT_MS;
     return {
         async searchUri (uri) {
@@ -119,24 +149,26 @@ export const WEB_RISK_SETTINGS_DOC = 'webRisk';
             const url = new URL((_options_endpoint = options.endpoint) != null ? _options_endpoint : WEB_RISK_SEARCH_ENDPOINT);
             for (const type of LINK_THREAT_TYPES)url.searchParams.append('threatTypes', type);
             url.searchParams.set('uri', uri);
-            const controller = new AbortController();
-            const timer = setTimeout(()=>controller.abort(), timeoutMs);
-            try {
-                const response = await request(url.toString(), {
+            // A real deadline over the request AND its body: the signal alone did
+            // not bound Next's patched fetch (AGL-3565). It still cancels the
+            // request where it can.
+            const answer = await boundedAwait(async (signal)=>{
+                var _options_fetch;
+                const response = await ((_options_fetch = options.fetch) != null ? _options_fetch : fetch)(url.toString(), {
                     method: 'GET',
                     headers: {
                         Authorization: `Bearer ${token}`,
                         Accept: 'application/json'
                     },
-                    signal: controller.signal
+                    signal
                 });
                 if (!response.ok) {
                     throw new WebRiskHttpError(response.status, await response.text().catch(()=>''));
                 }
                 return parseWebRiskSearch(await response.json());
-            } finally{
-                clearTimeout(timer);
-            }
+            }, timeoutMs, null, 'web-risk.search');
+            if (!answer) throw new Error(`Web Risk did not answer within ${timeoutMs} ms`);
+            return answer;
         }
     };
 }
@@ -146,10 +178,11 @@ export const WEB_RISK_SETTINGS_DOC = 'webRisk';
  */ async function serviceAccountAccessToken() {
     try {
         var _ref;
-        var _this;
         const credential = getApp().options.credential;
         if (!credential) return null;
-        return (_ref = (_this = await credential.getAccessToken()) == null ? void 0 : _this.access_token) != null ? _ref : null;
+        // A token exchange with Google, on a page render's path (AGL-3565).
+        const token = await boundedAwait(credential.getAccessToken(), WEB_RISK_TOKEN_DEADLINE_MS, null, 'web-risk.access-token');
+        return (_ref = token == null ? void 0 : token.access_token) != null ? _ref : null;
     } catch (unused) {
         return null;
     }
@@ -163,13 +196,13 @@ function activeClient() {
     });
     return defaultClient;
 }
-const memory = new Map();
-let switchMemo = null;
+/** Hosts and addresses share one memory: an address always holds `://`, a host never does. */ const memory = new Map();
+let settingsMemo = null;
 let backoffUntilMs = 0;
 let lastWarnAtMs = 0;
-/** Test seam: forget the caches, the switch, the back-off; optionally stand in a client. */ export function resetWebRiskForTests(client) {
+/** Test seam: forget the caches, the switches, the back-off; optionally stand in a client. */ export function resetWebRiskForTests(client) {
     memory.clear();
-    switchMemo = null;
+    settingsMemo = null;
     backoffUntilMs = 0;
     lastWarnAtMs = 0;
     clientOverride = client;
@@ -178,16 +211,31 @@ let lastWarnAtMs = 0;
 function warn(nowMs, message, error) {
     if (nowMs - lastWarnAtMs < WARN_EVERY_MS) return;
     lastWarnAtMs = nowMs;
-    console.warn(`[web-risk] ${message} — the host reads unknown, which is not evidence`, error != null ? error : '');
+    console.warn(`[web-risk] ${message} — it reads unknown, which is not evidence`, error != null ? error : '');
 }
-/** Is the lookup switched off? Read at most once a minute; an unreadable switch is on. */ async function switchedOff(nowMs) {
-    if (switchMemo && nowMs - switchMemo.atMs < SWITCH_TTL_MS) return switchMemo.off;
-    const off = await firebaseAdmin.app().firestore().collection(WEB_RISK_SETTINGS_COLLECTION).doc(WEB_RISK_SETTINGS_DOC).get().then((snapshot)=>snapshot.get('enabled') === false, ()=>false);
-    switchMemo = {
-        off,
+/**
+ * The switches, read at most once a minute. An unreadable document is on,
+ * in `'host'` mode; a mode that is not exactly `'url'` is `'host'`.
+ */ async function webRiskSettings(nowMs) {
+    if (settingsMemo && nowMs - settingsMemo.atMs < SWITCH_TTL_MS) return settingsMemo;
+    const settings = await firebaseAdmin.app().firestore().collection(WEB_RISK_SETTINGS_COLLECTION).doc(WEB_RISK_SETTINGS_DOC).get().then((snapshot)=>({
+            off: snapshot.get('enabled') === false,
+            mode: snapshot.get('lookupMode') === 'url' ? 'url' : 'host'
+        }), ()=>({
+            off: false,
+            mode: 'host'
+        }));
+    settingsMemo = _extends({}, settings, {
         atMs: nowMs
-    };
-    return off;
+    });
+    return settings;
+}
+/**
+ * The lookup mode in force (AGL-3459): `'url'` only while the lookup is on
+ * and `platformSettings/webRisk.lookupMode` is exactly `'url'`.
+ */ export async function webRiskLookupMode(nowMs = Date.now()) {
+    const settings = await webRiskSettings(nowMs);
+    return settings.off ? 'host' : settings.mode;
 }
 /**
  * Hosts that are the platform's own: every site's subdomain on the tenant
@@ -200,8 +248,15 @@ function warn(nowMs, message, error) {
 function isPlatformHost(host) {
     return platformReputationExclusions().some((domain)=>host === domain || host.endsWith(`.${domain}`));
 }
-function verdictFrom(data) {
+/** An address's document id in {@link WEB_RISK_URL_CACHE_COLLECTION}: an address holds `/`, a hash does not. */ export function webRiskUrlVerdictId(url) {
+    return createHash('sha256').update(url).digest('hex');
+}
+function isAddress(key) {
+    return key.includes('://');
+}
+/** A stored answer, or null. An address's must name the address it was asked for. */ function verdictFrom(data, url) {
     if (!data) return null;
+    if (url !== undefined && data['url'] !== url) return null;
     const threats = Array.isArray(data['threats']) ? LINK_THREAT_TYPES.filter((type)=>data['threats'].includes(type)) : [];
     const checkedAtMs = Number(data['checkedAtMs']);
     const expiresAtMs = Number(data['expiresAtMs']);
@@ -212,25 +267,31 @@ function verdictFrom(data) {
         expiresAtMs
     };
 }
-function remember(host, verdict, nowMs) {
-    memory.set(host, _extends({}, verdict, {
+function remember(key, verdict, nowMs) {
+    if (memory.size >= WEB_RISK_MEMORY_MAX_ENTRIES && !memory.has(key)) memory.clear();
+    memory.set(key, _extends({}, verdict, {
         memoryUntilMs: verdict.threats.length ? verdict.expiresAtMs : Math.min(verdict.expiresAtMs, nowMs + WEB_RISK_MEMORY_CLEAN_TTL_MS)
     }));
 }
+function fromMemory(key, nowMs) {
+    const known = memory.get(key);
+    return known && nowMs < known.memoryUntilMs ? known : null;
+}
 /**
- * The reputation of each host: listed, clean, or unknown. Platform hosts are
- * clean without asking. Never throws, and never waits longer than the
- * deadline: a call still running then finishes in the background and fills
- * the caches for the next asker.
+ * The reputation of each host — and, in `'url'` mode, of each address on a
+ * host that is not itself listed: listed, clean, or unknown. Platform hosts
+ * are clean without asking. Never throws, and never waits on the API longer
+ * than the deadline: a call still running then finishes in the background
+ * and fills the caches for the next asker.
  *
  * "Never throws" covers the store as well as the API: a missing app, a
  * settings read that fails before it is a promise, or a cache reference
  * that cannot be built all read every host as unknown. The page review
  * calls this before its own fail-closed `try`, so a throw here would take
  * down every page with a link off the site.
- */ export async function lookupHostReputation(hosts, options = {}) {
+ */ export async function lookupLinkReputation(hosts, options = {}) {
     try {
-        return await lookupHostReputationOrThrow(hosts, options);
+        return await lookupLinkReputationOrThrow(hosts, options);
     } catch (error) {
         var _options_nowMs;
         warn((_options_nowMs = options.nowMs) != null ? _options_nowMs : Date.now(), 'a lookup failed before it could answer', error);
@@ -245,8 +306,9 @@ function remember(host, verdict, nowMs) {
         };
     }
 }
-async function lookupHostReputationOrThrow(hosts, options) {
-    var _options_nowMs;
+async function lookupLinkReputationOrThrow(hosts, options) {
+    var _options_nowMs, _options_maxUrls;
+    var _options_urls;
     const nowMs = (_options_nowMs = options.nowMs) != null ? _options_nowMs : Date.now();
     const result = {
         hits: [],
@@ -264,119 +326,192 @@ async function lookupHostReputationOrThrow(hosts, options) {
         else result.unknown.push(host);
     }
     if (!wanted.length) return result;
-    const settle = (host, verdict)=>{
-        if (verdict.threats.length) result.hits.push({
-            host,
-            threats: verdict.threats
+    let urls = ((_options_urls = options.urls) == null ? void 0 : _options_urls.length) ? pickReputationUrls(options.urls, wanted, (_options_maxUrls = options.maxUrls) != null ? _options_maxUrls : MAX_REPUTATION_URLS_PER_LOOKUP) : [];
+    const hostOf = new Map(urls.map((url)=>{
+        var _ref;
+        var _normalizeReputationLink;
+        return [
+            url,
+            (_ref = (_normalizeReputationLink = normalizeReputationLink(url)) == null ? void 0 : _normalizeReputationLink.host) != null ? _ref : ''
+        ];
+    }));
+    // What is known before the API is asked, host or address.
+    const known = new Map();
+    const unknownTo = (keys)=>keys.filter((key)=>{
+            const verdict = fromMemory(key, nowMs);
+            if (verdict) known.set(key, verdict);
+            return !verdict;
         });
-        else result.clean.push(host);
-    };
-    // 1. This process's memory.
-    let missing = wanted.filter((host)=>{
-        const known = memory.get(host);
-        if (known && nowMs < known.memoryUntilMs) {
-            settle(host, known);
-            return false;
-        }
-        return true;
-    });
-    if (!missing.length) return result;
-    // 2. The switch. Off reads every remaining host as unknown.
-    if (await switchedOff(nowMs)) {
-        result.unknown.push(...missing);
-        return result;
+    // 1. This process's memory, for the hosts.
+    let missingHosts = unknownTo(wanted);
+    let missingUrls = [];
+    let mayAsk = true;
+    // 2. The switches — read only when a host is left to ask about, or
+    //    addresses wait on the mode. Off reads everything left as unknown.
+    if (missingHosts.length || urls.length) {
+        const settings = await webRiskSettings(nowMs);
+        if (settings.off || settings.mode !== 'url') urls = [];
+        if (settings.off) mayAsk = false;
+        // An address on a host already known to be listed adds nothing.
+        urls = urls.filter((url)=>{
+            var _hostOf_get;
+            var _known_get;
+            return !((_known_get = known.get((_hostOf_get = hostOf.get(url)) != null ? _hostOf_get : '')) == null ? void 0 : _known_get.threats.length);
+        });
+        // 1b. This process's memory, for the addresses.
+        missingUrls = unknownTo(urls);
     }
-    // 3. The store, in one round trip.
+    // 3. The store, hosts and addresses in one round trip.
     const firestore = firebaseAdmin.app().firestore();
-    const collection = firestore.collection(WEB_RISK_CACHE_COLLECTION);
-    try {
-        const snapshots = await firestore.getAll(...missing.map((host)=>collection.doc(host)));
-        const stored = new Map();
-        for (const snapshot of snapshots){
-            const verdict = verdictFrom(snapshot.exists ? snapshot.data() : undefined);
-            if (verdict && nowMs < verdict.expiresAtMs) stored.set(snapshot.id, verdict);
+    const hostCollection = firestore.collection(WEB_RISK_CACHE_COLLECTION);
+    const urlCollection = firestore.collection(WEB_RISK_URL_CACHE_COLLECTION);
+    if (mayAsk && (missingHosts.length || missingUrls.length)) {
+        const keys = [
+            ...missingHosts,
+            ...missingUrls
+        ];
+        try {
+            const snapshots = await firestore.getAll(...keys.map((key)=>isAddress(key) ? urlCollection.doc(webRiskUrlVerdictId(key)) : hostCollection.doc(key)));
+            snapshots.forEach((snapshot, index)=>{
+                const key = keys[index];
+                const data = snapshot.exists ? snapshot.data() : undefined;
+                const verdict = verdictFrom(data, isAddress(key) ? key : undefined);
+                if (!verdict || nowMs >= verdict.expiresAtMs) return;
+                remember(key, verdict, nowMs);
+                known.set(key, verdict);
+            });
+            missingHosts = missingHosts.filter((host)=>!known.has(host));
+            missingUrls = missingUrls.filter((url)=>!known.has(url));
+        } catch (error) {
+            warn(nowMs, 'the verdict cache could not be read', error);
         }
-        missing = missing.filter((host)=>{
-            const verdict = stored.get(host);
-            if (!verdict) return true;
-            remember(host, verdict, nowMs);
-            settle(host, verdict);
-            return false;
-        });
-    } catch (error) {
-        warn(nowMs, 'the verdict cache could not be read', error);
     }
-    if (!missing.length) return result;
-    // 4. The API, every remaining host at once, within the deadline.
+    // 4. The API, everything left at once, within the deadline: each host
+    //    first, then — unless the host itself is listed — its addresses.
+    const answered = new Map();
     const client = activeClient();
-    if (!client || nowMs < backoffUntilMs) {
-        result.unknown.push(...missing);
-        return result;
-    }
-    result.looked = missing.length;
-    const answers = new Map();
-    const calls = missing.map(async (host)=>{
-        try {
-            var _answer_expireTimeMs;
-            const answer = await client.searchUri(webRiskUriForHost(host));
-            const checkedAtMs = Date.now();
-            const verdict = {
-                threats: answer.threats,
-                checkedAtMs,
-                expiresAtMs: answer.threats.length ? (_answer_expireTimeMs = answer.expireTimeMs) != null ? _answer_expireTimeMs : checkedAtMs + WEB_RISK_LISTED_FALLBACK_TTL_MS : checkedAtMs + WEB_RISK_CLEAN_TTL_MS
-            };
-            answers.set(host, verdict);
-            remember(host, verdict, checkedAtMs);
-            await collection.doc(host).set(_extends({
-                host
-            }, verdict, {
-                listed: verdict.threats.length > 0
-            })).catch((error)=>warn(checkedAtMs, 'a verdict could not be cached', error));
-        } catch (error) {
-            if (error instanceof WebRiskHttpError && [
-                401,
-                403,
-                404
-            ].includes(error.status)) {
-                backoffUntilMs = Date.now() + REFUSED_BACKOFF_MS;
-                console.error('[web-risk] the API refused the credential — lookups paused for 15 minutes', error);
-            } else if (error instanceof WebRiskHttpError && error.status === 429) {
-                backoffUntilMs = Date.now() + THROTTLED_BACKOFF_MS;
-                warn(Date.now(), 'the API is rate limiting', error);
-            } else if (error instanceof WebRiskUnavailableError) {
-                backoffUntilMs = Date.now() + REFUSED_BACKOFF_MS;
-                warn(Date.now(), 'no credential to call the API with', error);
-            } else {
-                warn(Date.now(), `the lookup of ${host} failed or timed out`, error);
+    if (mayAsk && client && nowMs >= backoffUntilMs && (missingHosts.length || missingUrls.length)) {
+        var _options_deadlineMs;
+        let looked = 0;
+        const ask = async (key)=>{
+            if (Date.now() < backoffUntilMs) return null;
+            looked += 1;
+            try {
+                var _answer_expireTimeMs, _hostOf_get;
+                const answer = await client.searchUri(isAddress(key) ? key : webRiskUriForHost(key));
+                const checkedAtMs = Date.now();
+                const verdict = {
+                    threats: answer.threats,
+                    checkedAtMs,
+                    expiresAtMs: answer.threats.length ? (_answer_expireTimeMs = answer.expireTimeMs) != null ? _answer_expireTimeMs : checkedAtMs + WEB_RISK_LISTED_FALLBACK_TTL_MS : checkedAtMs + WEB_RISK_CLEAN_TTL_MS
+                };
+                answered.set(key, verdict);
+                remember(key, verdict, checkedAtMs);
+                const document = isAddress(key) ? urlCollection.doc(webRiskUrlVerdictId(key)).set(_extends({
+                    host: (_hostOf_get = hostOf.get(key)) != null ? _hostOf_get : '',
+                    url: key
+                }, verdict, {
+                    listed: verdict.threats.length > 0
+                })) : hostCollection.doc(key).set(_extends({
+                    host: key
+                }, verdict, {
+                    listed: verdict.threats.length > 0
+                }));
+                await document.catch((error)=>warn(checkedAtMs, 'a verdict could not be cached', error));
+                return verdict;
+            } catch (error) {
+                if (error instanceof WebRiskHttpError && [
+                    401,
+                    403,
+                    404
+                ].includes(error.status)) {
+                    backoffUntilMs = Date.now() + REFUSED_BACKOFF_MS;
+                    console.error('[web-risk] the API refused the credential — lookups paused for 15 minutes', error);
+                } else if (error instanceof WebRiskHttpError && error.status === 429) {
+                    backoffUntilMs = Date.now() + THROTTLED_BACKOFF_MS;
+                    warn(Date.now(), 'the API is rate limiting', error);
+                } else if (error instanceof WebRiskUnavailableError) {
+                    backoffUntilMs = Date.now() + REFUSED_BACKOFF_MS;
+                    warn(Date.now(), 'no credential to call the API with', error);
+                } else {
+                    warn(Date.now(), `the lookup of ${key} failed or timed out`, error);
+                }
+                return null;
             }
-        }
-    });
-    let timer;
-    await Promise.race([
-        Promise.allSettled(calls),
-        new Promise((resolve)=>{
-            var _options_deadlineMs;
-            timer = setTimeout(resolve, (_options_deadlineMs = options.deadlineMs) != null ? _options_deadlineMs : WEB_RISK_DEADLINE_MS);
-        })
-    ]);
-    if (timer) clearTimeout(timer);
-    const late = missing.filter((host)=>!answers.has(host));
-    for (const host of missing){
-        const verdict = answers.get(host);
-        if (verdict) settle(host, verdict);
+        };
+        const pipelines = wanted.map(async (host)=>{
+            var _known_get;
+            const hostVerdict = (_known_get = known.get(host)) != null ? _known_get : missingHosts.includes(host) ? await ask(host) : null;
+            if (hostVerdict == null ? void 0 : hostVerdict.threats.length) return;
+            await Promise.all(missingUrls.filter((url)=>hostOf.get(url) === host).map((url)=>ask(url)));
+        });
+        await boundedAwait(Promise.allSettled(pipelines), (_options_deadlineMs = options.deadlineMs) != null ? _options_deadlineMs : WEB_RISK_DEADLINE_MS, undefined, 'web-risk.lookup');
+        result.looked = looked;
+        const late = [
+            ...missingHosts,
+            ...missingUrls
+        ].filter((key)=>!answered.has(key)).length;
+        if (late) warn(Date.now(), `${late} host(s) or address(es) had no answer within the deadline`);
+    }
+    // 5. The answer. An address on a listed host is not reported: the host's
+    //    listing already names it.
+    const verdictOf = (key)=>{
+        var _known_get;
+        return (_known_get = known.get(key)) != null ? _known_get : answered.get(key);
+    };
+    for (const host of wanted){
+        const verdict = verdictOf(host);
+        if (!verdict) result.unknown.push(host);
+        else if (verdict.threats.length) result.hits.push({
+            host,
+            threats: verdict.threats
+        });
+        else result.clean.push(host);
+    }
+    for (const url of urls){
+        var _hostOf_get;
+        var _verdictOf;
+        const host = (_hostOf_get = hostOf.get(url)) != null ? _hostOf_get : '';
+        if ((_verdictOf = verdictOf(host)) == null ? void 0 : _verdictOf.threats.length) continue;
+        const verdict = verdictOf(url);
+        if (!verdict) result.unknown.push(url);
+        else if (verdict.threats.length) result.hits.push({
+            host,
+            url,
+            threats: verdict.threats
+        });
+        else result.clean.push(url);
     }
-    result.unknown.push(...late);
-    if (late.length) warn(Date.now(), `${late.length} host(s) had no answer within the deadline`);
     return result;
 }
+/**
+ * Delete the addresses' stored answers that expired more than
+ * {@link WEB_RISK_URL_VERDICT_GRACE_MS} ago (AGL-3459), at most `limit` at a
+ * time. The daily re-check calls it once per walk. A host's answers are not
+ * reaped: they are few and rewritten in place. Never throws.
+ */ export async function reapExpiredWebRiskUrlVerdicts(options = {}) {
+    var _options_nowMs, _options_firestore;
+    const nowMs = (_options_nowMs = options.nowMs) != null ? _options_nowMs : Date.now();
+    const firestore = (_options_firestore = options.firestore) != null ? _options_firestore : firebaseAdmin.app().firestore();
+    try {
+        var _options_limit;
+        const stale = await firestore.collection(WEB_RISK_URL_CACHE_COLLECTION).where('expiresAtMs', '<', nowMs - WEB_RISK_URL_VERDICT_GRACE_MS).limit((_options_limit = options.limit) != null ? _options_limit : 500).get();
+        await Promise.all(stale.docs.map((snapshot)=>snapshot.ref.delete()));
+        return stale.docs.length;
+    } catch (error) {
+        console.warn('[web-risk] expired address verdicts could not be deleted', error);
+        return 0;
+    }
+}
 /**
  * Puts this lookup behind the shared screen's reputation seam, so the send
  * seam (`screenTenantMessage`) asks it. Called by
  * `installOutboundScreenGate`, at that module's load.
  */ export function installLinkReputationLookup() {
     if (typeof setLinkReputationLookup !== 'function') return;
-    setLinkReputationLookup((hosts, options)=>lookupHostReputation(hosts, {
-            deadlineMs: options == null ? void 0 : options.deadlineMs
+    setLinkReputationLookup((hosts, options)=>lookupLinkReputation(hosts, {
+            deadlineMs: options == null ? void 0 : options.deadlineMs,
+            urls: options == null ? void 0 : options.urls
         }));
 }