@q32/signal-scanner 0.1.0

package/src/dynamic.ts

@@ -0,0 +1,273 @@
+// Dynamic JavaScript behavior analysis.
+//
+// Runs a page's inline scripts against an instrumented window/document where the
+// dangerous primitives (eval/Function, fetch/XHR/sendBeacon/WebSocket, location,
+// document.write/innerHTML, form.action, atob, cookies) are NEUTERED and
+// RECORDED rather than executed. What the script *tries* to do is the signal:
+// JS redirects (decloaking), exfil endpoints, injected credential forms,
+// decoded payloads. Recorded markup and eval'd code are re-fed through the
+// static scanner so existing rules light up on runtime-produced content.
+//
+// This module is isolate-agnostic on purpose. It exposes:
+//   - RECORDER_SOURCE: the recorder as self-contained source, for a caller to
+//     run inside whatever isolate it has (a CF Dynamic Worker, node isolated-vm,
+//     a node:vm context, ...). The lib knows nothing about those mechanisms.
+//   - runInstrumented(): an in-process default (compiles RECORDER_SOURCE here)
+//     for when no isolate boundary is needed.
+//   - analyzeDynamicWith(html, opts, evaluate): the generic seam — the caller
+//     passes an `evaluate` that produces a BehaviorReport however it likes.
+
+import { assessRedirect, createScanner, isAdOrAnalyticsHost, registrableDomainFor, type Finding, type Severity, type Confidence } from "./index";
+import type { RuleScoreModel } from "./rules/types";
+
+export interface NetworkAttempt {
+  kind: "fetch" | "xhr" | "beacon" | "websocket" | "script" | "image" | "form";
+  url: string;
+}
+
+export interface BehaviorReport {
+  redirects: string[];
+  network: NetworkAttempt[];
+  writes: string[];
+  evals: string[];
+  decoded: string[];
+  cookies: string[];
+  errors: string[];
+}
+
+export interface DynamicAnalysisOptions {
+  /** Base/page URL, used to resolve relative targets and classify off-origin. */
+  url?: string;
+}
+
+/** The caller supplies one of these: "run these scripts in an isolate, give me what they attempted." */
+export type IsolatedEvaluator = (scripts: string[], options: DynamicAnalysisOptions) => BehaviorReport | Promise<BehaviorReport>;
+
+const EMPTY_REPORT: BehaviorReport = { redirects: [], network: [], writes: [], evals: [], decoded: [], cookies: [], errors: [] };
+
+// Self-contained recorder. Evaluating this source defines `recordBehavior(scripts, url)`
+// which returns a BehaviorReport. It references only standard globals (URL, atob,
+// btoa, Proxy) available in any JS isolate — no imports, no transport, no
+// assumptions about how it is hosted. Inline scripts run via `new Function` with
+// the dangerous globals shadowed by recorder stubs (sloppy mode so `eval` /
+// `Function` can be shadowed as parameters).
+export const RECORDER_SOURCE = String.raw`
+function recordBehavior(scripts, url) {
+  var report = { redirects: [], network: [], writes: [], evals: [], decoded: [], cookies: [], errors: [] };
+  var resolve = function (v) { var raw = String(v == null ? "" : v); try { return url ? new URL(raw, url).toString() : raw; } catch (e) { return raw; } };
+  var recordEval = function (c) { report.evals.push(String(c)); return undefined; };
+  var FunctionStub = function () { var a = arguments; report.evals.push(String(a[a.length - 1] == null ? "" : a[a.length - 1])); return function () {}; };
+  var locationProxy = new Proxy({ href: url || "", assign: function (u) { report.redirects.push(resolve(u)); }, replace: function (u) { report.redirects.push(resolve(u)); }, reload: function () {}, toString: function () { return url || ""; } }, { set: function (t, p, val) { if (p === "href") report.redirects.push(resolve(val)); t[p] = val; return true; } });
+  var noop = function () {};
+  var makeElement = function (tag) {
+    var el = { tagName: String(tag).toUpperCase(), style: {}, children: [], attributes: {}, dataset: {}, classList: { add: noop, remove: noop, toggle: noop, contains: function () { return false; } } };
+    var s = "", a = "";
+    Object.defineProperty(el, "src", { get: function () { return s; }, set: function (v) { s = String(v); report.network.push({ kind: el.tagName === "IMG" ? "image" : "script", url: resolve(v) }); } });
+    Object.defineProperty(el, "action", { get: function () { return a; }, set: function (v) { a = String(v); report.network.push({ kind: "form", url: resolve(v) }); } });
+    Object.defineProperty(el, "innerHTML", { get: function () { return ""; }, set: function (v) { report.writes.push(String(v)); } });
+    Object.defineProperty(el, "outerHTML", { get: function () { return ""; }, set: function (v) { report.writes.push(String(v)); } });
+    el.insertAdjacentHTML = function (pos, html) { report.writes.push(String(html)); };
+    el.setAttribute = function (k, v) { if (k === "src") el.src = v; else if (k === "action") el.action = v; else el.attributes[k] = v; };
+    el.getAttribute = function (k) { return el.attributes[k] != null ? el.attributes[k] : null; };
+    el.appendChild = function (c) { return c; }; el.removeChild = function (c) { return c; }; el.remove = noop; el.addEventListener = noop; el.removeEventListener = noop;
+    // DOM queries on an element return instrumented elements too, so a chained
+    // injection (container.querySelector('.x').innerHTML = ...) still records.
+    el.querySelector = function () { return makeElement("div"); };
+    el.querySelectorAll = function () { return [makeElement("div")]; };
+    el.getElementsByTagName = function (t) { return [makeElement(t)]; };
+    el.getElementsByClassName = function () { return [makeElement("div")]; };
+    return el;
+  };
+  // getElementById/querySelector return INSTRUMENTED elements (not null), so the
+  // most common injection pattern — getElementById('x').innerHTML = '<form>...' —
+  // is recorded instead of throwing on null and aborting the whole script.
+  var documentShim = { write: function () { report.writes.push(Array.prototype.map.call(arguments, String).join("")); }, writeln: function () { report.writes.push(Array.prototype.map.call(arguments, String).join("")); }, createElement: function (t) { return makeElement(String(t)); }, getElementById: function () { return makeElement("div"); }, getElementsByTagName: function (t) { return [makeElement(String(t))]; }, getElementsByClassName: function () { return [makeElement("div")]; }, querySelector: function () { return makeElement("div"); }, querySelectorAll: function () { return [makeElement("div")]; }, addEventListener: noop, body: makeElement("body"), head: makeElement("head"), documentElement: makeElement("html"), location: locationProxy };
+  Object.defineProperty(documentShim, "cookie", { get: function () { return ""; }, set: function (v) { report.cookies.push(String(v)); } });
+  var safeAtob = function (v) { var out; try { out = atob(String(v)); } catch (e) { out = String(v); } report.decoded.push(out); return out; };
+  var safeBtoa = function (v) { try { return btoa(String(v)); } catch (e) { return String(v); } };
+  var win = {
+    document: documentShim, location: locationProxy,
+    navigator: { userAgent: "Mozilla/5.0", platform: "Win32", language: "en-US", sendBeacon: function (u) { report.network.push({ kind: "beacon", url: resolve(u) }); return true; } },
+    screen: { width: 1920, height: 1080 },
+    localStorage: { getItem: function () { return null; }, setItem: function () {}, removeItem: function () {} },
+    sessionStorage: { getItem: function () { return null; }, setItem: function () {}, removeItem: function () {} },
+    atob: safeAtob, btoa: safeBtoa,
+    fetch: function (u) { report.network.push({ kind: "fetch", url: resolve(u) }); return Promise.resolve({ ok: true, status: 200, json: function () { return Promise.resolve({}); }, text: function () { return Promise.resolve(""); } }); },
+    XMLHttpRequest: function () { return { open: function (m, u) { report.network.push({ kind: "xhr", url: resolve(u) }); }, send: function () {}, setRequestHeader: function () {}, addEventListener: function () {} }; },
+    WebSocket: function (u) { report.network.push({ kind: "websocket", url: resolve(u) }); return { send: function () {}, close: function () {} }; },
+    eval: recordEval, Function: FunctionStub,
+    setTimeout: function (fn) { if (typeof fn === "string") report.evals.push(fn); return 0; },
+    setInterval: function (fn) { if (typeof fn === "string") report.evals.push(fn); return 0; },
+    addEventListener: function () {}, console: { log: function () {}, warn: function () {}, error: function () {} }
+  };
+  win.window = win; win.self = win; win.globalThis = win; win.top = win;
+  var params = { window: win, self: win, globalThis: win, document: documentShim, location: locationProxy, navigator: win.navigator, fetch: win.fetch, XMLHttpRequest: win.XMLHttpRequest, WebSocket: win.WebSocket, eval: recordEval, Function: FunctionStub, atob: safeAtob, btoa: safeBtoa, setTimeout: win.setTimeout, setInterval: win.setInterval, localStorage: win.localStorage, console: win.console };
+  var keys = Object.keys(params), vals = keys.map(function (k) { return params[k]; });
+  var list = Array.isArray(scripts) ? scripts.slice(0, 64) : [];
+  for (var i = 0; i < list.length; i++) {
+    var script = list[i];
+    if (typeof script !== "string" || script.length > 262144) { report.errors.push("script skipped"); continue; }
+    try { Function.apply(null, keys.concat([script])).apply(null, vals); }
+    catch (e) { report.errors.push(e && e.message ? e.message : "script error"); }
+  }
+  return report;
+}
+`;
+
+let compiledRecorder: ((scripts: string[], url?: string) => BehaviorReport) | null = null;
+function inProcessRecorder(): (scripts: string[], url?: string) => BehaviorReport {
+  if (!compiledRecorder) {
+    // eslint-disable-next-line no-new-func
+    compiledRecorder = new Function(`${RECORDER_SOURCE}\nreturn recordBehavior;`)() as (scripts: string[], url?: string) => BehaviorReport;
+  }
+  return compiledRecorder;
+}
+
+/** Extract inline <script> bodies (no src) from HTML. */
+export function extractInlineScripts(html: string): string[] {
+  const scripts: string[] = [];
+  for (const match of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script>/gi)) {
+    const attrs = match[1] ?? "";
+    if (/\bsrc\s*=/i.test(attrs)) continue; // external scripts are fetched + scanned separately
+    if (/\btype\s*=\s*["']?(?:application\/json|application\/ld\+json|text\/template)/i.test(attrs)) continue;
+    const body = match[2]?.trim();
+    if (body) scripts.push(body);
+  }
+  return scripts;
+}
+
+// Source URLs of external scripts the page loads (<script src=...>). A renderer
+// fetches and runs these against the DOM, where externally-injected forms appear.
+export function extractScriptSources(html: string): string[] {
+  const sources: string[] = [];
+  for (const match of html.matchAll(/<script\b[^>]*\bsrc\s*=\s*["']([^"']+)["'][^>]*>/gi)) {
+    if (match[1]) sources.push(match[1]);
+  }
+  return [...new Set(sources)];
+}
+
+/**
+ * In-process default evaluator. Runs the recorder in THIS isolate (no boundary).
+ * Use analyzeDynamicWith with a caller-supplied evaluator when isolation matters.
+ */
+export function runInstrumented(scripts: string[], options: DynamicAnalysisOptions = {}): BehaviorReport {
+  try {
+    return inProcessRecorder()(scripts, options.url);
+  } catch (error) {
+    return { ...EMPTY_REPORT, errors: [error instanceof Error ? error.message : "recorder failed"] };
+  }
+}
+
+/** Full in-process pass: extract inline scripts, record behavior, turn it into findings. */
+export function analyzeDynamic(html: string, options: DynamicAnalysisOptions = {}): { report: BehaviorReport; findings: Finding[] } {
+  const report = runInstrumented(extractInlineScripts(html), options);
+  return { report, findings: behaviorFindings(report, options.url) };
+}
+
+/** Generic seam: the caller supplies how scripts are evaluated (in whatever isolate it has). */
+export async function analyzeDynamicWith(
+  html: string,
+  options: DynamicAnalysisOptions,
+  evaluate: IsolatedEvaluator
+): Promise<{ report: BehaviorReport; findings: Finding[] }> {
+  const scripts = extractInlineScripts(html);
+  const report = scripts.length ? await evaluate(scripts, options) : EMPTY_REPORT;
+  return { report, findings: behaviorFindings(report, options.url) };
+}
+
+const EXFIL_SCORE: RuleScoreModel = { base: 72, tags: ["exfiltration", "script"] };
+const OFFSITE_REQUEST_SCORE: RuleScoreModel = { base: 8, tags: ["script"] };
+const REDIRECT_SCORE: RuleScoreModel = { base: 45, tags: ["redirect", "script"] };
+// eval/Function/string-timer use is ubiquitous in legitimate bundles — weak
+// alone. The re-scan of what they produce (below) is where real convictions come from.
+const EVAL_SCORE: RuleScoreModel = { base: 12, tags: ["obfuscation", "script"], maxGroup: "dynamic-code" };
+
+/** Map recorded behavior to scanner findings, re-scanning injected markup and decoded/eval'd code. */
+export function behaviorFindings(report: BehaviorReport, baseUrl?: string): Finding[] {
+  const findings: Finding[] = [];
+  let i = 0;
+  const add = (severity: Severity, confidence: Confidence, model: RuleScoreModel, ruleId: string, title: string, description: string, location: string, metadata: Record<string, unknown>) => {
+    findings.push({ id: `${ruleId}:${i++}`, ruleId, severity, confidence, score: model.base, scoreModel: model, title, description, locationType: "javascript", locationValue: location, metadata });
+  };
+
+  for (const target of report.network) {
+    // An off-site runtime request is NOT exfiltration on its own — legit sites
+    // constantly fetch their own subdomains, CDNs, payment processors, analytics
+    // and APIs. Only convict (high) when the destination host ITSELF looks
+    // suspicious (shortener, suspicious TLD, punycode, IP literal, shared/
+    // generated host) — the actual sketchy-endpoint exfil pattern. A request to
+    // an ordinary off-site domain is recorded as a low-signal note, not a flag.
+    if (!isThirdPartyTarget(target.url, baseUrl)) continue;
+    const suspiciousDestination = baseUrl ? assessRedirect(baseUrl, target.url)?.destinationSuspicious ?? false : false;
+    if (suspiciousDestination) {
+      add("high", "high", EXFIL_SCORE, "runtime_offsite_exfil", `Runtime ${target.kind} to a suspicious off-site endpoint`, `Page JavaScript issued a ${target.kind} request to a suspicious unrelated host at runtime — a common credential/data exfiltration pattern.`, target.url, { kind: target.kind });
+    } else {
+      add("info", "low", OFFSITE_REQUEST_SCORE, "runtime_offsite_request", `Runtime ${target.kind} to an unrelated domain`, `Page JavaScript issued a ${target.kind} request to an unrelated (but not obviously suspicious) domain at runtime.`, target.url, { kind: target.kind });
+    }
+  }
+  for (const redirect of report.redirects) {
+    if (isThirdPartyTarget(redirect, baseUrl)) {
+      add("medium", "high", REDIRECT_SCORE, "runtime_offsite_redirect", "JavaScript navigated to an unrelated domain at runtime", "Page JavaScript set location to an unrelated domain — used to cloak content from scanners and route victims onward.", redirect, {});
+    }
+  }
+  if (report.evals.length) {
+    add("low", "medium", EVAL_SCORE, "runtime_dynamic_code", "Runtime dynamic code execution", `Page JavaScript invoked eval/Function/string-timer ${report.evals.length} time(s) at runtime.`, "eval", { count: report.evals.length });
+  }
+
+  // Re-scan runtime-produced content (injected markup, decoded blobs, eval'd
+  // code) through the static scanner so existing rules apply to it.
+  const derived = [...report.writes, ...report.decoded, ...report.evals];
+  for (const chunk of derived) {
+    if (!chunk || chunk.length < 8) continue;
+    const scanner = createScanner({ source: { url: baseUrl, contentType: "text/html" } });
+    scanner.feed(new TextEncoder().encode(chunk));
+    for (const finding of scanner.finish().findings) {
+      findings.push({ ...finding, id: `dyn.${finding.ruleId}:${i++}`, metadata: { ...finding.metadata, via: "dynamic_analysis" } });
+    }
+  }
+  return findings;
+}
+
+// URLs a page's JavaScript surfaced at runtime — redirect/navigation targets,
+// fetch/XHR/script/form endpoints, and any links embedded in markup the JS
+// injected (document.write / innerHTML). The crawler merges the same-origin
+// ones into its frontier so it CONTINUES into JS-revealed pages instead of
+// treating dynamic analysis as a dead end.
+export function discoveredUrlsFromBehavior(report: BehaviorReport, baseUrl?: string): string[] {
+  const urls = new Set<string>();
+  for (const redirect of report.redirects) if (redirect) urls.add(redirect);
+  for (const target of report.network) if (target.url) urls.add(target.url);
+  for (const chunk of report.writes) {
+    if (!chunk || chunk.length < 4) continue;
+    const scanner = createScanner({ source: { url: baseUrl, contentType: "text/html" } });
+    scanner.feed(new TextEncoder().encode(chunk));
+    for (const url of scanner.finish().urls) urls.add(url.normalized);
+  }
+  return [...urls];
+}
+
+// A runtime target counts only if it's a different registrable domain than the
+// page AND not a known ad/analytics/CDN host — so a site's own subdomains and
+// mainstream third parties (gstatic, analytics) don't read as exfil.
+function isThirdPartyTarget(url: string, baseUrl?: string): boolean {
+  let absolute: string;
+  let targetHost: string;
+  try {
+    const resolved = new URL(url, baseUrl);
+    if (resolved.protocol !== "http:" && resolved.protocol !== "https:") return false;
+    absolute = resolved.toString();
+    targetHost = resolved.hostname.toLowerCase();
+  } catch {
+    return false;
+  }
+  if (isAdOrAnalyticsHost(absolute)) return false;
+  if (!baseUrl) return true;
+  try {
+    const baseHost = new URL(baseUrl).hostname.toLowerCase();
+    const targetReg = registrableDomainFor(targetHost) ?? targetHost;
+    const baseReg = registrableDomainFor(baseHost) ?? baseHost;
+    return targetReg !== baseReg;
+  } catch {
+    return true;
+  }
+}

package/src/feeds.ts

@@ -0,0 +1,334 @@
+// Cached blocklist-feed index for the signal scanner.
+//
+// Runtime-agnostic: all persistence goes through an injected `IntelStorage`
+// (R2 in a Worker, the filesystem in a CLI, an in-memory map in tests). The
+// scanner never knows where bytes live.
+//
+// Feeds can be millions of entries, far too large to hold in a Worker, so the
+// index is sharded by a stable host bucket (`shardOf`) into small files. The
+// match path fetches only the few shards a scan actually needs.
+//
+// Evidence strength is a numeric score (the lib's native currency), decided at
+// ingest and encoded by which score-band shard family a host lands in — so
+// shard files stay compact host arrays. Recent/active/evidence-backed entries
+// get a high score; aged/weak entries a lower one. Matching returns the highest
+// band a host appears in; the caller turns that score into a finding severity
+// via the usual scoring helpers.
+
+export interface IntelStorage {
+  get(key: string): Promise<Uint8Array | null>;
+  put(key: string, value: Uint8Array): Promise<void>;
+  list(prefix: string): Promise<string[]>;
+  delete?(key: string): Promise<void>;
+}
+
+export interface FeedEntry {
+  host: string;
+  score: number;
+}
+
+export interface FeedMeta {
+  source?: string;
+  generatedAt?: string;
+}
+
+export interface FeedRecord extends FeedMeta {
+  version: string;
+  /** Distinct score bands present in this version, and the host count in each. */
+  bands: Record<string, number>;
+}
+
+export interface GlobalFeedManifest {
+  feeds: Record<string, FeedRecord>;
+}
+
+export interface CachedFeedMatch {
+  feedId: string;
+  host: string;
+  score: number;
+  source?: string;
+}
+
+/** Default score bands. Callers may use any integer band; these are conventions. */
+export const FEED_SCORE_ACTIVE = 90; // live / recent / evidence-backed
+export const FEED_SCORE_AGED = 55; // historical / weak / unverified
+
+const ROOT = "feeds";
+const GLOBAL_MANIFEST_KEY = `${ROOT}/manifest.json`;
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+/** All 256 shard prefixes ("00".."ff"); a staged build finalizes one per job. */
+export const SHARD_PREFIXES: string[] = Array.from({ length: 256 }, (_, i) => i.toString(16).padStart(2, "0"));
+
+/** Stable, synchronous host -> shard bucket. FNV-1a low byte; loader and matcher must agree. */
+export function shardOf(host: string): string {
+  let hash = 0x811c9dc5;
+  const lower = host.toLowerCase();
+  for (let i = 0; i < lower.length; i++) {
+    hash ^= lower.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193);
+  }
+  return ((hash >>> 0) & 0xff).toString(16).padStart(2, "0");
+}
+
+/** Score a feed entry from its evidence: active/recent => high, else aged/weak. */
+export function scoreFor(input: {
+  active?: boolean;
+  addedAt?: string | null;
+  recencyDays?: number;
+  now?: number;
+  activeScore?: number;
+  agedScore?: number;
+}): number {
+  const recencyDays = input.recencyDays ?? 90;
+  const now = input.now ?? Date.parse(new Date().toISOString());
+  const recent = input.addedAt ? now - Date.parse(input.addedAt) <= recencyDays * 86_400_000 : true;
+  return input.active !== false && recent ? input.activeScore ?? FEED_SCORE_ACTIVE : input.agedScore ?? FEED_SCORE_AGED;
+}
+
+// ---- Small feeds: full rebuild in one pass --------------------------------
+
+/** Rebuild a feed's shard index from a complete entry set (feeds that fit in memory). */
+export async function rebuildFeed(
+  storage: IntelStorage,
+  feedId: string,
+  version: string,
+  entries: FeedEntry[],
+  meta: FeedMeta = {}
+): Promise<FeedRecord> {
+  const buckets = bucketEntries(dedupeEntries(entries));
+  const bands: Record<string, number> = {};
+  for (const [band, prefixes] of buckets) {
+    for (const [prefix, hosts] of prefixes) {
+      await putJson(storage, shardKey(feedId, version, band, prefix), [...hosts]);
+      bands[band] = (bands[band] ?? 0) + hosts.size;
+    }
+  }
+  return finalizeFeed(storage, feedId, version, bands, meta);
+}
+
+// ---- Huge feeds: staged chunk + per-shard merge ---------------------------
+
+/** Write one download chunk's parsed entries to staging. Safe to run many in parallel. */
+export async function writeFeedChunk(
+  storage: IntelStorage,
+  feedId: string,
+  version: string,
+  chunkId: string,
+  entries: FeedEntry[]
+): Promise<void> {
+  const buckets = bucketEntries(entries);
+  for (const [band, prefixes] of buckets) {
+    for (const [prefix, hosts] of prefixes) {
+      await putJson(storage, stagingKey(feedId, version, chunkId, band, prefix), [...hosts]);
+    }
+  }
+}
+
+/** Merge every chunk's partials for one (band, prefix) into the final shard. One job per shard. */
+export async function finalizeFeedShard(
+  storage: IntelStorage,
+  feedId: string,
+  version: string,
+  band: number,
+  prefix: string
+): Promise<number> {
+  // Staging is keyed band/prefix/chunk, so this lists only the chunks for this
+  // one shard rather than scanning the whole staging tree.
+  const shardStaging = `${ROOT}/${feedId}/${version}/staging/${band}/${prefix}/`;
+  const merged = new Set<string>();
+  for (const key of await storage.list(shardStaging)) {
+    for (const host of await readArray(storage, key)) merged.add(host);
+  }
+  if (merged.size) await putJson(storage, shardKey(feedId, version, String(band), prefix), [...merged]);
+  return merged.size;
+}
+
+/** Publish the feed: write its manifest, register it globally, and sweep staging + old versions. */
+export async function finalizeFeed(
+  storage: IntelStorage,
+  feedId: string,
+  version: string,
+  bands: Record<string, number>,
+  meta: FeedMeta = {}
+): Promise<FeedRecord> {
+  const record: FeedRecord = {
+    version,
+    bands,
+    source: meta.source,
+    generatedAt: meta.generatedAt ?? new Date().toISOString()
+  };
+  await putJson(storage, `${ROOT}/${feedId}/${version}/manifest.json`, record);
+
+  const manifest = (await readJson<GlobalFeedManifest>(storage, GLOBAL_MANIFEST_KEY)) ?? { feeds: {} };
+  const previous = manifest.feeds[feedId]?.version;
+  manifest.feeds[feedId] = record;
+  await putJson(storage, GLOBAL_MANIFEST_KEY, manifest);
+
+  await sweep(storage, `${ROOT}/${feedId}/${version}/staging/`);
+  if (previous && previous !== version) await sweep(storage, `${ROOT}/${feedId}/${previous}/`);
+  return record;
+}
+
+// ---- Match path -----------------------------------------------------------
+
+/** Match candidate hosts against all published feeds, returning the highest score band per hit. */
+export async function matchCachedFeeds(storage: IntelStorage, hosts: string[]): Promise<CachedFeedMatch[]> {
+  const manifest = await readJson<GlobalFeedManifest>(storage, GLOBAL_MANIFEST_KEY);
+  if (!manifest) return [];
+  const uniqueHosts = [...new Set(hosts.map((host) => host.toLowerCase()).filter(Boolean))];
+  const shardCache = new Map<string, Set<string>>();
+  const matches: CachedFeedMatch[] = [];
+
+  for (const [feedId, record] of Object.entries(manifest.feeds)) {
+    const bands = Object.keys(record.bands)
+      .map(Number)
+      .sort((a, b) => b - a); // highest score first
+    for (const host of uniqueHosts) {
+      const prefix = shardOf(host);
+      for (const band of bands) {
+        const key = shardKey(feedId, record.version, String(band), prefix);
+        let set = shardCache.get(key);
+        if (!set) {
+          set = new Set(await readArray(storage, key));
+          shardCache.set(key, set);
+        }
+        if (set.has(host)) {
+          matches.push({ feedId, host, score: band, source: record.source });
+          break; // strongest band wins for this host
+        }
+      }
+    }
+  }
+  return matches;
+}
+
+// ---- Parsers (lib owns the format; the app handles byte/line chunking) -----
+
+/** Extract a lowercase host from a URL or bare host line; null for comments/blanks. */
+export function hostFromLine(line: string): string | null {
+  const trimmed = line.trim();
+  if (!trimmed || trimmed.startsWith("#") || trimmed.startsWith("!")) return null;
+  const candidate = trimmed.includes("://") ? trimmed : `http://${trimmed.split(/\s+/)[0]}`;
+  try {
+    const host = new URL(candidate).hostname.toLowerCase();
+    // Real domains/IPv4 always contain a dot; reject single-label junk lines.
+    return host && host.includes(".") ? host : null;
+  } catch {
+    return null;
+  }
+}
+
+/** OpenPhish community feed: one active phishing URL per line. */
+export function parseOpenPhishFeed(text: string, score = FEED_SCORE_ACTIVE): FeedEntry[] {
+  return dedupeEntries(linesOf(text).map(hostFromLine).filter(isHost).map((host) => ({ host, score })));
+}
+
+/** A bare domain/host blocklist (e.g. Phishing.Database lists) at a caller-chosen score. */
+export function parseHostList(text: string, score: number): FeedEntry[] {
+  return dedupeEntries(linesOf(text).map(hostFromLine).filter(isHost).map((host) => ({ host, score })));
+}
+
+/** URLhaus CSV (id,dateadded,url,url_status,...): online+recent scores high, else aged. */
+export function parseUrlhausCsv(text: string, opts: { recencyDays?: number; now?: number } = {}): FeedEntry[] {
+  const entries: FeedEntry[] = [];
+  for (const line of linesOf(text)) {
+    if (!line || line.startsWith("#")) continue;
+    const cols = parseCsvRow(line);
+    if (cols.length < 4) continue;
+    const host = hostFromLine(cols[2]);
+    if (!host) continue;
+    entries.push({ host, score: scoreFor({ active: cols[3] === "online", addedAt: cols[1], recencyDays: opts.recencyDays, now: opts.now }) });
+  }
+  return dedupeEntries(entries);
+}
+
+// ---- internals ------------------------------------------------------------
+
+function shardKey(feedId: string, version: string, band: string, prefix: string): string {
+  return `${ROOT}/${feedId}/${version}/${band}/${prefix}.json`;
+}
+
+function stagingKey(feedId: string, version: string, chunkId: string, band: string, prefix: string): string {
+  // band/prefix first so a single shard's chunks share a narrow list prefix.
+  return `${ROOT}/${feedId}/${version}/staging/${band}/${prefix}/${chunkId}.json`;
+}
+
+// score band (string) -> shard prefix -> hosts
+function bucketEntries(entries: FeedEntry[]): Map<string, Map<string, Set<string>>> {
+  const buckets = new Map<string, Map<string, Set<string>>>();
+  for (const entry of entries) {
+    const host = entry.host.toLowerCase();
+    if (!host) continue;
+    const band = String(entry.score);
+    let prefixes = buckets.get(band);
+    if (!prefixes) buckets.set(band, (prefixes = new Map()));
+    const prefix = shardOf(host);
+    let set = prefixes.get(prefix);
+    if (!set) prefixes.set(prefix, (set = new Set()));
+    set.add(host);
+  }
+  return buckets;
+}
+
+function dedupeEntries(entries: FeedEntry[]): FeedEntry[] {
+  // Keep the strongest score when a host appears more than once.
+  const scoreByHost = new Map<string, number>();
+  for (const { host, score } of entries) {
+    scoreByHost.set(host, Math.max(scoreByHost.get(host) ?? 0, score));
+  }
+  return [...scoreByHost].map(([host, score]) => ({ host, score }));
+}
+
+function isHost(value: string | null): value is string {
+  return typeof value === "string" && value.length > 0;
+}
+
+function linesOf(text: string): string[] {
+  return text.split(/\r?\n/);
+}
+
+function parseCsvRow(line: string): string[] {
+  const cols: string[] = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+    if (char === '"') {
+      if (inQuotes && line[i + 1] === '"') { current += '"'; i++; } else inQuotes = !inQuotes;
+    } else if (char === "," && !inQuotes) {
+      cols.push(current);
+      current = "";
+    } else {
+      current += char;
+    }
+  }
+  cols.push(current);
+  return cols;
+}
+
+async function putJson(storage: IntelStorage, key: string, value: unknown): Promise<void> {
+  await storage.put(key, encoder.encode(JSON.stringify(value)));
+}
+
+async function readJson<T>(storage: IntelStorage, key: string): Promise<T | null> {
+  const bytes = await storage.get(key);
+  if (!bytes) return null;
+  try {
+    return JSON.parse(decoder.decode(bytes)) as T;
+  } catch {
+    return null;
+  }
+}
+
+async function readArray(storage: IntelStorage, key: string): Promise<string[]> {
+  const value = await readJson<string[]>(storage, key);
+  return Array.isArray(value) ? value : [];
+}
+
+async function sweep(storage: IntelStorage, prefix: string): Promise<void> {
+  if (!storage.delete) return;
+  for (const key of await storage.list(prefix)) await storage.delete(key);
+}