@q32/signal-scanner 0.1.0 → 0.1.1

package/dist/feeds.js

@@ -0,0 +1,259 @@
+// Cached blocklist-feed index for the signal scanner.
+//
+// Runtime-agnostic: all persistence goes through an injected `IntelStorage`
+// (R2 in a Worker, the filesystem in a CLI, an in-memory map in tests). The
+// scanner never knows where bytes live.
+//
+// Feeds can be millions of entries, far too large to hold in a Worker, so the
+// index is sharded by a stable host bucket (`shardOf`) into small files. The
+// match path fetches only the few shards a scan actually needs.
+//
+// Evidence strength is a numeric score (the lib's native currency), decided at
+// ingest and encoded by which score-band shard family a host lands in — so
+// shard files stay compact host arrays. Recent/active/evidence-backed entries
+// get a high score; aged/weak entries a lower one. Matching returns the highest
+// band a host appears in; the caller turns that score into a finding severity
+// via the usual scoring helpers.
+/** Default score bands. Callers may use any integer band; these are conventions. */
+export const FEED_SCORE_ACTIVE = 90; // live / recent / evidence-backed
+export const FEED_SCORE_AGED = 55; // historical / weak / unverified
+const ROOT = "feeds";
+const GLOBAL_MANIFEST_KEY = `${ROOT}/manifest.json`;
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+/** All 256 shard prefixes ("00".."ff"); a staged build finalizes one per job. */
+export const SHARD_PREFIXES = Array.from({ length: 256 }, (_, i) => i.toString(16).padStart(2, "0"));
+/** Stable, synchronous host -> shard bucket. FNV-1a low byte; loader and matcher must agree. */
+export function shardOf(host) {
+    let hash = 0x811c9dc5;
+    const lower = host.toLowerCase();
+    for (let i = 0; i < lower.length; i++) {
+        hash ^= lower.charCodeAt(i);
+        hash = Math.imul(hash, 0x01000193);
+    }
+    return ((hash >>> 0) & 0xff).toString(16).padStart(2, "0");
+}
+/** Score a feed entry from its evidence: active/recent => high, else aged/weak. */
+export function scoreFor(input) {
+    const recencyDays = input.recencyDays ?? 90;
+    const now = input.now ?? Date.parse(new Date().toISOString());
+    const recent = input.addedAt ? now - Date.parse(input.addedAt) <= recencyDays * 86_400_000 : true;
+    return input.active !== false && recent ? input.activeScore ?? FEED_SCORE_ACTIVE : input.agedScore ?? FEED_SCORE_AGED;
+}
+// ---- Small feeds: full rebuild in one pass --------------------------------
+/** Rebuild a feed's shard index from a complete entry set (feeds that fit in memory). */
+export async function rebuildFeed(storage, feedId, version, entries, meta = {}) {
+    const buckets = bucketEntries(dedupeEntries(entries));
+    const bands = {};
+    for (const [band, prefixes] of buckets) {
+        for (const [prefix, hosts] of prefixes) {
+            await putJson(storage, shardKey(feedId, version, band, prefix), [...hosts]);
+            bands[band] = (bands[band] ?? 0) + hosts.size;
+        }
+    }
+    return finalizeFeed(storage, feedId, version, bands, meta);
+}
+// ---- Huge feeds: staged chunk + per-shard merge ---------------------------
+/** Write one download chunk's parsed entries to staging. Safe to run many in parallel. */
+export async function writeFeedChunk(storage, feedId, version, chunkId, entries) {
+    const buckets = bucketEntries(entries);
+    for (const [band, prefixes] of buckets) {
+        for (const [prefix, hosts] of prefixes) {
+            await putJson(storage, stagingKey(feedId, version, chunkId, band, prefix), [...hosts]);
+        }
+    }
+}
+/** Merge every chunk's partials for one (band, prefix) into the final shard. One job per shard. */
+export async function finalizeFeedShard(storage, feedId, version, band, prefix) {
+    // Staging is keyed band/prefix/chunk, so this lists only the chunks for this
+    // one shard rather than scanning the whole staging tree.
+    const shardStaging = `${ROOT}/${feedId}/${version}/staging/${band}/${prefix}/`;
+    const merged = new Set();
+    for (const key of await storage.list(shardStaging)) {
+        for (const host of await readArray(storage, key))
+            merged.add(host);
+    }
+    if (merged.size)
+        await putJson(storage, shardKey(feedId, version, String(band), prefix), [...merged]);
+    return merged.size;
+}
+/** Publish the feed: write its manifest, register it globally, and sweep staging + old versions. */
+export async function finalizeFeed(storage, feedId, version, bands, meta = {}) {
+    const record = {
+        version,
+        bands,
+        source: meta.source,
+        generatedAt: meta.generatedAt ?? new Date().toISOString()
+    };
+    await putJson(storage, `${ROOT}/${feedId}/${version}/manifest.json`, record);
+    const manifest = (await readJson(storage, GLOBAL_MANIFEST_KEY)) ?? { feeds: {} };
+    const previous = manifest.feeds[feedId]?.version;
+    manifest.feeds[feedId] = record;
+    await putJson(storage, GLOBAL_MANIFEST_KEY, manifest);
+    await sweep(storage, `${ROOT}/${feedId}/${version}/staging/`);
+    if (previous && previous !== version)
+        await sweep(storage, `${ROOT}/${feedId}/${previous}/`);
+    return record;
+}
+// ---- Match path -----------------------------------------------------------
+/** Match candidate hosts against all published feeds, returning the highest score band per hit. */
+export async function matchCachedFeeds(storage, hosts) {
+    const manifest = await readJson(storage, GLOBAL_MANIFEST_KEY);
+    if (!manifest)
+        return [];
+    const uniqueHosts = [...new Set(hosts.map((host) => host.toLowerCase()).filter(Boolean))];
+    const shardCache = new Map();
+    const matches = [];
+    for (const [feedId, record] of Object.entries(manifest.feeds)) {
+        const bands = Object.keys(record.bands)
+            .map(Number)
+            .sort((a, b) => b - a); // highest score first
+        for (const host of uniqueHosts) {
+            const prefix = shardOf(host);
+            for (const band of bands) {
+                const key = shardKey(feedId, record.version, String(band), prefix);
+                let set = shardCache.get(key);
+                if (!set) {
+                    set = new Set(await readArray(storage, key));
+                    shardCache.set(key, set);
+                }
+                if (set.has(host)) {
+                    matches.push({ feedId, host, score: band, source: record.source });
+                    break; // strongest band wins for this host
+                }
+            }
+        }
+    }
+    return matches;
+}
+// ---- Parsers (lib owns the format; the app handles byte/line chunking) -----
+/** Extract a lowercase host from a URL or bare host line; null for comments/blanks. */
+export function hostFromLine(line) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#") || trimmed.startsWith("!"))
+        return null;
+    const candidate = trimmed.includes("://") ? trimmed : `http://${trimmed.split(/\s+/)[0]}`;
+    try {
+        const host = new URL(candidate).hostname.toLowerCase();
+        // Real domains/IPv4 always contain a dot; reject single-label junk lines.
+        return host && host.includes(".") ? host : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** OpenPhish community feed: one active phishing URL per line. */
+export function parseOpenPhishFeed(text, score = FEED_SCORE_ACTIVE) {
+    return dedupeEntries(linesOf(text).map(hostFromLine).filter(isHost).map((host) => ({ host, score })));
+}
+/** A bare domain/host blocklist (e.g. Phishing.Database lists) at a caller-chosen score. */
+export function parseHostList(text, score) {
+    return dedupeEntries(linesOf(text).map(hostFromLine).filter(isHost).map((host) => ({ host, score })));
+}
+/** URLhaus CSV (id,dateadded,url,url_status,...): online+recent scores high, else aged. */
+export function parseUrlhausCsv(text, opts = {}) {
+    const entries = [];
+    for (const line of linesOf(text)) {
+        if (!line || line.startsWith("#"))
+            continue;
+        const cols = parseCsvRow(line);
+        if (cols.length < 4)
+            continue;
+        const host = hostFromLine(cols[2]);
+        if (!host)
+            continue;
+        entries.push({ host, score: scoreFor({ active: cols[3] === "online", addedAt: cols[1], recencyDays: opts.recencyDays, now: opts.now }) });
+    }
+    return dedupeEntries(entries);
+}
+// ---- internals ------------------------------------------------------------
+function shardKey(feedId, version, band, prefix) {
+    return `${ROOT}/${feedId}/${version}/${band}/${prefix}.json`;
+}
+function stagingKey(feedId, version, chunkId, band, prefix) {
+    // band/prefix first so a single shard's chunks share a narrow list prefix.
+    return `${ROOT}/${feedId}/${version}/staging/${band}/${prefix}/${chunkId}.json`;
+}
+// score band (string) -> shard prefix -> hosts
+function bucketEntries(entries) {
+    const buckets = new Map();
+    for (const entry of entries) {
+        const host = entry.host.toLowerCase();
+        if (!host)
+            continue;
+        const band = String(entry.score);
+        let prefixes = buckets.get(band);
+        if (!prefixes)
+            buckets.set(band, (prefixes = new Map()));
+        const prefix = shardOf(host);
+        let set = prefixes.get(prefix);
+        if (!set)
+            prefixes.set(prefix, (set = new Set()));
+        set.add(host);
+    }
+    return buckets;
+}
+function dedupeEntries(entries) {
+    // Keep the strongest score when a host appears more than once.
+    const scoreByHost = new Map();
+    for (const { host, score } of entries) {
+        scoreByHost.set(host, Math.max(scoreByHost.get(host) ?? 0, score));
+    }
+    return [...scoreByHost].map(([host, score]) => ({ host, score }));
+}
+function isHost(value) {
+    return typeof value === "string" && value.length > 0;
+}
+function linesOf(text) {
+    return text.split(/\r?\n/);
+}
+function parseCsvRow(line) {
+    const cols = [];
+    let current = "";
+    let inQuotes = false;
+    for (let i = 0; i < line.length; i++) {
+        const char = line[i];
+        if (char === '"') {
+            if (inQuotes && line[i + 1] === '"') {
+                current += '"';
+                i++;
+            }
+            else
+                inQuotes = !inQuotes;
+        }
+        else if (char === "," && !inQuotes) {
+            cols.push(current);
+            current = "";
+        }
+        else {
+            current += char;
+        }
+    }
+    cols.push(current);
+    return cols;
+}
+async function putJson(storage, key, value) {
+    await storage.put(key, encoder.encode(JSON.stringify(value)));
+}
+async function readJson(storage, key) {
+    const bytes = await storage.get(key);
+    if (!bytes)
+        return null;
+    try {
+        return JSON.parse(decoder.decode(bytes));
+    }
+    catch {
+        return null;
+    }
+}
+async function readArray(storage, key) {
+    const value = await readJson(storage, key);
+    return Array.isArray(value) ? value : [];
+}
+async function sweep(storage, prefix) {
+    if (!storage.delete)
+        return;
+    for (const key of await storage.list(prefix))
+        await storage.delete(key);
+}
+//# sourceMappingURL=feeds.js.map

package/dist/feeds.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"feeds.js","sourceRoot":"","sources":["../src/feeds.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,wCAAwC;AACxC,EAAE;AACF,8EAA8E;AAC9E,6EAA6E;AAC7E,gEAAgE;AAChE,EAAE;AACF,+EAA+E;AAC/E,2EAA2E;AAC3E,8EAA8E;AAC9E,gFAAgF;AAChF,8EAA8E;AAC9E,iCAAiC;AAoCjC,oFAAoF;AACpF,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC,CAAC,kCAAkC;AACvE,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC,CAAC,iCAAiC;AAEpE,MAAM,IAAI,GAAG,OAAO,CAAC;AACrB,MAAM,mBAAmB,GAAG,GAAG,IAAI,gBAAgB,CAAC;AAEpD,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAClC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAElC,iFAAiF;AACjF,MAAM,CAAC,MAAM,cAAc,GAAa,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AAE/G,gGAAgG;AAChG,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC7D,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,QAAQ,CAAC,KAOxB;IACC,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC;IAC5C,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAClG,OAAO,KAAK,CAAC,MAAM,KAAK,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,IAAI,iBAAiB,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,IAAI,eAAe,CAAC;AACxH,CAAC;AAED,8EAA8E;AAE9E,yFAAyF;AACzF,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAqB,EACrB,MAAc,EACd,OAAe,EACf,OAAoB,EACpB,OAAiB,EAAE;IAEnB,MAAM,OAAO,GAAG,aAAa,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IACtD,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,OAAO,EAAE,CAAC;QACvC,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;YACvC,MAAM,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;YAC5E,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AAC7D,CAAC;AAED,8EAA8E;AAE9E,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAqB,EACrB,MAAc,EACd,OAAe,EACf,OAAe,EACf,OAAoB;IAEpB,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,OAAO,EAAE,CAAC;QACvC,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;YACvC,MAAM,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;AACH,CAAC;AAED,mGAAmG;AACnG,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAqB,EACrB,MAAc,EACd,OAAe,EACf,IAAY,EACZ,MAAc;IAEd,6EAA6E;IAC7E,yDAAyD;IACzD,MAAM,YAAY,GAAG,GAAG,IAAI,IAAI,MAAM,IAAI,OAAO,YAAY,IAAI,IAAI,MAAM,GAAG,CAAC;IAC/E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,GAAG,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QACnD,KAAK,MAAM,IAAI,IAAI,MAAM,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,MAAM,CAAC,IAAI;QAAE,MAAM,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;IACtG,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,oGAAoG;AACpG,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAqB,EACrB,MAAc,EACd,OAAe,EACf,KAA6B,EAC7B,OAAiB,EAAE;IAEnB,MAAM,MAAM,GAAe;QACzB,OAAO;QACP,KAAK;QACL,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KAC1D,CAAC;IACF,MAAM,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,MAAM,IAAI,OAAO,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE7E,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,CAAqB,OAAO,EAAE,mBAAmB,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACrG,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjD,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IAChC,MAAM,OAAO,CAAC,OAAO,EAAE,mBAAmB,EAAE,QAAQ,CAAC,CAAC;IAEtD,MAAM,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,MAAM,IAAI,OAAO,WAAW,CAAC,CAAC;IAC9D,IAAI,QAAQ,IAAI,QAAQ,KAAK,OAAO;QAAE,MAAM,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,MAAM,IAAI,QAAQ,GAAG,CAAC,CAAC;IAC7F,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAE9E,mGAAmG;AACnG,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAqB,EAAE,KAAe;IAC3E,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAqB,OAAO,EAAE,mBAAmB,CAAC,CAAC;IAClF,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,UAAU,GAAG,IAAI,GAAG,EAAuB,CAAC;IAClD,MAAM,OAAO,GAAsB,EAAE,CAAC;IAEtC,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;aACpC,GAAG,CAAC,MAAM,CAAC;aACX,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,sBAAsB;QAChD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;gBACnE,IAAI,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;oBAC7C,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBAC3B,CAAC;gBACD,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBAClB,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;oBACnE,MAAM,CAAC,oCAAoC;gBAC7C,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+EAA+E;AAE/E,uFAAuF;AACvF,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAChF,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1F,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACvD,0EAA0E;QAC1E,OAAO,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,KAAK,GAAG,iBAAiB;IACxE,OAAO,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;AACxG,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,KAAa;IACvD,OAAO,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;AACxG,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,OAA+C,EAAE;IAC7F,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC9B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5I,CAAC;IACD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;AAChC,CAAC;AAED,8EAA8E;AAE9E,SAAS,QAAQ,CAAC,MAAc,EAAE,OAAe,EAAE,IAAY,EAAE,MAAc;IAC7E,OAAO,GAAG,IAAI,IAAI,MAAM,IAAI,OAAO,IAAI,IAAI,IAAI,MAAM,OAAO,CAAC;AAC/D,CAAC;AAED,SAAS,UAAU,CAAC,MAAc,EAAE,OAAe,EAAE,OAAe,EAAE,IAAY,EAAE,MAAc;IAChG,2EAA2E;IAC3E,OAAO,GAAG,IAAI,IAAI,MAAM,IAAI,OAAO,YAAY,IAAI,IAAI,MAAM,IAAI,OAAO,OAAO,CAAC;AAClF,CAAC;AAED,+CAA+C;AAC/C,SAAS,aAAa,CAAC,OAAoB;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoC,CAAC;IAC5D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,QAAQ;YAAE,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG;YAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QAClD,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,OAAoB;IACzC,+DAA+D;IAC/D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,OAAO,EAAE,CAAC;QACtC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,MAAM,CAAC,KAAoB;IAClC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,IAAI,QAAQ,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBAAC,OAAO,IAAI,GAAG,CAAC;gBAAC,CAAC,EAAE,CAAC;YAAC,CAAC;;gBAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC;QAC1F,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,GAAG,EAAE,CAAC;QACf,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,OAAqB,EAAE,GAAW,EAAE,KAAc;IACvE,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,QAAQ,CAAI,OAAqB,EAAE,GAAW;IAC3D,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAM,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,OAAqB,EAAE,GAAW;IACzD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAW,OAAO,EAAE,GAAG,CAAC,CAAC;IACrD,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,OAAqB,EAAE,MAAc;IACxD,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO;IAC5B,KAAK,MAAM,GAAG,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1E,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,110 @@
+import type { RuleScoreModel } from "./rules/types.js";
+export type ContentKind = "html" | "javascript" | "css" | "json" | "svg" | "text" | "unknown" | "archive" | "executable";
+export type Severity = "info" | "low" | "medium" | "high" | "critical";
+export type Confidence = "low" | "medium" | "high";
+export type Disposition = "allow" | "warn" | "review" | "block";
+export interface FetchRecord {
+    url: string;
+    finalUrl: string;
+    status: number;
+    contentType: string;
+    contentLength: number | null;
+    redirectChain: string[];
+    tls?: {
+        protocol?: string;
+        issuer?: string;
+        subject?: string;
+        validFrom?: string;
+        validTo?: string;
+    };
+}
+export interface ScannerSource {
+    url?: string;
+    finalUrl?: string;
+    filename?: string;
+    contentType?: string | null;
+    originUrl?: string;
+    tls?: TlsMetadata;
+}
+export interface TlsMetadata {
+    authorized?: boolean;
+    authorizationError?: string | null;
+    issuer?: string;
+    subject?: string;
+    validFrom?: string;
+    validTo?: string;
+    fingerprint256?: string;
+    serialNumber?: string;
+}
+export interface ArtifactRecord {
+    source: string;
+    artifactType: string;
+    parentOffset: number;
+    depth: number;
+    sha256?: string;
+    text: string;
+}
+export interface Finding {
+    id: string;
+    severity: Severity;
+    confidence: Confidence;
+    score: number;
+    scoreModel: RuleScoreModel;
+    title: string;
+    description: string;
+    locationType: "url" | "html" | "javascript" | "css" | "source" | "binary" | "decoded_artifact" | "aggregate";
+    locationValue: string;
+    ruleId: string;
+    metadata: Record<string, unknown>;
+}
+export interface ExtractedUrl {
+    raw: string;
+    normalized: string;
+    registrableDomain: string | null;
+    relation: "same-origin" | "same-site" | "subdomain" | "off-site" | "unknown";
+    scheme: string;
+    destinationType: "http" | "https" | "ip" | "private" | "localhost" | "url-shortener" | "other";
+    flags: string[];
+}
+export interface ScannerReport {
+    contentKind: ContentKind;
+    findings: Finding[];
+    urls: ExtractedUrl[];
+    artifacts: ArtifactRecord[];
+    score: number;
+    disposition: Disposition;
+    counters: Record<string, number>;
+}
+export interface Scanner {
+    feed(chunk: Uint8Array): Finding[];
+    finish(): ScannerReport;
+}
+interface ScannerOptions {
+    source?: ScannerSource;
+    maxWindowChars?: number;
+    maxDecodedBytes?: number;
+    maxDecodeDepth?: number;
+}
+export declare function createScanner(options?: ScannerOptions): Scanner;
+export declare function detectContentKind(input: {
+    contentType?: string | null;
+    filename?: string | null;
+    firstBytes?: Uint8Array;
+}): ContentKind;
+export declare function normalizeUrl(raw: string, base?: string): ExtractedUrl | null;
+export declare function scoreFindings(findings: Finding[]): number;
+export declare function dispositionForScore(score: number): Disposition;
+export declare function registrableDomainFor(host: string): string | null;
+export declare function isAdOrAnalyticsHost(normalizedUrl: string): boolean;
+export interface RedirectAssessment {
+    /** The redirect crossed to a different registrable domain (not just a subdomain hop). */
+    offSite: boolean;
+    /** The destination host itself looks suspicious (shortener, suspicious TLD, punycode, IP, shared/generated host). */
+    destinationSuspicious: boolean;
+    requestedRegistrable: string;
+    finalRegistrable: string;
+    destinationFlags: string[];
+}
+export declare function assessRedirect(requestedUrl: string, finalUrl: string): RedirectAssessment | null;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAkB,cAAc,EAAY,MAAM,kBAAkB,CAAC;AAEjF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC;AACzH,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AACvE,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACnD,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEhE,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,GAAG,CAAC,EAAE;QACJ,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,WAAW,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,UAAU,EAAE,UAAU,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,KAAK,GAAG,MAAM,GAAG,YAAY,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,kBAAkB,GAAG,WAAW,CAAC;IAC7G,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,aAAa,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,CAAC;IAC7E,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,eAAe,GAAG,OAAO,CAAC;IAC/F,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,IAAI,EAAE,YAAY,EAAE,CAAC;IACrB,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,OAAO;IACtB,IAAI,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,EAAE,CAAC;IACnC,MAAM,IAAI,aAAa,CAAC;CACzB;AAED,UAAU,cAAc;IACtB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAmCD,wBAAgB,aAAa,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAoEnE;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IACvC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB,GAAG,WAAW,CA+Bd;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAqC5E;AA0YD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,CA2BzD;AAWD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAK9D;AAkQD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAQhE;AAkHD,wBAAgB,mBAAmB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAOlE;AAuGD,MAAM,WAAW,kBAAkB;IACjC,yFAAyF;IACzF,OAAO,EAAE,OAAO,CAAC;IACjB,qHAAqH;IACrH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAOD,wBAAgB,cAAc,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAehG"}