@q32/signal-scanner 0.1.1 → 0.2.0

package/dist/render-isolate/polyfills.js

@@ -0,0 +1,41 @@
+// Web globals a bare V8 isolate lacks, installed BEFORE linkedom/render load.
+// (self/window are set by the host via context.eval before this bundle runs.)
+import "fast-text-encoding";
+import base64 from "base-64";
+import { URL, URLSearchParams } from "whatwg-url-without-unicode";
+const g = globalThis;
+if (!g.atob)
+    g.atob = (s) => base64.decode(String(s));
+if (!g.btoa)
+    g.btoa = (s) => base64.encode(String(s));
+if (!g.URL)
+    g.URL = URL;
+if (!g.URLSearchParams)
+    g.URLSearchParams = URLSearchParams;
+// Minimal Buffer shim (linkedom's entity decoder + a few runtime paths use it).
+// Built on the globals above; covers from(str|base64|bytes) + toString(enc).
+if (!g.Buffer) {
+    const toBinary = (bytes) => {
+        let out = "";
+        for (let i = 0; i < bytes.length; i += 8192)
+            out += String.fromCharCode.apply(null, bytes.subarray(i, i + 8192));
+        return out;
+    };
+    g.Buffer = {
+        from(input, enc) {
+            let bytes;
+            if (input instanceof Uint8Array)
+                bytes = input;
+            else if (enc === "base64")
+                bytes = Uint8Array.from(g.atob(String(input)), (c) => c.charCodeAt(0));
+            else
+                bytes = new TextEncoder().encode(String(input));
+            const view = bytes;
+            view.toString = (e) => (e === "binary" || e === "latin1" ? toBinary(bytes) : e === "base64" ? g.btoa(toBinary(bytes)) : new TextDecoder().decode(bytes));
+            return view;
+        },
+        alloc: (n) => new Uint8Array(n),
+        isBuffer: (x) => x instanceof Uint8Array
+    };
+}
+//# sourceMappingURL=polyfills.js.map

package/dist/render-isolate/polyfills.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"polyfills.js","sourceRoot":"","sources":["../../src/render-isolate/polyfills.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,8EAA8E;AAC9E,OAAO,oBAAoB,CAAC;AAC5B,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAE,GAAG,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,MAAM,CAAC,GAAG,UAAiB,CAAC;AAC5B,IAAI,CAAC,CAAC,CAAC,IAAI;IAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9D,IAAI,CAAC,CAAC,CAAC,IAAI;IAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9D,IAAI,CAAC,CAAC,CAAC,GAAG;IAAE,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC;AACxB,IAAI,CAAC,CAAC,CAAC,eAAe;IAAE,CAAC,CAAC,eAAe,GAAG,eAAe,CAAC;AAE5D,gFAAgF;AAChF,6EAA6E;AAC7E,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACd,MAAM,QAAQ,GAAG,CAAC,KAAiB,EAAU,EAAE;QAC7C,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI;YAAE,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAwB,CAAC,CAAC;QACxI,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,CAAC,CAAC,MAAM,GAAG;QACT,IAAI,CAAC,KAAc,EAAE,GAAY;YAC/B,IAAI,KAAiB,CAAC;YACtB,IAAI,KAAK,YAAY,UAAU;gBAAE,KAAK,GAAG,KAAK,CAAC;iBAC1C,IAAI,GAAG,KAAK,QAAQ;gBAAE,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;;gBACrG,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,KAA0D,CAAC;YACxE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAClK,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC;QACvC,QAAQ,EAAE,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,YAAY,UAAU;KAClD,CAAC;AACJ,CAAC"}

package/dist/render-isolate/run.d.ts

@@ -0,0 +1,3 @@
+import type { RenderInput, RenderResult } from "../render.js";
+export declare function renderInIsolate(input: RenderInput): Promise<RenderResult>;
+//# sourceMappingURL=run.d.ts.map

package/dist/render-isolate/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/render-isolate/run.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAkE9D,wBAAsB,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAoB/E"}

package/dist/render-isolate/run.js

@@ -0,0 +1,88 @@
+// CLI executor for renderDom: runs the render bundle inside a real isolated-vm
+// isolate. True isolation — the page's untrusted JS gets web-global polyfills and
+// a floor-dropping fetch, and cannot reach the host's fetch/process/fs. A fresh
+// context per page prevents cross-page contamination; the heavy bundles are
+// compiled once.
+//
+// `isolated-vm` and `esbuild` are optionalDependencies: a base install can still
+// run the static scan/crawl. They are loaded lazily here so the package imports
+// cleanly without them; if absent, renderInIsolate throws and the caller
+// (cli.ts) skips dynamic rendering for that page.
+import { createRequire } from "node:module";
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+const HERE = import.meta.dirname;
+const CALL_TIMEOUT_MS = 6000;
+// Punycode lives in a transitive (linkedom/whatwg-url); resolve it through the
+// module graph so it works whether or not node_modules is hoisted.
+const require = createRequire(import.meta.url);
+const PUNYCODE = require.resolve("punycode/punycode.es6.js");
+// Sibling bundle entry: `.js` once compiled into dist/, `.ts` when run from
+// source via tsx. esbuild bundles either.
+function entryPath(base) {
+    const js = resolve(HERE, `${base}.js`);
+    return existsSync(js) ? js : resolve(HERE, `${base}.ts`);
+}
+let depsWarned = false;
+async function loadDeps() {
+    try {
+        const [ivmMod, esbuildMod] = await Promise.all([import("isolated-vm"), import("esbuild")]);
+        return { ivm: ivmMod.default ?? ivmMod, build: esbuildMod.build };
+    }
+    catch (error) {
+        if (!depsWarned) {
+            depsWarned = true;
+            console.error("signal-scanner: dynamic rendering is disabled — install the optional " +
+                "dependencies `isolated-vm` and `esbuild` to enable it. Static analysis continues.");
+        }
+        throw error instanceof Error ? error : new Error(String(error));
+    }
+}
+// Typed as `any` so the build does not hard-depend on the optional deps' types.
+let ready = null;
+async function bundleOnce(build, base) {
+    const result = await build({
+        entryPoints: [entryPath(base)],
+        bundle: true,
+        format: "iife",
+        platform: "node",
+        treeShaking: false, // keep polyfill side-effects (sideEffects:false would drop them)
+        alias: { punycode: PUNYCODE },
+        write: false
+    });
+    return result.outputFiles[0].text;
+}
+async function init() {
+    if (!ready) {
+        ready = (async () => {
+            const { ivm, build } = await loadDeps();
+            const [polyCode, renderCode] = await Promise.all([bundleOnce(build, "polyfills"), bundleOnce(build, "entry")]);
+            const isolate = new ivm.Isolate({ memoryLimit: 256 });
+            const poly = await isolate.compileScript(polyCode);
+            const render = await isolate.compileScript(renderCode);
+            return { isolate, poly, render };
+        })();
+        // A failed init must not poison every later call (e.g. transient bundle error).
+        ready.catch(() => { ready = null; });
+    }
+    return ready;
+}
+export async function renderInIsolate(input) {
+    const { isolate, poly, render } = await init();
+    const context = await isolate.createContext();
+    try {
+        await context.global.set("globalThis", context.global.derefInto());
+        // self/window must exist before the polyfill bundle (fast-text-encoding
+        // detects them), and polyfills must run before the render bundle (linkedom's
+        // entity decoder reads atob/Buffer at module init).
+        await context.eval("globalThis.self = globalThis; globalThis.window = globalThis;");
+        await poly.run(context);
+        await render.run(context);
+        const out = await context.evalClosure("return JSON.stringify(globalThis.__renderAndScan($0))", [input], { arguments: { copy: true }, result: { copy: true }, timeout: CALL_TIMEOUT_MS });
+        return JSON.parse(out);
+    }
+    finally {
+        context.release();
+    }
+}
+//# sourceMappingURL=run.js.map

package/dist/render-isolate/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/render-isolate/run.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,kFAAkF;AAClF,gFAAgF;AAChF,4EAA4E;AAC5E,iBAAiB;AACjB,EAAE;AACF,iFAAiF;AACjF,gFAAgF;AAChF,yEAAyE;AACzE,kDAAkD;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;AACjC,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,+EAA+E;AAC/E,mEAAmE;AACnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAE7D,4EAA4E;AAC5E,0CAA0C;AAC1C,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC;IACvC,OAAO,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC;AAC3D,CAAC;AAED,IAAI,UAAU,GAAG,KAAK,CAAC;AACvB,KAAK,UAAU,QAAQ;IACrB,IAAI,CAAC;QACH,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3F,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM,EAAE,KAAK,EAAG,UAAkB,CAAC,KAAK,EAAE,CAAC;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,UAAU,GAAG,IAAI,CAAC;YAClB,OAAO,CAAC,KAAK,CACX,uEAAuE;gBACrE,mFAAmF,CACtF,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,IAAI,KAAK,GAA6D,IAAI,CAAC;AAE3E,KAAK,UAAU,UAAU,CAAC,KAAU,EAAE,IAAY;IAChD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC;QACzB,WAAW,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,EAAE,IAAI;QACZ,MAAM,EAAE,MAAM;QACd,QAAQ,EAAE,MAAM;QAChB,WAAW,EAAE,KAAK,EAAE,iEAAiE;QACrF,KAAK,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE;QAC7B,KAAK,EAAE,KAAK;KACb,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,CAAC,KAAK,IAAI,EAAE;YAClB,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,EAAE,CAAC;YACxC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;YAC/G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YACtD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC,CAAC,EAAE,CAAC;QACL,gFAAgF;QAChF,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAkB;IACtD,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC;IAC9C,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QACnE,wEAAwE;QACxE,6EAA6E;QAC7E,oDAAoD;QACpD,MAAM,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QACpF,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxB,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,WAAW,CACnC,uDAAuD,EACvD,CAAC,KAAK,CAAC,EACP,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAChF,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAa,CAAiB,CAAC;IACnD,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,OAAO,EAAE,CAAC;IACpB,CAAC;AACH,CAAC"}

package/package.json

@@ -1,25 +1,33 @@
 {
   "name": "@q32/signal-scanner",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "type": "module",
   "description": "Static web signal scanner with bounded streaming analyzers, URL extraction, rule packs, scoring, and normalized reports.",
-  "license": "MIT",
+  "license": "LGPL-3.0-or-later",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/q32llc/signal-scanner.git"
   },
+  "engines": {
+    "node": ">=20.11"
+  },
+  "bin": {
+    "signal-scanner": "dist/cli.js"
+  },
   "files": [
     "README.md",
-    "dist/",
-    "scripts/"
+    "COPYING",
+    "COPYING.LESSER",
+    "dist/"
   ],
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "prepack": "npm run build",
     "test": "bun test",
+    "test:isolate": "npm run build && node --test test/integration/isolate.mjs",
     "coverage": "bun test --coverage --coverage-reporter=lcov --coverage-dir=coverage && bun scripts/check-coverage.ts coverage/lcov.info 80",
     "coverage:report": "bun test --coverage",
-    "scan": "tsx scripts/scan.ts",
+    "scan": "tsx src/cli.ts",
     "eval": "NODE_USE_ENV_PROXY=1 HTTP_PROXY=\"${EVAL_PROXY_URL-}\" HTTPS_PROXY=\"${EVAL_PROXY_URL-}\" tsx --env-file-if-exists=.env scripts/eval.ts"
   },
   "exports": {
@@ -53,14 +61,17 @@
     "htmlparser2": "^10.1.0",
     "linkedom": "^0.18.12"
   },
-  "devDependencies": {
+  "optionalDependencies": {
     "base-64": "^1.0.0",
     "buffer": "^6.0.3",
     "esbuild": "^0.28.0",
     "fast-text-encoding": "^1.0.6",
     "isolated-vm": "^6.1.2",
-    "typescript": "^5.9.3",
-    "@types/node": "^24.0.0",
+    "punycode": "^2.3.1",
     "whatwg-url-without-unicode": "^8.0.0-3"
+  },
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "typescript": "^5.9.3"
   }
 }

package/scripts/check-coverage.ts

@@ -1,33 +0,0 @@
-import { readFileSync } from "node:fs";
-
-const [, , lcovPath, thresholdArg] = process.argv;
-const threshold = Number(thresholdArg ?? 80);
-
-if (!lcovPath || !Number.isFinite(threshold)) {
-  console.error("Usage: bun scripts/check-coverage.ts <lcov.info> <line-threshold-percent>");
-  process.exit(2);
-}
-
-const lcov = readFileSync(lcovPath, "utf8");
-let found = 0;
-let hit = 0;
-
-for (const line of lcov.split(/\r?\n/)) {
-  if (line.startsWith("LF:")) found += Number(line.slice(3));
-  else if (line.startsWith("LH:")) hit += Number(line.slice(3));
-}
-
-if (!found) {
-  console.error(`No line coverage data found in ${lcovPath}`);
-  process.exit(2);
-}
-
-const pct = (hit / found) * 100;
-const display = pct.toFixed(2);
-
-if (pct < threshold) {
-  console.error(`Line coverage ${display}% is below required ${threshold}%`);
-  process.exit(1);
-}
-
-console.log(`Line coverage ${display}% meets required ${threshold}%`);

package/scripts/eval.ts

@@ -1,311 +0,0 @@
-// Eval harness: run the homegrown scanner over a labeled corpus of known-good
-// and known-bad sites and measure how well it separates them.
-//
-//   npm run eval               # reuse cached bad list if fresh (<6h)
-//   npm run eval -- --refresh  # re-pull a fresh live bad list
-//
-// Known-good is the curated corpus/good.txt. Known-bad is pulled live from
-// OpenPhish + URLhaus (they go offline fast), probed for reachability, and
-// cached to corpus/.bad-cache.txt. The scan path is CLI heuristics only
-// (structural + content + dynamic JS) — NO threat-intel feeds — so this measures
-// the homegrown detector's own discriminative power, not feed lookups.
-
-import { readFile, writeFile } from "node:fs/promises";
-import { resolve } from "node:path";
-import { crawlTargets, DEFAULT_CRAWL_OPTIONS, type CrawlOptions } from "./scan";
-import { dispositionForScore } from "../src/index";
-
-const FLAG_THRESHOLD = 50; // score >= 50 => product surfaces suspicious/malicious
-const TARGET_BAD = 80;
-const SITE_CONCURRENCY = 6;
-const CACHE_PATH = resolve("corpus/.bad-cache.txt");
-const PHISHING_CACHE_PATH = resolve("corpus/.bad-phishing-cache.txt");
-const CACHE_TTL_MS = 6 * 60 * 60 * 1000;
-const MAX_FP_RATE = 0.05; // gate: at most 5% of good sites may be flagged
-
-// Bounded per-site crawl: landing page + a shallow hop is enough to judge, and
-// keeps a 160-site sweep tractable.
-const CRAWL: CrawlOptions = {
-  ...DEFAULT_CRAWL_OPTIONS,
-  maxUrls: 10,
-  maxDepth: 1,
-  parallel: 4,
-  robots: false,
-  timeoutMs: 8000
-};
-
-const BROWSER_UA = DEFAULT_CRAWL_OPTIONS.userAgent;
-
-interface SiteResult {
-  url: string;
-  label: "good" | "bad";
-  score: number;
-  disposition: string;
-  pagesScanned: number;
-  topFindings: Array<{ ruleId: string; score: number }>;
-  unreachable: boolean;
-}
-
-// The dynamic-analysis sandbox runs untrusted page JS; a stray rejection or
-// throw from one site must never abort a 160-site sweep. Per-site scanning is
-// already best-effort, so swallow these and keep going.
-process.on("unhandledRejection", () => {});
-process.on("uncaughtException", (error) => {
-  console.error("  (ignored uncaught error from sandbox):", error instanceof Error ? error.message : error);
-});
-
-async function main(): Promise<void> {
-  const refresh = process.argv.includes("--refresh");
-  // --phishing pulls a phishing-ONLY bad corpus (OpenPhish + Phishing.Database
-  // active links, no URLhaus malware binaries) to measure catch rate on
-  // malicious PAGES — where the web heuristics (credential forms, brand
-  // impersonation, cloaking) should actually shine.
-  const phishingOnly = process.argv.includes("--phishing");
-  // --live uses the curated, hand-verified corpus/phishing-live.txt (real
-  // credential-capture pages confirmed alive) instead of a noisy feed.
-  const live = process.argv.includes("--live");
-  // Egress: set EVAL_PROXY_URL (e.g. an unfiltered residential proxy) so the
-  // crawl + reachability probe leave via that proxy instead of the local
-  // network — necessary when an ISP filter (e.g. Spectrum Security Shield)
-  // intercepts known-malicious URLs and serves a block page, which would
-  // otherwise make every bad site look benign. The npm script maps it onto
-  // HTTP(S)_PROXY with NODE_USE_ENV_PROXY=1 (read at startup by node's fetch).
-  const proxy = process.env.EVAL_PROXY_URL || process.env.HTTPS_PROXY || "";
-  console.error(`egress: ${proxy ? "proxy " + redactProxy(proxy) : "direct (local network)"}`);
-
-  const good = await loadList("corpus/good.txt");
-  const bad = live ? await loadList("corpus/phishing-live.txt") : await loadBad(refresh, phishingOnly);
-  console.error(`corpus: ${good.length} good, ${bad.length} bad (${live ? "curated live" : phishingOnly ? "phishing feed" : "mixed feed"})`);
-
-  const labeled: Array<{ url: string; label: "good" | "bad" }> = [
-    ...good.map((url) => ({ url, label: "good" as const })),
-    ...bad.map((url) => ({ url, label: "bad" as const }))
-  ];
-
-  const results: SiteResult[] = [];
-  let done = 0;
-  await pool(labeled, SITE_CONCURRENCY, async ({ url, label }) => {
-    const result = await scanSite(url, label);
-    results.push(result);
-    done += 1;
-    if (done % 10 === 0) console.error(`  scanned ${done}/${labeled.length}`);
-  });
-
-  report(results);
-}
-
-async function scanSite(url: string, label: "good" | "bad"): Promise<SiteResult> {
-  try {
-    const reports = await crawlTargets([url], CRAWL);
-    const scored = reports.filter((r) => !r.error && r.report);
-    if (!scored.length) {
-      return { url, label, score: 0, disposition: "allow", pagesScanned: 0, topFindings: [], unreachable: true };
-    }
-    const worst = scored.reduce((a, b) => (b.report.score > a.report.score ? b : a));
-    const score = worst.report.score;
-    const topFindings = [...worst.report.findings]
-      .sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
-      .slice(0, 3)
-      .map((f) => ({ ruleId: f.ruleId, score: f.score ?? 0 }));
-    return { url, label, score, disposition: dispositionForScore(score), pagesScanned: scored.length, topFindings, unreachable: false };
-  } catch {
-    return { url, label, score: 0, disposition: "allow", pagesScanned: 0, topFindings: [], unreachable: true };
-  }
-}
-
-function report(results: SiteResult[]): void {
-  const reachable = results.filter((r) => !r.unreachable);
-  const good = reachable.filter((r) => r.label === "good");
-  const bad = reachable.filter((r) => r.label === "bad");
-  const flagged = (r: SiteResult) => r.score >= FLAG_THRESHOLD;
-
-  const fp = good.filter(flagged); // good, flagged => false positive
-  const tn = good.filter((r) => !flagged(r));
-  const tp = bad.filter(flagged); // bad, flagged => caught
-  const fn = bad.filter((r) => !flagged(r)); // bad, missed
-
-  const pct = (n: number, d: number) => (d ? `${((100 * n) / d).toFixed(1)}%` : "n/a");
-
-  const proxy = process.env.EVAL_PROXY_URL || process.env.HTTPS_PROXY || "";
-  console.log("\n================ SCANNER EVAL ================");
-  console.log(`egress: ${proxy ? "proxy " + redactProxy(proxy) : "direct (local network)"}`);
-  console.log(`unreachable (excluded): ${results.filter((r) => r.unreachable).length} / ${results.length}`);
-  console.log(`\nGood sites: ${good.length} reachable`);
-  console.log(`  flagged (FALSE POSITIVE): ${fp.length}  [${pct(fp.length, good.length)}]`);
-  console.log(`  clean (true negative):    ${tn.length}`);
-  console.log(`\nBad sites: ${bad.length} reachable`);
-  console.log(`  flagged (caught):         ${tp.length}  [recall ${pct(tp.length, bad.length)}]`);
-  console.log(`  missed (false negative):  ${fn.length}`);
-
-  console.log("\nScore distribution (count by band):");
-  console.log(`  band        good   bad`);
-  for (const [lo, hi] of [[0, 9], [10, 24], [25, 49], [50, 74], [75, 100]]) {
-    const g = good.filter((r) => r.score >= lo && r.score <= hi).length;
-    const b = bad.filter((r) => r.score >= lo && r.score <= hi).length;
-    const mark = lo >= FLAG_THRESHOLD ? " <-flag" : "";
-    console.log(`  ${String(lo).padStart(3)}-${String(hi).padEnd(3)}   ${String(g).padStart(5)} ${String(b).padStart(5)}${mark}`);
-  }
-  console.log(`  good: median ${median(good.map((r) => r.score))}, p90 ${percentile(good.map((r) => r.score), 90)}`);
-  console.log(`  bad:  median ${median(bad.map((r) => r.score))}, p90 ${percentile(bad.map((r) => r.score), 90)}`);
-
-  if (fp.length) {
-    console.log("\nFALSE POSITIVES (good sites flagged) — fix these:");
-    for (const r of fp.sort((a, b) => b.score - a.score)) {
-      console.log(`  [${r.score}] ${r.url}  ${r.topFindings.map((f) => `${f.ruleId}(${f.score})`).join(", ")}`);
-    }
-  }
-  if (fn.length) {
-    console.log("\nMISSED bad sites (score < flag threshold):");
-    for (const r of fn.sort((a, b) => b.score - a.score).slice(0, 25)) {
-      console.log(`  [${r.score}] ${r.url}  ${r.topFindings.map((f) => `${f.ruleId}(${f.score})`).join(", ") || "(no signal)"}`);
-    }
-    if (fn.length > 25) console.log(`  ... and ${fn.length - 25} more`);
-  }
-
-  const fpRate = good.length ? fp.length / good.length : 0;
-  const pass = fpRate <= MAX_FP_RATE;
-  console.log(`\nGATE: false-positive rate ${pct(fp.length, good.length)} (max ${MAX_FP_RATE * 100}%) => ${pass ? "PASS" : "FAIL"}`);
-  console.log("=============================================\n");
-  if (!pass) process.exitCode = 1;
-}
-
-// ---- known-bad corpus (live) --------------------------------------------
-
-async function loadBad(refresh: boolean, phishingOnly: boolean): Promise<string[]> {
-  const cachePath = phishingOnly ? PHISHING_CACHE_PATH : CACHE_PATH;
-  if (!refresh) {
-    const cached = await readCacheIfFresh(cachePath);
-    if (cached) {
-      console.error(`using cached bad list (${cached.length} urls)`);
-      return cached;
-    }
-  }
-  console.error(`pulling live bad URLs (${phishingOnly ? "phishing-only" : "mixed"}) ...`);
-  const candidates = shuffle(dedupe(await fetchBadCandidates(phishingOnly)));
-  console.error(`  ${candidates.length} candidates; probing reachability ...`);
-  const live = await probeReachable(candidates, TARGET_BAD);
-  await writeFile(cachePath, `# pulled ${new Date().toISOString()}\n${live.join("\n")}\n`, "utf8");
-  return live;
-}
-
-async function readCacheIfFresh(cachePath: string): Promise<string[] | null> {
-  try {
-    const text = await readFile(cachePath, "utf8");
-    const stamp = text.match(/# pulled (.+)/)?.[1];
-    if (!stamp || Date.now() - Date.parse(stamp) > CACHE_TTL_MS) return null;
-    const urls = parseList(text);
-    return urls.length ? urls : null;
-  } catch {
-    return null;
-  }
-}
-
-async function fetchBadCandidates(phishingOnly: boolean): Promise<string[]> {
-  const urls: string[] = [];
-  // OpenPhish community feed (public, ~hundreds of fresh phishing URLs).
-  try {
-    const res = await fetch("https://openphish.com/feed.txt", { signal: AbortSignal.timeout(15000) });
-    if (res.ok) urls.push(...parseList(await res.text()));
-  } catch (error) {
-    console.error("  openphish fetch failed:", error instanceof Error ? error.message : error);
-  }
-  if (phishingOnly) {
-    // Phishing.Database active links (public, large list of currently-active
-    // phishing URLs) — sampled, no auth.
-    try {
-      const res = await fetch("https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE.txt", { signal: AbortSignal.timeout(30000) });
-      if (res.ok) urls.push(...parseList(await res.text()).filter((u) => u.startsWith("http")).slice(0, 4000));
-    } catch (error) {
-      console.error("  phishing.database fetch failed:", error instanceof Error ? error.message : error);
-    }
-    return urls;
-  }
-  // URLhaus online URLs (malware distribution). Auth-Key used if present.
-  try {
-    const headers: Record<string, string> = {};
-    if (process.env.ABUSE_CH_AUTH_KEY) headers["Auth-Key"] = process.env.ABUSE_CH_AUTH_KEY;
-    const res = await fetch("https://urlhaus.abuse.ch/downloads/csv_online/", { headers, signal: AbortSignal.timeout(20000) });
-    if (res.ok) {
-      for (const line of (await res.text()).split("\n")) {
-        if (line.startsWith("#") || !line.trim()) continue;
-        const fields = line.split('","').map((f) => f.replace(/^"|"$/g, ""));
-        if (fields[3] === "online" && fields[2]?.startsWith("http")) urls.push(fields[2]);
-      }
-    }
-  } catch (error) {
-    console.error("  urlhaus fetch failed:", error instanceof Error ? error.message : error);
-  }
-  return urls;
-}
-
-async function probeReachable(candidates: string[], target: number): Promise<string[]> {
-  const live: string[] = [];
-  let i = 0;
-  await pool(candidates, 12, async (url) => {
-    if (live.length >= target) return;
-    try {
-      // A live phishing kit serves real content (200) at the URL itself. A
-      // taken-down one 404s or 301/302s to a park/block page — exclude those
-      // (status !== 200) so a dead corpus doesn't dilute recall. No body-size
-      // gate: a single <script> tag can be a complete phishing page.
-      const res = await fetch(url, { headers: { "user-agent": BROWSER_UA }, redirect: "manual", signal: AbortSignal.timeout(8000) });
-      if (res.status === 200 && live.length < target) live.push(url);
-    } catch {
-      // dead/unreachable — skip
-    }
-    i += 1;
-  });
-  return live.slice(0, target);
-}
-
-// ---- helpers -------------------------------------------------------------
-
-async function loadList(path: string): Promise<string[]> {
-  return parseList(await readFile(resolve(path), "utf8"));
-}
-function parseList(text: string): string[] {
-  return text.split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
-}
-function dedupe(values: string[]): string[] {
-  return [...new Set(values)];
-}
-function redactProxy(url: string): string {
-  try {
-    const u = new URL(url);
-    return `${u.hostname}:${u.port}`;
-  } catch {
-    return "set";
-  }
-}
-function shuffle<T>(values: T[]): T[] {
-  // Index-based jitter (no Math.random dependency needed for a rough mix).
-  return values
-    .map((v, i) => ({ v, k: (i * 2654435761) % values.length }))
-    .sort((a, b) => a.k - b.k)
-    .map((x) => x.v);
-}
-function median(values: number[]): number {
-  if (!values.length) return 0;
-  const s = [...values].sort((a, b) => a - b);
-  return s[Math.floor(s.length / 2)];
-}
-function percentile(values: number[], p: number): number {
-  if (!values.length) return 0;
-  const s = [...values].sort((a, b) => a - b);
-  return s[Math.min(s.length - 1, Math.floor((p / 100) * s.length))];
-}
-async function pool<T>(items: T[], concurrency: number, worker: (item: T) => Promise<void>): Promise<void> {
-  let index = 0;
-  const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
-    while (index < items.length) {
-      const item = items[index++];
-      await worker(item);
-    }
-  });
-  await Promise.all(runners);
-}
-
-main().catch((error) => {
-  console.error(error);
-  process.exit(1);
-});

package/scripts/render-isolate/entry.ts

@@ -1,2 +0,0 @@
-import { renderDom } from "../../src/render";
-(globalThis as any).__renderAndScan = renderDom;

package/scripts/render-isolate/polyfills.ts

@@ -1,33 +0,0 @@
-// Web globals a bare V8 isolate lacks, installed BEFORE linkedom/render load.
-// (self/window are set by the host via context.eval before this bundle runs.)
-import "fast-text-encoding";
-import base64 from "base-64";
-import { URL, URLSearchParams } from "whatwg-url-without-unicode";
-const g = globalThis as any;
-if (!g.atob) g.atob = (s: string) => base64.decode(String(s));
-if (!g.btoa) g.btoa = (s: string) => base64.encode(String(s));
-if (!g.URL) g.URL = URL;
-if (!g.URLSearchParams) g.URLSearchParams = URLSearchParams;
-
-// Minimal Buffer shim (linkedom's entity decoder + a few runtime paths use it).
-// Built on the globals above; covers from(str|base64|bytes) + toString(enc).
-if (!g.Buffer) {
-  const toBinary = (bytes: Uint8Array): string => {
-    let out = "";
-    for (let i = 0; i < bytes.length; i += 8192) out += String.fromCharCode.apply(null, bytes.subarray(i, i + 8192) as unknown as number[]);
-    return out;
-  };
-  g.Buffer = {
-    from(input: unknown, enc?: string): Uint8Array & { toString: (e?: string) => string } {
-      let bytes: Uint8Array;
-      if (input instanceof Uint8Array) bytes = input;
-      else if (enc === "base64") bytes = Uint8Array.from(g.atob(String(input)), (c: string) => c.charCodeAt(0));
-      else bytes = new TextEncoder().encode(String(input));
-      const view = bytes as Uint8Array & { toString: (e?: string) => string };
-      view.toString = (e?: string) => (e === "binary" || e === "latin1" ? toBinary(bytes) : e === "base64" ? g.btoa(toBinary(bytes)) : new TextDecoder().decode(bytes));
-      return view;
-    },
-    alloc: (n: number) => new Uint8Array(n),
-    isBuffer: (x: unknown) => x instanceof Uint8Array
-  };
-}

package/scripts/render-isolate/run.ts

@@ -1,63 +0,0 @@
-// CLI executor for renderDom: runs the render bundle inside a real isolated-vm
-// isolate. True isolation — the page's untrusted JS gets web-global polyfills and
-// a floor-dropping fetch, and cannot reach the host's fetch/process/fs. A fresh
-// context per page prevents cross-page contamination; the heavy bundles are
-// compiled once.
-import { resolve } from "node:path";
-import ivm from "isolated-vm";
-import { build } from "esbuild";
-import type { RenderInput, RenderResult } from "../../src/render";
-
-const HERE = import.meta.dirname;
-const PUNYCODE = resolve(HERE, "../../node_modules/punycode/punycode.es6.js");
-const CALL_TIMEOUT_MS = 6000;
-
-let ready: Promise<{ isolate: ivm.Isolate; poly: ivm.Script; render: ivm.Script }> | null = null;
-
-async function bundleOnce(entry: string): Promise<string> {
-  const result = await build({
-    entryPoints: [resolve(HERE, entry)],
-    bundle: true,
-    format: "iife",
-    platform: "node",
-    treeShaking: false, // keep polyfill side-effects (sideEffects:false would drop them)
-    alias: { punycode: PUNYCODE },
-    write: false
-  });
-  return result.outputFiles[0].text;
-}
-
-async function init() {
-  if (!ready) {
-    ready = (async () => {
-      const [polyCode, renderCode] = await Promise.all([bundleOnce("polyfills.ts"), bundleOnce("entry.ts")]);
-      const isolate = new ivm.Isolate({ memoryLimit: 256 });
-      const poly = await isolate.compileScript(polyCode);
-      const render = await isolate.compileScript(renderCode);
-      return { isolate, poly, render };
-    })();
-  }
-  return ready;
-}
-
-export async function renderInIsolate(input: RenderInput): Promise<RenderResult> {
-  const { isolate, poly, render } = await init();
-  const context = await isolate.createContext();
-  try {
-    await context.global.set("globalThis", context.global.derefInto());
-    // self/window must exist before the polyfill bundle (fast-text-encoding
-    // detects them), and polyfills must run before the render bundle (linkedom's
-    // entity decoder reads atob/Buffer at module init).
-    await context.eval("globalThis.self = globalThis; globalThis.window = globalThis;");
-    await poly.run(context);
-    await render.run(context);
-    const out = await context.evalClosure(
-      "return JSON.stringify(globalThis.__renderAndScan($0))",
-      [input],
-      { arguments: { copy: true }, result: { copy: true }, timeout: CALL_TIMEOUT_MS }
-    );
-    return JSON.parse(out as string) as RenderResult;
-  } finally {
-    context.release();
-  }
-}