@gajae-code/coding-agent 0.7.0 → 0.7.2

package/vendor/insane-search/engine/templates/playwright_real_chrome.js

@@ -0,0 +1,243 @@
+#!/usr/bin/env node
+/**
+ * Generic Playwright fetcher — real Chrome channel (not bundled Chromium).
+ *
+ * Usage (driven by engine/executor.py):
+ *   echo '{"url":"...", "profileDir":"/tmp/.p", "waitSelector":"article"}' | node playwright_real_chrome.js
+ *
+ * Outputs page HTML to stdout on success; errors to stderr with non-zero exit.
+ *
+ * NO-SITE-NAME RULE: this file must never branch on specific hostnames.
+ * All site specifics come from the JSON input (url, waitSelector).
+ *
+ * Dependencies (install once on target machine):
+ *   npm i -g playwright playwright-extra puppeteer-extra-plugin-stealth
+ *   npx playwright install chrome    # system Chrome binary
+ */
+
+const dns = require('dns').promises;
+const net = require('net');
+
+// Drain stdout fully before exiting. `process.exit()` can truncate a large
+// HTML payload because it does not wait for pending stdout I/O (Node docs).
+function writeStdoutAsync(payload) {
+  return new Promise((resolve, reject) => {
+    process.stdout.write(payload, (err) => (err ? reject(err) : resolve()));
+  });
+}
+
+// Structured envelope so the Python side can (a) validate on real status /
+// final URL and (b) bridge the browser-cleared cookies + UA into curl_cffi.
+async function buildEnvelope(ctx, page, html, resp, automation) {
+  let cookies = [];
+  try { cookies = (await ctx.cookies()).map((c) => ({ name: c.name, value: c.value, domain: c.domain })); } catch (_e) {}
+  let userAgent = '';
+  try { userAgent = await page.evaluate(() => navigator.userAgent); } catch (_e) {}
+  let finalUrl = '';
+  try { finalUrl = page.url(); } catch (_e) {}
+  let status = 0;
+  try { status = resp ? resp.status() : 0; } catch (_e) {}
+  return JSON.stringify({ html, finalUrl, status, cookies, userAgent, automation });
+}
+
+
+class UnsafeUrlError extends Error {
+  constructor(reason) {
+    super(`unsafe_url:${reason}`);
+    this.name = 'UnsafeUrlError';
+  }
+}
+
+function isBlockedHostname(hostname) {
+  const h = (hostname || '').toLowerCase().replace(/\.$/, '');
+  return !h || h === 'localhost' || h.endsWith('.localhost') || h.endsWith('.local') || h.endsWith('.internal') || h.endsWith('.home.arpa');
+}
+
+function isPrivateIPv4(address) {
+  const parts = address.split('.').map((part) => Number.parseInt(part, 10));
+  if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return true;
+  const [a, b] = parts;
+  return a === 0 || a === 10 || a === 127 || (a === 100 && b >= 64 && b <= 127) ||
+    (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) ||
+    (a === 192 && (b === 0 || b === 168)) || (a === 198 && (b === 18 || b === 19 || b === 51)) ||
+    (a === 203 && b === 0) || a >= 224;
+}
+
+function normalizeIPv4MappedIPv6(address) {
+  const lower = address.toLowerCase();
+  return lower.startsWith('::ffff:') ? lower.slice(7) : lower;
+}
+
+function isPrivateIPv6(address) {
+  const lower = address.toLowerCase();
+  const mapped = normalizeIPv4MappedIPv6(lower);
+  if (mapped !== lower && net.isIP(mapped) === 4) return isPrivateIPv4(mapped);
+  return lower === '::' || lower === '::1' || lower.startsWith('fc') || lower.startsWith('fd') ||
+    lower.startsWith('fe8') || lower.startsWith('fe9') || lower.startsWith('fea') || lower.startsWith('feb') ||
+    lower.startsWith('ff') || lower.startsWith('2001:db8') || lower.startsWith('::ffff:');
+}
+
+function isPrivateOrSpecialAddress(address) {
+  const normalized = normalizeIPv4MappedIPv6(address);
+  const family = net.isIP(normalized);
+  if (family === 4) return isPrivateIPv4(normalized);
+  if (family === 6) return isPrivateIPv6(normalized);
+  if (net.isIP(address) === 6) return isPrivateIPv6(address);
+  return true;
+}
+
+async function assertPublicHttpUrl(rawUrl) {
+  let parsed;
+  try { parsed = new URL(rawUrl); } catch (_e) { throw new UnsafeUrlError('invalid_url'); }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') throw new UnsafeUrlError(`scheme:${parsed.protocol || 'none'}`);
+  if (parsed.username || parsed.password) throw new UnsafeUrlError('credentials');
+  if (isBlockedHostname(parsed.hostname)) throw new UnsafeUrlError('internal_host');
+  if (net.isIP(parsed.hostname)) {
+    if (isPrivateOrSpecialAddress(parsed.hostname)) throw new UnsafeUrlError(`ip_blocked:${parsed.hostname}`);
+    return;
+  }
+  let records;
+  try { records = await dns.lookup(parsed.hostname, { all: true, verbatim: true }); }
+  catch (_e) { throw new UnsafeUrlError('resolve_failed'); }
+  if (!records.length) throw new UnsafeUrlError('resolve_empty');
+  const blocked = records.find((record) => isPrivateOrSpecialAddress(record.address));
+  if (blocked) throw new UnsafeUrlError(`resolves_internal:${parsed.hostname}->${blocked.address}`);
+}
+
+async function assertPagePublic(page, label) {
+  let current = '';
+  try { current = page.url(); } catch (_e) {}
+  await assertPublicHttpUrl(current);
+  return current;
+}
+
+async function readStdinJson() {
+  return await new Promise((resolve, reject) => {
+    let data = '';
+    process.stdin.on('data', (c) => (data += c));
+    process.stdin.on('end', () => {
+      try { resolve(JSON.parse(data || '{}')); }
+      catch (e) { reject(e); }
+    });
+    process.stdin.on('error', reject);
+  });
+}
+
+async function main() {
+  const args = await readStdinJson();
+  const url = args.url;
+  if (!url) { process.stderr.write('missing url\n'); process.exit(2); }
+  await assertPublicHttpUrl(url);
+
+  const profileDir = args.profileDir || '/tmp/.insane_pw_profile';
+  const waitSelector = args.waitSelector || null;
+  const timeoutMs = args.timeout || 60000;
+  const headless = args.headless ?? false;     // Akamai/etc detect headless
+  const viewport = args.viewport || { width: 1366, height: 900 };
+
+  let chromium;
+  let automation = 'playwright';
+  try {
+    // Patchright is a DROP-IN Playwright fork (same API) that closes the CDP
+    // Runtime.enable leak Cloudflare/DataDome now detect. Preferred when
+    // installed; it does its own patching, so NO stealth plugin is added.
+    // Additive only: if patchright is absent we fall back to exactly the
+    // previous playwright-extra(+stealth) → playwright behaviour.
+    ({ chromium } = require('patchright'));
+    automation = 'patchright';
+  } catch (_e0) {
+    try {
+      ({ chromium } = require('playwright-extra'));
+      const stealth = require('puppeteer-extra-plugin-stealth')();
+      chromium.use(stealth);
+      automation = 'playwright-extra+stealth';
+    } catch (_e) {
+      // Fallback to plain playwright (no stealth). Still uses channel:chrome.
+      ({ chromium } = require('playwright'));
+      automation = 'playwright';
+    }
+  }
+
+  let ctx;
+  try {
+    // Patchright official best practice: channel:'chrome', headless:false,
+    // no_viewport (JS: viewport:null), persistent context, and NO custom
+    // headers/UA/flags. We only override viewport for patchright; plain
+    // playwright keeps the fixed viewport it has always used.
+    const ctxOpts = { channel: 'chrome', headless };
+    if (automation === 'patchright') {
+      ctxOpts.viewport = null;     // == no_viewport=True (use real window size)
+    } else {
+      ctxOpts.viewport = viewport;
+    }
+    ctx = await chromium.launchPersistentContext(profileDir, ctxOpts);
+    const page = await ctx.newPage();
+    // Single shared deadline across warmup + main + reload navigations so the
+    // first nav can't eat the whole budget and starve the rest.
+    const deadline = Date.now() + timeoutMs;
+    const rem = (cap) => Math.max(1000, Math.min(cap || timeoutMs, deadline - Date.now()));
+
+    // Warmup hop: visit the site root first so Akamai-style bot managers
+    // can run their JS sensor and set a resolved session cookie. Direct
+    // landing on a search/deep URL is the classic first-hit rejection pattern.
+    // Use domcontentloaded (not networkidle) — many SPAs keep analytics/xhr
+    // open indefinitely and would hit the 90s timeout.
+    try {
+      const urlObj = new URL(url);
+      const rootUrl = `${urlObj.protocol}//${urlObj.host}/`;
+      if (rootUrl !== url) {
+        await assertPublicHttpUrl(rootUrl);
+        await page.goto(rootUrl, { waitUntil: 'domcontentloaded', timeout: rem(90000) });
+        await assertPagePublic(page, 'warmup');
+        await page.waitForTimeout(3500);   // let sensor JS finish
+      }
+    } catch (_e) {
+      if (_e && _e.name === 'UnsafeUrlError') throw _e;
+      // warmup is best-effort; continue even if it hiccups
+    }
+
+    // Main page — DOM loaded then give the sensor a moment.
+    let mainResp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: rem(90000) });
+    await assertPagePublic(page, 'main');
+    await page.waitForTimeout(2500);
+
+    if (waitSelector) {
+      try {
+        await page.waitForSelector(waitSelector, { timeout: rem(20000) });
+      } catch (_e) {
+        // Selector still missing — try one hard reload in case the first hit
+        // landed on a challenge page and the sensor has just cleared.
+        try {
+          mainResp = await page.reload({ waitUntil: 'domcontentloaded', timeout: rem(90000) });
+          await assertPagePublic(page, 'reload');
+          await page.waitForTimeout(2000);
+          try {
+            await page.waitForSelector(waitSelector, { timeout: rem(10000) });
+          } catch (_e2) {
+            // Still no luck — caller validates HTML anyway.
+          }
+        } catch (_e3) {
+          // reload failed — proceed with whatever we have
+        }
+      }
+    } else {
+      // Without a positive-proof selector, give the sensor a couple more seconds.
+      await page.waitForTimeout(2000);
+    }
+
+    await assertPagePublic(page, 'content');
+    const html = await page.content();
+    const payload = await buildEnvelope(ctx, page, html, mainResp, automation);
+    await writeStdoutAsync(payload);  // flush fully before any exit
+    process.exitCode = 0;
+    return;                           // let finally close ctx, then exit naturally
+  } catch (e) {
+    process.stderr.write(`${e.name || 'Error'}: ${e.message || e}\n`);
+    process.exitCode = 1;
+    return;
+  } finally {
+    try { if (ctx) await ctx.close(); } catch (_e) {}
+  }
+}
+
+main();

package/vendor/insane-search/engine/tests/test_hardening.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
+sys.path.insert(0, ROOT)
+
+from engine.fetch_chain import FetchResult  # noqa: E402
+from engine.transport import SessionPool  # noqa: E402
+
+
+class FetchResultJsonContractTest(unittest.TestCase):
+    def test_to_dict_omits_content_by_default_but_reports_length(self):
+        payload = FetchResult(ok=True, content="recovered").to_dict()
+        self.assertNotIn("content", payload)
+        self.assertEqual(payload["content_length"], len("recovered"))
+        self.assertFalse(payload["content_truncated"])
+
+    def test_to_dict_includes_bounded_content_for_cli_json(self):
+        payload = FetchResult(ok=True, content="abcdef").to_dict(include_content=True, content_limit=3)
+        self.assertEqual(payload["content"], "abc")
+        self.assertEqual(payload["content_length"], 6)
+        self.assertTrue(payload["content_truncated"])
+
+
+class RedirectSafetyTest(unittest.TestCase):
+    def test_transport_redirect_to_private_target_is_blocked(self):
+        class Resp:
+            status_code = 302
+            headers = {"Location": "http://127.0.0.1/private"}
+            text = ""
+            url = "https://public.example/redirect"
+
+        resp, err = SessionPool._fetch_following(lambda _url: Resp(), "https://public.example/redirect", False, 10, None)
+        self.assertIsNone(resp)
+        self.assertTrue(err.startswith("ssrf_redirect_blocked:"), err)
+
+    def test_playwright_templates_reject_private_initial_url_before_browser_launch(self):
+        template = os.path.join(ROOT, "engine", "templates", "playwright_real_chrome.js")
+        proc = subprocess.run(
+            ["node", template],
+            input='{"url":"http://127.0.0.1/private"}',
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        self.assertNotEqual(proc.returncode, 0)
+        self.assertIn("unsafe_url", proc.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/vendor/insane-search/engine/tests/test_smoke.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Smoke / regression test for the generic fetch chain.
+
+These tests hit real endpoints — mark as online / integration. They verify
+behaviour patterns, not content. No assertions on specific site brands.
+
+Run manually:
+    python3 engine/tests/test_smoke.py
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+
+# Allow running from anywhere.
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
+sys.path.insert(0, ROOT)
+
+from engine import fetch  # noqa: E402
+from engine.validators import validate, Verdict  # noqa: E402
+from engine.waf_detector import detect, _load_profiles  # noqa: E402
+from engine.url_transforms import iter_transformed  # noqa: E402
+
+
+# --- unit-level -------------------------------------------------------------
+def t_validator_tiny_body_is_challenge():
+    class R:
+        status_code = 200
+        text = "<html>short</html>"
+        headers = {}
+        cookies = type("C", (), {"jar": iter(())})()
+    vr = validate(R())
+    assert vr.verdict == Verdict.CHALLENGE, vr.verdict
+    assert any("tiny_body" in r for r in vr.reasons)
+    print("  ✓ tiny body → challenge")
+
+
+def t_validator_marker_is_challenge():
+    class R:
+        status_code = 200
+        text = "<html>" + ("x" * 5000) + " sec-if-cpt-container found </html>"
+        headers = {}
+        cookies = type("C", (), {"jar": iter(())})()
+    vr = validate(R())
+    assert vr.verdict == Verdict.CHALLENGE, vr.verdict
+    print("  ✓ challenge marker → challenge")
+
+
+def t_validator_weak_ok_without_selectors():
+    class R:
+        status_code = 200
+        text = "<html>" + ("x" * 5000) + "</html>"
+        headers = {}
+        cookies = type("C", (), {"jar": iter(())})()
+    vr = validate(R())
+    assert vr.verdict == Verdict.WEAK_OK, vr.verdict
+    print("  ✓ clean body w/o selectors → weak_ok")
+
+
+def t_validator_strong_ok_with_selectors():
+    class R:
+        status_code = 200
+        text = "<html><body>" + ("x" * 5000) + "<article>hello</article></body></html>"
+        headers = {}
+        cookies = type("C", (), {"jar": iter(())})()
+    vr = validate(R(), success_selectors=["article"])
+    assert vr.verdict == Verdict.STRONG_OK, vr.verdict
+    assert "article" in vr.matched_selectors
+    print("  ✓ selectors matched → strong_ok")
+
+
+def t_profiles_load():
+    p = _load_profiles()
+    for required in ("akamai_bot_manager", "cloudflare_turnstile", "unknown_challenge"):
+        assert required in p, f"missing profile: {required}"
+    print(f"  ✓ profiles loaded ({len(p)} keys)")
+
+
+def t_url_transforms():
+    # www → m
+    out = iter_transformed("https://www.example.com/a", ["original", "mobile_subdomain"])
+    urls = [u for _, u in out]
+    assert "https://www.example.com/a" in urls
+    assert "https://m.example.com/a" in urls, urls
+    # apex with am_prefix
+    out2 = iter_transformed("https://example.com/", ["original", "am_prefix"])
+    urls2 = [u for _, u in out2]
+    assert "https://m.example.com/" in urls2, urls2
+    print(f"  ✓ url_transforms produce expected forms")
+
+
+# --- online (network) -------------------------------------------------------
+def t_online_benign_site():
+    """A simple, usually-open site should pass probe directly when selectors provided."""
+    # example.com serves ~1.2KB content — below tiny_body threshold — but with
+    # success_selectors we trust caller's "content exists" definition.
+    r = fetch(
+        "https://example.com/",
+        success_selectors=["h1", "p"],
+        timeout=15,
+        max_attempts=3,
+        enable_playwright=False,
+    )
+    assert r.ok, f"{r.summary} | trace: {[a.verdict for a in r.trace]}"
+    assert r.verdict in ("strong_ok", "weak_ok"), r.verdict
+    print(f"  ✓ benign site → verdict={r.verdict} size={len(r.content)}")
+
+
+def t_online_trace_shape():
+    """Even on failure, trace should be populated and well-formed."""
+    r = fetch("https://httpbin.org/status/403", timeout=10, max_attempts=3, enable_playwright=False)
+    assert isinstance(r.trace, list) and len(r.trace) >= 1
+    for att in r.trace:
+        d = att.to_dict()
+        assert "phase" in d and "executor" in d and "verdict" in d
+    print(f"  ✓ httpbin 403 → trace_len={len(r.trace)} final={r.verdict}")
+
+
+ALL_TESTS = [
+    ("validator_tiny_body_is_challenge", t_validator_tiny_body_is_challenge),
+    ("validator_marker_is_challenge", t_validator_marker_is_challenge),
+    ("validator_weak_ok_without_selectors", t_validator_weak_ok_without_selectors),
+    ("validator_strong_ok_with_selectors", t_validator_strong_ok_with_selectors),
+    ("profiles_load", t_profiles_load),
+    ("url_transforms", t_url_transforms),
+    ("online_benign_site", t_online_benign_site),
+    ("online_trace_shape", t_online_trace_shape),
+]
+
+
+def main() -> int:
+    passed, failed = 0, 0
+    for name, fn in ALL_TESTS:
+        try:
+            print(f"[{name}]")
+            fn()
+            passed += 1
+        except AssertionError as e:
+            failed += 1
+            print(f"  ✗ FAIL: {e}")
+        except Exception as e:
+            failed += 1
+            print(f"  ✗ ERROR: {type(e).__name__}: {e}")
+    print(f"\n{passed} passed, {failed} failed")
+    return 0 if failed == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/vendor/insane-search/engine/tests/test_u1.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""U1 regression tests — validator v2 + diversity scheduler.
+
+Deterministic, network-free. Locks in the multi-AI-review fixes:
+  * grid diversity under a small cap (all TLS families + both transforms)
+  * avoid targets deprioritized, NOT deleted
+  * validator: small JSON ok, _abck-unresolved non-terminal, soft-marker
+    overridden by selector, status semantics.
+
+Run:  python3 engine/tests/test_u1.py
+"""
+from __future__ import annotations
+
+import os
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
+sys.path.insert(0, ROOT)
+
+from engine.validators import validate, Verdict  # noqa: E402
+from engine.waf_detector import _load_profiles  # noqa: E402
+from engine.fetch_chain import _build_plan, _family  # noqa: E402
+
+
+class _Ck:
+    def __init__(self, name, value):
+        self.name, self.value = name, value
+
+
+class _Jar:
+    def __init__(self, d):
+        self.jar = [_Ck(k, v) for k, v in d.items()]
+
+
+class _Resp:
+    def __init__(self, status=200, text="", headers=None, cookies=None):
+        self.status_code = status
+        self.text = text
+        self.headers = headers or {}
+        self.cookies = _Jar(cookies or {})
+
+
+class _Hit:
+    def __init__(self, pid):
+        self.profile_id = pid
+        self.confidence = 0.9
+        self.signals = []
+
+
+# ---------- scheduler ----------
+def t_scheduler_diversity_under_cap():
+    profiles = _load_profiles()
+    plan = _build_plan("https://www.example.com/p", [_Hit("akamai_bot_manager")],
+                       profiles, "auto", "safari", "self_root")
+    budget = 11  # max_attempts 12 - probe
+    head = plan[:budget]
+    fams = set(_family(c.impersonate) for c in head)
+    transforms = set(c.transform for c in head)
+    assert fams == {"safari", "safari_ios", "chrome", "chrome_android", "edge"}, fams
+    assert transforms == {"original", "mobile_subdomain"}, transforms
+    print(f"  ✓ first {budget} cover all families {sorted(fams)} + transforms {sorted(transforms)}")
+
+
+def t_scheduler_avoid_deprioritized_not_deleted():
+    profiles = _load_profiles()
+    plan = _build_plan("https://www.example.com/p", [_Hit("akamai_bot_manager")],
+                       profiles, "auto", "safari", "self_root")
+    imps = [c.impersonate for c in plan]
+    # chrome145/146 are in avoid; must still be present (exhaustive) but late.
+    assert "chrome145" in imps and "chrome146" in imps, "avoid targets were deleted!"
+    pos145 = min(i for i, x in enumerate(imps) if x == "chrome145")
+    early = imps[: len(imps) // 2]
+    assert "chrome145" not in early, "avoid target not deprioritized"
+    print(f"  ✓ avoid targets retained but late (chrome145 idx={pos145}/{len(imps)})")
+
+
+def t_scheduler_desktop_drops_mobile_transform():
+    profiles = _load_profiles()
+    plan = _build_plan("https://www.example.com/p", [_Hit("akamai_bot_manager")],
+                       profiles, "desktop", "safari", "self_root")
+    transforms = set(c.transform for c in plan)
+    fams = set(_family(c.impersonate) for c in plan)
+    assert "mobile_subdomain" not in transforms, transforms
+    assert "safari_ios" not in fams and "chrome_android" not in fams, fams
+    print(f"  ✓ desktop drops mobile transform & mobile TLS (transforms={sorted(transforms)})")
+
+
+# ---------- validator v2 ----------
+def t_validator_small_json_ok():
+    r = _Resp(200, '{"items":[{"id":1}],"total":1}', headers={"Content-Type": "application/json"})
+    v = validate(r)
+    assert v.verdict == Verdict.WEAK_OK, v.verdict
+    assert not (v.verdict == Verdict.CHALLENGE)
+    print(f"  ✓ small JSON → {v.verdict.value} (was challenge)")
+
+
+def t_validator_abck_unresolved_is_non_terminal():
+    r = _Resp(200, "<html>" + "x" * 5000 + "</html>", cookies={"_abck": "AA~-1~bb"})
+    v = validate(r)
+    assert v.verdict == Verdict.SUSPECT_OK, v.verdict
+    assert v.ok is False, "SUSPECT_OK must not count as terminal success"
+    print(f"  ✓ _abck unresolved → {v.verdict.value}, ok={v.ok} (was weak_ok/ok=True)")
+
+
+def t_validator_soft_marker_overridden_by_selector():
+    html = "<html><script>var s='captcha';</script><body>" + "x" * 5000 + "<main id='c'>real</main></body></html>"
+    v = validate(_Resp(200, html), success_selectors=["#c"])
+    assert v.verdict == Verdict.STRONG_OK, v.verdict
+    print(f"  ✓ 'captcha' word + matching selector → {v.verdict.value} (was challenge)")
+
+
+def t_validator_hard_marker_still_challenge():
+    v = validate(_Resp(200, "<html>" + "x" * 5000 + " sec-if-cpt-container </html>"))
+    assert v.verdict == Verdict.CHALLENGE, v.verdict
+    print(f"  ✓ hard marker still → {v.verdict.value}")
+
+
+def t_validator_status_semantics():
+    assert validate(_Resp(429, "slow down")).verdict == Verdict.RATE_LIMITED
+    assert validate(_Resp(401, "nope")).verdict == Verdict.AUTH_REQUIRED
+    assert validate(_Resp(404, "gone")).verdict == Verdict.NOT_FOUND
+    assert validate(_Resp(503, "later")).verdict == Verdict.BLOCKED
+    print("  ✓ status semantics 429/401/404/503 differentiated")
+
+
+def t_validator_byte_size_not_char_count():
+    # 1500 Korean chars = 1500 chars but 4500 bytes (>threshold) → not tiny.
+    body = "가" * 1500
+    v = validate(_Resp(200, body, headers={"Content-Type": "text/html"}))
+    # 4500 bytes ≥ 3000 → not tiny_body; no markers/selectors → weak_ok
+    assert v.body_size >= 3000, v.body_size
+    assert v.verdict == Verdict.WEAK_OK, (v.verdict, v.body_size)
+    print(f"  ✓ byte size counts UTF-8 bytes ({v.body_size}B from 1500 chars) → {v.verdict.value}")
+
+
+def t_validator_small_complete_page_is_weak_ok():
+    # example.com is a complete ~600B HTML document with real text — a small but
+    # genuine page must NOT be mislabelled a challenge stub (regression guard).
+    body = ('<!doctype html><html lang="en"><head><title>Example Domain</title>'
+            '</head><body><div><h1>Example Domain</h1><p>This domain is for use in '
+            'documentation examples without needing permission.</p>'
+            '<p><a href="https://iana.org/domains/example">Learn more</a></p>'
+            '</div></body></html>')
+    v = validate(_Resp(200, body, headers={"Content-Type": "text/html"}))
+    assert v.body_size < 3000, v.body_size
+    assert v.verdict == Verdict.WEAK_OK, (v.verdict, v.reasons)
+    print(f"  ✓ small complete page → {v.verdict.value} ({v.reasons})")
+
+
+def t_validator_small_script_stub_still_challenge():
+    # Script-only tiny body (no visible text) is still a suspicious stub.
+    body = '<html><head></head><body><script src="/cdn-cgi/challenge.js"></script></body></html>'
+    v = validate(_Resp(200, body, headers={"Content-Type": "text/html"}))
+    assert v.verdict == Verdict.CHALLENGE, (v.verdict, v.reasons)
+    print(f"  ✓ script-only tiny body → {v.verdict.value}")
+
+
+def t_validator_small_fragment_still_challenge():
+    # Incomplete fragment (no closing </html>/</body>) stays suspicious.
+    v = validate(_Resp(200, "<div>loading", headers={"Content-Type": "text/html"}))
+    assert v.verdict == Verdict.CHALLENGE, (v.verdict, v.reasons)
+    print(f"  ✓ incomplete fragment → {v.verdict.value}")
+
+
+ALL = [
+    ("scheduler_diversity_under_cap", t_scheduler_diversity_under_cap),
+    ("scheduler_avoid_deprioritized_not_deleted", t_scheduler_avoid_deprioritized_not_deleted),
+    ("scheduler_desktop_drops_mobile_transform", t_scheduler_desktop_drops_mobile_transform),
+    ("validator_small_json_ok", t_validator_small_json_ok),
+    ("validator_abck_unresolved_is_non_terminal", t_validator_abck_unresolved_is_non_terminal),
+    ("validator_soft_marker_overridden_by_selector", t_validator_soft_marker_overridden_by_selector),
+    ("validator_hard_marker_still_challenge", t_validator_hard_marker_still_challenge),
+    ("validator_status_semantics", t_validator_status_semantics),
+    ("validator_byte_size_not_char_count", t_validator_byte_size_not_char_count),
+    ("validator_small_complete_page_is_weak_ok", t_validator_small_complete_page_is_weak_ok),
+    ("validator_small_script_stub_still_challenge", t_validator_small_script_stub_still_challenge),
+    ("validator_small_fragment_still_challenge", t_validator_small_fragment_still_challenge),
+]
+
+
+def main() -> int:
+    p = f = 0
+    for name, fn in ALL:
+        try:
+            print(f"[{name}]")
+            fn()
+            p += 1
+        except AssertionError as e:
+            f += 1
+            print(f"  ✗ FAIL: {e}")
+        except Exception as e:
+            f += 1
+            print(f"  ✗ ERROR: {type(e).__name__}: {e}")
+    print(f"\n{p} passed, {f} failed")
+    return 0 if f == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())