lean-browser-use 0.0.0-stage → 0.1.0

package/reader/test_reader_2.py

@@ -0,0 +1,348 @@
+from reader_test_support import *
+
+@checks.mark.parametrize("chunked", [False, True])
+def test_proxy_http_frames_body_and_drops_pipelined_requests(monkeypatch, chunked):
+    monkeypatch.setattr(socket, "socket", REAL_SOCKET)
+    monkeypatch.setattr(socket, "getaddrinfo", REAL_GETADDRINFO)
+    received = []
+    body = b"4\r\ndata\r\n0\r\n\r\n" if chunked else b"data"
+
+    async def origin(source, client):
+        head = await source.readuntil(b"\r\n\r\n")
+        data = await source.readexactly(len(body))
+        try:
+            extra = await asyncio.wait_for(source.read(1), 0.05)
+        except asyncio.TimeoutError:
+            extra = b""
+        received.append((head, data, extra))
+        client.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
+        await client.drain()
+        client.close()
+        await client.wait_closed()
+
+    async def check():
+        server = await asyncio.start_server(origin, "127.0.0.1", 0)
+        async with server, helper["FilteringProxy"]() as proxy:
+            port = server.sockets[0].getsockname()[1]
+            url = f"http://127.0.0.1:{port}"
+            monkeypatch.setenv("READER_TEST_ALLOW_ORIGIN", url)
+            source, client = await asyncio.open_connection("127.0.0.1", proxy.port)
+            framing = "Transfer-Encoding: chunked" if chunked else "Content-Length: 4"
+            client.write((f"POST {url}/page?key=value HTTP/1.1\r\nHost: invalid.example\r\n"
+                          f"Proxy-Authorization: discard\r\nConnection: keep-alive\r\n{framing}\r\n\r\n").encode()
+                         + body + b"GET http://127.0.0.1:9/denied HTTP/1.1\r\n\r\n")
+            await client.drain()
+            assert (await source.read()).endswith(b"ok")
+            assert not proxy.blocked
+            head, data, extra = received[0]
+            assert head.startswith(b"POST /page?key=value HTTP/1.1\r\n")
+            assert f"Host: 127.0.0.1:{port}\r\n".encode() in head
+            assert b"Connection: close\r\n" in head
+            assert b"Proxy-Authorization" not in head
+            assert data == body and extra == b""
+            client.close()
+            await client.wait_closed()
+
+    asyncio.run(asyncio.wait_for(check(), 10))
+
+
+def test_proxy_connect_tunnels_bytes_unchanged(monkeypatch):
+    monkeypatch.setattr(socket, "socket", REAL_SOCKET)
+    received = []
+    addresses = []
+    connect = helper["pinned_connection"]
+    payload = b"\x16\x03\x01\x00\x10" + bytes(range(256)) + b"\r\n\r\n"
+    response = b"\x16\x03\x03" + payload[::-1]
+
+    def resolve(host, port, **kwargs):
+        assert host == "public.example"
+        return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (PUBLIC_IP, port))]
+
+    async def mapped_connection(address):
+        addresses.append(address)
+        # Map a checked public fixture address to the local byte-echo server.
+        assert address[4][0] == PUBLIC_IP
+        return await connect((*address[:4], ("127.0.0.1", address[4][1])))
+
+    monkeypatch.setattr(socket, "getaddrinfo", resolve)
+    monkeypatch.setitem(helper["FilteringProxy"].handle.__globals__, "pinned_connection", mapped_connection)
+
+    async def echo(source, client):
+        received.append(await source.read())
+        client.write(response)
+        await client.drain()
+        client.close()
+        await client.wait_closed()
+
+    async def check():
+        server = await asyncio.start_server(echo, "127.0.0.1", 0)
+        async with server, helper["FilteringProxy"]() as proxy:
+            port = server.sockets[0].getsockname()[1]
+            source, client = await asyncio.open_connection("127.0.0.1", proxy.port)
+            client.write(f"CONNECT public.example:{port} HTTP/1.1\r\n\r\n".encode())
+            await client.drain()
+            assert await source.readuntil(b"\r\n\r\n") == b"HTTP/1.1 200 Connection Established\r\n\r\n"
+            client.write(payload)
+            await client.drain()
+            client.write_eof()
+            assert await source.read() == response
+            assert received == [payload]
+            assert len(addresses) == 1 and not proxy.blocked
+            client.close()
+            await client.wait_closed()
+
+    asyncio.run(asyncio.wait_for(check(), 10))
+
+
+def test_browser_fetch_local_origin_and_script_rendering(local_http):
+    origin, _, pages, requests = local_http
+    pages["/page"] = (200, {}, '<p id="value">before</p><p>A public page with enough visible text for crawl4ai.</p>'
+                      '<script>document.querySelector("p").textContent="after"</script>')
+    page, method = reader.fetch(origin + "/page", preferred="crwl")
+    assert '<p id="value">after</p>' in page and method == "crwl"
+    assert (origin, "/page") in requests
+    assert all(host == origin for host, path in requests)
+
+
+@checks.mark.parametrize("destination", ["other-port", ".".join(['169', '254', '169', '254'])])
+@checks.mark.parametrize("kind", ["redirect", "image", "iframe", "websocket"])
+def test_browser_blocks_private_redirects_and_subresources(local_http, destination, kind):
+    origin, denied_origin, pages, requests = local_http
+    target = (denied_origin if destination == "other-port" else "http://" + ".".join(["169", "254", "169", "254"])) + "/denied"
+    if kind == "redirect":
+        pages["/page"] = (302, {"Location": target}, "")
+    elif kind == "websocket":
+        pages["/page"] = (200, {}, f'<p>allowed</p><script>new WebSocket("{target.replace("http:", "ws:")}")</script>')
+    else:
+        element = f'<img src="{target}">' if kind == "image" else f'<iframe src="{target}"></iframe>'
+        pages["/page"] = (200, {}, '<p>allowed</p>' + element)
+    with checks.raises(reader.ReaderError) as exc:
+        reader.fetch(origin + "/page", preferred="crwl")
+    assert exc.value.code == 10
+    assert "non-public" in str(exc.value)
+    assert requests == [(origin, "/page")]
+
+
+@checks.mark.parametrize("status", [301, 302, 303, 307, 308])
+def test_browser_follows_same_host_redirects(local_http, status):
+    origin, _, pages, requests = local_http
+    pages["/page"] = (status, {"Location": "/step"}, "")
+    pages["/step"] = (status, {"Location": origin + "/final"}, "")
+    pages["/final"] = (200, {}, '<p>Final page after the checked redirect chain.</p>'
+                       '<p>This public fixture contains enough visible text for crawl4ai.</p>')
+    page, method = reader.fetch(origin + "/page", preferred="crwl")
+    assert "Final page" in page and method == "crwl"
+    assert requests == [(origin, path) for path in ("/page", "/step", "/final")]
+
+
+def test_browser_follows_public_host_redirect_with_stub_resolver(local_http, tmp_path):
+    origin, _, pages, requests = local_http
+    port = origin.rsplit(":", 1)[1]
+    target = f"http://public.example:{port}/final"
+    pages["/page"] = (302, {"Location": target}, "")
+    pages["/final"] = (200, {}, '<p>Public host redirect followed through the filtering proxy.</p>')
+    # Stub only DNS and the physical connection to the public fixture address.
+    # The real proxy and public-address check run unchanged in the browser child.
+    script = f'''import runpy, socket
+helper = runpy.run_path({str(ROOT / "crawl_helper.py")!r})
+PUBLIC_IP = {PUBLIC_IP!r}
+resolve = socket.getaddrinfo
+def fixture_dns(host, port, *args, **kwargs):
+    if host == "public.example":
+        return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (PUBLIC_IP, port))]
+    assert host in ("127.0.0.1", "localhost"), host
+    return resolve(host, port, *args, **kwargs)
+socket.getaddrinfo = fixture_dns
+connect = helper["pinned_connection"]
+async def fixture_connection(address):
+    if address[4][0] == PUBLIC_IP:
+        address = (*address[:4], ("127.0.0.1", address[4][1]))
+    return await connect(address)
+helper["FilteringProxy"].handle.__globals__["pinned_connection"] = fixture_connection
+raise SystemExit(helper["main"]())
+'''
+    result = subprocess.run([reader.crawl4ai_python(), "-c", script, "fetch"],
+                            input=json.dumps({"url": origin + "/page"}), capture_output=True,
+                            text=True, timeout=45, env=reader.crwl_env(tmp_path))
+    assert result.returncode == 0, result.stderr[-2000:]
+    page = json.loads(result.stdout)
+    assert "Public host redirect followed" in page["html"]
+    assert page["redirected_url"] == target
+    assert requests == [(origin, "/page"), (origin, "/final")]
+
+
+def test_browser_follows_same_host_subresource_redirects(local_http):
+    origin, _, pages, requests = local_http
+    pages["/page"] = (200, {}, '<p id="value">before</p><script src="/redirect"></script>'
+                      '<p>This public fixture contains enough visible text for crawl4ai.</p>')
+    pages["/redirect"] = (302, {"Location": "/script"}, "")
+    pages["/script"] = (200, {"Content-Type": "application/javascript"},
+                        'document.querySelector("p").textContent="loaded";')
+    page, _ = reader.fetch(origin + "/page", preferred="crwl")
+    assert '<p id="value">loaded</p>' in page
+    assert (origin, "/script") in requests
+
+
+@checks.mark.parametrize("navigation", [False, True])
+def test_browser_checks_script_navigation_destination(local_http, navigation):
+    origin, _, pages, requests = local_http
+    target = origin if navigation else origin.replace("127.0.0.1", "localhost")
+    pages["/page"] = (200, {}, f'<script>location.replace("{target}/final")</script>')
+    pages["/final"] = (200, {}, '<p>Final page after the checked script navigation.</p>'
+                       '<p>This public fixture contains enough visible text for crawl4ai.</p>')
+    if navigation:
+        page, _ = reader.fetch(origin + "/page", preferred="crwl")
+        assert "Final page" in page
+        assert (origin, "/final") in requests
+    else:
+        with checks.raises(reader.ReaderError, match="non-public") as exc:
+            reader.fetch(origin + "/page", preferred="crwl")
+        assert exc.value.code == 10
+        assert requests == [(origin, "/page")]
+
+
+@checks.mark.parametrize("kind", ["redirect", "image", "iframe"])
+def test_browser_stops_private_later_redirect_hop(local_http, kind):
+    origin, denied_origin, pages, requests = local_http
+    element = '<img src="/redirect">' if kind == "image" else '<iframe src="/redirect"></iframe>'
+    pages["/page"] = ((302, {"Location": "/redirect"}, "") if kind == "redirect" else
+                      (200, {}, element))
+    pages["/redirect"] = (302, {"Location": denied_origin + "/denied"}, "")
+    with checks.raises(reader.ReaderError, match="non-public") as exc:
+        reader.fetch(origin + "/page", preferred="crwl")
+    assert exc.value.code == 10
+    assert (origin, "/redirect") in requests
+    assert all(path != "/denied" for host, path in requests)
+
+
+def test_test_origin_override_is_exact_and_opt_in(local_http, monkeypatch):
+    origin, denied_origin, _, _ = local_http
+    reader.checked_address(origin + "/allowed")
+    for target in (origin.replace("127.0.0.1", "localhost"), origin.replace("http:", "https:"),
+                   denied_origin, "http://" + ".".join(["169", "254", "169", "254"])):
+        with checks.raises(reader.ReaderError) as exc:
+            reader.checked_address(target)
+        assert exc.value.code == 2
+    monkeypatch.delenv("READER_TEST_ALLOW_ORIGIN")
+    with checks.raises(reader.ReaderError) as exc:
+        reader.checked_address(origin)
+    assert exc.value.code == 2
+
+
+def test_css_hostile_html_extracts_without_browser_or_requests(local_http, monkeypatch):
+    origin, _, _, requests = local_http
+    # Browser startup fails immediately if the CSS helper tries to launch it.
+    monkeypatch.setenv("PLAYWRIGHT_BROWSERS_PATH", "/reader-no-browser-for-css")
+    spec = {"type": "css", "schema": {"name": "sample", "baseSelector": "p",
+            "fields": [{"name": "value", "selector": "span", "type": "text"}]}}
+    page = (f'<p><span>extracted</span></p><script>fetch("{origin}/script")</script>'
+            f'<img src="{origin}/image"><img src="http://127.0.0.1:9/">')
+    assert reader.extract(spec, page) == [{"value": "extracted"}]
+    assert requests == []
+
+
+@checks.mark.parametrize("wrapper", ["shebang", "env", "exec", "missing"])
+def test_crawl4ai_python_resolution(monkeypatch, tmp_path, wrapper):
+    monkeypatch.delenv("READER_CRAWL4AI_PYTHON", raising=False)
+    executable = tmp_path / "crwl"
+    python = str(tmp_path / "python with spaces") if wrapper == "exec" else sys.executable
+    if wrapper == "shebang":
+        executable.write_text(f"#!{python}\n")
+    elif wrapper == "env":
+        executable.write_text("#!/usr/bin/env python3\n")
+        python = reader.shutil.which("python3")
+    elif wrapper == "exec":
+        executable.write_text(f'''#!/bin/sh\n\x27\x27\x27exec\x27 "{python}" "$0" "$@"\n\x27 \x27\x27\x27\n''')
+    else:
+        python = str(ROOT / ".venv/bin/python")
+    monkeypatch.setattr(reader, "crwl_binary", lambda: str(executable))
+    assert reader.crawl4ai_python() == python
+    monkeypatch.setenv("READER_CRAWL4AI_PYTHON", "/custom/python")
+    assert reader.crawl4ai_python() == "/custom/python"
+
+
+def test_compile_embedded_miss_then_css(monkeypatch):
+    monkeypatch.setenv("STUB_MODE", "pricing")
+    page = '<script type="application/ld+json">{"@type":"BreadcrumbList"}</script><table></table>'
+    monkeypatch.setattr(reader, "extract", lambda spec, html: [{"model": "Claude", "input": 2, "output": 10}])
+    spec, _ = reader.compile_spec("prices", "https://public.example", {}, "model prices", None, page)
+    assert spec["type"] == "css"
+    assert len(calls()) == 2
+    assert "embedded_json" in calls()[0]
+    assert "html" in calls()[1]
+
+
+def test_params_encoding_and_usage():
+    assert reader.render_url("https://public.example/{name}", {"name": "@scope/a?x=y"}) == "https://public.example/%40scope%2Fa%3Fx%3Dy"
+    assert reader.parse_params(["name=react", "name=zod", "x=a=b"]) == {"name": "zod", "x": "a=b"}
+    for operation in (lambda: reader.render_url("https://public.example/{missing}", {}),
+                      lambda: reader.parse_params(["bad"]), lambda: reader.spec_path("../secret")):
+        with checks.raises(reader.ReaderError) as exc:
+            operation()
+        assert exc.value.code == 2
+
+
+def test_cli_compile_run_ls(monkeypatch, capsys):
+    monkeypatch.setattr(reader, "fetch", lambda url, **kwargs: (html("zod" if url.endswith("zod") else "react"), "crwl"))
+    assert reader.main(["compile", "npm-package", "--url", "https://www.npmjs.com/package/{name}",
+                        "--want", "package fields", "--param", "name=react"]) == 0
+    assert json.loads(capsys.readouterr().out)[0]["name"] == "react"
+    assert reader.load("npm-package")["fetch_method"] == "crwl"
+    assert reader.main(["run", "npm-package", "--param", "name=zod"]) == 0
+    output = json.loads(capsys.readouterr().out)
+    assert output["rows"][0]["name"] == "zod"
+    assert output["meta"] == {"fetch_method": "crwl"}
+    assert reader.main(["ls"]) == 0
+    listing = capsys.readouterr().out
+    assert all(key in listing for key in ("npm-package", "embedded_json", "compiled_at", "last_ok", "healed_at"))
+
+
+def test_compile_without_example_warns_once_across_refinements(monkeypatch, capsys):
+    monkeypatch.setenv("STUB_MODE", "refine")
+    monkeypatch.setattr(reader, "fetch", lambda url, **kwargs: (html(), "urllib"))
+    assert reader.main(["compile", "npm-package", "--url", "https://public.example", "--want", "package fields"]) == 0
+    captured = capsys.readouterr()
+    assert json.loads(captured.out)[0]["name"] == "react"
+    assert len(calls()) == 2
+    assert captured.err.splitlines() == [
+        "reader: warning: wrong-but-plausible values cannot be detected without an example row; "
+        "use --example to anchor the compile"]
+
+
+@checks.mark.parametrize("url", ["file:///etc/passwd", "http://127.0.0.1"])
+def test_cli_compile_and_run_refuse_targets(url):
+    env = {**os.environ, "READER_COMPILE_CMD": "reader-must-not-be-called"}
+    compiled = subprocess.run([sys.executable, str(ROOT / "reader"), "compile", "bad", "--url", url,
+                               "--want", "name"], env=env, capture_output=True, text=True)
+    assert compiled.returncode == 2 and compiled.stdout == ""
+    spec, _ = compile_npm()
+    spec["url_template"] = url
+    reader.save(spec)
+    run = subprocess.run([sys.executable, str(ROOT / "reader"), "run", "npm-package"],
+                         env=env, capture_output=True, text=True)
+    assert run.returncode == 2 and run.stdout == ""
+
+
+def test_cli_invalid_example_and_corrupt_spec(capsys):
+    assert reader.main(["compile", "bad", "--url", "https://public.example", "--want", "name", "--example", "42"]) == 2
+    path = reader.spec_path("corrupt")
+    path.parent.mkdir(parents=True)
+    path.write_text("not JSON")
+    assert reader.main(["run", "corrupt"]) == 2
+    assert "cannot load spec" in capsys.readouterr().err
+
+
+def test_candidate_rejects_computed_code_and_required_omissions():
+    candidate = json.loads((DATA / "pricing-schema.json").read_text())
+    candidate["schema"]["fields"][0]["expression"] = "__import__('os')"
+    with checks.raises(ValueError, match="computed"):
+        reader.validate_candidate(candidate, None)
+    candidate = json.loads((DATA / "npm-schema.json").read_text())
+    candidate["required"] = ["missing"]
+    with checks.raises(ValueError, match="required"):
+        reader.validate_candidate(candidate, None)
+
+
+
+def load_tests(loader, suite, pattern):
+    return checks.suite(globals())

package/reader/test_reader_3.py

@@ -0,0 +1,254 @@
+from reader_test_support import *
+
+@checks.mark.parametrize("transform", [[], "unknown", {"regex": "("}, {"regex": "(x)", "group": 3},
+                                        {"regex": 42}, {"number": "yes"}, {"group": 1}])
+def test_candidate_rejects_malformed_transforms(transform):
+    candidate = json.loads((DATA / "npm-schema.json").read_text())
+    candidate["transforms"] = {"name": transform}
+    with checks.raises(ValueError):
+        reader.validate_candidate(candidate, None)
+
+
+def test_heal_preserves_required_fields_without_example(monkeypatch):
+    spec, _ = compile_npm()
+    spec["example"] = None
+    spec["paths"]["license"] = "broken"
+    reader.save(spec)
+    monkeypatch.setattr(reader, "fetch", lambda url, **kwargs: (html(), "urllib"))
+    candidate = {"type": "embedded_json", "source": "__context__", "rows": "context",
+                 "paths": {"name": "packument.name"}, "required": ["name"]}
+    compiler = mock.Mock(return_value=subprocess.CompletedProcess([], 0, json.dumps(candidate), ""))
+    monkeypatch.setattr(reader.subprocess, "run", compiler)
+    with checks.raises(reader.ReaderError) as exc:
+        reader.run_spec(spec, {}, False)
+    assert exc.value.code == 12
+    assert compiler.call_count == 4
+    assert reader.load("npm-package")["required"] == spec["required"]
+
+
+@checks.mark.parametrize("command", ["", " ", "'unclosed"])
+def test_invalid_compile_command_exits_11(monkeypatch, command):
+    monkeypatch.setenv("READER_COMPILE_CMD", command)
+    with checks.raises(reader.ReaderError) as exc:
+        compile_npm()
+    assert exc.value.code == 11
+
+
+def test_stored_spec_requires_object():
+    path = reader.spec_path("array")
+    path.parent.mkdir(parents=True)
+    path.write_text("[]")
+    with checks.raises(reader.ReaderError) as exc:
+        reader.load("array")
+    assert exc.value.code == 2
+
+
+@checks.mark.parametrize("key,value", [("example", "bad"), ("example", []), ("example", [42]),
+                                       ("params", []), ("want", None), ("fetch_method", "unknown")])
+def test_stored_spec_validates_metadata(key, value):
+    spec, _ = compile_npm()
+    spec[key] = value
+    reader.save(spec)
+    with checks.raises(reader.ReaderError) as exc:
+        reader.load("npm-package")
+    assert exc.value.code == 2
+
+
+def test_other_compile_command_keeps_stdin(monkeypatch):
+    command = mock.Mock(return_value=subprocess.CompletedProcess([], 0, "{}", ""))
+    monkeypatch.setattr(reader.subprocess, "run", command)
+    reader.invoke_compile("custom-compiler --flag", "é" * 60000)
+    assert command.call_args.args[0] == ["custom-compiler", "--flag"]
+    assert command.call_args.kwargs["input"] == "é" * 60000
+
+
+@checks.mark.parametrize("output, expected", [
+    ('lane ready\n{"paths":{"x":"value with } and \\\" quotes"},"required":["x"]}\ndone',
+     {"paths": {"x": 'value with } and " quotes'}, "required": ["x"]}),
+    ('status {"ignore":true}\n```json\n{"type":"css"}\n```\ndone', {"type": "css"}),
+    ('prefix {not JSON}\n{"need_html":true}\n{"later":true}', {"need_html": True}),
+])
+def test_schema_json_from_mixed_stdout(output, expected):
+    assert reader.json_object(output) == expected
+
+
+@checks.mark.parametrize("status", [401, 403, 429])
+def test_blocked_http_falls_back_to_crwl(monkeypatch, crwl_fetch_stub, status):
+    body = io.BytesIO(b"denied")
+    urllib = mock.Mock(side_effect=HTTPError("https://www.npmjs.com/package/react", status, "blocked", {}, body))
+    monkeypatch.setattr(reader, "fetch_urllib", urllib)
+    page, method = reader.fetch("https://www.npmjs.com/package/react")
+    assert page == html() and method == "crwl"
+    assert body.closed
+    assert len(crwl_fetch_stub.read_text().splitlines()) == 1
+
+
+@checks.mark.parametrize("page", ['<title>Just a moment...</title>',
+                                  '<title>Attention Required! | Cloudflare</title>',
+                                  '<script src="/cdn-cgi/challenge-platform/h/g/orchestrate/chl_page/v1"></script>',
+                                  '<div>Please enable JS and disable any ad blocker</div>'])
+def test_bot_wall_falls_back_to_crwl(monkeypatch, crwl_fetch_stub, page):
+    monkeypatch.setattr(reader, "fetch_urllib", lambda url: page)
+    assert reader.fetch("https://www.npmjs.com/package/zod") == (html("zod"), "crwl")
+
+
+def test_cloudflare_detection_script_preserves_valid_page(monkeypatch):
+    page = html() + '<script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script>'
+    monkeypatch.setattr(reader, "fetch_urllib", lambda url: page)
+    crwl = mock.Mock(side_effect=AssertionError("valid page needs no browser retry"))
+    monkeypatch.setattr(reader, "fetch_crwl", crwl)
+    assert reader.fetch("https://www.npmjs.com/package/react") == (page, "urllib")
+    crwl.assert_not_called()
+
+
+def test_blocked_compile_run_metadata_and_crwl_preference(monkeypatch, crwl_fetch_stub, capsys):
+    urllib = mock.Mock(side_effect=HTTPError("https://www.npmjs.com/package/react", 403, "blocked", {}, None))
+    monkeypatch.setattr(reader, "fetch_urllib", urllib)
+    assert reader.main(["compile", "npm-package", "--url", "https://www.npmjs.com/package/{name}",
+                        "--want", "package fields", "--param", "name=react"]) == 0
+    capsys.readouterr()
+    assert reader.load("npm-package")["fetch_method"] == "crwl"
+    assert urllib.call_count == 1 and len(calls()) == 1
+    for package in ("react", "zod"):
+        assert reader.main(["run", "npm-package", "--param", "name=" + package]) == 0
+        output = json.loads(capsys.readouterr().out)
+        assert output["meta"] == {"fetch_method": "crwl"}
+        assert output["rows"][0]["name"] == package
+        assert reader.failures(reader.load("npm-package"), output["rows"]) == []
+    assert urllib.call_count == 1 and len(calls()) == 1
+    assert len(crwl_fetch_stub.read_text().splitlines()) == 3
+
+
+def test_missing_crwl_fetch_is_exit_10_and_skips_compile(monkeypatch, capsys):
+    monkeypatch.setattr(reader, "fetch_urllib", mock.Mock(side_effect=HTTPError("https://public.example", 403, "blocked", {}, None)))
+    monkeypatch.setattr(reader, "checked_address", lambda url: public_address())
+    monkeypatch.setenv("READER_CRAWL4AI_PYTHON", "reader-missing-crawl-python")
+    assert reader.main(["compile", "blocked", "--url", "https://public.example", "--want", "fields"]) == 10
+    output = capsys.readouterr()
+    assert output.out == "" and "crwl" in output.err
+    assert calls() == []
+
+
+@checks.mark.parametrize("status", [404, 500])
+def test_other_http_errors_skip_crwl(monkeypatch, status):
+    monkeypatch.setattr(reader, "fetch_urllib", mock.Mock(side_effect=HTTPError("https://public.example", status, "error", {}, None)))
+    crwl = mock.Mock(side_effect=AssertionError("no browser retry"))
+    monkeypatch.setattr(reader, "fetch_crwl", crwl)
+    with checks.raises(reader.ReaderError) as exc:
+        reader.fetch("https://public.example")
+    assert exc.value.code == 10
+    crwl.assert_not_called()
+
+
+@checks.mark.parametrize("result", [
+    {"html": "<p>ok</p>", "success": False},
+    {"html": "<p>denied</p>", "status_code": 403},
+    {"html": "<title>Just a moment...</title>"}, {}, {"html": ""},
+    {"html": "x" * (reader.FETCH_CAP + 1)},
+])
+def test_crwl_bad_pages_exit_10(monkeypatch, result):
+    monkeypatch.setattr(reader, "checked_address", lambda url: public_address())
+    monkeypatch.setattr(reader.subprocess, "run", mock.Mock(return_value=subprocess.CompletedProcess([], 0, json.dumps(result), "")))
+    with checks.raises(reader.ReaderError) as exc:
+        reader.fetch("https://public.example", preferred="crwl")
+    assert exc.value.code == 10 and "crwl" in str(exc.value)
+
+
+@checks.mark.parametrize("failure", [subprocess.CompletedProcess([], 1, "", ""),
+                                    subprocess.CompletedProcess([], 0, "invalid JSON", ""),
+                                    subprocess.TimeoutExpired("crwl", 90)])
+def test_crwl_command_failures_exit_10(monkeypatch, failure):
+    monkeypatch.setattr(reader, "checked_address", lambda url: public_address())
+    command = mock.Mock(side_effect=failure) if isinstance(failure, Exception) else mock.Mock(return_value=failure)
+    monkeypatch.setattr(reader.subprocess, "run", command)
+    with checks.raises(reader.ReaderError) as exc:
+        reader.fetch("https://public.example", preferred="crwl")
+    assert exc.value.code == 10 and "crwl" in str(exc.value)
+
+
+@checks.mark.parametrize("url", ["file:///etc/passwd", "http://127.0.0.1"])
+def test_crwl_first_refuses_targets_before_subprocess(monkeypatch, url):
+    socket.getaddrinfo.side_effect = None
+    socket.getaddrinfo.return_value = [public_address("127.0.0.1")]
+    command = mock.Mock(side_effect=AssertionError("refused target cannot reach crwl"))
+    monkeypatch.setattr(reader.subprocess, "run", command)
+    with checks.raises(reader.ReaderError) as exc:
+        reader.fetch(url, preferred="crwl")
+    assert exc.value.code == 2
+    command.assert_not_called()
+
+
+def test_crwl_reported_private_redirect_is_refused(monkeypatch):
+    socket.getaddrinfo.side_effect = None
+    socket.getaddrinfo.side_effect = [[public_address()], [public_address("127.0.0.1")]]
+    result = {"html": "<p>private</p>", "redirected_url": "http://127.0.0.1"}
+    monkeypatch.setattr(reader.subprocess, "run", mock.Mock(return_value=subprocess.CompletedProcess([], 0, json.dumps(result), "")))
+    with checks.raises(reader.ReaderError) as exc:
+        reader.fetch("https://public.example", preferred="crwl")
+    assert exc.value.code == 2
+
+
+def test_legacy_spec_uses_urllib_then_remembers_crwl(monkeypatch):
+    spec, _ = compile_npm()
+    spec.pop("fetch_method")
+    reader.save(spec)
+    fetch = mock.Mock(return_value=(html(), "crwl"))
+    monkeypatch.setattr(reader, "fetch", fetch)
+    reader.run_spec(reader.load("npm-package"), {}, False)
+    fetch.assert_called_once_with("https://www.npmjs.com/package/react", preferred="urllib")
+    assert reader.load("npm-package")["fetch_method"] == "crwl"
+
+
+def test_bundled_npm_spec_crwl_runs(monkeypatch):
+    spec = json.loads((ROOT / "specs/npm-package.json").read_text())
+    reader.save(spec)
+    def live_shape(url):
+        package = url.rsplit("/", 1)[-1]
+        embedded, _ = reader.page_data(html(package))
+        context = embedded["__context__"]["context"]
+        context["packument"].update(
+            version=context["packument"]["dist-tags"]["latest"],
+            license=context["packageVersion"]["license"],
+            repository=context["packageVersion"]["repository"]["url"])
+        context["downloads"] = [{"downloads": context["downloads"]["weeklyDownloads"]}]
+        return '<script>window.__context__ = ' + json.dumps(embedded["__context__"]) + ';</script>'
+
+    crwl = mock.Mock(side_effect=live_shape)
+    monkeypatch.setattr(reader, "fetch_crwl", crwl)
+    monkeypatch.setattr(reader, "fetch_urllib", mock.Mock(side_effect=AssertionError("bundled spec prefers crwl")))
+    for package in ("react", "zod"):
+        result = reader.run_spec(reader.load("npm-package"), {"name": package}, False)
+        assert result["rows"][0]["name"] == package
+        assert result["rows"][0]["license"] == "MIT"
+        assert type(result["rows"][0]["weekly_downloads"]) is int
+        assert reader.failures(spec, result["rows"]) == []
+        assert result["meta"] == {"fetch_method": "crwl"}
+    assert calls() == []
+    assert crwl.call_count == 2
+
+
+@checks.mark.parametrize("package_license", ["MIT", "4.6.7"])
+def test_bundled_npm_spec_captured_zod_context(monkeypatch, package_license):
+    # Public data excerpt from crwl's live zod HTML, captured 2026-10-23.
+    context = {
+        "packument": {"name": "zod", "version": "4.6.7", "license": "MIT",
+                      "repository": "https://github.com/colinhacks/zod"},
+        "packageVersion": {"name": "zod", "version": "4.6.7", "license": package_license},
+        "downloads": [{"downloads": 216094796, "label": "2026-10-09 to 2026-10-15"},
+                      {"downloads": 144621339, "label": "2026-10-16 to 2026-10-22"}],
+    }
+    page = '<script>window.__context__ = ' + json.dumps({"context": context}) + ';</script>'
+    spec = json.loads((ROOT / "specs/npm-package.json").read_text())
+    reader.save(spec)
+    monkeypatch.setattr(reader, "fetch", lambda url, **kwargs: (page, "crwl"))
+    output = reader.run_spec(reader.load("npm-package"), {"name": "zod"}, False)
+    assert output["rows"] == [{"name": "zod", "latest_version": "4.6.7",
+                               "weekly_downloads": 144621339, "license": "MIT",
+                               "repository_url": "https://github.com/colinhacks/zod"}]
+    assert type(output["rows"][0]["weekly_downloads"]) is int
+    assert calls() == []
+
+
+
+def load_tests(loader, suite, pattern):
+    return checks.suite(globals())

package/reader/test_reader_contract.py

@@ -0,0 +1,57 @@
+"""Compile-command usage and bundled-spec offline contracts."""
+from reader_test_support import *
+
+
+def test_unset_compile_command_exits_two(monkeypatch, capsys):
+    monkeypatch.delenv('READER_COMPILE_CMD')
+    assert reader.main(['compile', 'sample', '--url', 'https://example.com', '--want', 'title']) == 2
+    message = capsys.readouterr().err
+    assert 'READER_COMPILE_CMD' in message
+    assert "export READER_COMPILE_CMD='claude -p'" in message
+    socket.getaddrinfo.assert_not_called()
+
+
+def test_unset_command_self_heal_and_valid_run(monkeypatch):
+    spec, _ = compile_npm()
+    monkeypatch.delenv('READER_COMPILE_CMD')
+    monkeypatch.setattr(reader, 'fetch', lambda url, **kwargs: (html(), 'urllib'))
+    assert reader.run_spec(spec, {}, False)['rows'][0]['name'] == 'react'
+    spec['paths']['license'] = 'absent'
+    with checks.raises(reader.ReaderError) as error:
+        reader.run_spec(spec, {}, False)
+    assert error.value.code == 2
+
+
+def test_bundled_pricing_spec_offline_contract(monkeypatch):
+    spec = json.loads((ROOT / 'specs/claude-pricing.json').read_text())
+    page = (DATA / 'claude-pricing.html').read_text()
+    # A stdlib HTML parser checks the captured rows before mocking the CSS engine.
+    class Table(reader.HTMLParser):
+        def __init__(self):
+            super().__init__()
+            self.rows, self.cells, self.cell, self.retired = [], [], None, False
+        def handle_starttag(self, tag, attrs):
+            if tag == 'tr': self.cells, self.retired = [], False
+            if tag == 'td': self.cell = ''
+            if tag == 'button' and 'Retired' in dict(attrs).get('aria-label', ''): self.retired = True
+        def handle_data(self, data):
+            if self.cell is not None: self.cell += data
+        def handle_endtag(self, tag):
+            if tag == 'td': self.cells.append(self.cell); self.cell = None
+            if tag == 'tr' and len(self.cells) == len(spec['schema']['fields']) and not self.retired:
+                self.rows.append(dict(zip([f['name'] for f in spec['schema']['fields']], self.cells)))
+    parser = Table(); parser.feed(page)
+    def helper(operation, payload, code):
+        assert operation == 'css'
+        assert payload['schema'] == spec['schema']
+        assert payload['html'] == page
+        return {'rows': parser.rows}
+    monkeypatch.setattr(reader, 'crawl_helper', helper)
+    rows = reader.extract(spec, page)
+    assert reader.failures(spec, rows) == []
+    assert len(rows) == len(spec['sample_output'])
+    assert {row['model'] for row in rows} == {row['model'] for row in spec['sample_output']}
+
+
+def load_tests(loader, suite, pattern):
+    return checks.suite(globals())