lean-browser-use 0.0.0-stage → 0.1.0

package/reader/reader_test_harness.py

@@ -0,0 +1,173 @@
+"""Small unittest fixture adapter for the retained regression functions."""
+from contextlib import ExitStack, redirect_stdout, redirect_stderr
+import inspect
+import io
+import itertools
+import os
+from pathlib import Path
+import re
+import socket
+import sys
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest import mock
+
+fixtures = {}
+
+
+def fixture(fn=None, autouse=False):
+    def decorate(function):
+        fixtures[function.__name__] = (function, autouse)
+        return function
+    return decorate(fn) if fn else decorate
+
+
+class Mark:
+    def parametrize(self, names, values):
+        def decorate(function):
+            existing = getattr(function, '_parameters', [])
+            function._parameters = existing + [(names.replace(' ', '').split(','), values)]
+            return function
+        return decorate
+
+
+mark = Mark()
+skip = lambda reason: (_ for _ in ()).throw(unittest.SkipTest(reason))
+
+
+class raises:
+    def __init__(self, exception, match=None):
+        self.exception, self.match = exception, match
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, kind, value, tb):
+        if kind is None:
+            raise AssertionError('Expected exception')
+        if not isinstance(value, self.exception):
+            return False
+        if self.match and not re.search(self.match, str(value)):
+            raise AssertionError('Exception text did not match')
+        self.value = value
+        return True
+
+
+class MonkeyPatch:
+    def __init__(self, stack):
+        self.stack = stack
+
+    def setenv(self, name, value):
+        self.stack.enter_context(mock.patch.dict(os.environ, {name: str(value)}))
+
+    def delenv(self, name, raising=True):
+        if raising and name not in os.environ:
+            raise KeyError(name)
+        self.stack.enter_context(mock.patch.dict(os.environ))
+        os.environ.pop(name, None)
+
+    def setattr(self, obj, name, value):
+        original = getattr(obj, name)
+        # The command facade re-exports functions from focused reader modules.
+        # Patch their imported bindings too, keeping the original injection tests.
+        if getattr(obj, '__name__', '') == 'page_reader':
+            for module_name, module in list(sys.modules.items()):
+                if module_name in ('reader_fetch', 'reader_crawl', 'reader_extract', 'reader_compile', 'reader_runtime') and getattr(module, name, object()) is original:
+                    self.stack.enter_context(mock.patch.object(module, name, value))
+        self.stack.enter_context(mock.patch.object(obj, name, value))
+
+    def setitem(self, mapping, key, value):
+        self.stack.enter_context(mock.patch.dict(mapping, {key: value}))
+
+
+class Capture:
+    def __init__(self, stack):
+        self.out, self.err = io.StringIO(), io.StringIO()
+        stack.enter_context(redirect_stdout(self.out))
+        stack.enter_context(redirect_stderr(self.err))
+
+    def readouterr(self):
+        result = SimpleNamespace(out=self.out.getvalue(), err=self.err.getvalue())
+        for stream in (self.out, self.err):
+            stream.seek(0)
+            stream.truncate()
+        return result
+
+
+def crawler_available():
+    try:
+        import importlib.util
+        return importlib.util.find_spec('crawl4ai') is not None
+    except (ImportError, ValueError):
+        return False
+
+
+def local_sockets_available():
+    try:
+        with socket.socket() as sock:
+            sock.bind(('127.0.0.1', 0))
+        return True
+    except PermissionError:
+        return False
+
+
+class RegressionCase(unittest.TestCase):
+    def __init__(self, function, parameters, label):
+        super().__init__('runTest')
+        self.function, self.parameters, self.label = function, parameters, label
+
+    def id(self):
+        return self.label
+
+    def __str__(self):
+        return self.label
+
+    def runTest(self):
+        name = self.function.__name__
+        needs_crawler = name.startswith('test_browser_') or name in (
+            'test_real_crawl4ai_pricing_snapshot', 'test_compile_css_refinement_carries_failing_rows',
+            'test_css_hostile_html_extracts_without_browser_or_requests')
+        if needs_crawler and not crawler_available():
+            self.skipTest('optional crawl4ai/browser dependency is absent; offline contracts still run')
+        needs_socket = name.startswith('test_proxy_') or 'local_http' in inspect.signature(self.function).parameters
+        if needs_socket and not local_sockets_available():
+            self.skipTest('sandbox blocks local sockets; public-address unit checks still run')
+        with ExitStack() as stack:
+            directory = stack.enter_context(tempfile.TemporaryDirectory(prefix='lbu-reader-test-'))
+            cache = {'tmp_path': Path(directory), 'monkeypatch': MonkeyPatch(stack), 'capsys': Capture(stack)}
+            cache.update(self.parameters)
+
+            def resolve(key):
+                if key in cache:
+                    return cache[key]
+                factory = fixtures[key][0]
+                value = factory(**{arg: resolve(arg) for arg in inspect.signature(factory).parameters})
+                if inspect.isgenerator(value):
+                    generator = value
+                    value = next(generator)
+                    stack.callback(lambda: next(generator, None))
+                cache[key] = value
+                return value
+
+            for key, (_, automatic) in fixtures.items():
+                if automatic:
+                    resolve(key)
+            self.function(**{arg: resolve(arg) for arg in inspect.signature(self.function).parameters})
+
+
+def suite(namespace):
+    cases = []
+    for name, function in namespace.items():
+        if not name.startswith('test_') or not callable(function):
+            continue
+        parameters = [{}]
+        for names, values in getattr(function, '_parameters', []):
+            expanded = []
+            for previous, value in itertools.product(parameters, values):
+                values_tuple = value if len(names) > 1 else (value,)
+                expanded.append({**previous, **dict(zip(names, values_tuple))})
+            parameters = expanded
+        for index, values in enumerate(parameters):
+            cases.append(RegressionCase(function, values, f'{name}[{index}]'))
+    return unittest.TestSuite(cases)

package/reader/reader_test_support.py

@@ -0,0 +1,214 @@
+"""Offline reader regression checks using the standard library."""
+
+
+import asyncio
+
+
+import importlib.machinery
+
+
+import importlib.util
+
+
+import hashlib
+
+
+import io
+
+
+import json
+
+
+import os
+
+
+from pathlib import Path
+
+
+import runpy
+
+
+import shlex
+
+
+import socket
+
+
+import subprocess
+
+
+import sys
+
+
+import threading
+
+
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+
+from unittest import mock
+
+
+from urllib.error import HTTPError
+
+
+from urllib.request import Request
+
+
+import reader_test_harness as checks
+
+
+ROOT = Path(__file__).resolve().parent
+
+
+DATA = ROOT / "testdata"
+
+
+REAL_SOCKET = socket.socket
+
+
+REAL_GETADDRINFO = socket.getaddrinfo
+
+
+loader = importlib.machinery.SourceFileLoader("page_reader", str(ROOT / "reader"))
+
+
+module_spec = importlib.util.spec_from_loader(loader.name, loader)
+
+
+reader = importlib.util.module_from_spec(module_spec)
+
+
+loader.exec_module(reader)
+
+
+helper = runpy.run_path(str(ROOT / "crawl_helper.py"))
+
+
+@checks.fixture(autouse=True)
+def offline(monkeypatch, tmp_path):
+    monkeypatch.setenv("READER_SPECS_DIR", str(tmp_path / "specs"))
+    monkeypatch.setenv("READER_STATE_DIR", str(tmp_path / "state"))
+    monkeypatch.setenv("READER_COMPILE_CMD", shlex.join([sys.executable, str(DATA / "compile_stub.py")]))
+    monkeypatch.setenv("STUB_CALL_LOG", str(tmp_path / "calls.jsonl"))
+    monkeypatch.setenv("STUB_MODE", "npm")
+    # Any accidental live network access fails the offline suite.
+    monkeypatch.setattr(socket, "getaddrinfo", mock.Mock(side_effect=AssertionError("offline DNS")))
+    monkeypatch.setattr(socket, "socket", mock.Mock(side_effect=AssertionError("offline socket")))
+
+
+def calls():
+    path = Path(os.environ["STUB_CALL_LOG"])
+    return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
+
+
+def html(package="react"):
+    return (DATA / f"npm-{package}.html").read_text()
+
+
+def compile_npm():
+    return reader.compile_spec("npm-package", "https://www.npmjs.com/package/{name}",
+                               {"name": "react"}, "package name, latest version, weekly downloads, license, repository URL",
+                               {"name": "react", "latest_version": "19.2.0", "weekly_downloads": 42000000,
+                                "license": "MIT", "repository_url": "git+https://github.com/facebook/react.git"}, html())
+
+
+PUBLIC_IP = ".".join(["93", "184", "216", "34"])
+
+
+def public_address(ip=PUBLIC_IP):
+    return (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (ip, 443))
+
+
+@checks.fixture
+def anchor_compile(monkeypatch, tmp_path):
+    schema = {"type": "embedded_json", "source": "__NEXT_DATA__", "rows": "models",
+              "paths": {"model": "model", "input": "input", "output": "output"},
+              "required": ["model", "input", "output"]}
+    command = tmp_path / "anchor_compile.py"
+    command.write_text('''import json, os, sys
+payload = json.loads(sys.stdin.read().split("INPUT JSON:\\n", 1)[1])
+with open(os.environ["STUB_CALL_LOG"], "a") as stream:
+    stream.write(json.dumps(payload) + "\\n")
+schema = json.loads(sys.argv[1])
+if not payload.get("validation_failures") or os.environ["STUB_MODE"] == "anchor-broken":
+    schema["transforms"] = {"model": {"regex": r"(Claude .* [0-9]+)"}}
+print(json.dumps(schema))
+''')
+    monkeypatch.setenv("READER_COMPILE_CMD", shlex.join([sys.executable, str(command), json.dumps(schema)]))
+    rows = [{"model": "Claude Fable 5.1", "input": 10, "output": 50},
+            {"model": "Claude Sonnet 5.5", "input": 2, "output": 10}]
+    page = '<script id="__NEXT_DATA__">' + json.dumps({"models": rows}) + '</script>'
+    monkeypatch.setattr(reader, "fetch", lambda url, **kwargs: (page, "urllib"))
+    return rows
+
+
+@checks.fixture
+def local_http(monkeypatch):
+    # Only these integration tests open sockets, all against local servers.
+    monkeypatch.setattr(socket, "socket", REAL_SOCKET)
+    monkeypatch.setattr(socket, "getaddrinfo", REAL_GETADDRINFO)
+    pages = {}
+    requests = []
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_GET(self):
+            requests.append((f"http://127.0.0.1:{self.server.server_address[1]}", self.path))
+            status, headers, body = pages.get(self.path, (200, {}, "<p>private</p>"))
+            self.send_response(status)
+            self.send_header("Content-Type", "text/html; charset=utf-8")
+            for key, value in headers.items():
+                self.send_header(key, value)
+            self.send_header("Content-Length", str(len(body.encode())))
+            self.end_headers()
+            self.wfile.write(body.encode())
+
+        def log_message(self, *args):
+            pass
+
+    allowed = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    port = allowed.server_address[1]
+    denied = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    servers = [allowed, denied]
+    threads = [threading.Thread(target=server.serve_forever, daemon=True) for server in servers]
+    for thread in threads:
+        thread.start()
+    origin = f"http://127.0.0.1:{port}"
+    denied_origin = f"http://127.0.0.1:{denied.server_address[1]}"
+    monkeypatch.setenv("READER_TEST_ALLOW_ORIGIN", origin)
+    monkeypatch.setenv("INSTABOX_HEADLESS", "1")
+    try:
+        yield origin, denied_origin, pages, requests
+    finally:
+        for server in servers:
+            server.shutdown()
+            server.server_close()
+        for thread in threads:
+            thread.join(timeout=5)
+
+
+@checks.fixture
+def crwl_fetch_stub(monkeypatch, tmp_path):
+    log = tmp_path / "crwl-calls.jsonl"
+    executable = tmp_path / "crawl-python"
+    executable.write_text(f'''#!{sys.executable}
+import json
+import os
+from pathlib import Path
+import sys
+assert sys.argv[2:] == ["fetch"]
+request = json.load(sys.stdin)
+with open(os.environ["CRWL_FETCH_LOG"], "a") as stream:
+    stream.write(json.dumps(request) + "\\n")
+package = request["url"].rsplit("/", 1)[-1]
+page = (Path({str(DATA)!r}) / ("npm-" + package + ".html")).read_text()
+print("crwl: browser ready")
+print(json.dumps({{"html": page, "success": True, "status_code": 200, "url": request["url"]}}))
+''')
+    executable.chmod(0o755)
+    monkeypatch.setenv("READER_CRAWL4AI_PYTHON", str(executable))
+    monkeypatch.setattr(reader, "checked_address", lambda url: public_address())
+    monkeypatch.setenv("CRWL_FETCH_LOG", str(log))
+    return log
+
+

package/reader/specs/claude-pricing.json

@@ -0,0 +1,223 @@
+{
+  "type": "css",
+  "schema": {
+    "name": "claude_pricing",
+    "baseSelector": "#content-container table:has(colgroup[span=\"3\"]) > tbody > tr:has(td[rowspan]):not(:has(button[aria-label*=\"Retired\"]))",
+    "fields": [
+      {
+        "name": "model",
+        "selector": "td:nth-child(1)",
+        "type": "text"
+      },
+      {
+        "name": "input",
+        "selector": "td:nth-child(2)",
+        "type": "text"
+      },
+      {
+        "name": "output",
+        "selector": "td:nth-child(3)",
+        "type": "text"
+      },
+      {
+        "name": "cache_write_5m",
+        "selector": "td:nth-child(4)",
+        "type": "text"
+      },
+      {
+        "name": "cache_write_1h",
+        "selector": "td:nth-child(5)",
+        "type": "text"
+      },
+      {
+        "name": "cache_read",
+        "selector": "td:nth-child(6)",
+        "type": "text"
+      }
+    ]
+  },
+  "required": [
+    "model",
+    "input",
+    "output"
+  ],
+  "transforms": {
+    "model": {
+      "regex": "(Claude\\s+\\w+\\s+\\d+(?:\\.\\d+)?)"
+    },
+    "input": {
+      "regex": "\\$([\\d.]+)",
+      "number": true
+    },
+    "output": {
+      "regex": "\\$([\\d.]+)",
+      "number": true
+    },
+    "cache_write_5m": {
+      "regex": "\\$([\\d.]+)",
+      "number": true
+    },
+    "cache_write_1h": {
+      "regex": "\\$([\\d.]+)",
+      "number": true
+    },
+    "cache_read": {
+      "regex": "\\$([\\d.]+)",
+      "number": true
+    }
+  },
+  "example": {
+    "model": "Claude Sonnet 5.5",
+    "input": 2,
+    "output": 10,
+    "cache_write_5m": 2.5,
+    "cache_write_1h": 4,
+    "cache_read": 0.1
+  },
+  "name": "claude-pricing",
+  "url_template": "https://platform.claude.com/docs/en/about-claude/pricing",
+  "params": {},
+  "want": "One row for each current Claude model with model name, numeric input and output USD per million tokens, cache write 5m and 1h and cache read prices where shown. Exclude retired models and use baseline context pricing.",
+  "sample_output": [
+    {
+      "model": "Claude Fable 5.1",
+      "input": 10.0,
+      "output": 50.0,
+      "cache_write_5m": 12.5,
+      "cache_write_1h": 20.0,
+      "cache_read": 0.25
+    },
+    {
+      "model": "Claude Opus 5.5",
+      "input": 4.0,
+      "output": 20.0,
+      "cache_write_5m": 5.0,
+      "cache_write_1h": 8.0,
+      "cache_read": 0.2
+    },
+    {
+      "model": "Claude Sonnet 5.5",
+      "input": 2.0,
+      "output": 10.0,
+      "cache_write_5m": 2.5,
+      "cache_write_1h": 4.0,
+      "cache_read": 0.1
+    },
+    {
+      "model": "Claude Haiku 5.5",
+      "input": 0.1,
+      "output": 0.5,
+      "cache_write_5m": 0.125,
+      "cache_write_1h": 0.2,
+      "cache_read": 0.01
+    },
+    {
+      "model": "Claude Mythos 5.1",
+      "input": 10.0,
+      "output": 50.0,
+      "cache_write_5m": 12.5,
+      "cache_write_1h": 20.0,
+      "cache_read": 0.25
+    },
+    {
+      "model": "Claude Fable 5",
+      "input": 10.0,
+      "output": 50.0,
+      "cache_write_5m": 12.5,
+      "cache_write_1h": 20.0,
+      "cache_read": 1.0
+    },
+    {
+      "model": "Claude Mythos 5",
+      "input": 10.0,
+      "output": 50.0,
+      "cache_write_5m": 12.5,
+      "cache_write_1h": 20.0,
+      "cache_read": 1.0
+    },
+    {
+      "model": "Claude Opus 5",
+      "input": 5.0,
+      "output": 25.0,
+      "cache_write_5m": 6.25,
+      "cache_write_1h": 10.0,
+      "cache_read": 0.5
+    },
+    {
+      "model": "Claude Opus 4.8",
+      "input": 5.0,
+      "output": 25.0,
+      "cache_write_5m": 6.25,
+      "cache_write_1h": 10.0,
+      "cache_read": 0.5
+    },
+    {
+      "model": "Claude Opus 4.7",
+      "input": 5.0,
+      "output": 25.0,
+      "cache_write_5m": 6.25,
+      "cache_write_1h": 10.0,
+      "cache_read": 0.5
+    },
+    {
+      "model": "Claude Opus 4.6",
+      "input": 5.0,
+      "output": 25.0,
+      "cache_write_5m": 6.25,
+      "cache_write_1h": 10.0,
+      "cache_read": 0.5
+    },
+    {
+      "model": "Claude Opus 4.5",
+      "input": 5.0,
+      "output": 25.0,
+      "cache_write_5m": 6.25,
+      "cache_write_1h": 10.0,
+      "cache_read": 0.5
+    },
+    {
+      "model": "Claude Sonnet 5",
+      "input": 2.0,
+      "output": 10.0,
+      "cache_write_5m": 2.5,
+      "cache_write_1h": 4.0,
+      "cache_read": 0.2
+    },
+    {
+      "model": "Claude Sonnet 4.6",
+      "input": 3.0,
+      "output": 15.0,
+      "cache_write_5m": 3.75,
+      "cache_write_1h": 6.0,
+      "cache_read": 0.3
+    },
+    {
+      "model": "Claude Sonnet 4.5",
+      "input": 3.0,
+      "output": 15.0,
+      "cache_write_5m": 3.75,
+      "cache_write_1h": 6.0,
+      "cache_read": 0.3
+    },
+    {
+      "model": "Claude Haiku 4.5",
+      "input": 1.0,
+      "output": 5.0,
+      "cache_write_5m": 1.25,
+      "cache_write_1h": 2.0,
+      "cache_read": 0.1
+    }
+  ],
+  "compiled_at": "2026-10-23T09:13:35+00:00",
+  "compiled_by": "python3 reader/testdata/compile_stub.py",
+  "crawl4ai_version": "0.9.2",
+  "page_hash": "580850369340da5cce9998a0b9f7f6c1dca3e15fc0280e596a0d1da301669a4c",
+  "last_ok": "2026-10-23T09:14:26+00:00",
+  "healed_at": null,
+  "source_discovery": {
+    "overview_url": "https://platform.claude.com/docs/en/models/overview.md",
+    "pricing_url": "https://platform.claude.com/docs/en/about-claude/pricing",
+    "checked_at": "2026-10-23",
+    "note": "Pricing link followed from official overview. Live public page compiled and validated with the documented compile-command fixture, not a live AI lane."
+  }
+}

package/reader/specs/npm-package.json

@@ -0,0 +1,67 @@
+{
+  "type": "embedded_json",
+  "source": "__context__",
+  "rows": "context",
+  "paths": {
+    "name": "packument.name",
+    "latest_version": "packument.version",
+    "weekly_downloads": "downloads.-1.downloads",
+    "license": "packument.license",
+    "repository_url": "packument.repository"
+  },
+  "required": [
+    "name",
+    "latest_version",
+    "weekly_downloads",
+    "license",
+    "repository_url"
+  ],
+  "transforms": {
+    "weekly_downloads": "number"
+  },
+  "example": {
+    "name": "react",
+    "latest_version": "19.2.0",
+    "weekly_downloads": 42000000,
+    "license": "MIT",
+    "repository_url": "git+https://github.com/facebook/react.git"
+  },
+  "name": "npm-package",
+  "url_template": "https://www.npmjs.com/package/{name}",
+  "params": {
+    "name": "react"
+  },
+  "want": "Return package name, latest version, numeric weekly downloads, license, and repository URL.",
+  "sample_output": [
+    {
+      "name": "react",
+      "latest_version": "19.2.0",
+      "weekly_downloads": 42000000,
+      "license": "MIT",
+      "repository_url": "git+https://github.com/facebook/react.git"
+    }
+  ],
+  "compiled_at": "2026-10-23T09:14:06+00:00",
+  "compiled_by": "python3 reader/testdata/compile_stub.py",
+  "fetch_method": "crwl",
+  "crawl4ai_version": "0.9.2",
+  "page_hash": "92237ff233695cbf21545d7fddf95cc51981e1f2874e26310c8522e2eb8d126d",
+  "last_ok": "2026-10-23T09:14:06+00:00",
+  "healed_at": null,
+  "verification": {
+    "status": "live_zod_snapshot",
+    "url": "https://www.npmjs.com/package/zod",
+    "verified_at": "2026-10-23T09:45:06+00:00",
+    "page_hash": "9c860f2f365e31322abab76710ab3c85694cef021d13b5c8fc1e4ec61c820890",
+    "sample_output": [
+      {
+        "name": "zod",
+        "latest_version": "4.6.7",
+        "weekly_downloads": 144621339,
+        "license": "MIT",
+        "repository_url": "https://github.com/colinhacks/zod"
+      }
+    ],
+    "note": "Paths manually corrected against the public zod HTML fetched with bundled crwl -o all. Original compiled_by and compile sample retained. React live fetch and a real compile lane were not rerun in this fix."
+  }
+}