@allansantos-dev/smart-tool 0.9.7 → 0.9.9

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## 0.9.9 - beta
+
+- `web_fetch` renders JavaScript pages with [Moli](https://github.com/lexmount/moli) first (a headless browser built
+  for agents: layout and paint only when a page needs them; 43 MB download, SHA-256 checked) and starts Chromium
+  through Crawl4AI only when Moli fails or comes back thin. On 21 pages HTTP could not read: Moli 1.6 s and 90 MB
+  peak (median) against 4.2 s and 581 MB; Moli read 14, Chromium 16, both together 17 (only Chromium read YouTube,
+  Google Maps and Airbnb). With several agent sessions reading at once, measured on N simultaneous rendered reads:
+  aggregate peak memory 448 → 35 MB (1), 1,788 → 194 MB (3), 2,901 → 634 MB (5), 4.4-4.7 GB → 1.0-1.4 GB (10), and
+  the median read time halved (10 at once: 5.0 s → 2.5 s). Pages only Chromium reads now take Moli's 1-4 s more.
+  The `web_fetch` metric records which engine read the page and Moli's error when it fell back.
+
+## 0.9.8 - beta
+
+- An update could keep running the previous code: npm dates every file in the package 1985-10-26, the installer kept
+  that date when copying, and Python reuses a compiled `.pyc` when the source's date and size match, so a file changed
+  without changing size (a version bump, a one-character fix) went on running the old bytecode. Reproduced on a real
+  reinstall: `version.py` said 0.9.6 and the daemon served 0.9.8. Copied and restored files now get the current time,
+  and the program's `__pycache__` folders are cleared after copying and after a rollback (two copies within the same
+  second keep the same date). The update to this version already clears any stale bytecode left by earlier ones.
+  Reported by the agent-launcher session, which had the same bug.
+
 ## 0.9.7 - beta
 
 The hook decides code searches with a fixed rule instead of the router model.

package/NOTICE

@@ -15,6 +15,7 @@ Python packages (installation venv, from PyPI)
 
 Browser runtime (web_adapters/browser/.venv, from PyPI and the projects' download servers)
 - camoufox Python package (MIT) and the Camoufox browser, a Firefox fork (MPL-2.0), fetched by `camoufox fetch`
+- Moli (Apache-2.0 OR MIT), https://github.com/lexmount/moli: renders JavaScript pages for web_fetch; downloaded at install time from its GitHub release (SHA-256 checked), with its own license files
 - crawl4ai (Apache-2.0), https://github.com/unclecode/crawl4ai: page rendering is powered by Crawl4AI
 - Playwright (Apache-2.0) and the Chromium build it installs (BSD-style, see https://www.chromium.org)
 

package/README.md

@@ -146,4 +146,4 @@ user` / `codex mcp remove smart-tool`, then delete `%LOCALAPPDATA%\Programs\Smar
 
 Apache License 2.0. See `LICENSE` and `NOTICE` for third-party components.
 
-Page rendering is powered by [Crawl4AI](https://github.com/unclecode/crawl4ai) (Apache-2.0).
+Page rendering is powered by [Moli](https://github.com/lexmount/moli) (Apache-2.0 / MIT), with [Crawl4AI](https://github.com/unclecode/crawl4ai) (Apache-2.0) as its fallback.

package/install.py

@@ -197,6 +197,7 @@ def copy_files(target, files, backup, changed, removed):
         changed.append(rel)
         dest.parent.mkdir(parents=True, exist_ok=True)
         shutil.copy2(source, dest)
+        os.utime(dest)
     for rel in sorted(previous - set(files)):
         if (target / rel).is_file():
             (backup / rel).parent.mkdir(parents=True, exist_ok=True)
@@ -210,16 +211,31 @@ def copy_files(target, files, backup, changed, removed):
                                               indent=1), encoding="utf-8")
 
 
+def clear_bytecode(target):
+    """Deletes the program's __pycache__ folders (the virtual environment's are left alone). npm writes the same
+    mtime (1985-10-26) into every tarball entry, and Python's bytecode check compares only the source's mtime and
+    size: a file changed without changing size kept running the old code after an update (reproduced on 2026-10-10:
+    version.py said 0.9.6, the daemon served 0.9.8)."""
+    for current, dirs, _files in os.walk(target):
+        dirs[:] = [d for d in dirs if d != ".venv"]
+        if "__pycache__" in dirs:
+            shutil.rmtree(Path(current) / "__pycache__", ignore_errors=True)
+            dirs.remove("__pycache__")
+
+
 def restore(target, backup, changed, removed):
     for rel in changed:
         if (backup / rel).is_file():
             shutil.copy2(backup / rel, target / rel)
+            os.utime(target / rel)
         else:
             (target / rel).unlink(missing_ok=True)
     for rel in removed:
         if (backup / rel).is_file():
             (target / rel).parent.mkdir(parents=True, exist_ok=True)
             shutil.move(backup / rel, target / rel)
+            os.utime(target / rel)
+    clear_bytecode(target)
     if (backup / MANIFEST).is_file():
         shutil.copy2(backup / MANIFEST, target / MANIFEST)
     else:
@@ -319,6 +335,7 @@ def main():
     changed, removed, frozen, deps_touched = [], [], None, False
     try:
         copy_files(target, files, backup, changed, removed)
+        clear_bytecode(target)
         print(f"Files changed: {len(changed)}; removed as obsolete: {len(removed)}")
         if not args.skip_deps:
             frozen = freeze(python)

package/install_runtime.py

@@ -31,6 +31,14 @@ TYPESCRIPT_PARSER = NODE_DIR / "node_modules" / "typescript" / "lib" / "typescri
 JAVA_PARSER = NODE_DIR / 'node_modules' / 'java-parser' / 'src' / 'index.js'
 ANGULAR_PARSER = NODE_DIR / 'node_modules' / '@angular' / 'compiler' / 'fesm2022' / 'compiler.mjs'
 NODE_RUNTIME = NODE_DIR / ".node-runtime"
+# Moli renders the JavaScript pages web_fetch cannot read over HTTP; Chromium (Crawl4AI) stays as its fallback.
+# Measured on 2026-10-10 over 21 such pages: 1.6 s and 90 MB peak (median) against 4.2 s and 581 MB for Chromium.
+MOLI_VERSION = "1.1.15"
+MOLI_FOLDER = f"moli-v{MOLI_VERSION}-x86_64-pc-windows-msvc"
+MOLI_URL = f"https://github.com/lexmount/moli/releases/download/v{MOLI_VERSION}/moli-x86_64-pc-windows-msvc.zip"
+MOLI_SHA256 = "64c00ce02d6db8e1c22c55e88fbcaaf3e7517e930f24c25ff46a457b421040c8"
+MOLI_DIR = BROWSER_DIR / "moli"
+MOLI_EXE = MOLI_DIR / "moli.exe"
 
 
 def _node_supported(node):
@@ -58,6 +66,49 @@ def _node_commands():
     return (node, npm) if node and npm and _node_supported(node) else (None, None)
 
 
+def _download_verified(url, expected_sha256, path):
+    """Downloads url to path and fails when its SHA-256 differs from the published one."""
+    digest = hashlib.sha256()
+    with urllib.request.urlopen(url, timeout=60) as response, path.open("wb") as out:
+        while block := response.read(1024 * 1024):
+            digest.update(block)
+            out.write(block)
+    if digest.hexdigest().lower() != expected_sha256.lower():
+        raise RuntimeError(f"SHA-256 of {url} does not match the published checksum.")
+
+
+def _extract_folder(zip_path, top, destination):
+    """Extracts the files under the archive's single top folder into destination, refusing any other path."""
+    destination.mkdir()
+    with zipfile.ZipFile(zip_path) as zf:
+        for member in zf.infolist():
+            parts = Path(member.filename.replace("\\", "/")).parts
+            if parts == (top,) and member.is_dir():
+                continue
+            if len(parts) < 2 or parts[0] != top or ".." in parts:
+                raise RuntimeError(f"{zip_path.name} contains an unexpected path: {member.filename}")
+            target = destination.joinpath(*parts[1:])
+            if member.is_dir():
+                target.mkdir(parents=True, exist_ok=True)
+            else:
+                target.parent.mkdir(parents=True, exist_ok=True)
+                with zf.open(member) as source, target.open("wb") as out:
+                    shutil.copyfileobj(source, out)
+
+
+def _replace_folder(unpacked, final, temp_dir):
+    """Swaps final for unpacked, putting the previous folder back if the swap fails."""
+    previous = temp_dir / "previous"
+    if final.exists():
+        os.replace(final, previous)
+    try:
+        os.replace(unpacked, final)
+    except OSError:
+        if previous.exists() and not final.exists():
+            os.replace(previous, final)
+        raise
+
+
 def _install_node():
     """Baixa o Node LTS oficial e confere SHA-256 antes de extrair."""
     print("Downloading Node LTS for the Smart Tool web runtime...", flush=True)
@@ -79,43 +130,38 @@ def _install_node():
     with tempfile.TemporaryDirectory(prefix="node-setup-", dir=NODE_DIR) as temp:
         temp_dir = Path(temp)
         zip_path = temp_dir / archive
-        digest = hashlib.sha256()
-        with urllib.request.urlopen(base + archive, timeout=60) as response, zip_path.open("wb") as out:
-            while block := response.read(1024 * 1024):
-                digest.update(block)
-                out.write(block)
-        if digest.hexdigest().lower() != expected.lower():
-            raise RuntimeError("Node SHA-256 checksum does not match the official index.")
+        _download_verified(base + archive, expected, zip_path)
         unpacked = temp_dir / "runtime"
-        unpacked.mkdir()
-        with zipfile.ZipFile(zip_path) as zf:
-            for member in zf.infolist():
-                parts = Path(member.filename.replace("\\", "/")).parts
-                if parts == (f"node-{version}-win-x64",) and member.is_dir():
-                    continue
-                if len(parts) < 2 or parts[0] != f"node-{version}-win-x64" or ".." in parts:
-                    raise RuntimeError("Node archive contains an unexpected path.")
-                destination = unpacked.joinpath(*parts[1:])
-                if member.is_dir():
-                    destination.mkdir(parents=True, exist_ok=True)
-                else:
-                    destination.parent.mkdir(parents=True, exist_ok=True)
-                    with zf.open(member) as source, destination.open("wb") as target:
-                        shutil.copyfileobj(source, target)
+        _extract_folder(zip_path, f"node-{version}-win-x64", unpacked)
         if not (unpacked / "node.exe").is_file() or not (unpacked / "npm.cmd").is_file():
             raise RuntimeError("Node package does not contain the expected executables.")
-        previous = temp_dir / "previous-node-runtime"
-        if NODE_RUNTIME.exists():
-            os.replace(NODE_RUNTIME, previous)
-        try:
-            os.replace(unpacked, NODE_RUNTIME)
-        except OSError:
-            if previous.exists() and not NODE_RUNTIME.exists():
-                os.replace(previous, NODE_RUNTIME)
-            raise
+        _replace_folder(unpacked, NODE_RUNTIME, temp_dir)
     print(f"Node {version} installed inside Smart Tool.")
 
 
+def _moli_ready():
+    try:
+        return MOLI_EXE.is_file() and (MOLI_DIR / "VERSION").read_text(encoding="utf-8").strip() == MOLI_VERSION
+    except OSError:
+        return False
+
+
+def _install_moli():
+    """Downloads the pinned Moli release from GitHub, checks its SHA-256 and unpacks it into web_adapters/browser."""
+    print(f"Downloading Moli {MOLI_VERSION} (structured-first headless browser, 43 MB)...", flush=True)
+    BROWSER_DIR.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="moli-setup-", dir=BROWSER_DIR) as temp:
+        temp_dir = Path(temp)
+        zip_path = temp_dir / "moli.zip"
+        _download_verified(MOLI_URL, MOLI_SHA256, zip_path)
+        unpacked = temp_dir / "moli"
+        _extract_folder(zip_path, MOLI_FOLDER, unpacked)
+        if not (unpacked / "moli.exe").is_file():
+            raise RuntimeError("The Moli package does not contain moli.exe.")
+        _replace_folder(unpacked, MOLI_DIR, temp_dir)
+    print(f"Moli {MOLI_VERSION} installed inside Smart Tool.")
+
+
 def _run(args, *, cwd=None, label="Web dependency", idle_s=600, stream=False, attempts=1):
     """Runs a step with UTF-8 output and closed stdin (a prompt fails at once instead of hanging); it is stopped only
     after idle_s without any output, and stream shows the output live."""
@@ -197,6 +243,7 @@ def _check():
         "browser_packages": False,
         "camoufox_browser": False,
         "chromium_browser": False,
+        "moli_browser": _moli_ready(),
     }
     if status["browser_python"]:
         try:
@@ -286,10 +333,12 @@ def install(reuse_browser_runtime=None):
              idle_s=180, stream=True, attempts=3)
         _run([str(BROWSER_PYTHON), "-m", "playwright", "install", "chromium"],
              label="Chromium browser", idle_s=180, stream=True, attempts=3)
+    if not _moli_ready():
+        _install_moli()
     status = _check()
     if not all(status.values()):
         raise RuntimeError(f"Incomplete web runtime: {status}")
-    print("Smart Tool runtime installed: Node, open-websearch, TypeScript, Java, Angular, Camoufox and Crawl4AI.")
+    print("Smart Tool runtime installed: Node, open-websearch, TypeScript, Java, Angular, Camoufox, Crawl4AI and Moli.")
 
 
 def main():

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@allansantos-dev/smart-tool",
-  "version": "0.9.7",
+  "version": "0.9.9",
   "description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
   "license": "Apache-2.0",
   "author": "Allan Santos",

package/smart_tool_daemon.py

@@ -37,6 +37,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 import capabilities
 import client_hooks
+import install_runtime
 import integration_metrics
 import config
 import freshness
@@ -3187,17 +3188,52 @@ def _tool_result(text, is_error=False):
 
 WEB_FETCH_TIMEOUT_S = 90
 WEB_FETCH_RENDER_TIMEOUT_S = 60
+MOLI_RENDER_TIMEOUT_S = 25
+# Engine of the last render on this request's thread: web_fetch.read_page calls _render_page on the handler's thread.
+_RENDER = threading.local()
 WEB_FETCH_VECTOR_ROOT = os.path.join(paths.DATA_DIR, "web-fetch")
 
 
+def _moli_markdown(url, timeout):
+    """Markdown of a JavaScript page rendered by Moli (layout and paint only when the page needs them)."""
+    if not install_runtime.MOLI_EXE.is_file():
+        raise RuntimeError(f"Moli is not installed ({install_runtime.MOLI_EXE}); reinstall Smart Tool.")
+    done = subprocess.run([str(install_runtime.MOLI_EXE), "fetch", "--dump", "markdown", url], capture_output=True,
+                          timeout=timeout, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
+    if done.returncode:
+        raise RuntimeError(f"moli exited {done.returncode}: {done.stderr.decode('utf-8', 'replace').strip()[-300:]}")
+    return done.stdout.decode("utf-8", "replace")
+
+
 def _render_page(url, deadline):
+    """Moli first; Chromium through Crawl4AI only when Moli fails or comes back thin. Measured on 2026-10-10 over 21
+    pages HTTP could not read: Moli 1.6 s and 90 MB peak (median), Chromium 4.2 s and 581 MB; Moli read 14, Chromium
+    16, both together 17 (only Chromium read YouTube, Google Maps and Airbnb). The engine that answered goes to the
+    web_fetch metric through _RENDER."""
     left = deadline - time.monotonic()
     if left < 5:
         return ""
+    _RENDER.engine, _RENDER.moli_error = None, None
+    try:
+        markdown = _moli_markdown(url, min(MOLI_RENDER_TIMEOUT_S, left - 2))
+    except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:
+        markdown, _RENDER.moli_error = "", f"{type(exc).__name__}: {exc}"[:200]
+        _log(f"web_fetch Moli render failed, trying Chromium: {_RENDER.moli_error}")
+    if not web_fetch.thin(markdown):
+        _RENDER.engine = "moli"
+        return markdown
+    left = deadline - time.monotonic()
+    if left < 5:
+        return markdown
     raw = _run_search_script(CAMOUFOX_VENV_PYTHON, CRAWL4AI_SEARCH_SCRIPT, [url, "--max-pages", "1"],
                              min(WEB_FETCH_RENDER_TIMEOUT_S, left))
     pages = raw.get("pages") if isinstance(raw, dict) else None
-    return (pages[0].get("markdown") or "") if pages else ""
+    chromium = (pages[0].get("markdown") or "") if pages else ""
+    if len(chromium.strip()) > len(markdown.strip()):
+        _RENDER.engine = "chromium"
+        return chromium
+    _RENDER.engine = "moli"
+    return markdown
 
 
 def _web_fetch_embed(texts, deadline):
@@ -3243,6 +3279,7 @@ def _handle_web_fetch(arguments):
         if problem:
             raise RuntimeError(f"Gateway model unavailable ({problem}).")
         deadline = started + WEB_FETCH_TIMEOUT_S
+        _RENDER.engine, _RENDER.moli_error = None, None
         page = web_fetch.read_page(url, deadline, _render_page)
         selected, pieces = web_fetch.select(page["text"], prompt, lambda texts: _web_fetch_embed(texts, deadline))
         text = web_fetch.answer(config.load_config()["router_model"], prompt, {**page, "text": selected}, deadline)
@@ -3252,6 +3289,7 @@ def _handle_web_fetch(arguments):
         raise RuntimeError(f"web_fetch failed: {str(exc).rstrip('.')}. The native WebFetch is allowed for this URL for "
                            f"{web_fetch.FAILED_TTL_S // 60} min.") from None
     _metric("web_fetch", cached=page["cached"], rendered=page["rendered"], render_error=page.get("render_error"),
+            render_engine=_RENDER.engine if page["rendered"] else None, moli_error=_RENDER.moli_error,
             page_chars=page["chars"], read_chars=len(selected), pieces=pieces,
             answer_chars=len(text), elapsed_s=round(time.monotonic() - started, 2))
     origin = "from cache" if page["cached"] else "rendered in the browser" if page["rendered"] else "downloaded now"

package/version.py

@@ -1,7 +1,7 @@
 """Single source of the Smart Tool version and of the User-Agent sent to public APIs."""
 import re
 
-VERSION = "0.9.7"
+VERSION = "0.9.9"
 _CONTACT_RE = re.compile(r"^(?:[^\s@()<>;]+@[^\s@()<>;]+\.[^\s@()<>;]+|https?://[^\s()<>;]+)$")