donsetch 5.0.0-beta.3 → 5.1.0

package/README.md

@@ -2,7 +2,7 @@
 
 **The web, for AI agents.** Fetch, search, crawl and screenshot from one local binary. Zero API keys, zero accounts.
 
-**v5 beta:** a new fetch engine. No browser in the fetch path: a wall is answered by the right client identity, the site's own challenge program run in-process, or the site's own public interface.
+**v5:** a new fetch engine. No browser in the fetch path: a wall is answered by the right client identity, the site's own challenge program run in-process, or the site's own public interface.
 
 [![Release](https://img.shields.io/github/v/release/dondai44423/donsetch?color=00d4aa&style=flat-square)](https://github.com/dondai44423/donsetch/releases)
 [![CI](https://img.shields.io/github/actions/workflow/status/dondai44423/donsetch/ci.yml?label=CI&style=flat-square)](https://github.com/dondai44423/donsetch/actions/workflows/ci.yml)
@@ -103,7 +103,7 @@ donsetch update
 | `web_fetch` | Any URL as clean markdown, or only the part that answers: `must_contain` probes, `focus` passages, `toc` → `section`. Walls answered in-process, never by a browser. PDFs with OCR and per-page confidence, site adapters, page memory, batches of up to 12. |
 | `web_search` | Keyless multi-engine search: 10+ backends, consensus plus semantic reranking, query-aware official-source placement. Ranked URLs and snippets, never a scraped article dump. |
 | `web_crawl` | Best-first same-domain crawl. Sitemap plus frontier, `focus` ranking, elastic pacing, resume tokens, honest stop reasons. |
-| `web_screenshot` | Rendered PNG of any URL through DonSeTch's browser (used only for screenshots and login), with the usual URL safety guards. |
+| `web_screenshot` | Rendered PNG of any URL through DonSeTch's browser (used only for screenshots), with the usual URL safety guards. |
 
 MCP results default to self-contained text: leading `[meta]` JSON followed by source evidence, including for unknown client bridges. PNGs remain image blocks.
 
@@ -115,7 +115,7 @@ Every failure is structured: a stable `code` (`wall.challenge`, `guard.ssrf`, `d
 - **PDFs done properly.** A custom PDFium FFI, no Python subprocess. Tables and borders come from the rendered bitmap, text from the glyph stream, and the trust audit flags exactly the regions that need OCR.
 - **Crawl with manners.** Per-host adaptive pacing that honors `Retry-After` and robots `Crawl-delay`, cross-process politeness so two crawls do not double a site's rate, near-duplicate detection, resume tokens that survive restarts, and honest stop reasons.
 - **Fetch that answers honestly.** `content_ok`, `thin`, `changed` with section diffs, and `must_contain` says `unverified` instead of a false "not found" when part of a page was unread. No fake success.
-- **Walls named, not guessed.** A bot wall names its vendor and what it asks for; a login wall is a declared `wall.auth` (fetch carries no session in v5; session support is planned).
+- **Walls named, not guessed.** A bot wall names its vendor and what it asks for; a login wall is a declared `wall.auth` (the fetch carries no session; authenticated work belongs to a browser driver).
 - **Works everywhere.** Linux, macOS, Windows. npm, Homebrew, the Pi agent as a native extension, and the DeepSeek Harness as a first-class plugin.
 
 ## Sponsors

package/install.js

@@ -6,6 +6,7 @@
 
 const http = require('http');
 const https = require('https');
+const net = require('net');
 const tls = require('tls');
 const crypto = require('crypto');
 const fs = require('fs');
@@ -159,15 +160,30 @@ function requestDirect(target) {
   });
 }
 
+// Basic credentials from the proxy URL's user:pass@ part, or null when
+// it has none. The URL parser keeps them percent-encoded.
+function proxyAuthorization(proxy) {
+  if (!proxy.username && !proxy.password) return null;
+  const decode = (part) => {
+    try { return decodeURIComponent(part); } catch (_) { return part; }
+  };
+  const pair = `${decode(proxy.username)}:${decode(proxy.password)}`;
+  return `Basic ${Buffer.from(pair).toString('base64')}`;
+}
+
 function requestThroughProxy(target, proxy) {
   return new Promise((resolve, reject) => {
+    const authority = `${target.hostname}:${target.port || 443}`;
+    const connectHeaders = { Host: authority };
+    const authorization = proxyAuthorization(proxy);
+    if (authorization) connectHeaders['Proxy-Authorization'] = authorization;
     const connectTransport = proxy.protocol === 'https:' ? https : http;
     const proxyReq = connectTransport.request({
       hostname: proxy.hostname,
       port: proxy.port || (proxy.protocol === 'https:' ? 443 : 80),
       method: 'CONNECT',
-      path: `${target.hostname}:${target.port || 443}`,
-      headers: { Host: `${target.hostname}:${target.port || 443}` },
+      path: authority,
+      headers: connectHeaders,
     });
     proxyReq.once('connect', (response, socket) => {
       if (response.statusCode !== 200) {
@@ -184,10 +200,17 @@ function requestThroughProxy(target, proxy) {
           Accept: 'application/octet-stream',
           'User-Agent': 'donsetch-npm-installer',
         },
-        agent: false,
+        // No `agent` key: Node uses createConnection only when no agent
+        // is in use, and `agent: false` means a fresh default Agent,
+        // which dials the host directly and leaves the tunnel unused.
+        // Without an agent the request also closes its connection, and
+        // with it the tunnel, once the response ends.
+        // The certificate is verified (tls.connect's default) against
+        // the release host; SNI takes host names only (RFC 6066 3).
         createConnection: () => tls.connect({
           socket,
-          servername: target.hostname,
+          host: target.hostname,
+          servername: net.isIP(target.hostname) ? undefined : target.hostname,
         }),
       }, resolve);
       req.setTimeout(REQUEST_TIMEOUT_MS, () => req.destroy(new Error('request timeout')));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "donsetch",
-  "version": "5.0.0-beta.3",
+  "version": "5.1.0",
   "description": "MCP server that gives AI agents the web: fetch, search, crawl and screenshot from one local Rust binary. Zero API keys, zero accounts, Chrome-true TLS.",
   "license": "AGPL-3.0-only",
   "author": "Bishesh Bhandari",

package/pi-extension.ts

@@ -199,8 +199,8 @@ function startServer(): Promise<void> {
       }
     });
 
-    // The daemon writes diagnostics to stderr (Xvfb warnings,
-    // fallback notices, ghost telemetry). Pi's TUI surfaces stderr
+    // The daemon writes diagnostics to stderr (config warnings,
+    // rerank and OCR notices). Pi's TUI surfaces stderr
     // as user-facing output: raw forwarding used to pop non-fatal
     // warnings up (issue #95). Gate it:
     // - DONSETCH_DEBUG / DEBUG set: forward everything (dev mode)
@@ -449,14 +449,12 @@ function getSearchProvider(sc: any, dbg: any): string {
 }
 
 /**
- * Extract the fetch source label from model state with a
- * client-only diagnostics fallback. Compact MCP contracts keep
- * transport telemetry in _meta["com.donsetch/fetch-debug"], so the
- * tier can live there; older servers still expose it on
- * structuredContent.
- * Returns "cache" when tier is "1(warm)" (warm cookies, not a
- * fresh fetch), "ghost" when tier starts with "2" (browser
- * escalation), or "" for a normal tier-1 fetch.
+ * Extract the fetch source label from the `tier` field that servers
+ * before 5.0 report, in _meta["com.donsetch/fetch-debug"] or on
+ * structuredContent. Returns "cache" when tier is "1(warm)" (warm
+ * cookies, not a fresh fetch), "ghost" when tier starts with "2"
+ * (their browser tier), or "" otherwise. A 5.x server sends no
+ * tier and has no browser tier, so the result is always "".
  */
 function getFetchSource(sc: any, dbg: any): string {
   const tier = String(