@bacnh85/pi-web 0.6.0 → 0.6.2

package/CHANGELOG.md

@@ -1,5 +1,38 @@
 # Changelog
 
+## 0.6.2 (2026-09-06)
+
+### Changed
+
+- `web_screenshot` now returns the PNG **inline as an image block**
+  (`ImageContent`) alongside the text summary, so multimodal models (GLM-5.3,
+  Claude, Gemini) actually see the screenshot instead of a base64 char
+  count. The "Data: base64 PNG (N chars)" line is gone; artifact/MIME/size
+  summary unchanged. Inspired by zcode-plugins video2code's vision-in-the-loop.
+  Regression-tested in `test/unit/screenshot.test.ts` (fetch stubbed — no
+  daemon needed): image block present + base64-text line absent; text-only
+  fallback when the daemon returns no screenshot.
+
+  Daemon `success:false` responses (HTTP 200) now surface `error_message` as
+  a tool error instead of returning a silently empty screenshot result.
+
+## 0.6.1 (2026-08-30)
+
+### Changed
+
+- Trimmed static prompt overhead ~385 tokens/turn: compressed the injected
+  `Web Tool Routing` guidance block (1,247 -> 281 chars, same routing table
+  + backend rules), cut all 7 tools' promptGuidelines to <=2 unique lines,
+  and shortened web_crawl/web_screenshot schema descriptions. One
+  hook.test.ts assertion updated to the compressed phrasing. No tool,
+  parameter, or default changed.
+
+### Changed (2026-08-19)
+
+- `web_extract` agy backend default model updated to `gemini-3.7-flash-medium`
+  — the current Flash generation in agy 1.1.x (3.6 is still served, this just
+  follows the latest).
+
 ## 0.6.0 (2026-08-07)
 
 ### Features

package/README.md

@@ -142,7 +142,7 @@ web_crawl url="https://example.com" mode=light poll=true    # Poll for completio
 
 ### `web_screenshot` — Page screenshot
 
-Captures a full-page PNG screenshot using Crawl4AI. Returns base64-encoded PNG.
+Captures a full-page PNG screenshot using Crawl4AI. Returns the PNG inline as an image block (multimodal models see it); text summary includes artifact/MIME/size.
 
 ```
 web_screenshot url="https://example.com"

package/extensions/index.ts

@@ -57,27 +57,14 @@ const crawl4aiControlSchema = {
 // the guidance travels with the package and disappears when pi-web is absent.
 const WEB_ROUTING_GUIDANCE = `## Web Tool Routing (pi-web)
 
-The pi-web extension provides 7 unified tools that auto-select the best backend:
+- **web_search** — web search (auto: SearXNG → Brave → Firecrawl; force via \`backend\`, tune via \`engines\`).
+- **web_extract** — URL → markdown (auto: static JSDOM → dynamic Firecrawl → full Crawl4AI → agy; force via \`mode\`; prompt+schema for JSON extraction).
+- **web_map** — discover site URLs (Firecrawl Map).
+- **web_crawl** — multi-page crawl: \`mode: "light"\` (Firecrawl, url) or \`mode: "full"\` (Crawl4AI, urls[]).
+- **web_screenshot** / **web_pdf** — page capture (Crawl4AI).
+- **web_status** — provider config + health.
 
-- **\`web_search\`** — Search the web (auto: SearXNG → Brave → Firecrawl). Use
-  \`backend\` for explicit control, \`engines\` for SearXNG tuning.
-- **\`web_extract\`** — Extract readable content from a URL (auto: static JSDOM
-  → dynamic Firecrawl → full Crawl4AI → agy model-backed). Use \`mode\` for
-  explicit control.
-- **\`web_map\`** — Discover URLs from a site (Firecrawl Map).
-- **\`web_crawl\`** — Crawl multiple pages. \`mode: "light"\` (Firecrawl) or
-  \`mode: "full"\` (Crawl4AI).
-- **\`web_screenshot\`** / **\`web_pdf\`** — Visual/page capture (Crawl4AI).
-- **\`web_status\`** — Check provider configuration and server health.
-
-Backend selection rules:
-
-- Firecrawl Search has poor semantic accuracy on domain-specific queries; prefer
-  SearXNG or Brave for precision (force via \`backend\`).
-- Firecrawl Scrape fails on bot-protected sites (e.g. Ansible docs); Crawl4AI
-  handles those (force via \`mode: "full"\`), and agy (Gemini/Claude read_url)
-  handles the rest as a last-resort fallback (force via \`mode: "agy"\`).
-- Always cite source URLs when web results materially support an answer.`;
+Rules: Firecrawl Search is weak on domain-specific queries — prefer SearXNG/Brave; Firecrawl Scrape fails on bot-protected sites — use Crawl4AI (\`mode: "full"\`) then agy (\`mode: "agy"\`); cite source URLs.`;
 
 
 // ---------------------------------------------------------------------------
@@ -92,13 +79,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
     description:
       "Search the web. Auto-selects backends: SearXNG, Brave, Firecrawl.",
     promptSnippet: "Search current web results",
-    promptGuidelines: [
-      "Source discovery, docs, facts, and general search.",
-      "Broad discovery uses SearXNG; precision/site/docs use Brave; Firecrawl is fallback.",
-      "Force backend via backend:'brave'|'searxng' for poor auto results.",
-      "Use engines='google,github' for SearXNG tuning.",
-      "Cite source URLs.",
-    ],
+    promptGuidelines: ["Source discovery, docs, facts. Precision/site/docs → Brave via backend:'brave'; tune SearXNG via engines.", "Cite source URLs."],
     parameters: Type.Object({
       query: Type.String(),
       count: Type.Optional(Type.Number({ default: 5 })),
@@ -140,14 +121,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
     description:
       "Extract readable content from a URL. Auto mode: static\u2192dynamic\u2192full\u2192agy.",
     promptSnippet: "Extract readable webpage content as markdown",
-    promptGuidelines: [
-      "Clean markdown from a known URL.",
-      "mode: 'static' (no API key, JSDOM), 'dynamic' (Firecrawl JS), 'full' (Crawl4AI), 'agy' (Gemini/Claude via agy).",
-      "'auto' tries static\u2192dynamic\u2192full\u2192agy; see diagnostics for fallback chain.",
-      "mode: 'agy' uses agy's native read_url for bot-protected/JS-heavy pages \u2014 last-resort fallback in auto.",
-      "Use prompt+schema for structured JSON extraction (dynamic/agy modes).",
-      "Cite the source URL.",
-    ],
+    promptGuidelines: ["Markdown from a known URL; prompt+schema for structured JSON extraction.", "Cite the source URL."],
     parameters: Type.Object({
       url: Type.String(),
       mode: Type.Optional(Type.Union(
@@ -191,11 +165,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
     description:
       "Discover site URLs via Firecrawl Map.",
     promptSnippet: "Map site URLs",
-    promptGuidelines: [
-      "Discover site URLs before crawling. Prefer web_extract for small jobs.",
-      "Best on base domains. sitemap:'only' for sub-path discovery.",
-      "Keep limits small unless broad discovery is requested.",
-    ],
+    promptGuidelines: ["URL discovery before crawling; prefer web_extract for small jobs."],
     parameters: Type.Object({
       url: Type.String(),
       limit: Type.Optional(Type.Number({ default: 100 })),
@@ -234,21 +204,17 @@ export default function piWebExtension(pi: ExtensionAPI) {
     description:
       "Crawl pages. Firecrawl 'light' or Crawl4AI 'full' headless mode.",
     promptSnippet: "Crawl a small site section",
-    promptGuidelines: [
-      "Prefer web_map + web_extract over crawl for small jobs.",
-      "'light'=Firecrawl (url param), 'full'=Crawl4AI (urls[] param).",
-      "Keep limit low (default 10).",
-    ],
+    promptGuidelines: ["'light'=Firecrawl (url), 'full'=Crawl4AI (urls[]). Prefer web_map + web_extract for small jobs."],
     parameters: Type.Object({
-      url: Type.Optional(Type.String({ description: "URL for Firecrawl-style crawl (mode:'light')." })),
-      urls: Type.Optional(Type.Array(Type.String(), { description: "URLs for Crawl4AI-style crawl (mode:'full'), up to 100." })),
+      url: Type.Optional(Type.String({ description: "URL for mode:'light' (Firecrawl)." })),
+      urls: Type.Optional(Type.Array(Type.String(), { description: "URLs for mode:'full' (Crawl4AI), up to 100." })),
       mode: Type.Optional(Type.Union([Type.Literal("light"), Type.Literal("full")], { default: "light", description: "'light'(Firecrawl) or 'full'(Crawl4AI)." })),
       limit: Type.Optional(Type.Number({ default: 10 })),
-      include_paths: Type.Optional(Type.String({ description: "Comma-separated paths to include (Firecrawl mode)." })),
-      exclude_paths: Type.Optional(Type.String({ description: "Comma-separated paths to exclude (Firecrawl mode)." })),
-      poll: Type.Optional(Type.Boolean({ default: false, description: "Poll for completion (Firecrawl mode)." })),
-      browser_config: Type.Optional(Type.Any({ description: "BrowserConfig JSON (full mode only)." })),
-      crawler_config: Type.Optional(Type.Any({ description: "CrawlerRunConfig JSON (full mode only)." })),
+      include_paths: Type.Optional(Type.String({ description: "Comma-separated include paths (light mode)." })),
+      exclude_paths: Type.Optional(Type.String({ description: "Comma-separated exclude paths (light mode)." })),
+      poll: Type.Optional(Type.Boolean({ default: false, description: "Poll until completion (light mode)." })),
+      browser_config: Type.Optional(Type.Any({ description: "BrowserConfig JSON (full mode)." })),
+      crawler_config: Type.Optional(Type.Any({ description: "CrawlerRunConfig JSON (full mode)." })),
       content_chars: Type.Optional(Type.Number({ default: 20000 })),
       ...firecrawlControlSchema,
       ...crawl4aiControlSchema,
@@ -320,16 +286,13 @@ export default function piWebExtension(pi: ExtensionAPI) {
     name: "web_screenshot",
     label: "Web Page Screenshot",
     description:
-      "Full-page PNG screenshot via Crawl4AI.",
+      "Full-page PNG screenshot via Crawl4AI. The PNG is returned inline as an image block.",
     promptSnippet: "Screenshot a webpage",
-    promptGuidelines: [
-      "Use when web_extract fails on JS-heavy or bot-protected pages.",
-      "wait_for (default 2s) delays capture for dynamic content.",
-    ],
+    promptGuidelines: ["Full-page PNG returned inline (multimodal models see it); use when web_extract fails on JS-heavy pages, or to visually inspect a built UI."],
     parameters: Type.Object({
       url: Type.String(),
       wait_for: Type.Optional(Type.Number({ default: 2, description: "Seconds to wait before capture." })),
-      wait_for_images: Type.Optional(Type.Boolean({ default: false, description: "Wait for images before capture." })),
+      wait_for_images: Type.Optional(Type.Boolean({ default: false })),
       ...crawl4aiControlSchema,
       ...sharedControlSchema,
     }),
@@ -342,16 +305,23 @@ export default function piWebExtension(pi: ExtensionAPI) {
         params.wait_for_images as boolean | undefined,
         signal,
       );
+      if (result.success === false) {
+        throw new Error(String(result.error_message ?? "Crawl4AI screenshot failed"));
+      }
       const screenshot = result.screenshot as string | undefined;
       const artifactUrl = result.url as string | undefined;
       const mime = result.mime as string | undefined;
       const size = result.size as number | undefined;
       let text = `Screenshot: ${params.url}\n`;
-      if (screenshot) text += `Data: base64 PNG (${screenshot.length} chars)\n`;
       if (artifactUrl) text += `Artifact: ${artifactUrl}\n`;
       if (mime) text += `MIME: ${mime}\n`;
       if (size) text += `Size: ${size} bytes\n`;
-      return { content: [{ type: "text" as const, text: truncateText(text) }], details: { ...result, url: params.url } };
+      // Return the PNG as a real image block so multimodal models see it.
+      const content: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> = [
+        { type: "text", text: truncateText(text) },
+      ];
+      if (screenshot) content.push({ type: "image", data: screenshot, mimeType: mime || "image/png" });
+      return { content, details: { ...result, url: params.url } };
     },
   });
 
@@ -362,10 +332,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
     description:
       "PDF document via Crawl4AI.",
     promptSnippet: "PDF a webpage",
-    promptGuidelines: [
-      "Printable or archivable page snapshot.",
-      "Returns base64 PDF string.",
-    ],
+    promptGuidelines: ["Printable/archivable page snapshot; returns base64 PDF."],
     parameters: Type.Object({
       url: Type.String(),
       ...crawl4aiControlSchema,
@@ -392,11 +359,7 @@ export default function piWebExtension(pi: ExtensionAPI) {
     description:
       "Show web provider config status without printing secrets.",
     promptSnippet: "Check web provider config and server status",
-    promptGuidelines: [
-      "Check which backends are configured and their server status.",
-      "Never prints secrets — reports only presence and source.",
-      "apiKeyFound:false is normal for self-hosted Firecrawl; check ready field.",
-    ],
+    promptGuidelines: ["Reports backend presence/health; never prints secrets."],
     parameters: Type.Object({}),
     async execute(_id: string, _params: Record<string, unknown>, signal: AbortSignal, _onUpdate: unknown, ctx: any) {
       const cwd = cwdFromContext(ctx);

package/extensions/lib/agy.ts

@@ -17,7 +17,7 @@ const cp = _require("node:child_process") as typeof import("node:child_process")
 const AGY_FETCH_TIMEOUT_MS = 90_000; // agy needs time for model call + web fetch
 const AGY_PROBE_TIMEOUT_MS = 5_000;
 const AGY_MAX_OUTPUT_BYTES = 200_000; // bound output to protect Pi context
-export const AGY_MODEL = "gemini-3.6-flash-medium"; // ponytail: fixed default; users needing model control use agy_execute
+export const AGY_MODEL = "gemini-3.7-flash-medium"; // ponytail: fixed default; users needing model control use agy_execute
 
 // Cache install status with a TTL — spawnSync blocks the event loop up to
 // AGY_PROBE_TIMEOUT_MS, and web_status/extract can call this repeatedly.

package/extensions/lib/content.ts

@@ -59,6 +59,16 @@ export async function fetchReadableContent(
     signal: signalWithTimeout(timeoutMs, signal),
   });
   if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  // Raw text/JSON payloads (raw.githubusercontent.com, JSON APIs) — Readability
+  // shreds them to nothing. Pass through verbatim. text/html and text/xml keep
+  // the Readability path — they're ordinary web pages. Session mining: 9/40
+  // static extract failures were raw-text/JSON shapes.
+  const contentType = (response.headers.get("content-type") ?? "").split(";")[0].trim();
+  if ((contentType.startsWith("text/") && contentType !== "text/html" && contentType !== "text/xml") || contentType === "application/json") {
+    const body = await response.text();
+    const markdown = contentType === "application/json" ? "```json\n" + body + "\n```" : body;
+    return { title: "", markdown: markdown.slice(0, 20000) };
+  }
   const html = await response.text();
   const deps = loadReadableContentDependencies();
   const dom = new deps.JSDOM(html, { url });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bacnh85/pi-web",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "Pi extension for web search, page extraction, Firecrawl scraping/crawling, and Crawl4AI headless browser crawling.",
   "type": "module",
   "license": "MIT",
@@ -59,7 +59,13 @@
     "@types/node": "^26.0.1",
     "@types/turndown": "^5.0.6",
     "chai": "^6.2.2",
-    "mocha": "^11.7.6",
+    "mocha": "^11.8.0",
     "tsx": "^4.22.4"
+  },
+  "overrides": {
+    "serialize-javascript@>=5.0.0 <7.0.5": "^7.0.5",
+    "js-yaml@>=4.0.0 <4.3.1": "^4.3.1",
+    "brace-expansion@>=2.0.0 <2.1.4": "^2.1.4",
+    "diff": "^8.0.3"
   }
 }