@gr8ful/spf 0.11.2 → 0.12.0

package/assets/defaults/spf.config.yaml

@@ -246,6 +246,7 @@ agents:
       - ls
       - bash
       - write
+      - webfetch                   # check current docs for a stack/tool the repo uses, rather than guessing from training data
 
   # No tester agent: running the suite is a known command, so it is a kind="code"
   # phase over core/quality.ts. See SKILL.md hard rule 8.
@@ -265,6 +266,7 @@ agents:
       - ls
       - bash
       - write
+      - webfetch                   # look up a possible tech stack/tool/3rd-party API's current docs before decomposing around it
 
   - name: reviewer
     model: openai/gpt-5.6-terra

package/assets/prompts/refiner/system.md

@@ -11,6 +11,7 @@ Decompose a product spec into a feature/story-or-bug tree of vertical slices the
 - You inherit the operator's shell environment — their PATH, toolchains and credentials are already live. Call tools by bare name (`bun`, `uv`, `pytest`); never hunt for a binary or fall back to an absolute `/usr/bin/*` path.
 - Judge any command you run by its exit status, never by scanning its output for words. `error` or `not found` inside passing output is text, not a failure.
 - Write your working notes to `<context_handoff_dir>/refine_plan.md` before emitting your Report JSON.
+- `webfetch` fetches a URL's current content — use it to check a specific library/API/tool's actual current docs when a slice depends on one you're not certain about, rather than decomposing around stale training-data assumptions. It fetches a page you name; it cannot search the web for one.
 
 ## Grounding: tie every slice to real code
 

package/assets/prompts/scout/system.md

@@ -12,6 +12,7 @@ Find and report where things live. Change nothing.
 - Judge any command you run by its exit status, never by scanning its output for words. `error` or `not found` inside passing output is text, not a failure.
 - Write your findings to `<context_handoff_dir>/scout_findings.md` for agents that follow.
 - If you find nothing, say so plainly — an empty finding is a valid finding.
+- `webfetch` fetches a URL's current content when the repo touches a library, API, or tool you're not certain about — use it to check, don't rely on stale training-data knowledge of a fast-moving stack. Not for general research: fetch a specific doc/reference page, not a search query.
 
 ## Subagents
 

package/assets/skill/references/config.md

@@ -564,11 +564,17 @@ session** — a joined run starts that agent fresh instead of resuming.
 | `grep` | search file contents |
 | `glob` (alias: `find`) | find files by pattern |
 | `ls` | recognized name, **no built-in on either backend** — harmless to list, never mounts |
+| `webfetch` | fetch a URL over HTTP(S), return its content as plain text — e.g. a library/API's current docs |
 
 These names are canonical across backends — a roster entry never says
 which; each backend module (`agent_flue.ts`, `agent_cc.ts`) maps them to
 its own tool vocabulary (Flue's lowercase functions, Claude Code's
-capitalized `Read`/`Bash`/...).
+capitalized `Read`/`Bash`/...). `webfetch` maps to Claude Code's own native
+`WebFetch` on that backend; Flue has no such built-in, so on `flue` it's a
+custom tool that runs `curl`/`wget` through the agent's own `Sandbox.exec()`
+— which means a remote sandbox's `egress` policy (see `sandbox` below)
+governs it exactly like any other `bash`-issued network call, with no
+separate rule to configure.
 
 **Resolution order:** an agent's own `tools` wins → else `defaults.tools` →
 else unset (all tools usable). An empty list is a tool-less agent, and it

package/dist/core/agent_cc.js

@@ -213,6 +213,15 @@ const TOOL_NAME_MAP = {
     bash: "Bash",
     grep: "Grep",
     glob: "Glob",
+    // CC's own built-in — fetches a URL and returns its content as markdown,
+    // already runs headlessly under --dangerously-skip-permissions like every
+    // other tool here. No sandbox/egress layer of SPF's own to route through:
+    // CC's own process makes the request directly, same as a bash `curl` an
+    // agent with `bash` could already issue — see agent_flue.ts's
+    // `createWebFetchTool` for the equivalent on the flue backend, where no
+    // native fetch tool exists and one has to route through the Sandbox
+    // instead.
+    webfetch: "WebFetch",
 };
 const TOOL_ALIASES = { find: "glob" };
 const DROPPED_TOOLS = new Set(["ls"]);

package/dist/core/agent_flue.d.ts

@@ -22,7 +22,8 @@
  * `run()`'s signature deliberately mirrors the old agent_pi.ts `run()` so
  * agents.ts's `send()` closure changes only its imports and field names.
  */
-import { type ConversationStreamChunk } from "@flue/runtime";
+import * as v from "valibot";
+import { type ConversationStreamChunk, type Sandbox } from "@flue/runtime";
 import type { AgentRequest, AgentResult } from "./data_types.ts";
 /**
  * Folds Flue's `tool-input` + `tool-output`/`tool-output-error` chunk pair
@@ -37,6 +38,45 @@ export declare class ToolCallTracker {
     observe(chunk: ConversationStreamChunk): Record<string, any> | null;
     private finish;
 }
+/**
+ * Strip `<script>`/`<style>` blocks and tags, decode the handful of entities
+ * a real docs page actually uses, and collapse whitespace. NOT a real
+ * HTML-to-markdown renderer — Claude Code's native `WebFetch` (the
+ * claude_code backend) does that; Flue has nothing equivalent built in, and
+ * a full renderer is more than this needs. A JSON/plain-text response
+ * passes through essentially unchanged, since none of these patterns match
+ * it — same "minimal, not a full parser" trade `jira_provider.ts`'s
+ * `adfToText` makes for ADF.
+ */
+export declare function stripHtml(text: string): string;
+/**
+ * The flue backend's stand-in for Claude Code's native `WebFetch` tool —
+ * `@flue/runtime` ships no such built-in (its only tool factories are
+ * read/write/edit/bash/grep/glob), so this is a custom `defineTool()`.
+ *
+ * Runs the actual request THROUGH `env.exec()` rather than calling Node's
+ * own `fetch()` directly — deliberately: `exec()` is the one universal
+ * primitive every `Sandbox` implements (`local`, and the remote
+ * `opensandbox`/`cloudflare` backends via `core/sandbox.ts`), so a command
+ * run through it executes INSIDE whichever sandbox the agent is actually
+ * using. A remote sandbox's own `egress` policy (`sandbox_opensandbox.ts`'s
+ * `networkPolicy`) then governs this exactly as it already governs every
+ * `bash` call — no separate egress rule needed for this tool. Calling
+ * `fetch()` here instead would silently bypass that policy by making the
+ * request from SPF's own orchestrator process rather than the sandbox.
+ *
+ * `curl`, with a `wget` fallback in the SAME command for portability across
+ * whatever base image a remote sandbox happens to ship — both are close to
+ * universal, but neither is guaranteed; a sandbox image with neither
+ * surfaces that plainly as a failed tool call (exit code + stderr handed
+ * back to the model), not a hang. Only `http`/`https` are accepted — this is
+ * also what stops a `file://` URL from turning "fetch a page" into "read an
+ * arbitrary local file" on the `local` sandbox, where `exec()` runs with the
+ * operator's own full filesystem access.
+ */
+export declare function createWebFetchTool(env: Sandbox): import("@flue/runtime").ToolDefinition<v.ObjectSchema<{
+    readonly url: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.NonEmptyAction<string, "url is required">]>;
+}, undefined>, undefined, false, false>;
 /** Used by agents.validate() so a typo'd tool name fails before anything spawns. */
 export declare function isKnownToolName(name: string): boolean;
 /**

package/dist/core/agent_flue.js

@@ -22,7 +22,8 @@
  * `run()`'s signature deliberately mirrors the old agent_pi.ts `run()` so
  * agents.ts's `send()` closure changes only its imports and field names.
  */
-import { AgentRunError, createBashTool, createEditTool, createGlobTool, createGrepTool, createReadTool, createWriteTool, init, observe, useDataWriter, useModel, useSandbox, useTool, } from "@flue/runtime";
+import * as v from "valibot";
+import { AgentRunError, createBashTool, createEditTool, createGlobTool, createGrepTool, createReadTool, createWriteTool, defineTool, init, observe, useDataWriter, useModel, useSandbox, useTool, } from "@flue/runtime";
 import { local, sqlite, start } from "@flue/runtime/node";
 import { UsageBreakdown, makeAgentResult } from "./data_types.js";
 import { registerOllamaModel } from "./ollama_provider.js";
@@ -127,6 +128,102 @@ export class ToolCallTracker {
         return record;
     }
 }
+// ── webfetch: Flue's stand-in for Claude Code's native WebFetch ─────────────
+const WEBFETCH_MAX_BYTES = 200_000; // a bounded chunk of a page, not the whole thing — matches RESULT_SNIPPET_CHARS's spirit
+const WEBFETCH_TIMEOUT_MS = 20_000;
+const WebFetchParams = v.object({ url: v.pipe(v.string(), v.nonEmpty("url is required")) });
+/** POSIX-safe single-quoting for a shell argument: closes and reopens the quote around any embedded `'`. */
+function shQuote(value) {
+    return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+/**
+ * Strip `<script>`/`<style>` blocks and tags, decode the handful of entities
+ * a real docs page actually uses, and collapse whitespace. NOT a real
+ * HTML-to-markdown renderer — Claude Code's native `WebFetch` (the
+ * claude_code backend) does that; Flue has nothing equivalent built in, and
+ * a full renderer is more than this needs. A JSON/plain-text response
+ * passes through essentially unchanged, since none of these patterns match
+ * it — same "minimal, not a full parser" trade `jira_provider.ts`'s
+ * `adfToText` makes for ADF.
+ */
+export function stripHtml(text) {
+    const cleaned = text
+        .replace(/<script[\s\S]*?<\/script>/gi, "")
+        .replace(/<style[\s\S]*?<\/style>/gi, "")
+        .replace(/<[^>]+>/g, " ")
+        .replace(/&nbsp;/g, " ")
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/[ \t]+/g, " ");
+    // Line-by-line trim BEFORE collapsing blank runs: a tag->" " substitution
+    // (above) routinely leaves a stray leading/trailing space on the line that
+    // used to hold a block-level tag's boundary — collapsing blank lines first
+    // would miss those, since they're not yet blank.
+    return cleaned
+        .split("\n")
+        .map((line) => line.trim())
+        .join("\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+/**
+ * The flue backend's stand-in for Claude Code's native `WebFetch` tool —
+ * `@flue/runtime` ships no such built-in (its only tool factories are
+ * read/write/edit/bash/grep/glob), so this is a custom `defineTool()`.
+ *
+ * Runs the actual request THROUGH `env.exec()` rather than calling Node's
+ * own `fetch()` directly — deliberately: `exec()` is the one universal
+ * primitive every `Sandbox` implements (`local`, and the remote
+ * `opensandbox`/`cloudflare` backends via `core/sandbox.ts`), so a command
+ * run through it executes INSIDE whichever sandbox the agent is actually
+ * using. A remote sandbox's own `egress` policy (`sandbox_opensandbox.ts`'s
+ * `networkPolicy`) then governs this exactly as it already governs every
+ * `bash` call — no separate egress rule needed for this tool. Calling
+ * `fetch()` here instead would silently bypass that policy by making the
+ * request from SPF's own orchestrator process rather than the sandbox.
+ *
+ * `curl`, with a `wget` fallback in the SAME command for portability across
+ * whatever base image a remote sandbox happens to ship — both are close to
+ * universal, but neither is guaranteed; a sandbox image with neither
+ * surfaces that plainly as a failed tool call (exit code + stderr handed
+ * back to the model), not a hang. Only `http`/`https` are accepted — this is
+ * also what stops a `file://` URL from turning "fetch a page" into "read an
+ * arbitrary local file" on the `local` sandbox, where `exec()` runs with the
+ * operator's own full filesystem access.
+ */
+export function createWebFetchTool(env) {
+    return defineTool({
+        name: "webfetch",
+        description: "Fetch a URL over HTTP(S) and return its content as plain text (HTML tags stripped). " +
+            "Use it to check current documentation for a library, API, or tool before relying on prior " +
+            "knowledge that may be stale or version-specific. GET only, no custom headers/auth, response " +
+            "truncated to a safe size.",
+        input: WebFetchParams,
+        run: async ({ data }) => {
+            let parsed;
+            try {
+                parsed = new URL(data.url);
+            }
+            catch {
+                return `webfetch: ${JSON.stringify(data.url)} is not a valid URL`;
+            }
+            if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+                return `webfetch: unsupported scheme ${JSON.stringify(parsed.protocol)} — only http/https are allowed`;
+            }
+            const url = shQuote(parsed.toString());
+            const command = `(curl -sS -L --max-time 20 --max-redirs 5 -A "spf-webfetch/1.0" ${url} ` +
+                `|| wget -qO- --timeout=20 ${url}) | head -c ${WEBFETCH_MAX_BYTES}`;
+            const result = await env.exec(command, { timeoutMs: WEBFETCH_TIMEOUT_MS });
+            if (result.exitCode !== 0 || !result.stdout.trim()) {
+                return `webfetch: fetching ${parsed.toString()} failed (exit ${result.exitCode}): ${(result.stderr || "no output").trim().slice(0, 2000)}`;
+            }
+            return stripHtml(result.stdout);
+        },
+    });
+}
 // ── tool-name resolution ─────────────────────────────────────────────────────
 const BUILTIN_TOOLS = {
     read: createReadTool,
@@ -135,6 +232,7 @@ const BUILTIN_TOOLS = {
     bash: createBashTool,
     grep: createGrepTool,
     glob: createGlobTool,
+    webfetch: createWebFetchTool,
 };
 // pi's vocabulary -> Flue's. "ls" has no Flue built-in (bash/glob cover it);
 // it is a KNOWN name that resolves to nothing, not an unknown one.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gr8ful/spf",
-  "version": "0.11.2",
+  "version": "0.12.0",
   "description": "Super Portable Factory — a global CLI for repeatable agents-plus-code workflows (ADWs)",
   "type": "module",
   "license": "MIT",