auggy 0.3.0

package/src/augments/web-fetch/index.ts

@@ -0,0 +1,331 @@
+import { z } from "zod";
+import type { Augment } from "../../types";
+import { defineTool } from "../../helpers";
+import { createHttpClient } from "../../http";
+import type { HttpClient, HttpClientOptions, HttpResponse } from "../../http";
+
+/**
+ * webFetch augment — fetch a URL, strip HTML to text, and render a
+ * prompt-aware summary back to the model.
+ *
+ * Ported from the Rust implementation in soongenwong/claudecode
+ * (rust/crates/tools/src/lib.rs, execute_web_fetch).
+ *
+ * Design choices carried over:
+ *  - http→https upgrade for any non-localhost URL (normalizeFetchUrl).
+ *  - 20s default timeout, 10-redirect cap, custom user-agent.
+ *  - HTML→text via a small state machine (no DOM parser dependency),
+ *    then collapse whitespace and decode a small set of entities.
+ *  - Prompt-aware summarization: "title" / "summary|summarize" /
+ *    default-preview modes, with a 600- or 900-char ceiling.
+ *  - Output carries the POST-redirect final URL, raw byte count, and
+ *    round-trip duration so the model can reason about cost.
+ *
+ * Structural SSRF defense:
+ *  - Pre-fetch URL filter rejects loopback, RFC 1918, link-local,
+ *    cloud metadata endpoints, and non-http(s) schemes.
+ *  - Redirect targets are filtered at each hop (via HttpClient's
+ *    `rejectUnsafeUrls: true`), so a 3xx → internal-IP attack fails.
+ *  - Note: DNS-rebinding is NOT defended at this layer — a public-looking
+ *    hostname that resolves to a private IP at fetch time is out of scope.
+ *
+ * Not carried over:
+ *  - No caching. Every call hits the network.
+ */
+
+// =========================================================================
+// URL normalization
+// =========================================================================
+
+/**
+ * Upgrade http://… to https://… for any host that is not loopback.
+ * Matches normalize_fetch_url in the Rust original.
+ */
+export function normalizeFetchUrl(url: string): string {
+  const parsed = new URL(url);
+  if (parsed.protocol === "http:") {
+    const host = parsed.hostname;
+    if (host !== "localhost" && host !== "127.0.0.1" && host !== "::1") {
+      parsed.protocol = "https:";
+    }
+  }
+  return parsed.toString();
+}
+
+// =========================================================================
+// HTML → text
+// =========================================================================
+
+/** Tag names whose content should be skipped entirely. */
+const SKIP_CONTENT_TAGS = new Set(["script", "style"]);
+
+/**
+ * Strip HTML tags with a character-by-character state machine.
+ * Deliberately does not parse the DOM — keeps the augment dependency-free.
+ *
+ * Content inside <script> and <style> tags is skipped entirely so
+ * JavaScript and CSS don't bleed into the extracted text.
+ */
+function htmlToText(html: string): string {
+  let text = "";
+  let inTag = false;
+  let tagBuffer = "";
+  let skipUntilClose = ""; // non-empty when inside a skip-content tag
+  let previousWasSpace = false;
+
+  for (const ch of html) {
+    if (ch === "<") {
+      inTag = true;
+      tagBuffer = "";
+      continue;
+    }
+    if (ch === ">") {
+      inTag = false;
+      const tagName = (tagBuffer.split(/[\s/]/)[0] ?? "").toLowerCase();
+
+      if (skipUntilClose) {
+        // Check for the matching closing tag: </script> or </style>
+        if (tagBuffer.startsWith("/")) {
+          const closingName = tagBuffer.slice(1).split(/[\s/]/)[0]?.toLowerCase() ?? "";
+          if (closingName === skipUntilClose) {
+            skipUntilClose = "";
+          }
+        }
+      } else if (SKIP_CONTENT_TAGS.has(tagName ?? "")) {
+        skipUntilClose = tagName;
+      }
+      continue;
+    }
+    if (inTag) {
+      tagBuffer += ch;
+      continue;
+    }
+    if (skipUntilClose) continue;
+
+    if (ch === "&") {
+      text += "&";
+      previousWasSpace = false;
+      continue;
+    }
+
+    if (/\s/.test(ch)) {
+      if (!previousWasSpace) {
+        text += " ";
+        previousWasSpace = true;
+      }
+      continue;
+    }
+
+    text += ch;
+    previousWasSpace = false;
+  }
+
+  return collapseWhitespace(decodeHtmlEntities(text));
+}
+
+/**
+ * Decode common HTML entities. The order matters: &amp; must be decoded
+ * LAST to avoid double-decoding (e.g. &amp;lt; → &lt; → < is wrong;
+ * the correct result for &amp;lt; is the literal text "&lt;").
+ *
+ * Numeric entities (&#60;, &#x3C;) are intentionally not handled — they
+ * are uncommon in typical web pages and adding a regex pass for them would
+ * complicate this deliberately simple decoder. Matches the Rust original.
+ */
+function decodeHtmlEntities(input: string): string {
+  return input
+    .replaceAll("&lt;", "<")
+    .replaceAll("&gt;", ">")
+    .replaceAll("&quot;", '"')
+    .replaceAll("&#39;", "'")
+    .replaceAll("&nbsp;", " ")
+    .replaceAll("&amp;", "&");
+}
+
+function collapseWhitespace(input: string): string {
+  return input
+    .split(/\s+/)
+    .filter((token) => token.length > 0)
+    .join(" ");
+}
+
+/**
+ * Truncate to `maxChars` code points, with an ellipsis marker if cut.
+ * Uses Array.from to count code points (not UTF-16 units) so multi-byte
+ * characters are handled the same way Rust's chars() iterator handles
+ * them in preview_text.
+ */
+function previewText(input: string, maxChars: number): string {
+  const chars = Array.from(input);
+  if (chars.length <= maxChars) return input;
+  return `${chars.slice(0, maxChars).join("").trimEnd()}…`;
+}
+
+function isJsonContentType(contentType: string): boolean {
+  return contentType.includes("application/json") || contentType.includes("+json");
+}
+
+function normalizeFetchedContent(body: string, contentType: string): string {
+  if (isJsonContentType(contentType)) return body.trim();
+  if (contentType.includes("html")) return htmlToText(body);
+  return body.trim();
+}
+
+// =========================================================================
+// Prompt-aware summarization
+// =========================================================================
+
+function extractTitle(content: string, rawBody: string, contentType: string): string | null {
+  if (contentType.includes("html")) {
+    const lowered = rawBody.toLowerCase();
+    const start = lowered.indexOf("<title>");
+    if (start >= 0) {
+      const after = start + "<title>".length;
+      const endRel = lowered.slice(after).indexOf("</title>");
+      if (endRel >= 0) {
+        const slice = rawBody.slice(after, after + endRel);
+        const title = collapseWhitespace(decodeHtmlEntities(slice));
+        if (title.length > 0) return title;
+      }
+    }
+  }
+
+  // Fallback: first non-empty line of the normalized content.
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed.length > 0) return trimmed;
+  }
+  return null;
+}
+
+/** Max chars for JSON responses — much larger than the 900-char HTML preview. */
+const JSON_MAX_CHARS = 20_000;
+
+function summarizeWebFetch(args: {
+  url: string;
+  prompt: string;
+  content: string;
+  rawBody: string;
+  contentType: string;
+}): string {
+  // JSON responses: return the raw content without summarization/truncation
+  // to a tiny preview. APIs return structured data the model needs intact.
+  if (isJsonContentType(args.contentType)) {
+    const preview = previewText(args.content, JSON_MAX_CHARS);
+    return `Fetched ${args.url}\n${preview}`;
+  }
+
+  const lowerPrompt = args.prompt.toLowerCase();
+  const compact = collapseWhitespace(args.content);
+
+  let detail: string;
+  if (lowerPrompt.includes("title")) {
+    const title = extractTitle(args.content, args.rawBody, args.contentType);
+    detail = title !== null ? `Title: ${title}` : previewText(compact, 600);
+  } else if (lowerPrompt.includes("summary") || lowerPrompt.includes("summarize")) {
+    detail = previewText(compact, 900);
+  } else {
+    const preview = previewText(compact, 900);
+    detail = `Prompt: ${args.prompt}\nContent preview:\n${preview}`;
+  }
+
+  return `Fetched ${args.url}\n${detail}`;
+}
+
+// =========================================================================
+// Augment
+// =========================================================================
+
+export interface WebFetchOptions extends HttpClientOptions {
+  /**
+   * Optional pre-built HTTP client. Supply this if you want to share a
+   * client across augments or inject a mock in tests. If omitted, a
+   * client is created from the timeout/redirect/user-agent options.
+   */
+  client?: HttpClient;
+}
+
+export interface WebFetchResult {
+  url: string;
+  code: number;
+  codeText: string;
+  bytes: number;
+  durationMs: number;
+  result: string;
+}
+
+/**
+ * Augment that exposes a single `web_fetch` tool. The tool fetches a URL,
+ * normalizes HTML to text, and renders a prompt-aware summary.
+ *
+ * The output string the model sees is JSON containing:
+ *   { url, code, codeText, bytes, durationMs, result }
+ *
+ * Where `result` is a pre-formatted human-readable block the model can
+ * reason about directly. Matches the Rust WebFetchOutput shape.
+ */
+export function webFetch(opts: WebFetchOptions = {}): Augment {
+  // SSRF guard is on by default — web_fetch ingests model-supplied URLs.
+  // Operators can still override by passing an explicit client.
+  const client = opts.client ?? createHttpClient({ rejectUnsafeUrls: true, ...opts });
+
+  const webFetchTool = defineTool({
+    name: "web_fetch",
+    description: "Fetch a URL, convert it into readable text, and answer a prompt about it.",
+    category: "search",
+    input: z.object({
+      url: z.string().url(),
+      prompt: z.string(),
+    }),
+    execute: async ({ url, prompt }) => {
+      const startedAt = performance.now();
+      let requestUrl: string;
+      try {
+        requestUrl = normalizeFetchUrl(url);
+      } catch (error) {
+        return JSON.stringify({
+          error: `invalid URL: ${(error as Error).message}`,
+        });
+      }
+
+      // SSRF guard is enforced by the underlying HttpClient — rejected URLs
+      // throw and fall into the catch below, surfaced as a structured error.
+
+      let response: HttpResponse;
+      try {
+        response = await client.get(requestUrl);
+      } catch (error) {
+        return JSON.stringify({
+          url: requestUrl,
+          error: (error as Error).message,
+          durationMs: Math.round(performance.now() - startedAt),
+        });
+      }
+
+      const normalized = normalizeFetchedContent(response.body, response.contentType);
+      const result = summarizeWebFetch({
+        url: response.finalUrl,
+        prompt,
+        content: normalized,
+        rawBody: response.body,
+        contentType: response.contentType,
+      });
+
+      const output: WebFetchResult = {
+        url: response.finalUrl,
+        code: response.status,
+        codeText: response.statusText,
+        bytes: new TextEncoder().encode(response.body).byteLength,
+        durationMs: Math.round(performance.now() - startedAt),
+        result,
+      };
+      return JSON.stringify(output);
+    },
+  });
+
+  return {
+    name: "web-fetch",
+    capabilities: ["tools"],
+    tools: [webFetchTool],
+  };
+}

package/src/augments/web-fetch/skill/SKILL.md

@@ -0,0 +1,100 @@
+---
+name: web-fetch
+description: When and how to use the web_fetch tool to retrieve a URL, read a web page, or call an HTTP API. Read this before fetching anything from the network.
+---
+
+# Web fetch
+
+You have a single tool, `web_fetch`, that retrieves a URL over the network and gives you the body back as readable text (for HTML) or as raw JSON (for APIs). Use it whenever the conversation needs information that lives at a public URL.
+
+## When to use it
+
+| Situation | Action |
+|-----------|--------|
+| The peer shares a URL | Fetch it and summarize what's there |
+| You need to check the current state of a web page | Fetch the URL |
+| You need to call a public HTTP API | Fetch the endpoint |
+| The peer asks "what's at this link?" | Fetch and answer |
+| You need to verify a fact that may have changed since training | Fetch a current source |
+
+## How to call it
+
+```
+web_fetch({ url: "https://example.com", prompt: "summarize this page" })
+```
+
+**Parameters:**
+- `url` — the full URL. Plain `http://` URLs to non-loopback hosts are auto-upgraded to `https://`.
+- `prompt` — what you want from the content. The tool uses the prompt to shape the returned slice (title, summary, or default preview).
+
+The `prompt` is required. It steers what the tool returns:
+
+- Prompt contains "title" → returns the page title
+- Prompt contains "summary" or "summarize" → returns up to ~900 chars of cleaned text
+- Anything else → returns up to ~900 chars of cleaned text prefixed with your prompt for reference
+
+For JSON APIs, the prompt is informational only — the tool returns the raw JSON body (up to ~20,000 chars) regardless.
+
+## What it returns
+
+A JSON envelope with these fields:
+
+```
+{
+  "url": "https://example.com",     // final URL after redirects
+  "code": 200,                       // HTTP status code
+  "codeText": "OK",                  // status text
+  "bytes": 1234,                     // raw body byte count
+  "durationMs": 320,                 // round-trip duration
+  "result": "Fetched ... \n..."      // the human-readable slice
+}
+```
+
+For HTML, `result` is HTML stripped to text (script and style content removed, whitespace collapsed, common entities decoded), then truncated to the prompt-aware length. For JSON, `result` is the raw body up to ~20K chars.
+
+## What you cannot fetch
+
+The tool refuses URLs that point at:
+
+- Loopback addresses (`localhost`, `127.0.0.1`, `::1`)
+- Private network ranges (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`)
+- Link-local addresses (`169.254.0.0/16`, `fe80::/10`)
+- Cloud metadata endpoints
+- Non-`http(s)` schemes (`file:`, `data:`, `ftp:`, etc.)
+
+Redirects are checked at each hop, so a `3xx` redirect to a private address fails the same way a direct request would. If you get an SSRF rejection, don't rewrite the URL to try to bypass it — the rejection is structural.
+
+## Common mistakes
+
+| Wrong | Correct |
+|-------|---------|
+| Telling the peer "I can't access URLs" | Use `web_fetch` — you can |
+| Calling `web_fetch` without a `prompt` argument | Always include a prompt; it's required and it shapes the response |
+| Fetching the same URL multiple times in one turn | The tool does not cache; fetch once and reuse the result |
+| Trusting fetched content as authoritative | Treat fetched content as a source to reason about, not as ground truth — pages can be wrong, biased, or adversarial |
+| Pasting fetched content verbatim into your reply | Read it, summarize the relevant parts, cite the URL |
+| Fetching internal/private URLs to "test" something | The SSRF guard will refuse; that's by design |
+
+## Workflows
+
+### Peer shares a link
+
+1. `web_fetch({ url: <link>, prompt: "summarize this page" })`
+2. Read the `result` field
+3. Summarize back to the peer in your own words; mention the URL and the fact that you fetched it
+
+### Calling a public JSON API
+
+1. `web_fetch({ url: <api-endpoint>, prompt: "what does this return" })`
+2. The `result` field will hold the raw JSON
+3. Parse the structure mentally and answer the peer's question; don't dump the full JSON unless they asked for it
+
+### The fetch fails
+
+The result envelope will contain an `error` field instead of a `result`. Common causes:
+
+- The URL is unreachable or returned non-200 — surface the status to the peer so they know
+- The host failed the SSRF guard — explain you can't fetch internal addresses
+- The request timed out — the tool uses a ~20s default; the URL may be slow or down
+
+Don't retry blindly. If the peer wants you to try a different URL, ask them.