@yagni-app/code-staging 0.3.4-staging.1156.1 → 0.3.5-staging.1160.1

package/dist/extension/index.js

@@ -8,6 +8,7 @@ import { redactCommand } from "./redact.js";
 import { formatGuardianSubtotal, GUARDIAN_MODEL_TIER, makeGuardianState, resolveGuardianLimits, reviewCommand, deriveGuardianTimeoutMs } from "./permission/guardian.js";
 import { makeAskAdvisorTool, registerAdviseCommand } from "./askAdvisorTool.js";
 import { makeAskYagniTool } from "./askYagniTool.js";
+import { makeWebFetchTool } from "./webFetchTool.js";
 import { makeFileTicketTool, makeUpdateTicketStatusTool } from "./ticketTools.js";
 import { makeReviewBusinessMatchTool } from "./reviewTool.js";
 import { registerCmuxBridge } from "./cmux/index.js";
@@ -159,6 +160,9 @@ export async function registerYagni(pi, deps = {}) {
     // (flywheel-attributed records send dedupe: true). Run 7.
     const flywheelState = makeFlywheelState();
     pi.registerTool(makeAskYagniTool({ ...toolOpts, flywheel: flywheelState, getRepo: () => sessionRepo }));
+    // WebFetch (YAG-578): read an arbitrary URL as clean markdown + a
+    // standard-tier extraction, replacing the bash + curl + python dance.
+    pi.registerTool(makeWebFetchTool(toolOpts));
     // Ticket write-back (spec 2026-08-09): explicit user-intent writes to the
     // workspace tracker, attributed to the developer via per-user credentials.
     if (!evalMode) {

package/dist/extension/webFetch.d.ts

@@ -0,0 +1,85 @@
+/**
+ * WebFetch core: fetch + convert + extract, adapted from Claude Code's
+ * WebFetchTool (src/tools/WebFetchTool/utils.ts). Pure and pi-free so it is
+ * fully unit-testable against fixtures, mirroring resilientFetch.ts.
+ *
+ * Three stages:
+ *   1. FETCH    an arbitrary URL with a PLAIN fetch impl (deliberately NOT the
+ *               authed fetch — see the token-leak note below).
+ *   2. CONVERT  HTML → markdown via turndown; non-HTML passes through raw.
+ *   3. EXTRACT  a `standard`-tier completion against the user's prompt.
+ *
+ * SECURITY (token leak): the extension's `makeAuthedFetch` transparently
+ * re-sends a second request WITH a fresh `Bearer $YAGNI_TOKEN` whenever the
+ * first returns 401 (tokenProvider.ts). Routing that wrapper at an arbitrary
+ * host would leak the token to any site that answers 401. The URL fetch must
+ * therefore use a plain global fetch, never opts.fetchImpl.
+ */
+/** Upper bound on the markdown handed to the extraction model (Claude's cap). */
+export declare const MAX_MARKDOWN_LENGTH = 100000;
+/** Upper bound on the raw HTTP body we will accept (Claude's PSR cap). */
+export declare const MAX_HTTP_CONTENT_LENGTH: number;
+/** Per-request wall clock for the URL fetch (Claude's 60s). */
+export declare const FETCH_TIMEOUT_MS = 60000;
+/** Same-host redirect hop cap (Claude's 10, matching common client defaults). */
+export declare const MAX_REDIRECTS = 10;
+/** The marker the tool returns when a redirect jumps host. */
+export interface RedirectInfo {
+    type: "redirect";
+    originalUrl: string;
+    redirectUrl: string;
+    statusCode: number;
+}
+export interface FetchedContent {
+    content: string;
+    bytes: number;
+    code: number;
+    codeText: string;
+    contentType: string;
+}
+/**
+ * Reject URLs that could exfiltrate credentials or reach the user's own
+ * machine/network. Ported from Claude's validateURL: >2k chars, embedded
+ * userinfo, and single-label (non-public) hostnames are all refused.
+ */
+export declare function validateUrl(url: string): boolean;
+/**
+ * Whether a redirect may be followed: same protocol/port, no userinfo, and the
+ * host differs only by an optional leading `www.`. Ported from Claude.
+ */
+export declare function isPermittedRedirect(originalUrl: string, redirectUrl: string): boolean;
+export interface FetchMarkdownOptions {
+    /** Plain global fetch for the arbitrary URL (never an authed wrapper). */
+    fetchImpl?: typeof fetch;
+    signal?: AbortSignal;
+}
+/**
+ * Fetch `url` (following only same-host redirects) and convert the body to
+ * markdown. Returns the converted content, or a {@link RedirectInfo} when the
+ * response routes to a different host (the caller re-invokes with the new URL).
+ */
+export declare function fetchMarkdown(url: string, opts?: FetchMarkdownOptions): Promise<FetchedContent | RedirectInfo>;
+export interface ExtractionOptions {
+    baseUrl: string;
+    getToken: () => string | undefined;
+    /** The authed fetch (makeAuthedFetch) for the extraction call. */
+    fetchImpl?: typeof fetch;
+    attribution: () => Record<string, string>;
+    signal?: AbortSignal;
+}
+/**
+ * Build the extraction prompt: the fetched content plus the user's prompt plus
+ * the "answer only from the content" guardrails, mirroring Claude's
+ * makeSecondaryModelPrompt.
+ */
+export declare function makeExtractionPrompt(markdownContent: string, prompt: string): string;
+export interface ExtractResult {
+    text: string;
+}
+/**
+ * Run the extraction: one `standard`-tier completion against the metric proxy
+ * (the same /v1/chat/completions surface pi's `yagni` provider drives). The
+ * content was already fetched locally; only this summarization rides the proxy.
+ */
+export declare function extract(markdownContent: string, prompt: string, opts: ExtractionOptions): Promise<ExtractResult>;
+//# sourceMappingURL=webFetch.d.ts.map

package/dist/extension/webFetch.js

@@ -0,0 +1,192 @@
+/**
+ * WebFetch core: fetch + convert + extract, adapted from Claude Code's
+ * WebFetchTool (src/tools/WebFetchTool/utils.ts). Pure and pi-free so it is
+ * fully unit-testable against fixtures, mirroring resilientFetch.ts.
+ *
+ * Three stages:
+ *   1. FETCH    an arbitrary URL with a PLAIN fetch impl (deliberately NOT the
+ *               authed fetch — see the token-leak note below).
+ *   2. CONVERT  HTML → markdown via turndown; non-HTML passes through raw.
+ *   3. EXTRACT  a `standard`-tier completion against the user's prompt.
+ *
+ * SECURITY (token leak): the extension's `makeAuthedFetch` transparently
+ * re-sends a second request WITH a fresh `Bearer $YAGNI_TOKEN` whenever the
+ * first returns 401 (tokenProvider.ts). Routing that wrapper at an arbitrary
+ * host would leak the token to any site that answers 401. The URL fetch must
+ * therefore use a plain global fetch, never opts.fetchImpl.
+ */
+/** Upper bound on the markdown handed to the extraction model (Claude's cap). */
+export const MAX_MARKDOWN_LENGTH = 100_000;
+/** Upper bound on the raw HTTP body we will accept (Claude's PSR cap). */
+export const MAX_HTTP_CONTENT_LENGTH = 10 * 1024 * 1024;
+/** Per-request wall clock for the URL fetch (Claude's 60s). */
+export const FETCH_TIMEOUT_MS = 60_000;
+/** Same-host redirect hop cap (Claude's 10, matching common client defaults). */
+export const MAX_REDIRECTS = 10;
+/** URLs longer than this are rejected as a data-exfil surface (Claude). */
+const MAX_URL_LENGTH = 2000;
+/**
+ * Reject URLs that could exfiltrate credentials or reach the user's own
+ * machine/network. Ported from Claude's validateURL: >2k chars, embedded
+ * userinfo, and single-label (non-public) hostnames are all refused.
+ */
+export function validateUrl(url) {
+    if (url.length > MAX_URL_LENGTH)
+        return false;
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return false;
+    }
+    if (parsed.username || parsed.password)
+        return false;
+    const parts = parsed.hostname.split(".");
+    if (parts.length < 2)
+        return false;
+    return true;
+}
+/**
+ * Whether a redirect may be followed: same protocol/port, no userinfo, and the
+ * host differs only by an optional leading `www.`. Ported from Claude.
+ */
+export function isPermittedRedirect(originalUrl, redirectUrl) {
+    try {
+        const orig = new URL(originalUrl);
+        const redir = new URL(redirectUrl);
+        if (redir.protocol !== orig.protocol)
+            return false;
+        if (redir.port !== orig.port)
+            return false;
+        if (redir.username || redir.password)
+            return false;
+        const stripWww = (host) => host.replace(/^www\./, "");
+        return stripWww(orig.hostname) === stripWww(redir.hostname);
+    }
+    catch {
+        return false;
+    }
+}
+/** Lazily import turndown once, on first HTML fetch. */
+let turndownService;
+async function getTurndown() {
+    if (!turndownService) {
+        const Turndown = (await import("turndown")).default;
+        turndownService = new Turndown();
+    }
+    return turndownService;
+}
+/**
+ * Fetch `url` (following only same-host redirects) and convert the body to
+ * markdown. Returns the converted content, or a {@link RedirectInfo} when the
+ * response routes to a different host (the caller re-invokes with the new URL).
+ */
+export async function fetchMarkdown(url, opts = {}) {
+    if (!validateUrl(url))
+        throw new Error(`Invalid URL: "${url}"`);
+    const response = await fetchWithPermittedRedirects(url, opts, 0);
+    if ("type" in response)
+        return response;
+    const raw = Buffer.from(response.data);
+    const contentType = response.headers?.["content-type"] ?? "";
+    const bytes = raw.length;
+    const text = raw.toString("utf-8");
+    let content;
+    if (contentType.includes("text/html")) {
+        content = (await getTurndown()).turndown(text);
+    }
+    else {
+        content = text;
+    }
+    if (content.length > MAX_MARKDOWN_LENGTH) {
+        content = `${content.slice(0, MAX_MARKDOWN_LENGTH)}\n\n[Content truncated due to length...]`;
+    }
+    return {
+        content,
+        bytes,
+        code: response.status,
+        codeText: response.statusText,
+        contentType,
+    };
+}
+/** Low-level fetch with manual, same-host-only redirect following. */
+async function fetchWithPermittedRedirects(url, opts, depth) {
+    if (depth > MAX_REDIRECTS) {
+        throw new Error(`Too many redirects (exceeded ${MAX_REDIRECTS})`);
+    }
+    const signal = opts.signal;
+    const response = await (opts.fetchImpl ?? fetch)(url, {
+        method: "GET",
+        redirect: "manual",
+        signal,
+        headers: {
+            Accept: "text/markdown, text/html, */*",
+            "User-Agent": "YAGNI-Code (web-fetch)",
+        },
+    });
+    if ([301, 302, 307, 308].includes(response.status)) {
+        const location = response.headers.get("location");
+        if (!location)
+            throw new Error("Redirect missing Location header");
+        const redirectUrl = new URL(location, url).toString();
+        if (!isPermittedRedirect(url, redirectUrl)) {
+            return { type: "redirect", originalUrl: url, redirectUrl, statusCode: response.status };
+        }
+        return fetchWithPermittedRedirects(redirectUrl, opts, depth + 1);
+    }
+    const contentLength = Number(response.headers.get("content-length") ?? "0");
+    if (Number.isFinite(contentLength) && contentLength > MAX_HTTP_CONTENT_LENGTH) {
+        throw new Error(`Response exceeds ${MAX_HTTP_CONTENT_LENGTH} bytes`);
+    }
+    const data = await response.arrayBuffer();
+    if (data.byteLength > MAX_HTTP_CONTENT_LENGTH) {
+        throw new Error(`Response exceeds ${MAX_HTTP_CONTENT_LENGTH} bytes`);
+    }
+    const headers = {};
+    response.headers.forEach((value, key) => {
+        headers[key] = value;
+    });
+    return { data, headers, status: response.status, statusText: response.statusText };
+}
+/**
+ * Build the extraction prompt: the fetched content plus the user's prompt plus
+ * the "answer only from the content" guardrails, mirroring Claude's
+ * makeSecondaryModelPrompt.
+ */
+export function makeExtractionPrompt(markdownContent, prompt) {
+    const guidelines = "Provide a concise response based only on the content above. In your response:\n" +
+        " - Enforce a strict 125-character maximum for quotes from any source document.\n" +
+        " - Use quotation marks for exact language; any language outside the quotation should never be word-for-word the same.";
+    return `Web page content:\n---\n${markdownContent}\n---\n\n${prompt}\n\n${guidelines}`;
+}
+/**
+ * Run the extraction: one `standard`-tier completion against the metric proxy
+ * (the same /v1/chat/completions surface pi's `yagni` provider drives). The
+ * content was already fetched locally; only this summarization rides the proxy.
+ */
+export async function extract(markdownContent, prompt, opts) {
+    const res = await (opts.fetchImpl ?? fetch)(`${opts.baseUrl}/v1/chat/completions`, {
+        method: "POST",
+        headers: {
+            "content-type": "application/json",
+            authorization: `Bearer ${opts.getToken() ?? ""}`,
+            ...opts.attribution(),
+        },
+        body: JSON.stringify({
+            model: "standard",
+            messages: [{ role: "user", content: makeExtractionPrompt(markdownContent, prompt) }],
+        }),
+        signal: opts.signal,
+    });
+    if (!res.ok) {
+        throw new Error(`web_fetch extraction failed: HTTP ${res.status}`);
+    }
+    const data = (await res.json());
+    const text = data.choices?.[0]?.message?.content;
+    if (typeof text !== "string") {
+        throw new Error("web_fetch extraction returned no content");
+    }
+    return { text };
+}
+//# sourceMappingURL=webFetch.js.map

package/dist/extension/webFetchTool.d.ts

@@ -0,0 +1,34 @@
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { makeExtractionPrompt, MAX_MARKDOWN_LENGTH } from "./webFetch.js";
+export interface MakeWebFetchToolOptions {
+    baseUrl: string;
+    getToken: () => string | undefined;
+    /**
+     * The authed fetch (makeAuthedFetch) used for the EXTRACTION call only. It is
+     * deliberately never used for the arbitrary URL fetch — see the token-leak
+     * note in webFetch.ts.
+     */
+    fetchImpl?: typeof fetch;
+    /**
+     * The PLAIN fetch used for the arbitrary URL fetch (defaults to global fetch,
+     * never the authed wrapper). Injectable so tests can stub the network without
+     * risking a real outbound request.
+     */
+    webFetchImpl?: typeof fetch;
+}
+declare const parameters: Type.TObject<{
+    url: Type.TString;
+    prompt: Type.TString;
+}>;
+/**
+ * Build the `web_fetch` tool definition — YAG-578.
+ *
+ * Fetches a URL, converts HTML to markdown, then extracts the requested
+ * information with a `standard`-tier completion. Read-only, client-side; it is
+ * the tidy replacement for a `bash` + `curl` + `python` fetch-and-strip dance.
+ */
+export declare function makeWebFetchTool(opts: MakeWebFetchToolOptions): ToolDefinition<typeof parameters, undefined>;
+/** Re-exported for tests that assemble the extraction prompt. */
+export { MAX_MARKDOWN_LENGTH, makeExtractionPrompt };
+//# sourceMappingURL=webFetchTool.d.ts.map

package/dist/extension/webFetchTool.js

@@ -0,0 +1,104 @@
+import { Container, Text } from "@earendil-works/pi-tui";
+import { Type } from "typebox";
+import { attributionHeaders } from "./config.js";
+import { markdownOrPlain } from "./subagentRender.js";
+import { extract, fetchMarkdown, makeExtractionPrompt, MAX_MARKDOWN_LENGTH, } from "./webFetch.js";
+const parameters = Type.Object({
+    url: Type.String({ description: "The URL to fetch content from" }),
+    prompt: Type.String({ description: "The prompt to run on the fetched content" }),
+});
+/** Collapsed preview length, in lines. */
+const PREVIEW_LINES = 8;
+function clipLine(text, max) {
+    const collapsed = text.replace(/\s+/g, " ").trim();
+    if (collapsed.length <= max)
+        return collapsed;
+    return `${collapsed.slice(0, max - 1)}…`;
+}
+function hostOf(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Build the `web_fetch` tool definition — YAG-578.
+ *
+ * Fetches a URL, converts HTML to markdown, then extracts the requested
+ * information with a `standard`-tier completion. Read-only, client-side; it is
+ * the tidy replacement for a `bash` + `curl` + `python` fetch-and-strip dance.
+ */
+export function makeWebFetchTool(opts) {
+    return {
+        name: "web_fetch",
+        label: "Web fetch",
+        description: "Fetch a URL, convert HTML to markdown, and extract the requested information " +
+            "using a small model. Read-only. Use this to read an arbitrary web page or docs " +
+            "URL (an API reference, a changelog, a docs page) instead of reaching for bash + curl. " +
+            "It will fail for authenticated or private URLs (Google Docs, Jira, GitHub, etc.) — " +
+            "use a specialized connector or MCP tool for those.",
+        promptSnippet: "web_fetch: fetch a URL, convert to markdown, and summarize against a prompt (read-only).",
+        promptGuidelines: [
+            "Use web_fetch to read a URL's content instead of bash + curl when you need a page or a summary of it.",
+        ],
+        parameters,
+        renderCall(args, theme) {
+            const t = theme;
+            const url = clipLine(args?.url ?? "…", 80);
+            const prompt = args?.prompt ? clipLine(args.prompt, 60) : "";
+            let text = `${t.fg("toolTitle", t.bold("web_fetch"))} ${t.fg("accent", url)}`;
+            if (prompt)
+                text += ` — ${t.fg("dim", prompt)}`;
+            return new Text(text, 0, 0);
+        },
+        renderResult(result, { expanded, isPartial }, theme) {
+            const t = theme;
+            const content = result.content.find((c) => c.type === "text");
+            const text = content && "text" in content ? content.text : "";
+            if (isPartial)
+                return new Text(t.fg("muted", text || "Fetching…"), 0, 0);
+            if (expanded) {
+                const container = new Container();
+                container.addChild(markdownOrPlain(text || "(no content)", t));
+                return container;
+            }
+            const lines = text.trim().split("\n");
+            const out = lines.slice(0, PREVIEW_LINES).map((l) => t.fg("toolOutput", l));
+            if (lines.length > PREVIEW_LINES)
+                out.push(t.fg("muted", "  (ctrl+o to expand)"));
+            return new Text(out.join("\n"), 0, 0);
+        },
+        async execute(_toolCallId, params, signal, onUpdate) {
+            onUpdate?.({ content: [{ type: "text", text: "Fetching…" }], details: undefined });
+            const fetched = await fetchMarkdown(params.url, { fetchImpl: opts.webFetchImpl, signal });
+            // A cross-host redirect is not auto-followed; ask the model to re-call.
+            if (fetched.type === "redirect") {
+                const r = fetched;
+                const statusText = r.statusCode === 301 ? "Moved Permanently"
+                    : r.statusCode === 308 ? "Permanent Redirect"
+                        : r.statusCode === 307 ? "Temporary Redirect"
+                            : "Found";
+                const message = `REDIRECT DETECTED: The URL redirects to a different host.\n\n` +
+                    `Original URL: ${r.originalUrl}\nRedirect URL: ${r.redirectUrl}\n` +
+                    `Status: ${r.statusCode} ${statusText}\n\n` +
+                    `To complete your request, call web_fetch again with these parameters:\n` +
+                    `- url: "${r.redirectUrl}"\n- prompt: "${params.prompt}"`;
+                return { content: [{ type: "text", text: message }], details: undefined };
+            }
+            const content = fetched.content;
+            const { text } = await extract(content, params.prompt, {
+                baseUrl: opts.baseUrl,
+                getToken: opts.getToken,
+                fetchImpl: opts.fetchImpl,
+                attribution: () => attributionHeaders(),
+                signal,
+            });
+            return { content: [{ type: "text", text }], details: undefined };
+        },
+    };
+}
+/** Re-exported for tests that assemble the extraction prompt. */
+export { MAX_MARKDOWN_LENGTH, makeExtractionPrompt };
+//# sourceMappingURL=webFetchTool.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yagni-app/code-staging",
-  "version": "0.3.4-staging.1156.1",
+  "version": "0.3.5-staging.1160.1",
   "description": "YAGNI Code: a terminal coding agent that already knows your company. One YAGNI login routes the model and grounds the agent in your team's context.",
   "license": "SEE LICENSE IN LICENSE.md",
   "author": "YAGNI, Inc. <jack@yagni.app> (https://yagni.app)",
@@ -37,7 +37,8 @@
     "@earendil-works/pi-coding-agent": "0.84.1",
     "@earendil-works/pi-tui": "0.84.1",
     "smol-toml": "^1.8.0",
+    "turndown": "^7.2.4",
     "typebox": "^1.3.15"
   },
-  "yagniSourceSha": "28861a8fed4d500c1391588cc71597d6ff0fa43a"
+  "yagniSourceSha": "cb341f87d13165d17246c8c335200c69982bd60b"
 }