@oh-my-pi/pi-coding-agent 6.9.69 → 8.0.0

package/src/{core/tools/web-fetch.ts → tools/fetch.ts}

@@ -1,5 +1,4 @@
-import { rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
+import { mkdir, rm } from "node:fs/promises";
 import * as path from "node:path";
 import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core";
 import type { Component } from "@oh-my-pi/pi-tui";
@@ -8,26 +7,26 @@ import { ptree } from "@oh-my-pi/pi-utils";
 import { type Static, Type } from "@sinclair/typebox";
 import { nanoid } from "nanoid";
 import { parse as parseHtml } from "node-html-parser";
-import { type Theme, theme } from "../../modes/interactive/theme/theme";
-import webFetchDescription from "../../prompts/tools/web-fetch.md" with { type: "text" };
-import { ensureTool } from "../../utils/tools-manager";
-import type { RenderResultOptions } from "../custom-tools/types";
-import { renderPromptTemplate } from "../prompt-templates";
+import { renderPromptTemplate } from "$c/config/prompt-templates";
+import type { RenderResultOptions } from "$c/extensibility/custom-tools/types";
+import { type Theme, theme } from "$c/modes/theme/theme";
+import fetchDescription from "$c/prompts/tools/fetch.md" with { type: "text" };
+import type { OutputMeta } from "$c/tools/output-meta";
+import { ToolAbortError } from "$c/tools/tool-errors";
+import { ensureTool } from "$c/utils/tools-manager";
+import { specialHandlers } from "$c/web/scrapers/index";
+import type { RenderResult } from "$c/web/scrapers/types";
+import { finalizeOutput, loadPage, MAX_OUTPUT_CHARS } from "$c/web/scrapers/types";
+import { convertWithMarkitdown, fetchBinary } from "$c/web/scrapers/utils";
 import type { ToolSession } from "./index";
+import { applyListLimit } from "./list-limit";
 import { formatExpandHint } from "./render-utils";
-import { specialHandlers } from "./web-scrapers/index";
-import type { RenderResult } from "./web-scrapers/types";
-import { finalizeOutput, loadPage } from "./web-scrapers/types";
-import { convertWithMarkitdown, fetchBinary } from "./web-scrapers/utils";
+import { toolResult } from "./tool-result";
 
 // =============================================================================
 // Types and Constants
 // =============================================================================
 
-const MIN_TIMEOUT = 1_000;
-const DEFAULT_TIMEOUT = 20_000;
-const MAX_TIMEOUT = 45_000;
-
 // Convertible document types (markitdown supported)
 const CONVERTIBLE_MIMES = new Set([
 	"application/pdf",
@@ -246,7 +245,7 @@ async function tryMdSuffix(url: string, timeout: number, signal?: AbortSignal):
 		if (signal?.aborted) {
 			return null;
 		}
-		const result = await loadPage(candidate, { timeout: Math.min(timeout, MAX_TIMEOUT), signal });
+		const result = await loadPage(candidate, { timeout, signal });
 		if (result.ok && result.content.trim().length > 100 && !looksLikeHtml(result.content)) {
 			return result.content;
 		}
@@ -448,9 +447,10 @@ function parseFeedToMarkdown(content: string, maxItems = 10): string {
 async function renderHtmlToText(
 	html: string,
 	timeout: number,
+	scratchDir: string,
 ): Promise<{ content: string; ok: boolean; method: string }> {
-	const tmpDir = tmpdir();
-	const tmpFile = path.join(tmpDir, `omp-${nanoid()}.html`);
+	await mkdir(scratchDir, { recursive: true });
+	const tmpFile = path.join(scratchDir, `omp-${nanoid()}.html`);
 
 	try {
 		await Bun.write(tmpFile, html);
@@ -532,7 +532,7 @@ function formatJson(content: string): string {
 async function handleSpecialUrls(url: string, timeout: number, signal?: AbortSignal): Promise<RenderResult | null> {
 	for (const handler of specialHandlers) {
 		if (signal?.aborted) {
-			throw new Error("Operation aborted");
+			throw new ToolAbortError();
 		}
 		const result = await handler(url, timeout, signal);
 		if (result) return result;
@@ -550,13 +550,14 @@ async function handleSpecialUrls(url: string, timeout: number, signal?: AbortSig
 async function renderUrl(
 	url: string,
 	timeout: number,
-	raw: boolean = false,
+	raw: boolean,
+	scratchDir: string,
 	signal?: AbortSignal,
 ): Promise<RenderResult> {
 	const notes: string[] = [];
 	const fetchedAt = new Date().toISOString();
 	if (signal?.aborted) {
-		throw new Error("Operation aborted");
+		throw new ToolAbortError();
 	}
 
 	// Handle internal protocol URLs (e.g., pi-internal://) - return empty
@@ -586,7 +587,7 @@ async function renderUrl(
 	// Step 2: Fetch page
 	const response = await loadPage(url, { timeout, signal });
 	if (signal?.aborted) {
-		throw new Error("Operation aborted");
+		throw new ToolAbortError();
 	}
 	if (!response.ok) {
 		return {
@@ -787,11 +788,11 @@ async function renderUrl(
 		}
 
 		if (signal?.aborted) {
-			throw new Error("Operation aborted");
+			throw new ToolAbortError();
 		}
 
 		// Step 6: Render HTML with lynx or html2text
-		const htmlResult = await renderHtmlToText(rawContent, timeout);
+		const htmlResult = await renderHtmlToText(rawContent, timeout, scratchDir);
 		if (!htmlResult.ok) {
 			notes.push("html rendering failed (lynx/html2text unavailable)");
 			const output = finalizeOutput(rawContent);
@@ -871,66 +872,68 @@ async function renderUrl(
 // Tool Definition
 // =============================================================================
 
-const webFetchSchema = Type.Object({
-	url: Type.String({ description: "The URL to fetch and render" }),
-	timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (default: 20, max: 120)" })),
-	raw: Type.Optional(
-		Type.Boolean({ description: "Return raw content without site-specific rendering or LLM-friendly transforms" }),
-	),
+const fetchSchema = Type.Object({
+	url: Type.String({ description: "URL to fetch" }),
+	timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (default: 20)" })),
+	raw: Type.Optional(Type.Boolean({ description: "Return raw HTML without transforms" })),
 });
 
-export interface WebFetchToolDetails {
+export interface FetchToolDetails {
 	url: string;
 	finalUrl: string;
 	contentType: string;
 	method: string;
 	truncated: boolean;
 	notes: string[];
+	meta?: OutputMeta;
 }
 
-export class WebFetchTool implements AgentTool<typeof webFetchSchema, WebFetchToolDetails> {
-	public readonly name = "web_fetch";
-	public readonly label = "Web Fetch";
+export class FetchTool implements AgentTool<typeof fetchSchema, FetchToolDetails> {
+	public readonly name = "fetch";
+	public readonly label = "Fetch";
 	public readonly description: string;
-	public readonly parameters = webFetchSchema;
+	public readonly parameters = fetchSchema;
+	private readonly session: ToolSession;
 
-	constructor(_session: ToolSession) {
-		this.description = renderPromptTemplate(webFetchDescription);
+	constructor(session: ToolSession) {
+		this.session = session;
+		this.description = renderPromptTemplate(fetchDescription);
 	}
 
 	public async execute(
 		_toolCallId: string,
-		params: Static<typeof webFetchSchema>,
+		params: Static<typeof fetchSchema>,
 		signal?: AbortSignal,
-		_onUpdate?: AgentToolUpdateCallback<WebFetchToolDetails>,
+		_onUpdate?: AgentToolUpdateCallback<FetchToolDetails>,
 		_context?: AgentToolContext,
-	): Promise<AgentToolResult<WebFetchToolDetails>> {
-		const { url, timeout = DEFAULT_TIMEOUT, raw = false } = params;
+	): Promise<AgentToolResult<FetchToolDetails>> {
+		const { url, timeout: rawTimeout = 20, raw = false } = params;
+
+		// Auto-convert milliseconds to seconds if value > 1000 (16+ min is unreasonable)
+		const timeoutSec = rawTimeout > 1000 ? rawTimeout / 1000 : rawTimeout;
+
+		// Clamp to valid range (seconds)
+		const effectiveTimeout = Math.min(Math.max(timeoutSec, 1), 45);
 
 		if (signal?.aborted) {
-			throw new Error("Operation aborted");
+			throw new ToolAbortError();
 		}
 
-		// Clamp timeout
-		const effectiveTimeout = Math.min(Math.max(timeout, MIN_TIMEOUT), MAX_TIMEOUT);
-
-		const result = await renderUrl(url, effectiveTimeout, raw, signal);
+		const scratchDir = this.session.getArtifactsDir?.() ?? this.session.cwd;
+		const result = await renderUrl(url, effectiveTimeout, raw, scratchDir, signal);
 
 		// Format output
 		let output = "";
 		output += `URL: ${result.finalUrl}\n`;
 		output += `Content-Type: ${result.contentType}\n`;
 		output += `Method: ${result.method}\n`;
-		if (result.truncated) {
-			output += `Warning: Output was truncated\n`;
-		}
 		if (result.notes.length > 0) {
 			output += `Notes: ${result.notes.join("; ")}\n`;
 		}
 		output += `\n---\n\n`;
 		output += result.content;
 
-		const details: WebFetchToolDetails = {
+		const details: FetchToolDetails = {
 			url: result.url,
 			finalUrl: result.finalUrl,
 			contentType: result.contentType,
@@ -939,10 +942,21 @@ export class WebFetchTool implements AgentTool<typeof webFetchSchema, WebFetchTo
 			notes: result.notes,
 		};
 
-		return {
-			content: [{ type: "text", text: output }],
-			details,
-		};
+		const resultBuilder = toolResult(details).text(output).sourceUrl(result.finalUrl);
+		if (result.truncated) {
+			const outputLines = result.content.split("\n").length;
+			const outputBytes = Buffer.byteLength(result.content, "utf-8");
+			const totalBytes = Math.max(outputBytes + 1, MAX_OUTPUT_CHARS + 1);
+			const totalLines = outputLines + 1;
+			resultBuilder.truncationFromText(result.content, {
+				direction: "tail",
+				totalLines,
+				totalBytes,
+				maxBytes: MAX_OUTPUT_CHARS,
+			});
+		}
+
+		return resultBuilder.done();
 	}
 }
 
@@ -967,32 +981,26 @@ function getDomain(url: string): string {
 	}
 }
 
-/** Get first N lines of text as preview */
-function getPreviewLines(text: string, maxLines: number, maxLineLen: number, ellipsis: string): string[] {
-	const lines = text.split("\n").filter((l) => l.trim());
-	return lines.slice(0, maxLines).map((l) => truncate(l.trim(), maxLineLen, ellipsis));
-}
-
 /** Count non-empty lines */
 function countNonEmptyLines(text: string): number {
 	return text.split("\n").filter((l) => l.trim()).length;
 }
 
-/** Render web fetch call (URL preview) */
-export function renderWebFetchCall(
+/** Render fetch call (URL preview) */
+export function renderFetchCall(
 	args: { url: string; timeout?: number; raw?: boolean },
 	uiTheme: Theme = theme,
 ): Component {
 	const domain = getDomain(args.url);
 	const path = truncate(args.url.replace(/^https?:\/\/[^/]+/, ""), 50, uiTheme.format.ellipsis);
 	const icon = uiTheme.styledSymbol("status.pending", "muted");
-	const text = `${icon} ${uiTheme.fg("toolTitle", "Web Fetch")} ${uiTheme.fg("accent", domain)}${uiTheme.fg("dim", path)}`;
+	const text = `${icon} ${uiTheme.fg("toolTitle", "Fetch")} ${uiTheme.fg("accent", domain)}${uiTheme.fg("dim", path)}`;
 	return new Text(text, 0, 0);
 }
 
-/** Render web fetch result with tree-based layout */
-export function renderWebFetchResult(
-	result: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },
+/** Render fetch result with tree-based layout */
+export function renderFetchResult(
+	result: { content: Array<{ type: string; text?: string }>; details?: FetchToolDetails },
 	options: RenderResultOptions,
 	uiTheme: Theme = theme,
 ): Component {
@@ -1006,7 +1014,9 @@ export function renderWebFetchResult(
 	const domain = getDomain(details.finalUrl);
 	const hasRedirect = details.url !== details.finalUrl;
 	const hasNotes = details.notes.length > 0;
-	const statusIcon = details.truncated
+	const truncation = details.meta?.truncation;
+	const truncated = Boolean(details.truncated || truncation);
+	const statusIcon = truncated
 		? uiTheme.styledSymbol("status.warning", "warning")
 		: uiTheme.styledSymbol("status.success", "success");
 	const expandHint = formatExpandHint(uiTheme, expanded);
@@ -1021,6 +1031,7 @@ export function renderWebFetchResult(
 		: contentText;
 	const lineCount = countNonEmptyLines(contentBody);
 	const charCount = contentBody.trim().length;
+	const contentLines = contentBody.split("\n").filter((l) => l.trim());
 
 	if (!expanded) {
 		// Collapsed view: metadata + preview
@@ -1031,14 +1042,19 @@ export function renderWebFetchResult(
 		if (hasRedirect) {
 			metaLines.push(`${uiTheme.fg("muted", "Final URL:")} ${uiTheme.fg("mdLinkUrl", details.finalUrl)}`);
 		}
-		if (details.truncated) {
+		if (truncated) {
 			metaLines.push(uiTheme.fg("warning", `${uiTheme.status.warning} Output truncated`));
+			if (truncation?.artifactId) {
+				metaLines.push(uiTheme.fg("warning", `Full output: artifact://${truncation.artifactId}`));
+			}
 		}
 		if (hasNotes) {
 			metaLines.push(`${uiTheme.fg("muted", "Notes:")} ${details.notes.join("; ")}`);
 		}
 
-		const previewLines = getPreviewLines(contentBody, 3, 100, uiTheme.format.ellipsis);
+		const previewLimit = 3;
+		const previewList = applyListLimit(contentLines, { headLimit: previewLimit });
+		const previewLines = previewList.items.map((line) => truncate(line.trim(), 100, uiTheme.format.ellipsis));
 		const detailLines: string[] = [...metaLines];
 
 		if (previewLines.length === 0) {
@@ -1049,7 +1065,7 @@ export function renderWebFetchResult(
 			}
 		}
 
-		const remaining = Math.max(0, lineCount - previewLines.length);
+		const remaining = Math.max(0, contentLines.length - previewLines.length);
 		if (remaining > 0) {
 			detailLines.push(uiTheme.fg("muted", `${uiTheme.format.ellipsis} ${remaining} more lines`));
 		} else {
@@ -1074,8 +1090,11 @@ export function renderWebFetchResult(
 		const lineLabel = `${lineCount} line${lineCount === 1 ? "" : "s"}`;
 		metaLines.push(`${uiTheme.fg("muted", "Lines:")} ${lineLabel}`);
 		metaLines.push(`${uiTheme.fg("muted", "Chars:")} ${charCount}`);
-		if (details.truncated) {
+		if (truncated) {
 			metaLines.push(uiTheme.fg("warning", `${uiTheme.status.warning} Output truncated`));
+			if (truncation?.artifactId) {
+				metaLines.push(uiTheme.fg("warning", `Full output: artifact://${truncation.artifactId}`));
+			}
 		}
 		if (hasNotes) {
 			metaLines.push(`${uiTheme.fg("muted", "Notes:")} ${details.notes.join("; ")}`);
@@ -1089,8 +1108,10 @@ export function renderWebFetchResult(
 		}
 
 		text += `\n ${uiTheme.fg("dim", uiTheme.tree.last)} ${uiTheme.fg("accent", "Content Preview")}`;
-		const previewLines = getPreviewLines(contentBody, 12, 120, uiTheme.format.ellipsis);
-		const remaining = Math.max(0, lineCount - previewLines.length);
+		const previewLimit = 12;
+		const previewList = applyListLimit(contentLines, { headLimit: previewLimit });
+		const previewLines = previewList.items.map((line) => truncate(line.trim(), 120, uiTheme.format.ellipsis));
+		const remaining = Math.max(0, contentLines.length - previewLines.length);
 		const contentPrefix = uiTheme.fg("dim", " ");
 
 		if (previewLines.length === 0) {
@@ -1109,7 +1130,7 @@ export function renderWebFetchResult(
 	return new Text(text, 0, 0);
 }
 
-export const webFetchToolRenderer = {
-	renderCall: renderWebFetchCall,
-	renderResult: renderWebFetchResult,
+export const fetchToolRenderer = {
+	renderCall: renderFetchCall,
+	renderResult: renderFetchResult,
 };