@xl0/pi-lovely-web 0.1.2

package/extensions/lovely-web/providers/brave.ts

@@ -0,0 +1,103 @@
+import { requestJson } from "./http.js"
+import type { Provider, SearchResult } from "./types.js"
+
+const BASE_URL = "https://api.search.brave.com/res/v1"
+const DEFAULT_TIMEOUT_MS = 30_000
+
+interface BraveWebResult {
+	title: string
+	url: string
+	description: string
+}
+
+interface BraveWebResponse {
+	web?: { results?: BraveWebResult[] }
+}
+
+interface BraveNewsResult {
+	title: string
+	url: string
+	description: string
+}
+
+interface BraveNewsResponse {
+	results?: BraveNewsResult[]
+}
+
+interface BraveImageResult {
+	url: string
+	title?: string
+	description?: string
+}
+
+interface BraveImageResponse {
+	results?: BraveImageResult[]
+}
+
+function stripHtmlTags(text: string): string {
+	return text.replace(/<[^>]+>/g, "")
+}
+
+function fetchJson(url: string, apiKey: string, timeout: number, signal?: AbortSignal): Promise<unknown> {
+	return requestJson(
+		"Brave",
+		url,
+		{
+			headers: {
+				"X-Subscription-Token": apiKey,
+				Accept: "application/json"
+			}
+		},
+		timeout,
+		signal
+	)
+}
+
+function buildSearchUrl(source: string | undefined, query: string, count: number): string {
+	const encodedQuery = encodeURIComponent(query)
+	if (source === "news") {
+		return `${BASE_URL}/news/search?q=${encodedQuery}&count=${count}`
+	}
+	if (source === "images") {
+		return `${BASE_URL}/images/search?q=${encodedQuery}&count=${count}`
+	}
+	return `${BASE_URL}/web/search?q=${encodedQuery}&count=${count}`
+}
+
+export const braveProvider: Provider = {
+	id: "brave",
+	label: "Brave Search",
+	envApiKey: "BRAVE_API_KEY",
+
+	async search(apiKey, query, opts, signal) {
+		const url = buildSearchUrl(opts.source, query, opts.limit)
+		const raw = await fetchJson(url, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
+
+		let items: SearchResult[] = []
+
+		if (opts.source === "news") {
+			const data = raw as BraveNewsResponse
+			items = (data.results ?? []).map(item => ({
+				title: item.title || "Untitled",
+				url: item.url,
+				description: stripHtmlTags(item.description || "")
+			}))
+		} else if (opts.source === "images") {
+			const data = raw as BraveImageResponse
+			items = (data.results ?? []).map(item => ({
+				title: item.title || item.url,
+				url: item.url,
+				description: item.description || ""
+			}))
+		} else {
+			const data = raw as BraveWebResponse
+			items = (data.web?.results ?? []).map(item => ({
+				title: item.title || "Untitled",
+				url: item.url,
+				description: stripHtmlTags(item.description || "")
+			}))
+		}
+
+		return { results: items, raw }
+	}
+}

package/extensions/lovely-web/providers/exa.ts

@@ -0,0 +1,112 @@
+import { requestJson } from "./http.js"
+import type { Provider, SearchResult } from "./types.js"
+
+const BASE_URL = "https://api.exa.ai"
+const DEFAULT_TIMEOUT_MS = 30_000
+
+interface SearchBody {
+	query: string
+	numResults: number
+	type: string
+	contents: { summary: boolean }
+	category?: string
+}
+
+interface ContentsBody {
+	ids: string[]
+	text: boolean
+}
+
+interface ExaSearchResult {
+	title: string
+	url: string
+	summary?: string
+}
+
+interface ExaContentsResult {
+	id: string
+	url: string
+	title?: string
+	text?: string
+}
+
+function sourceToCategory(source?: string): string | undefined {
+	if (source === "news") return "news"
+	return undefined
+}
+
+function stripSummaryLabel(text: string): string {
+	return text.replace(/^Summary:\s*/i, "")
+}
+
+function fetchJson(url: string, body: unknown, apiKey: string, timeout: number, signal?: AbortSignal): Promise<unknown> {
+	return requestJson(
+		"Exa",
+		url,
+		{
+			method: "POST",
+			headers: {
+				"x-api-key": apiKey,
+				"Content-Type": "application/json"
+			},
+			body: JSON.stringify(body)
+		},
+		timeout,
+		signal
+	)
+}
+
+export const exaProvider: Provider = {
+	id: "exa",
+	label: "Exa",
+	envApiKey: "EXA_API_KEY",
+
+	async search(apiKey, query, opts, signal) {
+		const body: SearchBody = {
+			query,
+			numResults: opts.limit,
+			type: "auto",
+			contents: { summary: true }
+		}
+		const category = sourceToCategory(opts.source)
+		if (category) body.category = category
+
+		const raw = await fetchJson(`${BASE_URL}/search`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
+		const data = (raw as { results?: ExaSearchResult[] }).results ?? []
+
+		const results: SearchResult[] = data.map(item => {
+			const r: SearchResult = {
+				title: item.title || "Untitled",
+				url: item.url
+			}
+			if (item.summary) r.description = stripSummaryLabel(item.summary)
+			return r
+		})
+
+		return { results, raw }
+	},
+
+	async fetch(apiKey, url, opts, signal) {
+		const body: ContentsBody = {
+			ids: [url],
+			text: true
+		}
+
+		const raw = await fetchJson(`${BASE_URL}/contents`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
+		const results = (raw as { results?: ExaContentsResult[] }).results
+		const first = results?.[0]
+
+		if (!first?.text) {
+			const statuses = (raw as { statuses?: Array<{ id: string; status: string; error?: { tag: string } }> }).statuses
+			const status = statuses?.find(s => s.id === url)
+			const detail = status?.error?.tag ?? "no content"
+			throw new Error(`Exa could not fetch content for ${url}: ${detail}`)
+		}
+
+		return {
+			markdown: first.text.trim(),
+			metadata: first.title ? { title: first.title } : undefined,
+			raw
+		}
+	}
+}

package/extensions/lovely-web/providers/firecrawl.ts

@@ -0,0 +1,91 @@
+import { requestJson } from "./http.js"
+import type { Provider, SearchResult } from "./types.js"
+
+const BASE_URL = "https://api.firecrawl.dev/v1"
+const DEFAULT_TIMEOUT_MS = 30_000
+
+interface SearchBody {
+	query: string
+	limit: number
+	sources?: string[]
+}
+
+interface ScrapeBody {
+	url: string
+	formats: string[]
+	onlyMainContent: boolean
+	waitFor?: number
+}
+
+interface FirecrawlResponse {
+	success: boolean
+	data: unknown
+	error?: string
+}
+
+function throwIfError(response: unknown): asserts response is FirecrawlResponse {
+	if (!response || typeof response !== "object" || !("success" in response)) {
+		throw new Error("Unexpected Firecrawl response")
+	}
+	const r = response as FirecrawlResponse
+	if (r.success !== true) throw new Error(r.error || "Firecrawl request failed")
+}
+
+function fetchJson(url: string, body: unknown, apiKey: string, timeout: number, signal?: AbortSignal): Promise<unknown> {
+	return requestJson(
+		"Firecrawl",
+		url,
+		{
+			method: "POST",
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+				"Content-Type": "application/json"
+			},
+			body: JSON.stringify(body)
+		},
+		timeout,
+		signal
+	)
+}
+
+export const firecrawlProvider: Provider = {
+	id: "firecrawl",
+	label: "Firecrawl",
+	envApiKey: "FIRECRAWL_API_KEY",
+
+	async search(apiKey, query, opts, signal) {
+		const body: SearchBody = { query, limit: opts.limit }
+		if (opts.source) body.sources = [opts.source]
+
+		const raw = await fetchJson(`${BASE_URL}/search`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
+		throwIfError(raw)
+		const data = raw.data as Array<{ title: string; url: string; description: string }>
+
+		const results: SearchResult[] = data.map(item => {
+			const r: SearchResult = { title: item.title, url: item.url }
+			if (item.description) r.description = item.description
+			return r
+		})
+
+		return { results, raw }
+	},
+
+	async fetch(apiKey, url, opts, signal) {
+		const body: ScrapeBody = {
+			url,
+			formats: ["markdown"],
+			onlyMainContent: true
+		}
+		if (opts.waitFor !== undefined) body.waitFor = opts.waitFor
+
+		const raw = await fetchJson(`${BASE_URL}/scrape`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
+		throwIfError(raw)
+		const data = raw.data as { markdown: string; metadata?: unknown }
+
+		return {
+			markdown: data.markdown?.trim() || "No markdown content returned.",
+			metadata: data.metadata,
+			raw
+		}
+	}
+}

package/extensions/lovely-web/providers/http.ts

@@ -0,0 +1,21 @@
+export async function requestJson(label: string, url: string, init: RequestInit, timeout: number, signal?: AbortSignal): Promise<unknown> {
+	const controller = new AbortController()
+	const timer = setTimeout(() => controller.abort(), timeout)
+	const abort = () => controller.abort()
+	if (signal?.aborted) controller.abort()
+	else signal?.addEventListener("abort", abort, { once: true })
+
+	try {
+		const res = await fetch(url, { ...init, signal: controller.signal })
+
+		if (!res.ok) {
+			const text = await res.text()
+			throw new Error(`${label} request failed (${res.status}): ${text}`)
+		}
+
+		return res.json()
+	} finally {
+		clearTimeout(timer)
+		signal?.removeEventListener("abort", abort)
+	}
+}

package/extensions/lovely-web/providers/tavily.ts

@@ -0,0 +1,131 @@
+import { requestJson } from "./http.js"
+import type { Provider, SearchResult } from "./types.js"
+
+const BASE_URL = "https://api.tavily.com"
+const DEFAULT_TIMEOUT_MS = 30_000
+
+interface SearchBody {
+	query: string
+	max_results: number
+	search_depth: string
+	topic?: string
+}
+
+interface ExtractBody {
+	urls: string[]
+	extract_depth: string
+	format: string
+}
+
+interface TavilySearchResult {
+	title: string
+	url: string
+	content: string
+	raw_content?: string
+	score: number
+}
+
+interface TavilySearchResponse {
+	query: string
+	answer?: string
+	results?: TavilySearchResult[]
+	images?: unknown[]
+	response_time: number
+}
+
+interface TavilyExtractResult {
+	url: string
+	raw_content: string
+	images?: unknown[]
+	favicon?: string
+}
+
+interface TavilyExtractResponse {
+	results?: TavilyExtractResult[]
+	failed_results?: Array<{ url: string; error: string }>
+	response_time: number
+}
+
+function sourceToTopic(source?: string): string | undefined {
+	if (source === "news") return "news"
+	return undefined
+}
+
+function postJson(url: string, body: unknown, apiKey: string, timeout: number, signal?: AbortSignal): Promise<unknown> {
+	return requestJson(
+		"Tavily",
+		url,
+		{
+			method: "POST",
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+				"Content-Type": "application/json"
+			},
+			body: JSON.stringify(body)
+		},
+		timeout,
+		signal
+	)
+}
+
+export const tavilyProvider: Provider = {
+	id: "tavily",
+	label: "Tavily",
+	envApiKey: "TAVILY_API_KEY",
+
+	async search(apiKey, query, opts, signal) {
+		const body: SearchBody = {
+			query,
+			max_results: opts.limit,
+			search_depth: "basic"
+		}
+		const topic = sourceToTopic(opts.source)
+		if (topic) body.topic = topic
+
+		const raw = await postJson(`${BASE_URL}/search`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
+		const data = raw as TavilySearchResponse
+		const items = data.results ?? []
+
+		const results: SearchResult[] = items.map(item => {
+			const r: SearchResult = {
+				title: item.title || "Untitled",
+				url: item.url
+			}
+			if (item.content) r.description = item.content
+			return r
+		})
+
+		return { results, raw }
+	},
+
+	async fetch(apiKey, url, opts, signal) {
+		const body: ExtractBody = {
+			urls: [url],
+			extract_depth: "basic",
+			format: "markdown"
+		}
+
+		const raw = await postJson(`${BASE_URL}/extract`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
+		const data = raw as TavilyExtractResponse
+
+		const failed = data.failed_results?.find(f => f.url === url)
+		if (failed) {
+			throw new Error(`Tavily could not extract ${url}: ${failed.error}`)
+		}
+
+		const result = data.results?.find(r => r.url === url)
+		if (!result?.raw_content) {
+			throw new Error(`Tavily returned no content for ${url}`)
+		}
+
+		const metadata: { images?: unknown[]; favicon?: string } = {}
+		if (result.images?.length) metadata.images = result.images
+		if (result.favicon) metadata.favicon = result.favicon
+
+		return {
+			markdown: result.raw_content.trim(),
+			metadata: Object.keys(metadata).length ? metadata : undefined,
+			raw
+		}
+	}
+}

package/extensions/lovely-web/providers/types.ts

@@ -0,0 +1,31 @@
+export interface SearchResult {
+	title: string
+	url: string
+	description?: string
+	markdown?: string // populated for first result when fetchResult=true
+}
+
+export interface Provider {
+	readonly id: string
+	readonly label: string
+	readonly envApiKey: string
+	search(
+		apiKey: string,
+		query: string,
+		opts: { limit: number; source?: string; timeout?: number },
+		signal?: AbortSignal
+	): Promise<{ results: SearchResult[]; raw: unknown }>
+	fetch?(
+		apiKey: string,
+		url: string,
+		opts: { waitFor?: number; timeout?: number },
+		signal?: AbortSignal
+	): Promise<{ markdown: string; metadata?: unknown; raw: unknown }>
+}
+
+export interface WebToolsConfig {
+	webSearch?: { provider?: string; enabled?: boolean }
+	webFetch?: { provider?: string; enabled?: boolean }
+	webImage?: { enabled?: boolean; resize?: boolean; maxSize?: number }
+	webApiKeys?: Record<string, string>
+}

package/extensions/lovely-web/render.ts

@@ -0,0 +1,23 @@
+import type { Theme } from "@earendil-works/pi-coding-agent"
+import { Text } from "@earendil-works/pi-tui"
+
+const COLLAPSED_RESULT_LINES = 6
+
+export function renderTextResult(
+	result: { content: Array<{ type: string; text?: string }> },
+	expanded: boolean,
+	theme: Theme,
+	partialLabel: string
+) {
+	const content = result.content[0]
+	if (content?.type !== "text" || content.text === undefined) return new Text(theme.fg("error", "No text output"), 0, 0)
+	if (!content.text.trim()) return new Text(theme.fg("dim", partialLabel), 0, 0)
+
+	const lines = content.text.split("\n")
+	const shown = expanded ? lines : lines.slice(0, COLLAPSED_RESULT_LINES)
+	let text = shown.map(line => theme.fg("toolOutput", line)).join("\n")
+	if (!expanded && lines.length > COLLAPSED_RESULT_LINES) {
+		text += `\n${theme.fg("muted", `... ${lines.length - COLLAPSED_RESULT_LINES} more lines (ctrl-o to expand)`)}`
+	}
+	return new Text(text, 0, 0)
+}

package/extensions/lovely-web/tool-impl.ts

@@ -0,0 +1,95 @@
+import type { ImageContent, TextContent } from "@earendil-works/pi-ai"
+import { DEFAULT_TIMEOUT_MS, getProvider, resolveApiKey } from "./config.js"
+import { asErrorMessage, formatSearchOutput, stringify } from "./format.js"
+import type { WebToolsConfig } from "./providers/types.js"
+
+export interface ToolResult {
+	content: Array<TextContent | ImageContent>
+	details: unknown
+	isError?: boolean
+}
+
+export async function searchImpl(
+	config: WebToolsConfig,
+	params: { query: string; limit?: number; source?: string; fetchResult?: boolean },
+	signal?: AbortSignal,
+	onUpdate?: (result: ToolResult) => void
+): Promise<ToolResult> {
+	const searchProvider = getProvider("search", config)
+	const apiKey = resolveApiKey(searchProvider, config)
+
+	const searchResult = await searchProvider.search(
+		apiKey,
+		params.query,
+		{
+			limit: params.limit ?? 5,
+			timeout: DEFAULT_TIMEOUT_MS,
+			...(params.source !== undefined ? { source: params.source } : {})
+		},
+		signal
+	)
+
+	if (signal?.aborted) throw new Error("Search cancelled")
+
+	const shouldFetch = params.fetchResult ?? true
+	const first = searchResult.results[0]
+	if (shouldFetch && first?.url) {
+		onUpdate?.({
+			content: [{ type: "text" as const, text: `Fetching first result: ${first.url}` }],
+			details: undefined as unknown
+		})
+		try {
+			const fetchProvider = getProvider("fetch", config)
+			const fetchApiKey = resolveApiKey(fetchProvider, config)
+			const fetched = await fetchProvider.fetch(fetchApiKey, first.url, { timeout: DEFAULT_TIMEOUT_MS }, signal)
+
+			if (signal?.aborted) throw new Error("Search cancelled")
+			first.markdown = fetched.markdown
+			if (fetched.metadata) (first as { metadata?: unknown }).metadata = fetched.metadata
+		} catch (err) {
+			first.description = first.description || `[Fetch failed: ${asErrorMessage(err)}]`
+		}
+	}
+
+	const result: ToolResult = {
+		content: [{ type: "text" as const, text: formatSearchOutput(searchResult.results) }],
+		details: searchResult.raw
+	}
+	onUpdate?.(result)
+	return result
+}
+
+export async function fetchImpl(
+	config: WebToolsConfig,
+	params: { url: string; waitFor?: number; timeout?: number; includeMetadata?: boolean },
+	signal?: AbortSignal,
+	onUpdate?: (result: ToolResult) => void
+): Promise<ToolResult> {
+	const fetchProvider = getProvider("fetch", config)
+	const apiKey = resolveApiKey(fetchProvider, config)
+
+	const result = await fetchProvider.fetch(
+		apiKey,
+		params.url,
+		{
+			timeout: params.timeout ?? DEFAULT_TIMEOUT_MS,
+			...(params.waitFor !== undefined ? { waitFor: params.waitFor } : {})
+		},
+		signal
+	)
+
+	if (signal?.aborted) throw new Error("Fetch cancelled")
+
+	const warning =
+		["exa", "tavily"].includes(fetchProvider.id) && params.waitFor !== undefined
+			? `Warning: ${fetchProvider.label} ignores waitFor; request sent without any extra page-load delay.\n\n`
+			: ""
+	const metadata = params.includeMetadata && result.metadata ? `\n\nMetadata:\n${stringify(result.metadata)}` : ""
+
+	const toolResult: ToolResult = {
+		content: [{ type: "text" as const, text: `${warning}${result.markdown}${metadata}` }],
+		details: result.raw
+	}
+	onUpdate?.(toolResult)
+	return toolResult
+}