@xl0/pi-lovely-web 0.1.4 → 0.2.2

package/extensions/lovely-web/providers/exa.ts

@@ -1,8 +1,10 @@
+import { StringEnum } from "@earendil-works/pi-ai"
+import { Type } from "typebox"
+import { DEFAULT_TIMEOUT_MS } from "../constants.js"
 import { requestJson } from "./http.js"
 import type { Provider, SearchResult } from "./types.js"
 
 const BASE_URL = "https://api.exa.ai"
-const DEFAULT_TIMEOUT_MS = 30_000
 
 interface SearchBody {
 	query: string
@@ -10,11 +12,13 @@ interface SearchBody {
 	type: string
 	contents: { summary: boolean }
 	category?: string
+	userLocation?: string
 }
 
 interface ContentsBody {
-	ids: string[]
+	urls: string[]
 	text: boolean
+	maxAgeHours?: number
 }
 
 interface ExaSearchResult {
@@ -30,11 +34,6 @@ interface ExaContentsResult {
 	text?: string
 }
 
-function sourceToCategory(source?: string): string | undefined {
-	if (source === "news") return "news"
-	return undefined
-}
-
 function stripSummaryLabel(text: string): string {
 	return text.replace(/^Summary:\s*/i, "")
 }
@@ -60,6 +59,15 @@ export const exaProvider: Provider = {
 	id: "exa",
 	label: "Exa",
 	envApiKey: "EXA_API_KEY",
+	searchParameters: {
+		category: Type.Optional(StringEnum(["company", "people", "research paper", "news", "personal site", "financial report"])),
+		country: Type.Optional(Type.String({ description: "Two-letter ISO user location for result localization, e.g. US, DE, CO." }))
+	},
+	fetchParameters: {
+		maxAgeHours: Type.Optional(
+			Type.Integer({ description: "Maximum cache age in hours; 0 fetches fresh content, -1 uses cache only.", minimum: -1, maximum: 720 })
+		)
+	},
 
 	async search(apiKey, query, opts, signal) {
 		const body: SearchBody = {
@@ -68,8 +76,9 @@ export const exaProvider: Provider = {
 			type: "auto",
 			contents: { summary: true }
 		}
-		const category = sourceToCategory(opts.source)
-		if (category) body.category = category
+		if (opts.source) throw new Error("Exa search uses category, not source. Omit category for general web search, or set category: news.")
+		if (opts.category) body.category = opts.category
+		if (opts.country) body.userLocation = opts.country
 
 		const raw = await fetchJson(`${BASE_URL}/search`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
 		const data = (raw as { results?: ExaSearchResult[] }).results ?? []
@@ -88,9 +97,10 @@ export const exaProvider: Provider = {
 
 	async fetch(apiKey, url, opts, signal) {
 		const body: ContentsBody = {
-			ids: [url],
+			urls: [url],
 			text: true
 		}
+		if (opts.maxAgeHours !== undefined) body.maxAgeHours = opts.maxAgeHours
 
 		const raw = await fetchJson(`${BASE_URL}/contents`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
 		const results = (raw as { results?: ExaContentsResult[] }).results

package/extensions/lovely-web/providers/firecrawl.ts

@@ -1,13 +1,19 @@
+import { StringEnum } from "@earendil-works/pi-ai"
+import { Type } from "typebox"
+import { DEFAULT_TIMEOUT_MS } from "../constants.js"
 import { requestJson } from "./http.js"
 import type { Provider, SearchResult } from "./types.js"
 
-const BASE_URL = "https://api.firecrawl.dev/v1"
-const DEFAULT_TIMEOUT_MS = 30_000
+const BASE_URL = "https://api.firecrawl.dev/v2"
 
 interface SearchBody {
 	query: string
 	limit: number
 	sources?: string[]
+	categories?: string[]
+	location?: string
+	country?: string
+	tbs?: string
 }
 
 interface ScrapeBody {
@@ -52,18 +58,43 @@ export const firecrawlProvider: Provider = {
 	id: "firecrawl",
 	label: "Firecrawl",
 	envApiKey: "FIRECRAWL_API_KEY",
+	searchParameters: {
+		source: Type.Optional(StringEnum(["web", "news", "images"])),
+		category: Type.Optional(StringEnum(["github", "research", "pdf"])),
+		location: Type.Optional(Type.String({ description: "Geo-target results, e.g. Germany or San Francisco,California,United States." })),
+		country: Type.Optional(Type.String({ description: "ISO country code for geo-targeting, e.g. US, DE, CO." })),
+		tbs: Type.Optional(Type.String({ description: "Google-style time filter, e.g. qdr:d, qdr:w, qdr:m, qdr:y." }))
+	},
+	fetchParameters: {
+		waitFor: Type.Optional(Type.Integer({ description: "Extra delay in milliseconds before capturing page content.", minimum: 0 }))
+	},
 
 	async search(apiKey, query, opts, signal) {
 		const body: SearchBody = { query, limit: opts.limit }
 		if (opts.source) body.sources = [opts.source]
+		if (opts.category) body.categories = [opts.category]
+		if (opts.location) body.location = opts.location
+		if (opts.country) body.country = opts.country
+		if (opts.tbs) body.tbs = opts.tbs
 
 		const raw = await fetchJson(`${BASE_URL}/search`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
 		throwIfError(raw)
-		const data = raw.data as Array<{ title: string; url: string; description: string }>
+		const data = raw.data as {
+			web?: Array<{ title: string; url: string; description?: string; markdown?: string }>
+			news?: Array<{ title: string; url: string; snippet?: string; markdown?: string }>
+			images?: Array<{ title?: string; url: string; imageUrl?: string }>
+		}
+		const items = opts.source === "news" ? data.news || [] : opts.source === "images" ? data.images || [] : data.web || []
 
-		const results: SearchResult[] = data.map(item => {
-			const r: SearchResult = { title: item.title, url: item.url }
-			if (item.description) r.description = item.description
+		const results: SearchResult[] = items.map(item => {
+			const isImage = opts.source === "images"
+			const r: SearchResult = {
+				title: item.title || item.url,
+				url: isImage && "imageUrl" in item && item.imageUrl ? item.imageUrl : item.url
+			}
+			const description = "description" in item ? item.description : "snippet" in item ? item.snippet : undefined
+			if (description) r.description = description
+			if ("markdown" in item && item.markdown) r.markdown = item.markdown
 			return r
 		})
 

package/extensions/lovely-web/providers/tavily.ts

@@ -1,14 +1,20 @@
+import { StringEnum } from "@earendil-works/pi-ai"
+import { Type } from "typebox"
+import { DEFAULT_TIMEOUT_MS } from "../constants.js"
 import { requestJson } from "./http.js"
 import type { Provider, SearchResult } from "./types.js"
 
 const BASE_URL = "https://api.tavily.com"
-const DEFAULT_TIMEOUT_MS = 30_000
 
 interface SearchBody {
 	query: string
 	max_results: number
 	search_depth: string
 	topic?: string
+	time_range?: string
+	country?: string
+	include_images?: boolean
+	include_image_descriptions?: boolean
 }
 
 interface ExtractBody {
@@ -25,11 +31,16 @@ interface TavilySearchResult {
 	score: number
 }
 
+interface TavilyImageResult {
+	url: string
+	description?: string
+}
+
 interface TavilySearchResponse {
 	query: string
 	answer?: string
 	results?: TavilySearchResult[]
-	images?: unknown[]
+	images?: TavilyImageResult[]
 	response_time: number
 }
 
@@ -46,11 +57,6 @@ interface TavilyExtractResponse {
 	response_time: number
 }
 
-function sourceToTopic(source?: string): string | undefined {
-	if (source === "news") return "news"
-	return undefined
-}
-
 function postJson(url: string, body: unknown, apiKey: string, timeout: number, signal?: AbortSignal): Promise<unknown> {
 	return requestJson(
 		"Tavily",
@@ -72,6 +78,17 @@ export const tavilyProvider: Provider = {
 	id: "tavily",
 	label: "Tavily",
 	envApiKey: "TAVILY_API_KEY",
+	searchParameters: {
+		topic: Type.Optional(StringEnum(["general", "news", "finance"])),
+		includeImages: Type.Optional(Type.Boolean({ description: "Return query-related image URLs instead of page results." })),
+		country: Type.Optional(Type.String({ description: "Country name to boost general-topic results, e.g. colombia." })),
+		timeRange: Type.Optional(StringEnum(["day", "week", "month", "year", "d", "w", "m", "y"]))
+	},
+	fetchParameters: {
+		extractDepth: Type.Optional(
+			StringEnum(["basic", "advanced"], { description: "Advanced extraction retrieves more data such as tables; costs more." })
+		)
+	},
 
 	async search(apiKey, query, opts, signal) {
 		const body: SearchBody = {
@@ -79,11 +96,28 @@ export const tavilyProvider: Provider = {
 			max_results: opts.limit,
 			search_depth: "basic"
 		}
-		const topic = sourceToTopic(opts.source)
-		if (topic) body.topic = topic
+		if (opts.source) throw new Error("Tavily search uses topic/includeImages, not source.")
+		const topic = opts.topic
+		if (topic && topic !== "general") body.topic = topic
+		if (opts.timeRange) body.time_range = opts.timeRange
+		if (opts.country && topic !== "news" && topic !== "finance") body.country = opts.country.toLowerCase()
+		if (opts.includeImages) {
+			body.include_images = true
+			body.include_image_descriptions = true
+		}
 
 		const raw = await postJson(`${BASE_URL}/search`, body, apiKey, opts.timeout ?? DEFAULT_TIMEOUT_MS, signal)
 		const data = raw as TavilySearchResponse
+
+		if (opts.includeImages) {
+			const results: SearchResult[] = (data.images ?? []).map(item => ({
+				title: item.description || item.url,
+				url: item.url,
+				description: item.description || ""
+			}))
+			return { results, raw }
+		}
+
 		const items = data.results ?? []
 
 		const results: SearchResult[] = items.map(item => {
@@ -101,7 +135,7 @@ export const tavilyProvider: Provider = {
 	async fetch(apiKey, url, opts, signal) {
 		const body: ExtractBody = {
 			urls: [url],
-			extract_depth: "basic",
+			extract_depth: opts.extractDepth ?? "basic",
 			format: "markdown"
 		}
 

package/extensions/lovely-web/providers/types.ts

@@ -1,3 +1,5 @@
+import type { TSchema } from "typebox"
+
 export interface SearchResult {
 	title: string
 	url: string
@@ -5,27 +7,39 @@ export interface SearchResult {
 	markdown?: string // populated for first result when fetchResult=true
 }
 
+export interface SearchOptions {
+	limit: number
+	source?: string
+	timeout?: number
+	category?: string
+	location?: string
+	country?: string
+	tbs?: string
+	timeRange?: string
+	topic?: string
+	includeImages?: boolean
+	searchLang?: string
+	freshness?: string
+}
+
+export interface FetchOptions {
+	timeout?: number
+	waitFor?: number
+	maxAgeHours?: number
+	extractDepth?: string
+}
+
 export interface Provider {
 	readonly id: string
 	readonly label: string
 	readonly envApiKey: string
-	search(
-		apiKey: string,
-		query: string,
-		opts: { limit: number; source?: string; timeout?: number },
-		signal?: AbortSignal
-	): Promise<{ results: SearchResult[]; raw: unknown }>
+	readonly searchParameters?: Record<string, TSchema>
+	readonly fetchParameters?: Record<string, TSchema>
+	search(apiKey: string, query: string, opts: SearchOptions, signal?: AbortSignal): Promise<{ results: SearchResult[]; raw: unknown }>
 	fetch?(
 		apiKey: string,
 		url: string,
-		opts: { waitFor?: number; timeout?: number },
+		opts: FetchOptions,
 		signal?: AbortSignal
 	): Promise<{ markdown: string; metadata?: unknown; raw: unknown }>
 }
-
-export interface WebToolsConfig {
-	webSearch?: { provider?: string | null }
-	webFetch?: { provider?: string | null }
-	webImage?: { enabled?: boolean; resize?: boolean; maxSize?: number }
-	webApiKeys?: Record<string, string>
-}

package/extensions/lovely-web/render.ts

@@ -1,10 +1,11 @@
 import type { Theme } from "@earendil-works/pi-coding-agent"
 import { Text } from "@earendil-works/pi-tui"
+import { FIND_HIGHLIGHT_END, FIND_HIGHLIGHT_START } from "./find.js"
 
 const COLLAPSED_RESULT_LINES = 6
 
 export function renderTextResult(
-	result: { content: Array<{ type: string; text?: string }> },
+	result: { content: Array<{ type: string; text?: string }>; details?: unknown },
 	expanded: boolean,
 	theme: Theme,
 	partialLabel: string
@@ -13,11 +14,36 @@ export function renderTextResult(
 	if (content?.type !== "text" || content.text === undefined) return new Text(theme.fg("error", "No text output"), 0, 0)
 	if (!content.text.trim()) return new Text(theme.fg("dim", partialLabel), 0, 0)
 
-	const lines = content.text.split("\n")
+	const renderText = renderTextOverride(result.details) ?? content.text
+	const lines = renderText.split("\n")
 	const shown = expanded ? lines : lines.slice(0, COLLAPSED_RESULT_LINES)
-	let text = shown.map(line => theme.fg("toolOutput", line)).join("\n")
+	let text = shown.map(line => renderHighlightMarkers(line, theme)).join("\n")
 	if (!expanded && lines.length > COLLAPSED_RESULT_LINES) {
 		text += `\n${theme.fg("muted", `... ${lines.length - COLLAPSED_RESULT_LINES} more lines (ctrl-o to expand)`)}`
 	}
 	return new Text(text, 0, 0)
 }
+
+function renderTextOverride(details: unknown): string | undefined {
+	if (!details || typeof details !== "object") return undefined
+	const renderText = (details as { renderText?: unknown }).renderText
+	return typeof renderText === "string" ? renderText : undefined
+}
+
+function renderHighlightMarkers(line: string, theme: Theme): string {
+	let rest = line
+	let output = ""
+	while (true) {
+		const start = rest.indexOf(FIND_HIGHLIGHT_START)
+		if (start === -1) return output + theme.fg("toolOutput", stripHighlightMarkers(rest))
+		const end = rest.indexOf(FIND_HIGHLIGHT_END, start + FIND_HIGHLIGHT_START.length)
+		if (end === -1) return output + theme.fg("toolOutput", stripHighlightMarkers(rest))
+		output += theme.fg("toolOutput", rest.slice(0, start))
+		output += theme.bold(theme.fg("accent", rest.slice(start + FIND_HIGHLIGHT_START.length, end)))
+		rest = rest.slice(end + FIND_HIGHLIGHT_END.length)
+	}
+}
+
+function stripHighlightMarkers(text: string): string {
+	return text.replaceAll(FIND_HIGHLIGHT_START, "").replaceAll(FIND_HIGHLIGHT_END, "")
+}

package/extensions/lovely-web/smart.ts

@@ -0,0 +1,197 @@
+import { type Api, type AssistantMessage, completeSimple, type Model, type SimpleStreamOptions } from "@earendil-works/pi-ai"
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent"
+import type { WebToolsConfig } from "./config.js"
+
+const SMART_INPUT_FALLBACK_CHARS = 60_000 * 3
+
+export const SMART_SYSTEM_PROMPT = `Process one web_fetch result for a coding agent.
+Use only facts explicitly stated in the provided page text.
+Answer the smart query, not the whole page. Include neighboring/related sections only when needed for the requested task.
+
+Return concise Markdown in the format that best fits the query:
+- If the query asks for a specific format, use it.
+- For verbatim code, commands, schemas, examples, config, or error text, return the requested excerpt first, exactly as written, preserving indentation and punctuation.
+- For summaries, focus on what a coding agent needs: purpose, setup, APIs/options, steps, limits, caveats, security notes, migration/breaking changes, and gotchas that are directly stated.
+- For extraction/comparison, use bullet points, preserve exact names, signatures, flags, codes, amounts, dates, deadlines, defaults, and version/platform caveats.
+- For troubleshooting, include directly stated symptoms, causes, fixes/workarounds, and warnings.
+
+Grounding rules:
+- Include short exact quotes or source context for important claims unless the answer is only a verbatim excerpt.
+- There is one fetched page; use a heading like "Evidence (on this page):" for an evidence section. 
+- If requested information is absent or partially absent, say "Not found on page." Mention missing fields under a short Missing section when useful.
+- Use "None" only when the page explicitly supports it or the query only asks whether requested fields were found.
+- For legal/admin pages, state only what the page says and flag when exact amounts/codes require a calculator, form, or agency confirmation.`
+
+let warnedDefaultModel = false
+let warnedUnavailableModel = false
+let disabledForSession = false
+
+export interface SmartProcessInput {
+	query: string
+	resultText: string
+	sourceUrl?: string
+}
+
+export interface SmartProcessResult {
+	text: string
+	details: {
+		query: string
+		model: string
+		inputChars: number
+		originalInputChars: number
+		maxInputChars: number
+		truncated: boolean
+		stopReason: AssistantMessage["stopReason"]
+		usage: AssistantMessage["usage"]
+	}
+}
+
+export function resetSmartRuntimeState(): void {
+	warnedDefaultModel = false
+	warnedUnavailableModel = false
+	disabledForSession = false
+}
+
+export function resetSmartConfigState(): void {
+	warnedUnavailableModel = false
+	disabledForSession = false
+}
+
+function modelName(model: Model<Api>): string {
+	return `${model.provider}/${model.id}`
+}
+
+function warn(ctx: ExtensionContext, message: string): void {
+	ctx.ui.notify(message, "warning")
+}
+
+function disableSmart(ctx: ExtensionContext, reason: string): void {
+	disabledForSession = true
+	if (warnedUnavailableModel) return
+	warnedUnavailableModel = true
+	warn(ctx, `Lovely Web smart search disabled: ${reason}`)
+}
+
+function resolveConfiguredModel(ctx: ExtensionContext, configured: string): Model<Api> | undefined {
+	const separator = configured.indexOf("/")
+	if (separator <= 0 || separator === configured.length - 1) {
+		disableSmart(ctx, `configured model must be "provider/model", got "${configured}".`)
+		return undefined
+	}
+
+	const provider = configured.slice(0, separator)
+	const modelId = configured.slice(separator + 1)
+	const model = ctx.modelRegistry.getAvailable().find(model => model.provider === provider && model.id === modelId)
+	if (!model) {
+		disableSmart(ctx, `configured model unavailable: ${configured}`)
+		return undefined
+	}
+	if (!model.input.includes("text")) {
+		disableSmart(ctx, `configured model does not support text input: ${configured}`)
+		return undefined
+	}
+	return model
+}
+
+function resolveCurrentModel(ctx: ExtensionContext): Model<Api> | undefined {
+	const model = ctx.model
+	if (!model) {
+		disableSmart(ctx, "no current model is selected.")
+		return undefined
+	}
+	if (!model.input.includes("text")) {
+		disableSmart(ctx, `current model does not support text input: ${modelName(model)}.`)
+		return undefined
+	}
+	if (!warnedDefaultModel) {
+		warnedDefaultModel = true
+		warn(ctx, `Lovely Web smart search model not selected; using current model ${modelName(model)}.`)
+	}
+	return model
+}
+
+function resolveSmartModel(config: WebToolsConfig, ctx: ExtensionContext): Model<Api> | undefined {
+	if (!config.smartSearchEnabled || disabledForSession) return undefined
+	const configured = config.smartSearchModel.trim()
+	return configured ? resolveConfiguredModel(ctx, configured) : resolveCurrentModel(ctx)
+}
+
+export function validateSmartConfig(config: WebToolsConfig, ctx: ExtensionContext): boolean {
+	if (!config.smartSearchEnabled) return true
+	return resolveSmartModel(config, ctx) !== undefined
+}
+
+function truncateForModel(text: string, model: Model<Api>): { text: string; originalChars: number; maxChars: number; truncated: boolean } {
+	const maxChars = model.contextWindow > 0 ? Math.max(4000, Math.floor((model.contextWindow / 2) * 3)) : SMART_INPUT_FALLBACK_CHARS
+	if (text.length <= maxChars) return { text, originalChars: text.length, maxChars, truncated: false }
+	return { text: text.slice(0, maxChars), originalChars: text.length, maxChars, truncated: true }
+}
+
+function extractText(message: AssistantMessage): string {
+	return message.content
+		.filter((item): item is { type: "text"; text: string } => item.type === "text")
+		.map(item => item.text)
+		.join("\n")
+		.trim()
+}
+
+function smartSystemPrompt(config: WebToolsConfig): string {
+	return config.smartSearchSystemPrompt.trim() ? config.smartSearchSystemPrompt : SMART_SYSTEM_PROMPT
+}
+
+export async function smartProcess(
+	config: WebToolsConfig,
+	ctx: ExtensionContext,
+	input: SmartProcessInput,
+	signal?: AbortSignal
+): Promise<SmartProcessResult | undefined> {
+	const model = resolveSmartModel(config, ctx)
+	if (!model) return undefined
+
+	const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model)
+	if (!auth.ok) {
+		disableSmart(ctx, `model auth unavailable for ${modelName(model)}: ${auth.error}`)
+		return undefined
+	}
+
+	const maxTokens = Math.max(1, config.smartSearchMaxTokens)
+	const truncated = truncateForModel(input.resultText, model)
+	const sourceUrlText = input.sourceUrl ? `\n\nSource URL:\n${input.sourceUrl}` : ""
+	const prompt = `Smart query:\n${input.query}${sourceUrlText}\n\nResult text:\n${truncated.text}`
+	const options: SimpleStreamOptions = { maxTokens }
+	if (auth.apiKey !== undefined) options.apiKey = auth.apiKey
+	if (auth.headers !== undefined) options.headers = auth.headers
+	if (auth.env !== undefined) options.env = auth.env
+	if (signal !== undefined) options.signal = signal
+
+	const response = await completeSimple(
+		model,
+		{
+			systemPrompt: smartSystemPrompt(config),
+			messages: [{ role: "user", content: prompt, timestamp: Date.now() }]
+		},
+		options
+	)
+
+	if (response.stopReason === "error" || response.stopReason === "aborted") {
+		throw new Error(response.errorMessage || response.stopReason)
+	}
+
+	const answer = extractText(response) || "Not found in provided results."
+	const truncationNotice = truncated.truncated
+		? `\n\nNote: Smart input was truncated to ${truncated.text.length}/${truncated.originalChars} characters to stay within half of ${modelName(model)} context. Answer may omit trimmed content.`
+		: ""
+	return {
+		text: `${answer}${truncationNotice}`,
+		details: {
+			query: input.query,
+			model: modelName(model),
+			inputChars: truncated.text.length,
+			originalInputChars: truncated.originalChars,
+			maxInputChars: truncated.maxChars,
+			truncated: truncated.truncated,
+			stopReason: response.stopReason,
+			usage: response.usage
+		}
+	}
+}