@nghyane/arcane 0.1.12 → 0.1.14

package/src/web/search/index.ts

@@ -1,8 +1,8 @@
 /**
  * Unified Web Search Tool
  *
- * Single tool supporting Anthropic, Perplexity, Exa, Brave, Jina, Kimi, Gemini, Codex, Z.AI, and Synthetic
- * providers with provider-specific parameters exposed conditionally.
+ * Single tool supporting Anthropic, Perplexity, Exa, Brave, Jina, Kimi, Gemini, Codex, Z.AI, Synthetic,
+ * and grep.app providers with provider-specific parameters exposed conditionally.
  *
  * When EXA_API_KEY is available, additional specialized tools are exposed:
  * - web_search_deep: Natural language web search with synthesized results
@@ -14,16 +14,13 @@
 import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback } from "@nghyane/arcane-agent";
 import { StringEnum } from "@nghyane/arcane-ai";
 import { Type } from "@sinclair/typebox";
-import { renderPromptTemplate } from "../../config/prompt-templates";
 import { callExaTool, findApiKey as findExaKey, formatSearchResults, isSearchResponse } from "../../exa/mcp-client";
 import { renderExaCall, renderExaResult } from "../../exa/render";
 import type { ExaRenderDetails } from "../../exa/types";
 import type { CustomTool, CustomToolContext, RenderResultOptions } from "../../extensibility/custom-tools/types";
-import type { Theme } from "../../modes/theme/theme";
 import webSearchSystemPrompt from "../../prompts/system/web-search.md" with { type: "text" };
-import webSearchDescription from "../../prompts/tools/web-search.md" with { type: "text" };
-import type { ToolSession } from "../../tools";
-import { formatAge } from "../../tools/render-utils";
+import type { Theme } from "../../theme/theme";
+import { formatAge } from "../../ui/render-utils";
 import { getSearchProvider, resolveProviderChain, type SearchProvider } from "./provider";
 import { renderSearchCall, renderSearchResult, type SearchRenderDetails } from "./render";
 import type { SearchResponse } from "./types";
@@ -33,19 +30,22 @@ import { SearchProviderError } from "./types";
 export const webSearchSchema = Type.Object({
 	query: Type.String({ description: "Search query" }),
 	provider: Type.Optional(
-		StringEnum(
-			["auto", "exa", "brave", "jina", "kimi", "zai", "anthropic", "perplexity", "gemini", "codex", "synthetic"],
-			{
-				description: "Search provider (default: auto)",
-			},
-		),
+		StringEnum([
+			"auto",
+			"exa",
+			"brave",
+			"jina",
+			"kimi",
+			"zai",
+			"anthropic",
+			"perplexity",
+			"gemini",
+			"codex",
+			"synthetic",
+		]),
 	),
-	recency: Type.Optional(
-		StringEnum(["day", "week", "month", "year"], {
-			description: "Recency filter (Brave, Perplexity)",
-		}),
-	),
-	limit: Type.Optional(Type.Number({ description: "Max results to return" })),
+	recency: Type.Optional(StringEnum(["day", "week", "month", "year"], { description: "Filter results by recency" })),
+	limit: Type.Optional(Type.Number({ description: "Max number of results" })),
 });
 
 export type SearchParams = {
@@ -183,7 +183,10 @@ function formatForLLM(response: SearchResponse): string {
 async function executeSearch(
 	_toolCallId: string,
 	params: SearchParams,
-): Promise<{ content: Array<{ type: "text"; text: string }>; details: SearchRenderDetails }> {
+): Promise<{
+	content: Array<{ type: "text"; text: string }>;
+	details: SearchRenderDetails;
+}> {
 	const providers =
 		params.provider && params.provider !== "auto" && params.no_fallback
 			? (await getSearchProvider(params.provider).isAvailable())
@@ -234,16 +237,20 @@ async function executeSearch(
 
 	return {
 		content: [{ type: "text" as const, text: `Error: ${message}` }],
-		details: { response: { provider: lastProvider.id, sources: [] }, error: message },
+		details: {
+			response: { provider: lastProvider.id, sources: [] },
+			error: message,
+		},
 	};
 }
 
 /**
  * Execute a web search query for CLI/testing workflows.
  */
-export async function runSearchQuery(
-	params: SearchParams,
-): Promise<{ content: Array<{ type: "text"; text: string }>; details: SearchRenderDetails }> {
+export async function runSearchQuery(params: SearchParams): Promise<{
+	content: Array<{ type: "text"; text: string }>;
+	details: SearchRenderDetails;
+}> {
 	return executeSearch("cli-web-search", params);
 }
 
@@ -253,15 +260,13 @@ export async function runSearchQuery(
  * Supports Anthropic, Perplexity, Exa, Brave, Jina, Kimi, Gemini, Codex, Z.AI, and Synthetic providers with automatic fallback.
  * Session is accepted for interface consistency but not used.
  */
-export class SearchTool implements AgentTool<typeof webSearchSchema, SearchRenderDetails> {
+export class SearchTool implements AgentTool<typeof webSearchSchema, SearchRenderDetails, Theme> {
 	readonly name = "web_search";
 	readonly label = "Web Search";
-	readonly description: string;
+	readonly description = "Search the web";
 	readonly parameters = webSearchSchema;
-
-	constructor(_session: ToolSession) {
-		this.description = renderPromptTemplate(webSearchDescription);
-	}
+	readonly renderCall = renderSearchCall;
+	readonly renderResult = renderSearchResult;
 
 	async execute(
 		_toolCallId: string,
@@ -278,7 +283,7 @@ export class SearchTool implements AgentTool<typeof webSearchSchema, SearchRende
 export const webSearchCustomTool: CustomTool<typeof webSearchSchema, SearchRenderDetails> = {
 	name: "web_search",
 	label: "Web Search",
-	description: renderPromptTemplate(webSearchDescription),
+	description: "",
 	parameters: webSearchSchema,
 
 	async execute(toolCallId: string, params: SearchParams, _onUpdate, _ctx: CustomToolContext, _signal?: AbortSignal) {
@@ -307,36 +312,64 @@ const webSearchDeepSchema = Type.Object({
 		}),
 	),
 	include_domains: Type.Optional(
-		Type.Array(Type.String(), { description: "Only include results from these domains" }),
+		Type.Array(Type.String(), {
+			description: "Only include results from these domains",
+		}),
+	),
+	exclude_domains: Type.Optional(
+		Type.Array(Type.String(), {
+			description: "Exclude results from these domains",
+		}),
 	),
-	exclude_domains: Type.Optional(Type.Array(Type.String(), { description: "Exclude results from these domains" })),
 	start_published_date: Type.Optional(
-		Type.String({ description: "Filter results published after this date (ISO 8601)" }),
+		Type.String({
+			description: "Filter results published after this date (ISO 8601)",
+		}),
 	),
 	end_published_date: Type.Optional(
-		Type.String({ description: "Filter results published before this date (ISO 8601)" }),
+		Type.String({
+			description: "Filter results published before this date (ISO 8601)",
+		}),
 	),
 	num_results: Type.Optional(
-		Type.Number({ description: "Maximum results (default: 10, max: 100)", minimum: 1, maximum: 100 }),
+		Type.Number({
+			description: "Maximum results (default: 10, max: 100)",
+			minimum: 1,
+			maximum: 100,
+		}),
 	),
 });
 
 /** Schema for code context search */
 const webSearchCodeContextSchema = Type.Object({
 	query: Type.String({ description: "Code or technical search query" }),
-	code_context: Type.Optional(Type.String({ description: "Additional context about what you're looking for" })),
+	code_context: Type.Optional(
+		Type.String({
+			description: "Additional context about what you're looking for",
+		}),
+	),
 });
 
 /** Schema for URL crawling */
 const webSearchCrawlSchema = Type.Object({
 	url: Type.String({ description: "URL to crawl and extract content from" }),
-	text: Type.Optional(Type.Boolean({ description: "Include full page text content (default: false)" })),
-	highlights: Type.Optional(Type.Boolean({ description: "Include highlighted relevant snippets (default: false)" })),
+	text: Type.Optional(
+		Type.Boolean({
+			description: "Include full page text content (default: false)",
+		}),
+	),
+	highlights: Type.Optional(
+		Type.Boolean({
+			description: "Include highlighted relevant snippets (default: false)",
+		}),
+	),
 });
 
 /** Schema for LinkedIn search */
 const webSearchLinkedinSchema = Type.Object({
-	query: Type.String({ description: 'LinkedIn search query (e.g., "Software Engineer at OpenAI")' }),
+	query: Type.String({
+		description: 'LinkedIn search query (e.g., "Software Engineer at OpenAI")',
+	}),
 });
 
 /** Schema for company research */
@@ -349,7 +382,10 @@ async function executeExaTool(
 	mcpToolName: string,
 	params: Record<string, unknown>,
 	toolName: string,
-): Promise<{ content: Array<{ type: "text"; text: string }>; details: ExaRenderDetails }> {
+): Promise<{
+	content: Array<{ type: "text"; text: string }>;
+	details: ExaRenderDetails;
+}> {
 	try {
 		const apiKey = await findExaKey();
 		if (!apiKey) {
@@ -461,7 +497,7 @@ Parameters:
 	parameters: webSearchCrawlSchema,
 
 	async execute(_toolCallId, params, _onUpdate, _ctx, _signal) {
-		return executeExaTool("crawling", params as Record<string, unknown>, "web_search_crawl");
+		return executeExaTool("crawling_exa", params as Record<string, unknown>, "web_search_crawl");
 	},
 
 	renderCall(args, _options, theme) {
@@ -492,7 +528,7 @@ Parameters:
 	parameters: webSearchLinkedinSchema,
 
 	async execute(_toolCallId, params, _onUpdate, _ctx, _signal) {
-		return executeExaTool("linkedin_search", params as Record<string, unknown>, "web_search_linkedin");
+		return executeExaTool("linkedin_search_exa", params as Record<string, unknown>, "web_search_linkedin");
 	},
 
 	renderCall(args, _options, theme) {
@@ -522,7 +558,7 @@ Parameters:
 	parameters: webSearchCompanySchema,
 
 	async execute(_toolCallId, params, _onUpdate, _ctx, _signal) {
-		return executeExaTool("company_research", params as Record<string, unknown>, "web_search_company");
+		return executeExaTool("company_research_exa", params as Record<string, unknown>, "web_search_company");
 	},
 
 	renderCall(args, _options, theme) {
@@ -591,8 +627,8 @@ export async function hasExaSearch(): Promise<boolean> {
 	return exaKey !== null;
 }
 
-export {
-	getSearchProvider,
-	setPreferredSearchProvider,
-} from "./provider";
-export type { SearchProviderId as SearchProvider, SearchResponse } from "./types";
+export { getSearchProvider, setPreferredSearchProvider } from "./provider";
+export type {
+	SearchProviderId as SearchProvider,
+	SearchResponse,
+} from "./types";

package/src/web/search/provider.ts

@@ -56,22 +56,17 @@ export function setPreferredSearchProvider(provider: SearchProviderId | "auto"):
 export async function resolveProviderChain(
 	preferredProvider: SearchProviderId | "auto" = preferredProvId,
 ): Promise<SearchProvider[]> {
-	const providers: SearchProvider[] = [];
+	const order: SearchProviderId[] =
+		preferredProvider !== "auto"
+			? [preferredProvider, ...SEARCH_PROVIDER_ORDER.filter(id => id !== preferredProvider)]
+			: SEARCH_PROVIDER_ORDER;
 
-	if (preferredProvider !== "auto") {
-		if (await getSearchProvider(preferredProvider).isAvailable()) {
-			providers.push(getSearchProvider(preferredProvider));
-		}
-	}
+	const checks = await Promise.all(
+		order.map(async id => ({
+			provider: getSearchProvider(id),
+			available: await getSearchProvider(id).isAvailable(),
+		})),
+	);
 
-	for (const id of SEARCH_PROVIDER_ORDER) {
-		if (id === preferredProvider) continue;
-
-		const provider = getSearchProvider(id);
-		if (await provider.isAvailable()) {
-			providers.push(provider);
-		}
-	}
-
-	return providers;
+	return checks.filter(c => c.available).map(c => c.provider);
 }

package/src/web/search/providers/grep.ts

@@ -0,0 +1,160 @@
+/**
+ * grep.app Web Search Provider
+ *
+ * Cross-repo code search across all public GitHub repositories.
+ * No API key required. Supports regex and language filtering.
+ */
+
+import type { SearchSource } from "../types";
+
+const GREP_APP_URL = "https://grep.app/api/search";
+
+interface GrepAppHit {
+	repo: string;
+	path: string;
+	branch?: string;
+	total_matches?: string;
+	content?: {
+		snippet?: string;
+	};
+}
+
+interface GrepAppFacetBucket {
+	val: string;
+	count: number;
+}
+
+interface GrepAppResponse {
+	time?: number;
+	hits?: {
+		total?: number;
+		hits?: GrepAppHit[];
+	};
+	facets?: {
+		lang?: { buckets?: GrepAppFacetBucket[] };
+		repo?: { buckets?: GrepAppFacetBucket[] };
+		path?: { buckets?: GrepAppFacetBucket[] };
+	};
+}
+
+function stripHtml(html: string): string {
+	return html
+		.replace(/<mark[^>]*>/g, "")
+		.replace(/<\/mark>/g, "")
+		.replace(/<[^>]+>/g, "")
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&amp;/g, "&")
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'")
+		.trim();
+}
+
+function extractSnippet(hit: GrepAppHit): { lines: string[]; lineNumbers: number[] } {
+	const raw = hit.content?.snippet ?? "";
+	if (!raw) return { lines: [], lineNumbers: [] };
+
+	const lines: string[] = [];
+	const lineNumbers: number[] = [];
+
+	const table = raw.match(/<table[^>]*>(.*?)<\/table>/s);
+	if (!table) {
+		return { lines: [stripHtml(raw)], lineNumbers: [] };
+	}
+
+	const trBlocks = table[1].split(/<\/tr>/);
+	for (const tr of trBlocks) {
+		const lineMatch = tr.match(/data-line="(\d+)"/);
+		if (lineMatch) lineNumbers.push(Number.parseInt(lineMatch[1], 10));
+
+		const preMatch = tr.match(/<pre>(.*?)<\/pre>/s);
+		if (preMatch) lines.push(stripHtml(preMatch[1]));
+	}
+	return { lines, lineNumbers };
+}
+
+export interface SearchCodeParams {
+	query: string;
+	regexp?: boolean;
+	language?: string;
+	repo?: string;
+	limit?: number;
+	signal?: AbortSignal;
+}
+
+export interface SearchCodeSource extends SearchSource {
+	branch?: string;
+	lineNumbers?: number[];
+	matchCount?: number;
+}
+
+export interface SearchCodeResponse {
+	provider: "grep";
+	sources: SearchCodeSource[];
+	total?: number;
+	timeMs?: number;
+	topLanguages?: GrepAppFacetBucket[];
+	topRepos?: GrepAppFacetBucket[];
+}
+
+export class GrepAppProvider {
+	readonly id = "grep";
+	readonly label = "grep.app";
+
+	isAvailable(): boolean {
+		return true;
+	}
+
+	async search(params: SearchCodeParams): Promise<SearchCodeResponse> {
+		const url = new URL(GREP_APP_URL);
+		url.searchParams.set("q", params.query);
+		url.searchParams.set("regexp", params.regexp ? "true" : "false");
+		url.searchParams.set("case", "false");
+		if (params.language) {
+			url.searchParams.set("l", params.language);
+		}
+		if (params.repo) {
+			url.searchParams.set("r", params.repo);
+		}
+
+		const response = await fetch(url.toString(), {
+			signal: params.signal,
+			headers: { Accept: "application/json" },
+		});
+
+		if (!response.ok) {
+			const text = await response.text();
+			throw new Error(`grep.app error (${response.status}): ${text}`);
+		}
+
+		const data = (await response.json()) as GrepAppResponse;
+		const hits = data.hits?.hits ?? [];
+		const limit = params.limit ?? 10;
+		const sources: SearchCodeSource[] = [];
+
+		for (const hit of hits.slice(0, limit)) {
+			const { lines, lineNumbers } = extractSnippet(hit);
+			const branch = hit.branch ?? "HEAD";
+			const startLine = lineNumbers[0];
+			const lineAnchor = startLine ? `#L${startLine}` : "";
+
+			sources.push({
+				title: `${hit.repo}: ${hit.path}`,
+				url: `https://github.com/${hit.repo}/blob/${branch}/${hit.path}${lineAnchor}`,
+				snippet: lines.length > 0 ? lines.join("\n") : undefined,
+				branch,
+				lineNumbers: lineNumbers.length > 0 ? lineNumbers : undefined,
+				matchCount: hit.total_matches ? Number.parseInt(hit.total_matches, 10) : undefined,
+			});
+		}
+
+		return {
+			provider: "grep",
+			sources,
+			total: data.hits?.total,
+			timeMs: data.time,
+			topLanguages: data.facets?.lang?.buckets?.slice(0, 5),
+			topRepos: data.facets?.repo?.buckets?.slice(0, 5),
+		};
+	}
+}