@nghyane/arcane 0.1.0

package/src/web/search/providers/codex.ts

@@ -0,0 +1,377 @@
+/**
+ * OpenAI Codex Web Search Provider
+ *
+ * Uses Codex's built-in web_search tool via the Responses API.
+ * Requires OAuth credentials stored in agent.db for provider "openai-codex".
+ * Returns synthesized answers with web search sources.
+ */
+import * as os from "node:os";
+import { readSseJson } from "@nghyane/arcane-utils";
+import { getAgentDbPath } from "@nghyane/arcane-utils/dirs";
+import packageJson from "../../../../package.json" with { type: "json" };
+import { AgentStorage } from "../../../session/agent-storage";
+import type { SearchResponse, SearchSource } from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+
+const CODEX_BASE_URL = "https://chatgpt.com/backend-api";
+const CODEX_RESPONSES_PATH = "/codex/responses";
+const DEFAULT_MODEL = "gpt-5-codex-mini";
+const JWT_CLAIM_PATH = "https://api.openai.com/auth";
+const DEFAULT_INSTRUCTIONS =
+	"You are a helpful assistant with web search capabilities. Search the web to answer the user's question accurately and cite your sources.";
+
+export interface CodexSearchParams {
+	signal?: AbortSignal;
+	query: string;
+	system_prompt?: string;
+	num_results?: number;
+	/** Search context size: controls how much web content to include */
+	search_context_size?: "low" | "medium" | "high";
+}
+
+/** OAuth credential stored in agent.db */
+interface CodexOAuthCredential {
+	type: "oauth";
+	access: string;
+	refresh?: string;
+	expires: number;
+	accountId?: string;
+}
+
+/** JWT payload structure for extracting account ID */
+type JwtPayload = {
+	[key: string]: unknown;
+};
+
+/** Codex API response structure */
+interface CodexResponseItem {
+	type: string;
+	id?: string;
+	role?: string;
+	name?: string;
+	call_id?: string;
+	status?: string;
+	arguments?: string;
+	content?: CodexContentPart[];
+	summary?: Array<{ type: string; text: string }>;
+}
+
+interface CodexContentPart {
+	type: string;
+	text?: string;
+	annotations?: CodexAnnotation[];
+}
+
+interface CodexAnnotation {
+	type: string;
+	url?: string;
+	title?: string;
+	start_index?: number;
+	end_index?: number;
+}
+
+interface CodexUsage {
+	input_tokens?: number;
+	output_tokens?: number;
+	total_tokens?: number;
+	input_tokens_details?: { cached_tokens?: number };
+}
+
+interface CodexResponse {
+	id?: string;
+	model?: string;
+	status?: string;
+	usage?: CodexUsage;
+}
+
+/**
+ * Decodes a JWT token and extracts the payload.
+ * @param token - JWT token string
+ * @returns Decoded payload, or null if parsing fails
+ */
+function decodeJwt(token: string): JwtPayload | null {
+	try {
+		const parts = token.split(".");
+		if (parts.length !== 3) return null;
+		const payload = parts[1] ?? "";
+		const decoded = Buffer.from(payload, "base64").toString("utf-8");
+		return JSON.parse(decoded) as JwtPayload;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Extracts account ID from a Codex access token.
+ * @param accessToken - JWT access token
+ * @returns Account ID string, or null if not found
+ */
+function getAccountId(accessToken: string): string | null {
+	const payload = decodeJwt(accessToken);
+	const auth = payload?.[JWT_CLAIM_PATH] as { chatgpt_account_id?: string } | undefined;
+	const accountId = auth?.chatgpt_account_id;
+	return typeof accountId === "string" && accountId.length > 0 ? accountId : null;
+}
+
+/**
+ * Finds valid Codex OAuth credentials from agent.db.
+ * Checks agent credentials and returns the first non-expired credential.
+ * @returns OAuth credential with access token and account ID, or null if none found
+ */
+async function findCodexAuth(): Promise<{ accessToken: string; accountId: string } | null> {
+	const expiryBuffer = 5 * 60 * 1000; // 5 minutes
+	const now = Date.now();
+
+	try {
+		const storage = await AgentStorage.open(getAgentDbPath());
+		const records = storage.listAuthCredentials("openai-codex");
+
+		for (const record of records) {
+			const credential = record.credential;
+			if (credential.type !== "oauth") continue;
+
+			const oauthCred = credential as CodexOAuthCredential;
+			if (!oauthCred.access) continue;
+			if (oauthCred.expires <= now + expiryBuffer) continue;
+
+			const accountId = oauthCred.accountId ?? getAccountId(oauthCred.access);
+			if (!accountId) continue;
+
+			return { accessToken: oauthCred.access, accountId };
+		}
+	} catch {
+		return null;
+	}
+
+	return null;
+}
+
+/**
+ * Builds HTTP headers for Codex API requests.
+ * @param accessToken - OAuth access token
+ * @param accountId - ChatGPT account ID
+ * @returns Headers object for fetch requests
+ */
+function buildCodexHeaders(accessToken: string, accountId: string): Record<string, string> {
+	return {
+		Authorization: `Bearer ${accessToken}`,
+		"chatgpt-account-id": accountId,
+		"OpenAI-Beta": "responses=experimental",
+		originator: "arcane",
+		"User-Agent": `pi/${packageJson.version} (${os.platform()} ${os.release()}; ${os.arch()})`,
+		Accept: "text/event-stream",
+		"Content-Type": "application/json",
+	};
+}
+
+/**
+ * Calls the Codex Responses API with web search tool enabled.
+ * Streams the response and collects all events.
+ * @param auth - Authentication info (access token and account ID)
+ * @param query - Search query from the user
+ * @param options - Search options including system prompt and context size
+ * @returns Parsed response with answer, sources, and usage
+ * @throws {SearchProviderError} If the API request fails
+ */
+async function callCodexSearch(
+	auth: { accessToken: string; accountId: string },
+	query: string,
+	options: { signal?: AbortSignal; systemPrompt?: string; searchContextSize?: "low" | "medium" | "high" },
+): Promise<{
+	answer: string;
+	sources: SearchSource[];
+	model: string;
+	requestId: string;
+	usage?: { inputTokens: number; outputTokens: number; totalTokens: number };
+}> {
+	const url = `${CODEX_BASE_URL}${CODEX_RESPONSES_PATH}`;
+	const headers = buildCodexHeaders(auth.accessToken, auth.accountId);
+
+	const body: Record<string, unknown> = {
+		model: DEFAULT_MODEL,
+		stream: true,
+		store: false,
+		input: [
+			{
+				type: "message",
+				role: "user",
+				content: [{ type: "input_text", text: query }],
+			},
+		],
+		tools: [
+			{
+				type: "web_search",
+				search_context_size: options.searchContextSize ?? "high",
+			},
+		],
+		instructions: options.systemPrompt ?? DEFAULT_INSTRUCTIONS,
+	};
+
+	const response = await fetch(url, {
+		method: "POST",
+		headers,
+		body: JSON.stringify(body),
+	});
+
+	if (!response.ok) {
+		const errorText = await response.text();
+		throw new SearchProviderError("codex", `Codex API error (${response.status}): ${errorText}`, response.status);
+	}
+
+	if (!response.body) {
+		throw new SearchProviderError("codex", "Codex API returned no response body", 500);
+	}
+
+	// Parse SSE stream
+	const answerParts: string[] = [];
+	const sources: SearchSource[] = [];
+	let model = DEFAULT_MODEL;
+	let requestId = "";
+	let usage: { inputTokens: number; outputTokens: number; totalTokens: number } | undefined;
+
+	for await (const rawEvent of readSseJson<Record<string, unknown>>(response.body, options.signal)) {
+		const eventType = typeof rawEvent.type === "string" ? rawEvent.type : "";
+		if (!eventType) continue;
+
+		if (eventType === "response.output_item.done") {
+			const item = rawEvent.item as CodexResponseItem | undefined;
+			if (!item) continue;
+
+			// Handle text message content and extract sources from annotations
+			if (item.type === "message" && item.content) {
+				for (const part of item.content) {
+					if (part.type === "output_text" && part.text) {
+						answerParts.push(part.text);
+
+						// Extract sources from url_citation annotations
+						if (part.annotations) {
+							for (const annotation of part.annotations) {
+								if (annotation.type === "url_citation" && annotation.url) {
+									// Deduplicate by URL
+									if (!sources.some(s => s.url === annotation.url)) {
+										sources.push({
+											title: annotation.title ?? annotation.url,
+											url: annotation.url,
+										});
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+
+			// Handle reasoning summary as part of answer
+			if (item.type === "reasoning" && item.summary) {
+				for (const part of item.summary) {
+					if (part.type === "summary_text" && part.text) {
+						answerParts.push(part.text);
+					}
+				}
+			}
+		} else if (eventType === "response.completed" || eventType === "response.done") {
+			const resp = (rawEvent as { response?: CodexResponse }).response;
+			if (resp) {
+				if (resp.model) model = resp.model;
+				if (resp.id) requestId = resp.id;
+				if (resp.usage) {
+					const cachedTokens = resp.usage.input_tokens_details?.cached_tokens ?? 0;
+					usage = {
+						inputTokens: (resp.usage.input_tokens ?? 0) - cachedTokens,
+						outputTokens: resp.usage.output_tokens ?? 0,
+						totalTokens: resp.usage.total_tokens ?? 0,
+					};
+				}
+			}
+		} else if (eventType === "error") {
+			const code = (rawEvent as { code?: string }).code ?? "";
+			const message = (rawEvent as { message?: string }).message ?? "Unknown error";
+			throw new SearchProviderError("codex", `Codex error (${code}): ${message}`, 500);
+		} else if (eventType === "response.failed") {
+			const resp = (rawEvent as { response?: { error?: { message?: string } } }).response;
+			const errorMessage = resp?.error?.message ?? "Request failed";
+			throw new SearchProviderError("codex", `Codex request failed: ${errorMessage}`, 500);
+		}
+	}
+
+	return {
+		answer: answerParts.join("\n\n"),
+		sources,
+		model,
+		requestId,
+		usage,
+	};
+}
+
+/**
+ * Executes a web search using OpenAI Codex's built-in web search tool.
+ * Requires OAuth credentials stored in agent.db for provider "openai-codex".
+ * @param params - Search parameters including query and optional settings
+ * @returns Search response with synthesized answer, sources, and usage
+ * @throws {Error} If no Codex OAuth credentials are configured
+ */
+export async function searchCodex(params: CodexSearchParams): Promise<SearchResponse> {
+	const auth = await findCodexAuth();
+	if (!auth) {
+		throw new Error(
+			"No Codex OAuth credentials found. Login with 'arc /login openai-codex' to enable Codex web search.",
+		);
+	}
+
+	const result = await callCodexSearch(auth, params.query, {
+		systemPrompt: params.system_prompt,
+		searchContextSize: params.search_context_size ?? "high",
+	});
+
+	let sources = result.sources;
+
+	// Apply num_results limit if specified
+	if (params.num_results && sources.length > params.num_results) {
+		sources = sources.slice(0, params.num_results);
+	}
+
+	return {
+		provider: "codex",
+		answer: result.answer || undefined,
+		sources,
+		usage: result.usage
+			? {
+					inputTokens: result.usage.inputTokens,
+					outputTokens: result.usage.outputTokens,
+					totalTokens: result.usage.totalTokens,
+				}
+			: undefined,
+		model: result.model,
+		requestId: result.requestId,
+	};
+}
+
+/**
+ * Checks if Codex web search is available.
+ * @returns True if valid OAuth credentials exist for openai-codex
+ */
+export async function hasCodexSearch(): Promise<boolean> {
+	const auth = await findCodexAuth();
+	return auth !== null;
+}
+
+/** Search provider for OpenAI Codex web search. */
+export class CodexProvider extends SearchProvider {
+	readonly id = "codex";
+	readonly label = "Codex";
+
+	isAvailable(): Promise<boolean> {
+		return Promise.resolve(hasCodexSearch());
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchCodex({
+			signal: params.signal,
+			query: params.query,
+			system_prompt: params.systemPrompt,
+			num_results: params.numSearchResults ?? params.limit,
+		});
+	}
+}

package/src/web/search/providers/exa.ts

@@ -0,0 +1,158 @@
+/**
+ * Exa Web Search Provider
+ *
+ * High-quality neural search via Exa Search API.
+ * Returns structured search results with optional content extraction.
+ */
+import { getEnvApiKey } from "@nghyane/arcane-ai";
+import { findApiKey as findExaKey } from "../../../exa/mcp-client";
+import type { SearchResponse, SearchSource } from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+
+const EXA_API_URL = "https://api.exa.ai/search";
+
+type ExaSearchType = "neural" | "fast" | "auto" | "deep";
+
+type ExaSearchParamType = ExaSearchType | "keyword";
+
+export interface ExaSearchParams {
+	query: string;
+	num_results?: number;
+	type?: ExaSearchParamType;
+	include_domains?: string[];
+	exclude_domains?: string[];
+	start_published_date?: string;
+	end_published_date?: string;
+}
+
+interface ExaSearchResult {
+	title?: string | null;
+	url?: string | null;
+	author?: string | null;
+	publishedDate?: string | null;
+	text?: string | null;
+	highlights?: string[] | null;
+}
+
+interface ExaSearchResponse {
+	requestId?: string;
+	resolvedSearchType?: string;
+	results?: ExaSearchResult[];
+	costDollars?: { total: number };
+	searchTime?: number;
+}
+
+function normalizeSearchType(type: ExaSearchParamType | undefined): ExaSearchType {
+	if (!type) return "auto";
+	if (type === "keyword") return "fast";
+	return type;
+}
+
+/** Call Exa Search API */
+async function callExaSearch(apiKey: string, params: ExaSearchParams): Promise<ExaSearchResponse> {
+	const body: Record<string, unknown> = {
+		query: params.query,
+		numResults: params.num_results ?? 10,
+		type: normalizeSearchType(params.type),
+	};
+
+	if (params.include_domains?.length) {
+		body.includeDomains = params.include_domains;
+	}
+	if (params.exclude_domains?.length) {
+		body.excludeDomains = params.exclude_domains;
+	}
+	if (params.start_published_date) {
+		body.startPublishedDate = params.start_published_date;
+	}
+	if (params.end_published_date) {
+		body.endPublishedDate = params.end_published_date;
+	}
+
+	const response = await fetch(EXA_API_URL, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			"x-api-key": apiKey,
+		},
+		body: JSON.stringify(body),
+	});
+
+	if (!response.ok) {
+		const errorText = await response.text();
+		throw new SearchProviderError("exa", `Exa API error (${response.status}): ${errorText}`, response.status);
+	}
+
+	return response.json() as Promise<ExaSearchResponse>;
+}
+
+/** Calculate age in seconds from ISO date string */
+function dateToAgeSeconds(dateStr: string | null | undefined): number | undefined {
+	if (!dateStr) return undefined;
+	try {
+		const date = new Date(dateStr);
+		if (Number.isNaN(date.getTime())) return undefined;
+		return Math.floor((Date.now() - date.getTime()) / 1000);
+	} catch {
+		return undefined;
+	}
+}
+
+/** Execute Exa web search */
+export async function searchExa(params: ExaSearchParams): Promise<SearchResponse> {
+	const apiKey = getEnvApiKey("exa");
+	if (!apiKey) {
+		throw new Error("EXA_API_KEY not found. Set it in environment or .env file.");
+	}
+
+	const response = await callExaSearch(apiKey, params);
+
+	// Convert to unified SearchResponse
+	const sources: SearchSource[] = [];
+
+	if (response.results) {
+		for (const result of response.results) {
+			if (!result.url) continue;
+			sources.push({
+				title: result.title ?? result.url,
+				url: result.url,
+				snippet: result.text ?? result.highlights?.join(" ") ?? undefined,
+				publishedDate: result.publishedDate ?? undefined,
+				ageSeconds: dateToAgeSeconds(result.publishedDate ?? undefined),
+				author: result.author ?? undefined,
+			});
+		}
+	}
+
+	// Apply num_results limit if specified
+	const limitedSources = params.num_results ? sources.slice(0, params.num_results) : sources;
+
+	return {
+		provider: "exa",
+		sources: limitedSources,
+		requestId: response.requestId,
+	};
+}
+
+/** Search provider for Exa. */
+export class ExaProvider extends SearchProvider {
+	readonly id = "exa";
+	readonly label = "Exa";
+
+	isAvailable(): boolean {
+		try {
+			return !!findExaKey();
+		} catch {
+			return false;
+		}
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchExa({
+			query: params.query,
+			num_results: params.numSearchResults ?? params.limit,
+		});
+	}
+}