@f5-sales-demo/xcsh 19.51.2

package/src/web/search/providers/codex.ts

@@ -0,0 +1,402 @@
+/**
+ * OpenAI Codex Web Search Provider
+ *
+ * Uses Codex's built-in web_search tool via the Responses API.
+ * Requires OAuth credentials stored in agent.db for provider "openai-codex".
+ * Returns synthesized answers with web search sources.
+ */
+import * as os from "node:os";
+import { $env, getAgentDbPath, readSseJson } from "@f5xc-salesdemos/pi-utils";
+import packageJson from "../../../../package.json" with { type: "json" };
+import { AgentStorage } from "../../../session/agent-storage";
+import type { SearchResponse, SearchSource } from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+
+const CODEX_BASE_URL = "https://chatgpt.com/backend-api";
+const CODEX_RESPONSES_PATH = "/codex/responses";
+const DEFAULT_MODEL = "gpt-5-codex-mini";
+const JWT_CLAIM_PATH = "https://api.openai.com/auth";
+const DEFAULT_INSTRUCTIONS =
+	"You are a helpful assistant with web search capabilities. Search the web to answer the user's question accurately and cite your sources.";
+
+function getModel(): string {
+	const configuredModel = $env.PI_CODEX_WEB_SEARCH_MODEL?.trim();
+	return configuredModel ? configuredModel : DEFAULT_MODEL;
+}
+
+export interface CodexSearchParams {
+	signal?: AbortSignal;
+	query: string;
+	system_prompt?: string;
+	num_results?: number;
+	/** Search context size: controls how much web content to include */
+	search_context_size?: "low" | "medium" | "high";
+}
+
+/** OAuth credential stored in agent.db */
+interface CodexOAuthCredential {
+	type: "oauth";
+	access: string;
+	refresh?: string;
+	expires: number;
+	accountId?: string;
+}
+
+/** JWT payload structure for extracting account ID */
+type JwtPayload = {
+	[key: string]: unknown;
+};
+
+/** Codex API response structure */
+interface CodexResponseItem {
+	type: string;
+	id?: string;
+	role?: string;
+	name?: string;
+	call_id?: string;
+	status?: string;
+	arguments?: string;
+	content?: CodexContentPart[];
+	summary?: Array<{ type: string; text: string }>;
+}
+
+interface CodexContentPart {
+	type: string;
+	text?: string;
+	annotations?: CodexAnnotation[];
+}
+
+interface CodexAnnotation {
+	type: string;
+	url?: string;
+	title?: string;
+	start_index?: number;
+	end_index?: number;
+}
+
+interface CodexUsage {
+	input_tokens?: number;
+	output_tokens?: number;
+	total_tokens?: number;
+	input_tokens_details?: { cached_tokens?: number };
+}
+
+interface CodexResponse {
+	id?: string;
+	model?: string;
+	status?: string;
+	usage?: CodexUsage;
+}
+
+function isImagePlaceholderAnswer(text: string): boolean {
+	return text.trim().toLowerCase() === "(see attached image)";
+}
+
+/**
+ * Decodes a JWT token and extracts the payload.
+ * @param token - JWT token string
+ * @returns Decoded payload, or null if parsing fails
+ */
+function decodeJwt(token: string): JwtPayload | null {
+	try {
+		const parts = token.split(".");
+		if (parts.length !== 3) return null;
+		const payload = parts[1] ?? "";
+		const decoded = Buffer.from(payload, "base64").toString("utf-8");
+		return JSON.parse(decoded) as JwtPayload;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Extracts account ID from a Codex access token.
+ * @param accessToken - JWT access token
+ * @returns Account ID string, or null if not found
+ */
+function getAccountId(accessToken: string): string | null {
+	const payload = decodeJwt(accessToken);
+	const auth = payload?.[JWT_CLAIM_PATH] as { chatgpt_account_id?: string } | undefined;
+	const accountId = auth?.chatgpt_account_id;
+	return typeof accountId === "string" && accountId.length > 0 ? accountId : null;
+}
+
+/**
+ * Finds valid Codex OAuth credentials from agent.db.
+ * Checks agent credentials and returns the first non-expired credential.
+ * @returns OAuth credential with access token and account ID, or null if none found
+ */
+async function findCodexAuth(): Promise<{ accessToken: string; accountId: string } | null> {
+	const expiryBuffer = 5 * 60 * 1000; // 5 minutes
+	const now = Date.now();
+
+	try {
+		const storage = await AgentStorage.open(getAgentDbPath());
+		const records = storage.listAuthCredentials("openai-codex");
+
+		for (const record of records) {
+			const credential = record.credential;
+			if (credential.type !== "oauth") continue;
+
+			const oauthCred = credential as CodexOAuthCredential;
+			if (!oauthCred.access) continue;
+			if (oauthCred.expires <= now + expiryBuffer) continue;
+
+			const accountId = oauthCred.accountId ?? getAccountId(oauthCred.access);
+			if (!accountId) continue;
+
+			return { accessToken: oauthCred.access, accountId };
+		}
+	} catch {
+		return null;
+	}
+
+	return null;
+}
+
+/**
+ * Builds HTTP headers for Codex API requests.
+ * @param accessToken - OAuth access token
+ * @param accountId - ChatGPT account ID
+ * @returns Headers object for fetch requests
+ */
+function buildCodexHeaders(accessToken: string, accountId: string): Record<string, string> {
+	return {
+		Authorization: `Bearer ${accessToken}`,
+		"chatgpt-account-id": accountId,
+		"OpenAI-Beta": "responses=experimental",
+		originator: "pi",
+		"User-Agent": `pi/${packageJson.version} (${os.platform()} ${os.release()}; ${os.arch()})`,
+		Accept: "text/event-stream",
+		"Content-Type": "application/json",
+	};
+}
+
+/**
+ * Calls the Codex Responses API with web search tool enabled.
+ * Streams the response and collects all events.
+ * @param auth - Authentication info (access token and account ID)
+ * @param query - Search query from the user
+ * @param options - Search options including system prompt and context size
+ * @returns Parsed response with answer, sources, and usage
+ * @throws {SearchProviderError} If the API request fails
+ */
+async function callCodexSearch(
+	auth: { accessToken: string; accountId: string },
+	query: string,
+	options: { signal?: AbortSignal; systemPrompt?: string; searchContextSize?: "low" | "medium" | "high" },
+): Promise<{
+	answer: string;
+	sources: SearchSource[];
+	model: string;
+	requestId: string;
+	usage?: { inputTokens: number; outputTokens: number; totalTokens: number };
+}> {
+	const url = `${CODEX_BASE_URL}${CODEX_RESPONSES_PATH}`;
+	const headers = buildCodexHeaders(auth.accessToken, auth.accountId);
+
+	const requestedModel = getModel();
+
+	const body: Record<string, unknown> = {
+		model: requestedModel,
+		stream: true,
+		store: false,
+		input: [
+			{
+				type: "message",
+				role: "user",
+				content: [{ type: "input_text", text: query }],
+			},
+		],
+		tools: [
+			{
+				type: "web_search",
+				search_context_size: options.searchContextSize ?? "high",
+			},
+		],
+		instructions: options.systemPrompt ?? DEFAULT_INSTRUCTIONS,
+	};
+
+	const response = await fetch(url, {
+		method: "POST",
+		headers,
+		body: JSON.stringify(body),
+	});
+
+	if (!response.ok) {
+		const errorText = await response.text();
+		throw new SearchProviderError("codex", `Codex API error (${response.status}): ${errorText}`, response.status);
+	}
+
+	if (!response.body) {
+		throw new SearchProviderError("codex", "Codex API returned no response body", 500);
+	}
+
+	// Parse SSE stream
+	const answerParts: string[] = [];
+	const streamedAnswerParts: string[] = [];
+	const sources: SearchSource[] = [];
+	let model = requestedModel;
+	let requestId = "";
+	let usage: { inputTokens: number; outputTokens: number; totalTokens: number } | undefined;
+
+	for await (const rawEvent of readSseJson<Record<string, unknown>>(response.body, options.signal)) {
+		const eventType = typeof rawEvent.type === "string" ? rawEvent.type : "";
+		if (!eventType) continue;
+
+		if (eventType === "response.output_text.delta") {
+			const delta = typeof rawEvent.delta === "string" ? rawEvent.delta : "";
+			if (delta) {
+				streamedAnswerParts.push(delta);
+			}
+		} else if (eventType === "response.output_item.done") {
+			const item = rawEvent.item as CodexResponseItem | undefined;
+			if (!item) continue;
+
+			// Handle text message content and extract sources from annotations
+			if (item.type === "message" && item.content) {
+				for (const part of item.content) {
+					if (part.type === "output_text" && part.text) {
+						answerParts.push(part.text);
+
+						// Extract sources from url_citation annotations
+						if (part.annotations) {
+							for (const annotation of part.annotations) {
+								if (annotation.type === "url_citation" && annotation.url) {
+									// Deduplicate by URL
+									if (!sources.some(s => s.url === annotation.url)) {
+										sources.push({
+											title: annotation.title ?? annotation.url,
+											url: annotation.url,
+										});
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+
+			// Handle reasoning summary as part of answer
+			if (item.type === "reasoning" && item.summary) {
+				for (const part of item.summary) {
+					if (part.type === "summary_text" && part.text) {
+						answerParts.push(part.text);
+					}
+				}
+			}
+		} else if (eventType === "response.completed" || eventType === "response.done") {
+			const resp = (rawEvent as { response?: CodexResponse }).response;
+			if (resp) {
+				if (resp.model) model = resp.model;
+				if (resp.id) requestId = resp.id;
+				if (resp.usage) {
+					const cachedTokens = resp.usage.input_tokens_details?.cached_tokens ?? 0;
+					usage = {
+						inputTokens: (resp.usage.input_tokens ?? 0) - cachedTokens,
+						outputTokens: resp.usage.output_tokens ?? 0,
+						totalTokens: resp.usage.total_tokens ?? 0,
+					};
+				}
+			}
+		} else if (eventType === "error") {
+			const code = (rawEvent as { code?: string }).code ?? "";
+			const message = (rawEvent as { message?: string }).message ?? "Unknown error";
+			throw new SearchProviderError("codex", `Codex error (${code}): ${message}`, 500);
+		} else if (eventType === "response.failed") {
+			const resp = (rawEvent as { response?: { error?: { message?: string } } }).response;
+			const errorMessage = resp?.error?.message ?? "Request failed";
+			throw new SearchProviderError("codex", `Codex request failed: ${errorMessage}`, 500);
+		}
+	}
+
+	const finalAnswer = answerParts.join("\n\n").trim();
+	const streamedAnswer = streamedAnswerParts.join("").trim();
+	const answer =
+		finalAnswer.length > 0 && !isImagePlaceholderAnswer(finalAnswer)
+			? finalAnswer
+			: streamedAnswer.length > 0
+				? streamedAnswer
+				: finalAnswer;
+
+	return {
+		answer,
+		sources,
+		model,
+		requestId,
+		usage,
+	};
+}
+
+/**
+ * Executes a web search using OpenAI Codex's built-in web search tool.
+ * Requires OAuth credentials stored in agent.db for provider "openai-codex".
+ * @param params - Search parameters including query and optional settings
+ * @returns Search response with synthesized answer, sources, and usage
+ * @throws {Error} If no Codex OAuth credentials are configured
+ */
+export async function searchCodex(params: CodexSearchParams): Promise<SearchResponse> {
+	const auth = await findCodexAuth();
+	if (!auth) {
+		throw new Error(
+			"No Codex OAuth credentials found. Login with 'xcsh /login openai-codex' to enable Codex web search.",
+		);
+	}
+
+	const result = await callCodexSearch(auth, params.query, {
+		systemPrompt: params.system_prompt,
+		searchContextSize: params.search_context_size ?? "high",
+	});
+
+	let sources = result.sources;
+
+	// Apply num_results limit if specified
+	if (params.num_results && sources.length > params.num_results) {
+		sources = sources.slice(0, params.num_results);
+	}
+
+	return {
+		provider: "codex",
+		answer: result.answer || undefined,
+		sources,
+		usage: result.usage
+			? {
+					inputTokens: result.usage.inputTokens,
+					outputTokens: result.usage.outputTokens,
+					totalTokens: result.usage.totalTokens,
+				}
+			: undefined,
+		model: result.model,
+		requestId: result.requestId,
+	};
+}
+
+/**
+ * Checks if Codex web search is available.
+ * @returns True if valid OAuth credentials exist for openai-codex
+ */
+export async function hasCodexSearch(): Promise<boolean> {
+	const auth = await findCodexAuth();
+	return auth !== null;
+}
+
+/** Search provider for OpenAI Codex web search. */
+export class CodexProvider extends SearchProvider {
+	readonly id = "codex";
+	readonly label = "Codex";
+
+	isAvailable(): Promise<boolean> {
+		return Promise.resolve(hasCodexSearch());
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchCodex({
+			signal: params.signal,
+			query: params.query,
+			system_prompt: params.systemPrompt,
+			num_results: params.numSearchResults ?? params.limit,
+		});
+	}
+}

package/src/web/search/providers/exa.ts

@@ -0,0 +1,264 @@
+/**
+ * Exa Web Search Provider
+ *
+ * High-quality neural search via Exa Search API.
+ * Returns structured search results with optional content extraction.
+ * Requests per-result summaries via `contents.summary` and synthesizes
+ * them into a combined `answer` string on the SearchResponse.
+ */
+import { getEnvApiKey } from "@f5xc-salesdemos/pi-ai";
+import { settings } from "../../../config/settings";
+import { callExaTool, findApiKey, isSearchResponse } from "../../../exa/mcp-client";
+import type { SearchResponse, SearchSource } from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import { dateToAgeSeconds } from "../utils";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+
+const EXA_API_URL = "https://api.exa.ai/search";
+
+type ExaSearchType = "neural" | "fast" | "auto" | "deep";
+
+type ExaSearchParamType = ExaSearchType | "keyword";
+
+export interface ExaSearchParams {
+	query: string;
+	num_results?: number;
+	type?: ExaSearchParamType;
+	include_domains?: string[];
+	exclude_domains?: string[];
+	start_published_date?: string;
+	end_published_date?: string;
+}
+
+interface ExaSearchResult {
+	title?: string | null;
+	url?: string | null;
+	author?: string | null;
+	publishedDate?: string | null;
+	text?: string | null;
+	highlights?: string[] | null;
+	summary?: string | null;
+}
+
+interface ExaSearchResponse {
+	requestId?: string;
+	resolvedSearchType?: string;
+	results?: ExaSearchResult[];
+	costDollars?: { total: number };
+	searchTime?: number;
+}
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+	if (typeof value !== "object" || value === null) return null;
+	return value as Record<string, unknown>;
+}
+
+function parseOptionalField(section: string, label: string): string | null | undefined {
+	const regex = new RegExp(`(?:^|\\n)${label}:\\s*([^\\n]*)`);
+	const match = section.match(regex);
+	if (!match) return undefined;
+	const value = match[1].trim();
+	return value.length > 0 ? value : null;
+}
+
+function parseTextField(section: string): string | null | undefined {
+	const match = section.match(/(?:^|\n)Text:\s*([\s\S]*)$/);
+	if (!match) return undefined;
+	const value = match[1].trim();
+	return value.length > 0 ? value : null;
+}
+
+function parseExaMcpTextPayload(payload: unknown): ExaSearchResponse | null {
+	const root = asRecord(payload);
+	if (!root) return null;
+
+	const content = root.content;
+	if (!Array.isArray(content)) return null;
+
+	const textBlocks = content
+		.map(item => {
+			const part = asRecord(item);
+			const text = typeof part?.text === "string" ? part.text : "";
+			return text.replace(/\r\n?/g, "\n").trim();
+		})
+		.filter(text => text.length > 0);
+
+	if (textBlocks.length === 0) return null;
+
+	const sections = textBlocks
+		.join("\n\n")
+		.split(/\n{2,}(?=Title:\s*[^\n]*(?:\n(?:URL|Author|Published Date|Text):))/)
+		.map(section => section.trim())
+		.filter(section => section.startsWith("Title:"));
+
+	const results: ExaSearchResult[] = [];
+	for (const section of sections) {
+		const title = parseOptionalField(section, "Title");
+		const url = parseOptionalField(section, "URL");
+		const author = parseOptionalField(section, "Author");
+		const publishedDate = parseOptionalField(section, "Published Date");
+		const text = parseTextField(section);
+
+		if (!title && !url && !text) continue;
+
+		results.push({
+			title: title ?? undefined,
+			url: url ?? undefined,
+			author: author ?? undefined,
+			publishedDate: publishedDate ?? undefined,
+			text: text ?? undefined,
+		});
+	}
+
+	if (results.length === 0) return null;
+	return { results };
+}
+
+export function normalizeSearchType(type: ExaSearchParamType | undefined): ExaSearchType {
+	if (!type) return "auto";
+	if (type === "keyword") return "fast";
+	return type;
+}
+
+/** Maximum number of per-result summaries to include in the synthesized answer. */
+const MAX_ANSWER_SUMMARIES = 3;
+
+/**
+ * Synthesize an answer string from per-result summaries returned by Exa.
+ * Returns `undefined` when no non-empty summaries are available so callers
+ * can leave `SearchResponse.answer` unset (matching other providers).
+ */
+export function synthesizeAnswer(results: ExaSearchResult[]): string | undefined {
+	const parts: string[] = [];
+	for (const r of results) {
+		if (parts.length >= MAX_ANSWER_SUMMARIES) break;
+		const summary = r.summary?.trim();
+		if (!summary) continue;
+		const title = r.title?.trim() || r.url || "Untitled";
+		parts.push(`**${title}**: ${summary}`);
+	}
+	return parts.length > 0 ? parts.join("\n\n") : undefined;
+}
+
+/** Build the request body for `callExaSearch`. Exported for testing. */
+export function buildExaRequestBody(params: ExaSearchParams): Record<string, unknown> {
+	const body: Record<string, unknown> = {
+		query: params.query,
+		numResults: params.num_results ?? 10,
+		type: normalizeSearchType(params.type),
+		contents: {
+			summary: { query: params.query },
+		},
+	};
+
+	if (params.include_domains?.length) {
+		body.includeDomains = params.include_domains;
+	}
+	if (params.exclude_domains?.length) {
+		body.excludeDomains = params.exclude_domains;
+	}
+	if (params.start_published_date) {
+		body.startPublishedDate = params.start_published_date;
+	}
+	if (params.end_published_date) {
+		body.endPublishedDate = params.end_published_date;
+	}
+
+	return body;
+}
+
+/** Call Exa Search API */
+async function callExaSearch(apiKey: string, params: ExaSearchParams): Promise<ExaSearchResponse> {
+	const body = buildExaRequestBody(params);
+
+	const response = await fetch(EXA_API_URL, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			"x-api-key": apiKey,
+		},
+		body: JSON.stringify(body),
+	});
+
+	if (!response.ok) {
+		const errorText = await response.text();
+		throw new SearchProviderError("exa", `Exa API error (${response.status}): ${errorText}`, response.status);
+	}
+
+	return response.json() as Promise<ExaSearchResponse>;
+}
+
+async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchResponse> {
+	const response = await callExaTool("web_search_exa", { ...params }, findApiKey());
+	if (isSearchResponse(response)) {
+		return response as ExaSearchResponse;
+	}
+
+	const parsed = parseExaMcpTextPayload(response);
+	if (parsed) {
+		return parsed;
+	}
+
+	throw new Error("Exa MCP search returned unexpected response shape.");
+}
+
+/** Execute Exa web search */
+export async function searchExa(params: ExaSearchParams): Promise<SearchResponse> {
+	const apiKey = getEnvApiKey("exa");
+	const response = apiKey ? await callExaSearch(apiKey, params) : await callExaMcpSearch(params);
+
+	// Convert to unified SearchResponse
+	const sources: SearchSource[] = [];
+
+	if (response.results) {
+		for (const result of response.results) {
+			if (!result.url) continue;
+			sources.push({
+				title: result.title ?? result.url,
+				url: result.url,
+				snippet: result.summary || result.text || result.highlights?.join(" ") || undefined,
+				publishedDate: result.publishedDate ?? undefined,
+				ageSeconds: dateToAgeSeconds(result.publishedDate ?? undefined),
+				author: result.author ?? undefined,
+			});
+		}
+	}
+
+	// Apply num_results limit if specified
+	const limitedSources = params.num_results ? sources.slice(0, params.num_results) : sources;
+
+	// Synthesize answer only from results that have a URL (same guard as sources loop)
+	const answer = response.results ? synthesizeAnswer(response.results.filter(r => !!r.url)) : undefined;
+
+	return {
+		provider: "exa",
+		answer,
+		sources: limitedSources,
+		requestId: response.requestId,
+	};
+}
+
+/** Search provider for Exa. */
+export class ExaProvider extends SearchProvider {
+	readonly id = "exa";
+	readonly label = "Exa";
+
+	isAvailable(): boolean {
+		try {
+			if (settings.get("exa.enabled") === false || settings.get("exa.enableSearch") === false) {
+				return false;
+			}
+		} catch {
+			// Settings not initialized; fall through to public MCP availability
+		}
+		return true;
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchExa({
+			query: params.query,
+			num_results: params.numSearchResults ?? params.limit,
+		});
+	}
+}