@f5xc-salesdemos/xcsh 2.2.0 → 14.0.2

package/src/web/search/providers/perplexity.ts

@@ -0,0 +1,546 @@
+/**
+ * Perplexity Web Search Provider
+ *
+ * Supports three auth modes:
+ * - Cookies (`PERPLEXITY_COOKIES`) via `www.perplexity.ai/rest/sse/perplexity_ask`
+ * - OAuth JWT (stored in `agent.db`) via `www.perplexity.ai/rest/sse/perplexity_ask`
+ * - API key (`PERPLEXITY_API_KEY`) via `api.perplexity.ai/chat/completions`
+ */
+
+import { getEnvApiKey } from "@f5xc-salesdemos/pi-ai";
+import { $env, getAgentDbPath, readSseJson } from "@f5xc-salesdemos/pi-utils";
+import { AgentStorage } from "../../../session/agent-storage";
+import type {
+	PerplexityMessageOutput,
+	PerplexityRequest,
+	PerplexityResponse,
+	SearchCitation,
+	SearchResponse,
+	SearchSource,
+} from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import { dateToAgeSeconds } from "../utils";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+
+const PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions";
+const PERPLEXITY_OAUTH_ASK_URL = "https://www.perplexity.ai/rest/sse/perplexity_ask";
+
+const DEFAULT_MAX_TOKENS = 8192;
+const DEFAULT_TEMPERATURE = 0.2;
+const DEFAULT_NUM_SEARCH_RESULTS = 10;
+const OAUTH_EXPIRY_BUFFER_MS = 5 * 60 * 1000;
+const OAUTH_API_VERSION = "2.18";
+const OAUTH_USER_AGENT = "Perplexity/641 CFNetwork/1568 Darwin/25.2.0";
+
+interface PerplexityOAuthCredential {
+	type: "oauth";
+	access: string;
+	expires: number;
+}
+
+type PerplexityAuth =
+	| {
+			type: "api_key";
+			token: string;
+	  }
+	| {
+			type: "oauth";
+			token: string;
+	  }
+	| {
+			type: "cookies";
+			cookies: string;
+	  };
+
+interface PerplexityOAuthStreamMarkdownBlock {
+	answer?: string;
+	chunks?: string[];
+	chunk_starting_offset?: number;
+}
+interface PerplexityOAuthStreamWebResult {
+	name?: string;
+	url?: string;
+	snippet?: string;
+	timestamp?: string;
+}
+
+interface PerplexityOAuthStreamWebResultBlock {
+	web_results?: PerplexityOAuthStreamWebResult[];
+}
+
+interface PerplexityOAuthStreamBlock {
+	intended_usage?: string;
+	markdown_block?: PerplexityOAuthStreamMarkdownBlock;
+	web_result_block?: PerplexityOAuthStreamWebResultBlock;
+}
+
+interface PerplexityOAuthStreamSource {
+	title?: string;
+	url?: string;
+	snippet?: string;
+	date?: string;
+}
+
+interface PerplexityOAuthStreamEvent {
+	status?: string;
+	final?: boolean;
+	text?: string;
+	blocks?: PerplexityOAuthStreamBlock[];
+	sources_list?: PerplexityOAuthStreamSource[];
+	error_code?: string;
+	error_message?: string;
+	display_model?: string;
+	uuid?: string;
+}
+
+function mergeMarkdownBlock(
+	existing: PerplexityOAuthStreamMarkdownBlock | undefined,
+	incoming: PerplexityOAuthStreamMarkdownBlock,
+): PerplexityOAuthStreamMarkdownBlock {
+	if (!existing) return { ...incoming };
+
+	const result: PerplexityOAuthStreamMarkdownBlock = { ...existing, ...incoming };
+	if (incoming.chunks?.length) {
+		const offset = incoming.chunk_starting_offset ?? 0;
+		const existingChunks = existing.chunks ?? [];
+		result.chunks = offset === 0 ? [...incoming.chunks] : [...existingChunks.slice(0, offset), ...incoming.chunks];
+	}
+
+	return result;
+}
+
+function mergeBlocks(
+	existing: PerplexityOAuthStreamBlock[],
+	incoming: PerplexityOAuthStreamBlock[],
+): PerplexityOAuthStreamBlock[] {
+	const blockMap = new Map<string, PerplexityOAuthStreamBlock>(
+		existing
+			.filter(block => typeof block.intended_usage === "string" && block.intended_usage.length > 0)
+			.map(block => [block.intended_usage as string, block]),
+	);
+
+	for (const block of incoming) {
+		if (!block.intended_usage) continue;
+		const prev = blockMap.get(block.intended_usage);
+		if (block.markdown_block) {
+			blockMap.set(block.intended_usage, {
+				...prev,
+				...block,
+				markdown_block: mergeMarkdownBlock(prev?.markdown_block, block.markdown_block),
+			});
+			continue;
+		}
+
+		blockMap.set(block.intended_usage, { ...prev, ...block });
+	}
+
+	return [...blockMap.values()];
+}
+
+function mergeOAuthEventSnapshot(
+	existing: PerplexityOAuthStreamEvent,
+	incoming: PerplexityOAuthStreamEvent,
+): PerplexityOAuthStreamEvent {
+	const merged: PerplexityOAuthStreamEvent = { ...existing, ...incoming };
+	if (incoming.blocks && incoming.blocks.length > 0) {
+		merged.blocks = mergeBlocks(existing.blocks ?? [], incoming.blocks);
+	} else {
+		merged.blocks = existing.blocks ?? [];
+	}
+
+	if (!merged.sources_list && existing.sources_list) {
+		merged.sources_list = existing.sources_list;
+	}
+
+	return merged;
+}
+export interface PerplexitySearchParams {
+	signal?: AbortSignal;
+	query: string;
+	system_prompt?: string;
+	search_recency_filter?: "hour" | "day" | "week" | "month" | "year";
+	num_results?: number;
+	/** Maximum output tokens. Defaults to 4096. */
+	max_tokens?: number;
+	/** Sampling temperature (0–1). Lower = more focused/factual. Defaults to 0.2. */
+	temperature?: number;
+	/** Number of search results to retrieve. Defaults to 10. */
+	num_search_results?: number;
+}
+
+/** Find PERPLEXITY_API_KEY from environment or .env files (also checks PPLX_API_KEY) */
+export function findApiKey(): string | null {
+	return getEnvApiKey("perplexity") ?? null;
+}
+
+async function findOAuthToken(): Promise<string | null> {
+	const now = Date.now();
+	try {
+		const storage = await AgentStorage.open(getAgentDbPath());
+		const records = storage.listAuthCredentials("perplexity");
+		for (const record of records) {
+			if (record.credential.type !== "oauth") continue;
+			const credential = record.credential as PerplexityOAuthCredential;
+			if (!credential.access) continue;
+			if (credential.expires <= now + OAUTH_EXPIRY_BUFFER_MS) continue;
+			return credential.access;
+		}
+	} catch {
+		return null;
+	}
+	return null;
+}
+
+async function findPerplexityAuth(): Promise<PerplexityAuth | null> {
+	// 1. PERPLEXITY_COOKIES env var
+	const cookies = $env.PERPLEXITY_COOKIES?.trim();
+	if (cookies) {
+		return { type: "cookies", cookies };
+	}
+	// 2. OAuth token from agent.db
+	const oauthToken = await findOAuthToken();
+	if (oauthToken) {
+		return { type: "oauth", token: oauthToken };
+	}
+	// 3. PERPLEXITY_API_KEY env var
+	const apiKey = findApiKey();
+	if (apiKey) {
+		return { type: "api_key", token: apiKey };
+	}
+	return null;
+}
+
+/** Call Perplexity API-key endpoint. */
+async function callPerplexityApi(
+	apiKey: string,
+	request: PerplexityRequest,
+	signal?: AbortSignal,
+): Promise<PerplexityResponse> {
+	const response = await fetch(PERPLEXITY_API_URL, {
+		method: "POST",
+		headers: {
+			Authorization: `Bearer ${apiKey}`,
+			"Content-Type": "application/json",
+		},
+		body: JSON.stringify(request),
+		signal,
+	});
+
+	if (!response.ok) {
+		const errorText = await response.text();
+		throw new SearchProviderError(
+			"perplexity",
+			`Perplexity API error (${response.status}): ${errorText}`,
+			response.status,
+		);
+	}
+
+	return response.json() as Promise<PerplexityResponse>;
+}
+
+function buildOAuthSources(event: PerplexityOAuthStreamEvent): SearchSource[] {
+	const results =
+		event.blocks?.find(block => block.intended_usage === "web_results")?.web_result_block?.web_results ?? [];
+
+	if (results.length > 0) {
+		return results
+			.filter(result => typeof result.url === "string" && result.url.length > 0)
+			.map(result => ({
+				title: result.name ?? result.url ?? "",
+				url: result.url ?? "",
+				snippet: result.snippet,
+				publishedDate: result.timestamp,
+				ageSeconds: dateToAgeSeconds(result.timestamp),
+			}));
+	}
+
+	return (event.sources_list ?? [])
+		.filter(source => typeof source.url === "string" && source.url.length > 0)
+		.map(source => ({
+			title: source.title ?? source.url ?? "",
+			url: source.url ?? "",
+			snippet: source.snippet,
+			publishedDate: source.date,
+			ageSeconds: dateToAgeSeconds(source.date),
+		}));
+}
+
+function buildOAuthAnswer(event: PerplexityOAuthStreamEvent): string {
+	if (!event.blocks?.length) {
+		return typeof event.text === "string" ? event.text : "";
+	}
+
+	const markdownBlock = event.blocks.find(
+		block => block.intended_usage?.includes("markdown") && block.markdown_block,
+	)?.markdown_block;
+	if (markdownBlock) {
+		if (Array.isArray(markdownBlock.chunks) && markdownBlock.chunks.length > 0) {
+			return markdownBlock.chunks.join("");
+		}
+		if (typeof markdownBlock.answer === "string" && markdownBlock.answer.length > 0) {
+			return markdownBlock.answer;
+		}
+	}
+
+	const textBlock = event.blocks.find(
+		block => block.intended_usage === "ask_text" && block.markdown_block,
+	)?.markdown_block;
+	if (textBlock) {
+		if (Array.isArray(textBlock.chunks) && textBlock.chunks.length > 0) {
+			return textBlock.chunks.join("");
+		}
+		if (typeof textBlock.answer === "string" && textBlock.answer.length > 0) {
+			return textBlock.answer;
+		}
+	}
+	if (typeof event.text === "string" && event.text.length > 0) {
+		return event.text;
+	}
+	return "";
+}
+
+async function callPerplexityOAuth(
+	auth: { type: "oauth"; token: string } | { type: "cookies"; cookies: string },
+	params: PerplexitySearchParams,
+): Promise<{ answer: string; sources: SearchSource[]; model?: string; requestId?: string }> {
+	const requestId = crypto.randomUUID();
+	const effectiveQuery = params.system_prompt ? `${params.system_prompt}\n\n${params.query}` : params.query;
+
+	const response = await fetch(PERPLEXITY_OAUTH_ASK_URL, {
+		method: "POST",
+		headers: {
+			...(auth.type === "cookies" ? { Cookie: auth.cookies } : { Authorization: `Bearer ${auth.token}` }),
+			"Content-Type": "application/json",
+			Accept: "text/event-stream",
+			Origin: "https://www.perplexity.ai",
+			Referer: "https://www.perplexity.ai/",
+			"User-Agent": OAUTH_USER_AGENT,
+			"X-App-ApiClient": "default",
+			"X-App-ApiVersion": OAUTH_API_VERSION,
+			"X-Perplexity-Request-Reason": "submit",
+			"X-Request-ID": requestId,
+		},
+		body: JSON.stringify({
+			query_str: effectiveQuery,
+			params: {
+				query_str: effectiveQuery,
+				search_focus: "internet",
+				mode: "copilot",
+				model_preference: "pplx_pro_upgraded",
+				sources: ["web"],
+				attachments: [],
+				frontend_uuid: crypto.randomUUID(),
+				frontend_context_uuid: crypto.randomUUID(),
+				version: OAUTH_API_VERSION,
+				language: "en-US",
+				timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+				search_recency_filter: params.search_recency_filter ?? null,
+				is_incognito: true,
+				use_schematized_api: true,
+				skip_search_enabled: true,
+			},
+		}),
+		signal: params.signal,
+	});
+
+	if (!response.ok) {
+		const errorText = await response.text();
+		throw new SearchProviderError(
+			"perplexity",
+			`Perplexity OAuth API error (${response.status}): ${errorText}`,
+			response.status,
+		);
+	}
+
+	if (!response.body) {
+		throw new SearchProviderError("perplexity", "Perplexity OAuth API returned no response body", 500);
+	}
+
+	let answer = "";
+	let model: string | undefined;
+	let finalRequestId: string | undefined;
+	const sourcesByUrl = new Map<string, SearchSource>();
+	let mergedEvent: PerplexityOAuthStreamEvent = { blocks: [] };
+
+	for await (const event of readSseJson<PerplexityOAuthStreamEvent>(response.body, params.signal)) {
+		if (event.error_code) {
+			const message = event.error_message ?? event.error_code;
+			throw new SearchProviderError("perplexity", `Perplexity OAuth stream error: ${message}`, 400);
+		}
+
+		mergedEvent = mergeOAuthEventSnapshot(mergedEvent, event);
+
+		const eventAnswer = buildOAuthAnswer(mergedEvent);
+		if (eventAnswer.length > 0) {
+			answer = eventAnswer;
+		}
+
+		for (const source of buildOAuthSources(mergedEvent)) {
+			sourcesByUrl.set(source.url, source);
+		}
+
+		if (mergedEvent.display_model) model = mergedEvent.display_model;
+		if (mergedEvent.uuid) finalRequestId = mergedEvent.uuid;
+		if (mergedEvent.final || mergedEvent.status === "COMPLETED") {
+			break;
+		}
+	}
+
+	return {
+		answer,
+		sources: [...sourcesByUrl.values()],
+		model,
+		requestId: finalRequestId ?? requestId,
+	};
+}
+
+function messageContentToText(content: PerplexityMessageOutput["content"]): string {
+	if (!content) return "";
+	if (typeof content === "string") return content;
+	return content.map(chunk => (chunk.type === "text" ? chunk.text : "")).join("");
+}
+
+/** Parse API response into unified SearchResponse */
+function parseResponse(response: PerplexityResponse): SearchResponse {
+	const messageContent = response.choices[0]?.message?.content ?? null;
+	const answer = messageContentToText(messageContent);
+
+	const sources: SearchSource[] = [];
+	const citations: SearchCitation[] = [];
+
+	const citationUrls = response.citations ?? [];
+	const searchResults = response.search_results ?? [];
+
+	if (citationUrls.length > 0) {
+		for (const url of citationUrls) {
+			const searchResult = searchResults.find(r => r.url === url);
+			sources.push({
+				title: searchResult?.title ?? url,
+				url,
+				snippet: searchResult?.snippet,
+				publishedDate: searchResult?.date ?? undefined,
+				ageSeconds: dateToAgeSeconds(searchResult?.date),
+			});
+			citations.push({
+				url,
+				title: searchResult?.title ?? url,
+			});
+		}
+	} else {
+		for (const searchResult of searchResults) {
+			sources.push({
+				title: searchResult.title ?? searchResult.url,
+				url: searchResult.url,
+				snippet: searchResult.snippet,
+				publishedDate: searchResult.date ?? undefined,
+				ageSeconds: dateToAgeSeconds(searchResult.date),
+			});
+		}
+	}
+
+	return {
+		provider: "perplexity",
+		answer: answer || undefined,
+		sources,
+		citations: citations.length > 0 ? citations : undefined,
+		usage: response.usage
+			? {
+					inputTokens: response.usage.prompt_tokens,
+					outputTokens: response.usage.completion_tokens,
+					totalTokens: response.usage.total_tokens,
+				}
+			: undefined,
+		model: response.model,
+		requestId: response.id,
+	};
+}
+
+function applySourceLimit(result: SearchResponse, limit?: number): SearchResponse {
+	if (limit && result.sources.length > limit) {
+		result.sources = result.sources.slice(0, limit);
+	}
+	return result;
+}
+
+/** Execute Perplexity web search */
+export async function searchPerplexity(params: PerplexitySearchParams): Promise<SearchResponse> {
+	const auth = await findPerplexityAuth();
+	if (!auth) {
+		throw new Error("Perplexity auth not found. Set PERPLEXITY_COOKIES, PERPLEXITY_API_KEY, or login via OAuth.");
+	}
+
+	if (auth.type === "oauth" || auth.type === "cookies") {
+		const oauthResult = await callPerplexityOAuth(auth, params);
+		return applySourceLimit(
+			{
+				provider: "perplexity",
+				answer: oauthResult.answer || undefined,
+				sources: oauthResult.sources,
+				model: oauthResult.model,
+				requestId: oauthResult.requestId,
+				authMode: "oauth",
+			},
+			params.num_results,
+		);
+	}
+
+	const systemPrompt = params.system_prompt;
+	const messages: PerplexityRequest["messages"] = [];
+	if (systemPrompt) {
+		messages.push({ role: "system", content: systemPrompt });
+	}
+	messages.push({ role: "user", content: params.query });
+
+	const request: PerplexityRequest = {
+		model: "sonar-pro",
+		messages,
+		max_tokens: params.max_tokens ?? DEFAULT_MAX_TOKENS,
+		temperature: params.temperature ?? DEFAULT_TEMPERATURE,
+		search_mode: "web",
+		num_search_results: params.num_search_results ?? DEFAULT_NUM_SEARCH_RESULTS,
+		web_search_options: {
+			search_type: "pro",
+			search_context_size: "medium",
+		},
+		enable_search_classifier: true,
+		reasoning_effort: "medium",
+		language_preference: "en",
+	};
+
+	if (params.search_recency_filter) {
+		request.search_recency_filter = params.search_recency_filter;
+	}
+
+	const response = await callPerplexityApi(auth.token, request, params.signal);
+	const result = parseResponse(response);
+	result.authMode = "api_key";
+	return applySourceLimit(result, params.num_results);
+}
+
+/** Search provider for Perplexity. */
+export class PerplexityProvider extends SearchProvider {
+	readonly id = "perplexity";
+	readonly label = "Perplexity";
+
+	async isAvailable() {
+		try {
+			return !!(await findPerplexityAuth());
+		} catch {
+			return false;
+		}
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchPerplexity({
+			signal: params.signal,
+			query: params.query,
+			temperature: params.temperature,
+			max_tokens: params.maxOutputTokens,
+			num_search_results: params.numSearchResults,
+			system_prompt: params.systemPrompt,
+			search_recency_filter: params.recency,
+			num_results: params.limit,
+		});
+	}
+}

package/src/web/search/providers/synthetic.ts

@@ -0,0 +1,113 @@
+/**
+ * Synthetic Web Search Provider
+ *
+ * Uses Synthetic's zero-data-retention web search API for coding agents.
+ * Endpoint: POST https://api.synthetic.new/v2/search
+ */
+
+import { getEnvApiKey } from "@f5xc-salesdemos/pi-ai";
+import type { SearchResponse, SearchSource } from "../../../web/search/types";
+import { SearchProviderError } from "../../../web/search/types";
+import type { SearchParams } from "./base";
+import { SearchProvider } from "./base";
+import { findCredential } from "./utils";
+
+const SYNTHETIC_SEARCH_URL = "https://api.synthetic.new/v2/search";
+
+interface SyntheticSearchResult {
+	url: string;
+	title: string;
+	text?: string;
+	published?: string;
+}
+
+interface SyntheticSearchResponse {
+	results: SyntheticSearchResult[];
+}
+
+/** Find Synthetic API key from environment or agent.db credentials. */
+export async function findApiKey(): Promise<string | null> {
+	return findCredential(getEnvApiKey("synthetic"), "synthetic");
+}
+
+/** Call Synthetic search API. */
+async function callSyntheticSearch(
+	apiKey: string,
+	query: string,
+	signal?: AbortSignal,
+): Promise<SyntheticSearchResponse> {
+	const response = await fetch(SYNTHETIC_SEARCH_URL, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			Authorization: `Bearer ${apiKey}`,
+		},
+		body: JSON.stringify({ query }),
+		signal,
+	});
+
+	if (!response.ok) {
+		const errorText = await response.text();
+		throw new SearchProviderError(
+			"synthetic",
+			`Synthetic API error (${response.status}): ${errorText}`,
+			response.status,
+		);
+	}
+
+	return (await response.json()) as SyntheticSearchResponse;
+}
+
+/** Execute Synthetic web search. */
+export async function searchSynthetic(params: {
+	query: string;
+	num_results?: number;
+	signal?: AbortSignal;
+}): Promise<SearchResponse> {
+	const apiKey = await findApiKey();
+	if (!apiKey) {
+		throw new Error("Synthetic credentials not found. Set SYNTHETIC_API_KEY or login with 'xcsh /login synthetic'.");
+	}
+
+	const data = await callSyntheticSearch(apiKey, params.query, params.signal);
+	const sources: SearchSource[] = [];
+
+	for (const result of data.results ?? []) {
+		if (!result.url) continue;
+		sources.push({
+			title: result.title ?? result.url,
+			url: result.url,
+			snippet: result.text ?? undefined,
+			publishedDate: result.published ?? undefined,
+		});
+	}
+
+	const limitedSources = params.num_results ? sources.slice(0, params.num_results) : sources;
+
+	return {
+		provider: "synthetic",
+		sources: limitedSources,
+	};
+}
+
+/** Search provider for Synthetic. */
+export class SyntheticProvider extends SearchProvider {
+	readonly id = "synthetic";
+	readonly label = "Synthetic";
+
+	async isAvailable(): Promise<boolean> {
+		try {
+			return !!(await findApiKey());
+		} catch {
+			return false;
+		}
+	}
+
+	search(params: SearchParams): Promise<SearchResponse> {
+		return searchSynthetic({
+			query: params.query,
+			num_results: params.numSearchResults ?? params.limit,
+			signal: params.signal,
+		});
+	}
+}