@oh-my-pi/pi-coding-agent 13.10.1 → 13.11.1

package/src/web/parallel.ts

@@ -0,0 +1,346 @@
+import { getEnvApiKey } from "@oh-my-pi/pi-ai";
+import { findCredential } from "./search/providers/utils";
+
+const PARALLEL_API_URL = "https://api.parallel.ai";
+const PARALLEL_SEARCH_URL = `${PARALLEL_API_URL}/v1beta/search`;
+const PARALLEL_EXTRACT_URL = `${PARALLEL_API_URL}/v1beta/extract`;
+const PARALLEL_BETA_HEADER = "search-extract-2025-10-10";
+
+export interface ParallelUsageItem {
+	name?: string;
+	count?: number;
+}
+
+export interface ParallelSearchSource {
+	title: string;
+	url: string;
+	snippet?: string;
+	publishedDate?: string;
+	excerpts: string[];
+}
+
+export interface ParallelSearchResult {
+	requestId: string;
+	sources: ParallelSearchSource[];
+	warnings: string[];
+	usage: ParallelUsageItem[];
+}
+
+export interface ParallelExtractDocument {
+	url: string;
+	title?: string;
+	publishedDate?: string;
+	excerpts: string[];
+	fullContent?: string;
+}
+
+export interface ParallelExtractErrorEntry {
+	url: string;
+	errorType?: string;
+	httpStatusCode?: number;
+	content?: string;
+}
+
+export interface ParallelExtractResult {
+	requestId: string;
+	results: ParallelExtractDocument[];
+	errors: ParallelExtractErrorEntry[];
+	warnings: string[];
+	usage: ParallelUsageItem[];
+}
+
+export interface ParallelSearchOptions {
+	mode?: "fast" | "research";
+	maxCharsPerResult?: number;
+	signal?: AbortSignal;
+}
+
+export interface ParallelExtractOptions {
+	objective?: string;
+	searchQueries?: string[];
+	excerpts?: boolean;
+	fullContent?: boolean;
+	signal?: AbortSignal;
+}
+
+export class ParallelApiError extends Error {
+	readonly statusCode?: number;
+
+	constructor(message: string, statusCode?: number) {
+		super(message);
+		this.name = "ParallelApiError";
+		this.statusCode = statusCode;
+	}
+}
+
+export async function findParallelApiKey(): Promise<string | null> {
+	return findCredential(getEnvApiKey("parallel"), "parallel");
+}
+
+export function getParallelExtractContent(document: ParallelExtractDocument): string {
+	const excerptContent = document.excerpts
+		.filter(excerpt => excerpt.trim().length > 0)
+		.join("\n\n")
+		.trim();
+	if (excerptContent.length > 0) {
+		return excerptContent;
+	}
+
+	return document.fullContent?.trim() ?? "";
+}
+
+function isObject(value: unknown): value is object {
+	return typeof value === "object" && value !== null;
+}
+
+function getOwnValue(value: object, key: string): unknown {
+	return Object.getOwnPropertyDescriptor(value, key)?.value;
+}
+
+function getString(value: object, key: string): string | undefined {
+	const candidate = getOwnValue(value, key);
+	return typeof candidate === "string" ? candidate : undefined;
+}
+
+function getNumber(value: object, key: string): number | undefined {
+	const candidate = getOwnValue(value, key);
+	return typeof candidate === "number" && Number.isFinite(candidate) ? candidate : undefined;
+}
+
+function getObjectArray(value: object, key: string): object[] {
+	const candidate = getOwnValue(value, key);
+	if (!Array.isArray(candidate)) return [];
+	return candidate.filter(isObject);
+}
+
+function getStringArray(value: object, key: string): string[] {
+	const candidate = getOwnValue(value, key);
+	if (!Array.isArray(candidate)) return [];
+	return candidate.filter(item => typeof item === "string");
+}
+
+function extractParallelErrorMessage(payload: unknown): string | null {
+	if (!isObject(payload)) return null;
+
+	const directMessage = getString(payload, "message") ?? getString(payload, "detail") ?? getString(payload, "error");
+	if (directMessage && directMessage.trim().length > 0) {
+		return directMessage.trim();
+	}
+
+	const errorObject = getOwnValue(payload, "error");
+	if (isObject(errorObject)) {
+		const nestedMessage = getString(errorObject, "message") ?? getString(errorObject, "detail");
+		if (nestedMessage && nestedMessage.trim().length > 0) {
+			return nestedMessage.trim();
+		}
+	}
+
+	return null;
+}
+
+function createParallelApiError(statusCode: number, detail?: string): ParallelApiError {
+	return new ParallelApiError(
+		detail ? `Parallel API error (${statusCode}): ${detail}` : `Parallel API error (${statusCode})`,
+		statusCode,
+	);
+}
+
+function parseParallelErrorResponse(statusCode: number, responseText: string): ParallelApiError {
+	const trimmedResponseText = responseText.trim();
+	if (trimmedResponseText.length === 0) {
+		return createParallelApiError(statusCode);
+	}
+
+	try {
+		const payload: unknown = JSON.parse(trimmedResponseText);
+		return createParallelApiError(statusCode, extractParallelErrorMessage(payload) ?? trimmedResponseText);
+	} catch {
+		return createParallelApiError(statusCode, trimmedResponseText);
+	}
+}
+
+function getAuthHeaders(apiKey: string): {
+	Accept: string;
+	"Content-Type": string;
+	"x-api-key": string;
+	"parallel-beta": string;
+} {
+	return {
+		Accept: "application/json",
+		"Content-Type": "application/json",
+		"x-api-key": apiKey,
+		"parallel-beta": PARALLEL_BETA_HEADER,
+	};
+}
+
+function normalizeSearchMode(mode: ParallelSearchOptions["mode"]): "fast" | "one-shot" {
+	return mode === "research" ? "one-shot" : "fast";
+}
+
+function parseUsageItems(payload: unknown): ParallelUsageItem[] {
+	if (!Array.isArray(payload)) return [];
+
+	const usageItems: ParallelUsageItem[] = [];
+	for (const item of payload) {
+		if (!isObject(item)) continue;
+		usageItems.push({
+			name: getString(item, "name"),
+			count: getNumber(item, "count"),
+		});
+	}
+	return usageItems;
+}
+
+function parseWarnings(payload: unknown): string[] {
+	if (!Array.isArray(payload)) return [];
+
+	const warnings: string[] = [];
+	for (const item of payload) {
+		if (typeof item === "string") {
+			warnings.push(item);
+			continue;
+		}
+		if (!isObject(item)) continue;
+		const message = getString(item, "message") ?? getString(item, "warning");
+		if (message) {
+			warnings.push(message);
+		}
+	}
+	return warnings;
+}
+
+function parseSearchPayload(payload: unknown): ParallelSearchResult {
+	if (!isObject(payload)) {
+		throw new ParallelApiError("Parallel search returned an invalid response payload.");
+	}
+
+	const requestId = getString(payload, "search_id") ?? "";
+	const rawResults = getObjectArray(payload, "results");
+	const sources: ParallelSearchSource[] = [];
+
+	for (const item of rawResults) {
+		const url = getString(item, "url");
+		if (!url) continue;
+
+		const excerpts = getStringArray(item, "excerpts");
+		const snippet = excerpts.length > 0 ? excerpts.join("\n\n") : undefined;
+		sources.push({
+			title: getString(item, "title") ?? url,
+			url,
+			snippet,
+			publishedDate: getString(item, "publish_date"),
+			excerpts,
+		});
+	}
+
+	return {
+		requestId,
+		sources,
+		warnings: parseWarnings(getOwnValue(payload, "warnings")),
+		usage: parseUsageItems(getOwnValue(payload, "usage")),
+	};
+}
+
+function parseExtractPayload(payload: unknown): ParallelExtractResult {
+	if (!isObject(payload)) {
+		throw new ParallelApiError("Parallel extract returned an invalid response payload.");
+	}
+
+	const requestId = getString(payload, "extract_id") ?? "";
+	const resultItems: ParallelExtractDocument[] = [];
+	for (const item of getObjectArray(payload, "results")) {
+		const url = getString(item, "url");
+		if (!url) continue;
+		resultItems.push({
+			url,
+			title: getString(item, "title"),
+			publishedDate: getString(item, "publish_date"),
+			excerpts: getStringArray(item, "excerpts"),
+			fullContent: getString(item, "full_content"),
+		});
+	}
+
+	const errors: ParallelExtractErrorEntry[] = [];
+	for (const item of getObjectArray(payload, "errors")) {
+		const url = getString(item, "url");
+		if (!url) continue;
+		errors.push({
+			url,
+			errorType: getString(item, "error_type"),
+			httpStatusCode: getNumber(item, "http_status_code"),
+			content: getString(item, "content"),
+		});
+	}
+
+	return {
+		requestId,
+		results: resultItems,
+		errors,
+		warnings: parseWarnings(getOwnValue(payload, "warnings")),
+		usage: parseUsageItems(getOwnValue(payload, "usage")),
+	};
+}
+
+export async function searchWithParallel(
+	objective: string,
+	queries: string[],
+	options: ParallelSearchOptions = {},
+): Promise<ParallelSearchResult> {
+	const apiKey = await findParallelApiKey();
+	if (!apiKey) {
+		throw new ParallelApiError(
+			"Parallel credentials not found. Set PARALLEL_API_KEY or login with 'omp /login parallel'.",
+		);
+	}
+
+	const response = await fetch(PARALLEL_SEARCH_URL, {
+		method: "POST",
+		headers: getAuthHeaders(apiKey),
+		body: JSON.stringify({
+			objective,
+			search_queries: queries,
+			mode: normalizeSearchMode(options.mode),
+			excerpts: {
+				max_chars_per_result: options.maxCharsPerResult ?? 10_000,
+			},
+		}),
+		signal: options.signal,
+	});
+	if (!response.ok) {
+		throw parseParallelErrorResponse(response.status, await response.text());
+	}
+
+	const payload: unknown = await response.json();
+	return parseSearchPayload(payload);
+}
+
+export async function extractWithParallel(
+	urls: string[],
+	options: ParallelExtractOptions = {},
+): Promise<ParallelExtractResult> {
+	const apiKey = await findParallelApiKey();
+	if (!apiKey) {
+		throw new ParallelApiError(
+			"Parallel credentials not found. Set PARALLEL_API_KEY or login with 'omp /login parallel'.",
+		);
+	}
+
+	const response = await fetch(PARALLEL_EXTRACT_URL, {
+		method: "POST",
+		headers: getAuthHeaders(apiKey),
+		body: JSON.stringify({
+			urls,
+			objective: options.objective,
+			search_queries: options.searchQueries,
+			excerpts: options.excerpts ?? true,
+			full_content: options.fullContent ?? false,
+		}),
+		signal: options.signal,
+	});
+	if (!response.ok) {
+		throw parseParallelErrorResponse(response.status, await response.text());
+	}
+
+	const payload: unknown = await response.json();
+	return parseExtractPayload(payload);
+}

package/src/web/scrapers/youtube.ts

@@ -2,8 +2,10 @@ import * as fs from "node:fs/promises";
 import * as os from "node:os";
 import * as path from "node:path";
 import { ptree, Snowflake } from "@oh-my-pi/pi-utils";
+import { settings } from "../../config/settings";
 import { throwIfAborted } from "../../tools/tool-errors";
 import { ensureTool } from "../../utils/tools-manager";
+import { extractWithParallel, findParallelApiKey, getParallelExtractContent } from "../parallel";
 import type { RenderResult, SpecialHandler } from "./types";
 import { buildResult, formatMediaDuration, formatNumber } from "./types";
 
@@ -109,6 +111,33 @@ export const handleYouTube: SpecialHandler = async (
 	const notes: string[] = [];
 	const videoUrl = `https://www.youtube.com/watch?v=${yt.videoId}`;
 
+	// Prefer Parallel extract when credentials are available
+	if (settings.get("providers.parallelFetch") && (await findParallelApiKey())) {
+		try {
+			const parallelResult = await extractWithParallel([videoUrl], {
+				objective: "Extract the main content of this YouTube video page",
+				excerpts: true,
+				fullContent: false,
+				signal,
+			});
+			const firstDocument = parallelResult.results[0];
+			if (firstDocument) {
+				const content = getParallelExtractContent(firstDocument);
+				if (content.trim().length > 100) {
+					return buildResult(content, {
+						url,
+						finalUrl: videoUrl,
+						method: "parallel",
+						fetchedAt,
+						notes: ["Used Parallel extract for YouTube"],
+					});
+				}
+			}
+		} catch {
+			throwIfAborted(signal);
+		}
+	}
+
 	// Ensure yt-dlp is available (auto-download if missing)
 	const ytdlp = await ensureTool("yt-dlp", { signal, silent: true });
 	if (!ytdlp) {