@nklisch/pi-enhanced 0.1.0

package/node_modules/pi-web-access/parallel.ts

@@ -0,0 +1,360 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.ts";
+import type { ExtractedContent, ExtractOptions } from "./extract.ts";
+import type { SearchOptions, SearchResponse } from "./perplexity.ts";
+import { getWebSearchConfigPath } from "./utils.ts";
+
+const PARALLEL_SEARCH_URL = "https://api.parallel.ai/v1/search";
+const PARALLEL_EXTRACT_URL = "https://api.parallel.ai/v1/extract";
+const CONFIG_PATH = getWebSearchConfigPath();
+const MIN_PARALLEL_API_KEY_LENGTH = 8;
+const MIN_USEFUL_CONTENT = 500;
+const SEARCH_TIMEOUT_MS = 60_000;
+
+const PLACEHOLDER_API_KEY_DENYLIST = new Set([
+	"replace_with_your_parallel_api_key",
+	"parallel_api_key",
+	"your-key",
+	"your-key-here",
+	"your-api-key-here",
+	"dummy",
+	"placeholder",
+	"changeme",
+	"insert-your-key",
+	"insert-your-key-here",
+	"api-key",
+	"xxx",
+]);
+
+interface WebSearchConfig {
+	parallelApiKey?: unknown;
+}
+
+interface V1WebSearchResult {
+	url: string;
+	title?: string | null;
+	publish_date?: string | null;
+	excerpts?: string[];
+}
+
+interface V1ExtractResult {
+	url: string;
+	title?: string | null;
+	publish_date?: string | null;
+	excerpts?: string[];
+	full_content?: string | null;
+}
+
+interface ParallelSearchOptions extends SearchOptions {
+	includeContent?: boolean;
+}
+
+let cachedConfig: WebSearchConfig | null = null;
+
+function loadConfig(): WebSearchConfig {
+	if (cachedConfig) return cachedConfig;
+	if (!existsSync(CONFIG_PATH)) {
+		cachedConfig = {};
+		return cachedConfig;
+	}
+
+	const raw = readFileSync(CONFIG_PATH, "utf-8");
+	try {
+		cachedConfig = JSON.parse(raw) as WebSearchConfig;
+		return cachedConfig;
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+	}
+}
+
+export function clearParallelConfigCache(): void {
+	cachedConfig = null;
+}
+
+function normalizeApiKey(value: unknown): string | null {
+	if (typeof value !== "string") return null;
+	const normalized = value.trim();
+	return normalized.length > 0 ? normalized : null;
+}
+
+function isPlaceholderApiKey(key: string): boolean {
+	const normalized = key.trim();
+	return normalized.length < MIN_PARALLEL_API_KEY_LENGTH || PLACEHOLDER_API_KEY_DENYLIST.has(normalized.toLowerCase());
+}
+
+function resolveApiKey(): string | null {
+	const envKey = normalizeApiKey(process.env.PARALLEL_API_KEY);
+	if (envKey && !isPlaceholderApiKey(envKey)) return envKey;
+
+	const configKey = normalizeApiKey(loadConfig().parallelApiKey);
+	if (configKey && !isPlaceholderApiKey(configKey)) return configKey;
+
+	return null;
+}
+
+function getApiKey(): string {
+	const key = resolveApiKey();
+	if (!key) {
+		throw new Error(
+			"Parallel API key not found. Either:\n" +
+			`  1. Create ${CONFIG_PATH} with { "parallelApiKey": "your-key" }\n` +
+			"  2. Set PARALLEL_API_KEY environment variable\n" +
+			"Get a key at https://platform.parallel.ai",
+		);
+	}
+	return key;
+}
+
+export function hasParallelApiKey(): boolean {
+	return !!resolveApiKey();
+}
+
+export function isParallelAvailable(): boolean {
+	return hasParallelApiKey();
+}
+
+function requestSignal(signal?: AbortSignal): AbortSignal {
+	const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+	return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+
+function errorMessage(err: unknown): string {
+	return err instanceof Error ? err.message : String(err);
+}
+
+function activityContext(
+	url: string,
+	body: Record<string, unknown>,
+): { type: "api" | "fetch"; query?: string; url?: string } {
+	if (typeof body.objective === "string" && body.objective.trim().length > 0) {
+		return { type: "api", query: body.objective };
+	}
+
+	const searchQueries = body.search_queries;
+	if (Array.isArray(searchQueries) && typeof searchQueries[0] === "string") {
+		return { type: "api", query: searchQueries[0] };
+	}
+
+	const urls = body.urls;
+	if (Array.isArray(urls) && typeof urls[0] === "string") {
+		return { type: "fetch", url: urls[0] };
+	}
+
+	return url.includes("/search") ? { type: "api", query: "Parallel search" } : { type: "fetch", url };
+}
+
+function recencyToAfterDate(filter: string): string {
+	const now = new Date();
+	const offsets: Record<string, number> = { day: 1, week: 7, month: 30, year: 365 };
+	const days = offsets[filter] ?? 0;
+	return new Date(now.getTime() - days * 86_400_000).toISOString().slice(0, 10);
+}
+
+function normalizeDomain(value: string): string | null {
+	let input = value.trim().toLowerCase();
+	if (!input) return null;
+	if (input.startsWith("-")) input = input.slice(1).trim();
+	if (!input) return null;
+	try {
+		const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+		input = parsed.hostname;
+	} catch {
+		input = input.split("/")[0]?.split(":")[0] ?? "";
+	}
+	input = input.replace(/^\.+|\.+$/g, "");
+	return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+
+function mapDomainFilter(domainFilter: string[] | undefined): { include_domains?: string[]; exclude_domains?: string[] } {
+	if (!domainFilter?.length) return {};
+	const include_domains: string[] = [];
+	const exclude_domains: string[] = [];
+	for (const raw of domainFilter) {
+		const domain = normalizeDomain(raw);
+		if (!domain) continue;
+		const target = raw.trim().startsWith("-") ? exclude_domains : include_domains;
+		if (!target.includes(domain)) target.push(domain);
+	}
+	return {
+		...(include_domains.length ? { include_domains } : {}),
+		...(exclude_domains.length ? { exclude_domains } : {}),
+	};
+}
+
+function buildSearchRequestBody(query: string, options: ParallelSearchOptions = {}): Record<string, unknown> {
+	const numResults = Math.max(1, Math.min(Math.floor(options.numResults ?? 5), 20));
+	const sourcePolicy = {
+		...mapDomainFilter(options.domainFilter),
+		...(options.recencyFilter ? { after_date: recencyToAfterDate(options.recencyFilter) } : {}),
+	};
+	return {
+		objective: query,
+		search_queries: [query],
+		advanced_settings: {
+			max_results: numResults,
+			...(Object.keys(sourcePolicy).length > 0 ? { source_policy: sourcePolicy } : {}),
+		},
+	};
+}
+
+function normalizeExcerpts(value: unknown): string[] {
+	if (!Array.isArray(value)) return [];
+	return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0);
+}
+
+function mapSearchResults(results: V1WebSearchResult[] | undefined): SearchResponse["results"] {
+	if (!Array.isArray(results)) return [];
+	const mapped: SearchResponse["results"] = [];
+	for (let i = 0; i < results.length; i++) {
+		const item = results[i];
+		if (!item?.url) continue;
+		const excerpts = normalizeExcerpts(item.excerpts);
+		mapped.push({
+			title: item.title || `Source ${i + 1}`,
+			url: item.url,
+			snippet: excerpts.length > 0 ? excerpts[0].replace(/\s+/g, " ").trim().slice(0, 200) : "",
+		});
+	}
+	return mapped;
+}
+
+function buildAnswerFromExcerpts(results: V1WebSearchResult[] | undefined): string {
+	if (!Array.isArray(results)) return "";
+	const parts: string[] = [];
+	for (let i = 0; i < results.length; i++) {
+		const item = results[i];
+		if (!item?.url) continue;
+		const excerpts = normalizeExcerpts(item.excerpts);
+		if (excerpts.length === 0) continue;
+		parts.push(`${excerpts.join(" ")}\nSource: ${item.title || `Source ${i + 1}`} (${item.url})`);
+	}
+	return parts.join("\n\n");
+}
+
+function mapInlineContent(results: V1WebSearchResult[] | undefined): ExtractedContent[] {
+	if (!Array.isArray(results)) return [];
+	return results.flatMap((result) => {
+		if (!result?.url) return [];
+		const excerpts = normalizeExcerpts(result.excerpts);
+		if (excerpts.length === 0) return [];
+		return [{ url: result.url, title: result.title || "", content: excerpts.join("\n\n"), error: null }];
+	});
+}
+
+function resolveExtractContent(result: V1ExtractResult): string {
+	const fullContent = typeof result.full_content === "string" ? result.full_content.trim() : "";
+	return fullContent.length > 0 ? fullContent : normalizeExcerpts(result.excerpts).join("\n\n");
+}
+
+function mapExtractResult(result: V1ExtractResult | undefined | null): ExtractedContent | null {
+	if (!result?.url) return null;
+	const content = resolveExtractContent(result);
+	if (content.length < MIN_USEFUL_CONTENT) return null;
+	return {
+		url: result.url,
+		title: typeof result.title === "string" ? result.title.trim() : "",
+		content,
+		error: null,
+	};
+}
+
+function buildExtractRequestBody(url: string, options: ExtractOptions = {}, fullContent = false): Record<string, unknown> {
+	const body: Record<string, unknown> = { urls: [url] };
+	const prompt = options.prompt?.trim();
+	if (prompt) body.objective = prompt;
+	if (fullContent) body.advanced_settings = { full_content: true };
+	return body;
+}
+
+function findExtractResult(results: V1ExtractResult[] | undefined, url: string): V1ExtractResult | undefined {
+	if (!Array.isArray(results)) return undefined;
+	return results.find(item => item?.url === url) ?? results[0];
+}
+
+function hasExtractUrlError(errors: unknown, url: string): boolean {
+	if (!Array.isArray(errors)) return false;
+	return errors.some((entry) => {
+		if (typeof entry === "string") return entry === url;
+		return typeof entry === "object" && entry !== null && (entry as { url?: unknown }).url === url;
+	});
+}
+
+async function fetchAndMapExtractResult(
+	url: string,
+	body: Record<string, unknown>,
+	signal?: AbortSignal,
+): Promise<{ mapped: ExtractedContent | null; result: V1ExtractResult | undefined }> {
+	const data = await parallelFetch(PARALLEL_EXTRACT_URL, body, signal);
+	if (hasExtractUrlError(data.errors, url)) return { mapped: null, result: undefined };
+	const result = findExtractResult(data.results as V1ExtractResult[] | undefined, url);
+	return { mapped: mapExtractResult(result), result };
+}
+
+export async function searchWithParallel(query: string, options: ParallelSearchOptions = {}): Promise<SearchResponse> {
+	const data = await parallelFetch(PARALLEL_SEARCH_URL, buildSearchRequestBody(query, options), options.signal);
+	const results = data.results as V1WebSearchResult[] | undefined;
+	const response: SearchResponse = {
+		answer: buildAnswerFromExcerpts(results),
+		results: mapSearchResults(results),
+	};
+	if (options.includeContent) {
+		const inlineContent = mapInlineContent(results);
+		if (inlineContent.length > 0) response.inlineContent = inlineContent;
+	}
+	return response;
+}
+
+export async function extractWithParallel(
+	url: string,
+	signal?: AbortSignal,
+	options: ExtractOptions = {},
+): Promise<ExtractedContent | null> {
+	const initial = await fetchAndMapExtractResult(url, buildExtractRequestBody(url, options), signal);
+	if (initial.mapped) return initial.mapped;
+	if (!initial.result || resolveExtractContent(initial.result).length >= MIN_USEFUL_CONTENT) return null;
+
+	const retry = await fetchAndMapExtractResult(url, buildExtractRequestBody(url, options, true), signal);
+	return retry.mapped;
+}
+
+async function parallelFetch(
+	url: string,
+	body: Record<string, unknown>,
+	signal?: AbortSignal,
+): Promise<Record<string, unknown>> {
+	const apiKey = getApiKey();
+	const activityId = activityMonitor.logStart(activityContext(url, body));
+	let response: Response;
+	try {
+		response = await fetch(url, {
+			method: "POST",
+			headers: {
+				"x-api-key": apiKey,
+				"Content-Type": "application/json",
+			},
+			body: JSON.stringify(body),
+			signal: requestSignal(signal),
+		});
+	} catch (err) {
+		const message = errorMessage(err);
+		if (message.toLowerCase().includes("abort")) activityMonitor.logComplete(activityId, 0);
+		else activityMonitor.logError(activityId, message);
+		throw err;
+	}
+
+	if (!response.ok) {
+		activityMonitor.logComplete(activityId, response.status);
+		const errorText = await response.text();
+		throw new Error(`Parallel API error ${response.status}: ${errorText.slice(0, 300)}`);
+	}
+
+	try {
+		const data = await response.json() as Record<string, unknown>;
+		activityMonitor.logComplete(activityId, response.status);
+		return data;
+	} catch (err) {
+		activityMonitor.logComplete(activityId, response.status);
+		throw new Error(`Parallel API returned invalid JSON: ${errorMessage(err)}`);
+	}
+}

package/node_modules/pi-web-access/pdf-extract.ts

@@ -0,0 +1,192 @@
+/**
+ * PDF Content Extractor
+ * 
+ * Extracts text from PDF files and saves to markdown.
+ * Uses unpdf (pdfjs-dist wrapper) for text extraction.
+ */
+
+import { getDocumentProxy } from "unpdf";
+import { writeFile, mkdir } from "node:fs/promises";
+import { join, basename } from "node:path";
+import { homedir } from "node:os";
+
+export interface PDFExtractResult {
+  title: string;
+  pages: number;
+  chars: number;
+  outputPath: string;
+}
+
+export interface PDFExtractOptions {
+  maxPages?: number;
+  outputDir?: string;
+  filename?: string;
+}
+
+const DEFAULT_MAX_PAGES = 100;
+const DEFAULT_OUTPUT_DIR = join(homedir(), "Downloads");
+
+/**
+ * Extract text from a PDF buffer and save to markdown file
+ */
+export async function extractPDFToMarkdown(
+  buffer: ArrayBuffer,
+  url: string,
+  options: PDFExtractOptions = {}
+): Promise<PDFExtractResult> {
+  const {
+    maxPages = DEFAULT_MAX_PAGES,
+    outputDir = DEFAULT_OUTPUT_DIR,
+    filename,
+  } = options;
+
+  const safeMaxPages = Number.isFinite(maxPages)
+    ? Math.max(1, Math.floor(maxPages))
+    : DEFAULT_MAX_PAGES;
+
+  const pdf = await getDocumentProxy(new Uint8Array(buffer));
+  const metadata = await pdf.getMetadata();
+  const metadataInfo = metadata.info && typeof metadata.info === "object"
+    ? metadata.info as Record<string, unknown>
+    : null;
+
+  // Extract title from metadata or URL
+  const metaTitle = typeof metadataInfo?.Title === "string" ? metadataInfo.Title : undefined;
+  const metaAuthor = typeof metadataInfo?.Author === "string" ? metadataInfo.Author : undefined;
+  const urlTitle = extractTitleFromURL(url);
+  const title = metaTitle?.trim() || urlTitle;
+
+  // Determine pages to extract
+  const pagesToExtract = Math.min(pdf.numPages, safeMaxPages);
+  const truncated = pdf.numPages > safeMaxPages;
+
+  // Extract text page by page for better structure
+  const pages: { pageNum: number; text: string }[] = [];
+  for (let i = 1; i <= pagesToExtract; i++) {
+    const page = await pdf.getPage(i);
+    const textContent = await page.getTextContent();
+    const pageText = textContent.items
+      .map((item: unknown) => {
+        const textItem = item as { str?: string };
+        return textItem.str || "";
+      })
+      .join(" ")
+      .replace(/\s+/g, " ")
+      .trim();
+    
+    if (pageText) {
+      pages.push({ pageNum: i, text: pageText });
+    }
+  }
+
+  // Build markdown content
+  const lines: string[] = [];
+  
+  // Header with metadata
+  lines.push(`# ${title}`);
+  lines.push("");
+  lines.push(`> Source: ${url}`);
+  lines.push(`> Pages: ${pdf.numPages}${truncated ? ` (extracted first ${pagesToExtract})` : ""}`);
+  if (metaAuthor) {
+    lines.push(`> Author: ${metaAuthor}`);
+  }
+  lines.push("");
+  lines.push("---");
+  lines.push("");
+
+  // Content with page markers
+  for (let i = 0; i < pages.length; i++) {
+    if (i > 0) {
+      lines.push("");
+      lines.push(`<!-- Page ${pages[i].pageNum} -->`);
+      lines.push("");
+    }
+    lines.push(pages[i].text);
+  }
+
+  if (truncated) {
+    lines.push("");
+    lines.push("---");
+    lines.push("");
+    lines.push(`*[Truncated: Only first ${pagesToExtract} of ${pdf.numPages} pages extracted]*`);
+  }
+
+  const content = lines.join("\n");
+
+  // Generate output filename
+  const outputFilename = filename || sanitizeFilename(title) + ".md";
+  const outputPath = join(outputDir, outputFilename);
+
+  // Ensure output directory exists
+  await mkdir(outputDir, { recursive: true });
+
+  // Write file
+  await writeFile(outputPath, content, "utf-8");
+
+  return {
+    title,
+    pages: pdf.numPages,
+    chars: content.length,
+    outputPath,
+  };
+}
+
+/**
+ * Extract a reasonable title from URL
+ */
+function extractTitleFromURL(url: string): string {
+  try {
+    const urlObj = new URL(url);
+    const pathname = urlObj.pathname;
+    
+    // Get filename without extension
+    let filename = basename(pathname, ".pdf");
+    
+    // Handle arxiv URLs: /pdf/1706.03762 → "arxiv-1706.03762"
+    if (urlObj.hostname.includes("arxiv.org")) {
+      const match = pathname.match(/\/(?:pdf|abs)\/(\d+\.\d+)/);
+      if (match) {
+        filename = `arxiv-${match[1]}`;
+      }
+    }
+    
+    // Clean up filename
+    filename = filename
+      .replace(/[_-]+/g, " ")
+      .replace(/\s+/g, " ")
+      .trim();
+    
+    return filename || "document";
+  } catch {
+    return "document";
+  }
+}
+
+/**
+ * Sanitize string for use as filename
+ */
+function sanitizeFilename(name: string): string {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, "")
+    .replace(/\s+/g, "-")
+    .replace(/-+/g, "-")
+    .slice(0, 100)
+    .replace(/^-|-$/g, "")
+    || "document";
+}
+
+/**
+ * Check if URL or content-type indicates a PDF
+ */
+export function isPDF(url: string, contentType?: string): boolean {
+  if (contentType?.includes("application/pdf")) {
+    return true;
+  }
+  try {
+    const urlObj = new URL(url);
+    return urlObj.pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return false;
+  }
+}