@bastani/atomic 0.9.16-alpha.2 → 0.9.16-alpha.3

package/dist/builtin/web-access/pdf-extract.ts

@@ -1,192 +0,0 @@
-/**
- * PDF Content Extractor
- * 
- * Extracts text from PDF files and saves to markdown.
- * Uses unpdf (pdfjs-dist wrapper) for text extraction.
- */
-
-import { getDocumentProxy } from "unpdf";
-import { writeFile, mkdir } from "node:fs/promises";
-import { join, basename } from "node:path";
-import { homedir } from "node:os";
-
-export interface PDFExtractResult {
-  title: string;
-  pages: number;
-  chars: number;
-  outputPath: string;
-}
-
-export interface PDFExtractOptions {
-  maxPages?: number;
-  outputDir?: string;
-  filename?: string;
-}
-
-const DEFAULT_MAX_PAGES = 100;
-const DEFAULT_OUTPUT_DIR = join(homedir(), "Downloads");
-
-/**
- * Extract text from a PDF buffer and save to markdown file
- */
-export async function extractPDFToMarkdown(
-  buffer: ArrayBuffer,
-  url: string,
-  options: PDFExtractOptions = {}
-): Promise<PDFExtractResult> {
-  const {
-    maxPages = DEFAULT_MAX_PAGES,
-    outputDir = DEFAULT_OUTPUT_DIR,
-    filename,
-  } = options;
-
-  const safeMaxPages = Number.isFinite(maxPages)
-    ? Math.max(1, Math.floor(maxPages))
-    : DEFAULT_MAX_PAGES;
-
-  const pdf = await getDocumentProxy(new Uint8Array(buffer));
-  const metadata = await pdf.getMetadata();
-  const metadataInfo = metadata.info && typeof metadata.info === "object"
-    ? metadata.info as Record<string, unknown>
-    : null;
-
-  // Extract title from metadata or URL
-  const metaTitle = typeof metadataInfo?.Title === "string" ? metadataInfo.Title : undefined;
-  const metaAuthor = typeof metadataInfo?.Author === "string" ? metadataInfo.Author : undefined;
-  const urlTitle = extractTitleFromURL(url);
-  const title = metaTitle?.trim() || urlTitle;
-
-  // Determine pages to extract
-  const pagesToExtract = Math.min(pdf.numPages, safeMaxPages);
-  const truncated = pdf.numPages > safeMaxPages;
-
-  // Extract text page by page for better structure
-  const pages: { pageNum: number; text: string }[] = [];
-  for (let i = 1; i <= pagesToExtract; i++) {
-    const page = await pdf.getPage(i);
-    const textContent = await page.getTextContent();
-    const pageText = textContent.items
-      .map((item: unknown) => {
-        const textItem = item as { str?: string };
-        return textItem.str || "";
-      })
-      .join(" ")
-      .replace(/\s+/g, " ")
-      .trim();
-    
-    if (pageText) {
-      pages.push({ pageNum: i, text: pageText });
-    }
-  }
-
-  // Build markdown content
-  const lines: string[] = [];
-  
-  // Header with metadata
-  lines.push(`# ${title}`);
-  lines.push("");
-  lines.push(`> Source: ${url}`);
-  lines.push(`> Pages: ${pdf.numPages}${truncated ? ` (extracted first ${pagesToExtract})` : ""}`);
-  if (metaAuthor) {
-    lines.push(`> Author: ${metaAuthor}`);
-  }
-  lines.push("");
-  lines.push("---");
-  lines.push("");
-
-  // Content with page markers
-  for (let i = 0; i < pages.length; i++) {
-    if (i > 0) {
-      lines.push("");
-      lines.push(`<!-- Page ${pages[i].pageNum} -->`);
-      lines.push("");
-    }
-    lines.push(pages[i].text);
-  }
-
-  if (truncated) {
-    lines.push("");
-    lines.push("---");
-    lines.push("");
-    lines.push(`*[Truncated: Only first ${pagesToExtract} of ${pdf.numPages} pages extracted]*`);
-  }
-
-  const content = lines.join("\n");
-
-  // Generate output filename
-  const outputFilename = filename || sanitizeFilename(title) + ".md";
-  const outputPath = join(outputDir, outputFilename);
-
-  // Ensure output directory exists
-  await mkdir(outputDir, { recursive: true });
-
-  // Write file
-  await writeFile(outputPath, content, "utf-8");
-
-  return {
-    title,
-    pages: pdf.numPages,
-    chars: content.length,
-    outputPath,
-  };
-}
-
-/**
- * Extract a reasonable title from URL
- */
-function extractTitleFromURL(url: string): string {
-  try {
-    const urlObj = new URL(url);
-    const pathname = urlObj.pathname;
-    
-    // Get filename without extension
-    let filename = basename(pathname, ".pdf");
-    
-    // Handle arxiv URLs: /pdf/1706.03762 → "arxiv-1706.03762"
-    if (urlObj.hostname === "arxiv.org" || urlObj.hostname.endsWith(".arxiv.org")) {
-      const match = pathname.match(/\/(?:pdf|abs)\/(\d+\.\d+)/);
-      if (match) {
-        filename = `arxiv-${match[1]}`;
-      }
-    }
-    
-    // Clean up filename
-    filename = filename
-      .replace(/[_-]+/g, " ")
-      .replace(/\s+/g, " ")
-      .trim();
-    
-    return filename || "document";
-  } catch {
-    return "document";
-  }
-}
-
-/**
- * Sanitize string for use as filename
- */
-function sanitizeFilename(name: string): string {
-  return name
-    .toLowerCase()
-    .replace(/[^a-z0-9\s-]/g, "")
-    .replace(/\s+/g, "-")
-    .replace(/-+/g, "-")
-    .slice(0, 100)
-    .replace(/^-|-$/g, "")
-    || "document";
-}
-
-/**
- * Check if URL or content-type indicates a PDF
- */
-export function isPDF(url: string, contentType?: string): boolean {
-  if (contentType?.includes("application/pdf")) {
-    return true;
-  }
-  try {
-    const urlObj = new URL(url);
-    return urlObj.pathname.toLowerCase().endsWith(".pdf");
-  } catch {
-    return false;
-  }
-}

package/dist/builtin/web-access/perplexity.ts

@@ -1,194 +0,0 @@
-import { existsSync, readFileSync } from "node:fs";
-import { activityMonitor } from "./activity.js";
-import type { ExtractedContent } from "./extract.js";
-import { findReadableConfigPath } from "./config-paths.ts";
-
-const PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions";
-const CONFIG_PATH = findReadableConfigPath();
-
-const RATE_LIMIT = {
-	maxRequests: 10,
-	windowMs: 60 * 1000,
-};
-
-const requestTimestamps: number[] = [];
-
-export interface SearchResult {
-	title: string;
-	url: string;
-	snippet: string;
-}
-
-export interface SearchResponse {
-	answer: string;
-	results: SearchResult[];
-	inlineContent?: ExtractedContent[];
-}
-
-export interface SearchOptions {
-	numResults?: number;
-	recencyFilter?: "day" | "week" | "month" | "year";
-	domainFilter?: string[];
-	signal?: AbortSignal;
-}
-
-interface WebSearchConfig {
-	perplexityApiKey?: unknown;
-}
-
-let cachedConfig: WebSearchConfig | null = null;
-
-function loadConfig(): WebSearchConfig {
-	if (cachedConfig) return cachedConfig;
-	if (!existsSync(CONFIG_PATH)) {
-		cachedConfig = {};
-		return cachedConfig;
-	}
-
-	const content = readFileSync(CONFIG_PATH, "utf-8");
-	try {
-		cachedConfig = JSON.parse(content) as WebSearchConfig;
-		return cachedConfig;
-	} catch (err) {
-		const message = err instanceof Error ? err.message : String(err);
-		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
-	}
-}
-
-function normalizeApiKey(value: unknown): string | null {
-	if (typeof value !== "string") return null;
-	const normalized = value.trim();
-	return normalized.length > 0 ? normalized : null;
-}
-
-function getApiKey(): string {
-	const config = loadConfig();
-	const key = normalizeApiKey(process.env.PERPLEXITY_API_KEY) ?? normalizeApiKey(config.perplexityApiKey);
-	if (!key) {
-		throw new Error(
-			"Perplexity API key not found. Either:\n" +
-			`  1. Create ${CONFIG_PATH} with { "perplexityApiKey": "your-key" }\n` +
-			"  2. Set PERPLEXITY_API_KEY environment variable\n" +
-			"Get a key at https://perplexity.ai/settings/api"
-		);
-	}
-	return key;
-}
-
-function checkRateLimit(): void {
-	const now = Date.now();
-	const windowStart = now - RATE_LIMIT.windowMs;
-
-	while (requestTimestamps.length > 0 && requestTimestamps[0] < windowStart) {
-		requestTimestamps.shift();
-	}
-
-	if (requestTimestamps.length >= RATE_LIMIT.maxRequests) {
-		const waitMs = requestTimestamps[0] + RATE_LIMIT.windowMs - now;
-		throw new Error(`Rate limited. Try again in ${Math.ceil(waitMs / 1000)}s`);
-	}
-
-	requestTimestamps.push(now);
-}
-
-function validateDomainFilter(domains: string[]): string[] {
-	return domains.filter((d) => {
-		const domain = d.startsWith("-") ? d.slice(1) : d;
-		return /^[a-zA-Z0-9][a-zA-Z0-9-_.]*\.[a-zA-Z]{2,}$/.test(domain);
-	});
-}
-
-export function isPerplexityAvailable(): boolean {
-	const config = loadConfig();
-	return !!(normalizeApiKey(process.env.PERPLEXITY_API_KEY) ?? normalizeApiKey(config.perplexityApiKey));
-}
-
-export async function searchWithPerplexity(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
-	checkRateLimit();
-
-	const activityId = activityMonitor.logStart({ type: "api", query });
-
-	activityMonitor.updateRateLimit({
-		used: requestTimestamps.length,
-		max: RATE_LIMIT.maxRequests,
-		oldestTimestamp: requestTimestamps[0] ?? null,
-		windowMs: RATE_LIMIT.windowMs,
-	});
-
-	const apiKey = getApiKey();
-	const numResults = Math.min(options.numResults ?? 5, 20);
-
-	const requestBody: Record<string, unknown> = {
-		model: "sonar",
-		messages: [{ role: "user", content: query }],
-		max_tokens: 1024,
-		return_related_questions: false,
-	};
-
-	if (options.recencyFilter) {
-		requestBody.search_recency_filter = options.recencyFilter;
-	}
-
-	if (options.domainFilter && options.domainFilter.length > 0) {
-		const validated = validateDomainFilter(options.domainFilter);
-		if (validated.length > 0) {
-			requestBody.search_domain_filter = validated;
-		}
-	}
-
-	let response: Response;
-	try {
-		response = await fetch(PERPLEXITY_API_URL, {
-			method: "POST",
-			headers: {
-				Authorization: `Bearer ${apiKey}`,
-				"Content-Type": "application/json",
-			},
-			body: JSON.stringify(requestBody),
-			signal: options.signal,
-		});
-	} catch (err) {
-		const message = err instanceof Error ? err.message : String(err);
-		if (message.toLowerCase().includes("abort")) {
-			activityMonitor.logComplete(activityId, 0);
-		} else {
-			activityMonitor.logError(activityId, message);
-		}
-		throw err;
-	}
-
-	if (!response.ok) {
-		activityMonitor.logComplete(activityId, response.status);
-		const errorText = await response.text();
-		throw new Error(`Perplexity API error ${response.status}: ${errorText}`);
-	}
-
-	let data: Record<string, unknown>;
-	try {
-		data = await response.json();
-	} catch (err) {
-		activityMonitor.logComplete(activityId, response.status);
-		const message = err instanceof Error ? err.message : String(err);
-		throw new Error(`Perplexity API returned invalid JSON: ${message}`);
-	}
-
-	const answer = (data.choices as Array<{ message?: { content?: string } }>)?.[0]?.message?.content || "";
-	const citations = Array.isArray(data.citations) ? data.citations : [];
-
-	const results: SearchResult[] = [];
-	for (let i = 0; i < Math.min(citations.length, numResults); i++) {
-		const citation = citations[i];
-		if (typeof citation === "string") {
-			results.push({ title: `Source ${i + 1}`, url: citation, snippet: "" });
-		} else if (citation && typeof citation === "object" && typeof citation.url === "string") {
-			results.push({
-				title: citation.title || `Source ${i + 1}`,
-				url: citation.url,
-				snippet: "",
-			});
-		}
-	}
-
-	activityMonitor.logComplete(activityId, response.status);
-	return { answer, results };
-}