@bastani/atomic 0.7.17-1 → 0.8.0-0

package/dist/builtin/web-access/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@bastani/web-access",
+  "version": "0.8.0-0",
+  "private": true,
+  "description": "Atomic extension for web search, URL fetching, GitHub repo cloning, PDF/video extraction.",
+  "contributors": [
+    "Nico Bailon",
+    "Alex Lavaee"
+  ],
+  "license": "MIT",
+  "type": "module",
+  "engines": {
+    "bun": ">=1.3.14"
+  },
+  "main": "./index.ts",
+  "types": "./index.ts",
+  "exports": {
+    ".": "./index.ts"
+  },
+  "files": [
+    "*.ts",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE"
+  ],
+  "pi": {
+    "extensions": [
+      "./index.ts"
+    ]
+  },
+  "peerDependencies": {
+    "@bastani/atomic": "*",
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*"
+  },
+  "peerDependenciesMeta": {
+    "@bastani/atomic": {
+      "optional": true
+    },
+    "@earendil-works/pi-coding-agent": {
+      "optional": true
+    },
+    "@earendil-works/pi-tui": {
+      "optional": true
+    }
+  },
+  "dependencies": {
+    "@mozilla/readability": "^0.5.0",
+    "linkedom": "^0.16.0",
+    "p-limit": "^6.1.0",
+    "turndown": "^7.2.0",
+    "unpdf": "^1.6.2"
+  }
+}

package/dist/builtin/web-access/pdf-extract.ts

@@ -0,0 +1,192 @@
+/**
+ * PDF Content Extractor
+ * 
+ * Extracts text from PDF files and saves to markdown.
+ * Uses unpdf (pdfjs-dist wrapper) for text extraction.
+ */
+
+import { getDocumentProxy } from "unpdf";
+import { writeFile, mkdir } from "node:fs/promises";
+import { join, basename } from "node:path";
+import { homedir } from "node:os";
+
+export interface PDFExtractResult {
+  title: string;
+  pages: number;
+  chars: number;
+  outputPath: string;
+}
+
+export interface PDFExtractOptions {
+  maxPages?: number;
+  outputDir?: string;
+  filename?: string;
+}
+
+const DEFAULT_MAX_PAGES = 100;
+const DEFAULT_OUTPUT_DIR = join(homedir(), "Downloads");
+
+/**
+ * Extract text from a PDF buffer and save to markdown file
+ */
+export async function extractPDFToMarkdown(
+  buffer: ArrayBuffer,
+  url: string,
+  options: PDFExtractOptions = {}
+): Promise<PDFExtractResult> {
+  const {
+    maxPages = DEFAULT_MAX_PAGES,
+    outputDir = DEFAULT_OUTPUT_DIR,
+    filename,
+  } = options;
+
+  const safeMaxPages = Number.isFinite(maxPages)
+    ? Math.max(1, Math.floor(maxPages))
+    : DEFAULT_MAX_PAGES;
+
+  const pdf = await getDocumentProxy(new Uint8Array(buffer));
+  const metadata = await pdf.getMetadata();
+  const metadataInfo = metadata.info && typeof metadata.info === "object"
+    ? metadata.info as Record<string, unknown>
+    : null;
+
+  // Extract title from metadata or URL
+  const metaTitle = typeof metadataInfo?.Title === "string" ? metadataInfo.Title : undefined;
+  const metaAuthor = typeof metadataInfo?.Author === "string" ? metadataInfo.Author : undefined;
+  const urlTitle = extractTitleFromURL(url);
+  const title = metaTitle?.trim() || urlTitle;
+
+  // Determine pages to extract
+  const pagesToExtract = Math.min(pdf.numPages, safeMaxPages);
+  const truncated = pdf.numPages > safeMaxPages;
+
+  // Extract text page by page for better structure
+  const pages: { pageNum: number; text: string }[] = [];
+  for (let i = 1; i <= pagesToExtract; i++) {
+    const page = await pdf.getPage(i);
+    const textContent = await page.getTextContent();
+    const pageText = textContent.items
+      .map((item: unknown) => {
+        const textItem = item as { str?: string };
+        return textItem.str || "";
+      })
+      .join(" ")
+      .replace(/\s+/g, " ")
+      .trim();
+    
+    if (pageText) {
+      pages.push({ pageNum: i, text: pageText });
+    }
+  }
+
+  // Build markdown content
+  const lines: string[] = [];
+  
+  // Header with metadata
+  lines.push(`# ${title}`);
+  lines.push("");
+  lines.push(`> Source: ${url}`);
+  lines.push(`> Pages: ${pdf.numPages}${truncated ? ` (extracted first ${pagesToExtract})` : ""}`);
+  if (metaAuthor) {
+    lines.push(`> Author: ${metaAuthor}`);
+  }
+  lines.push("");
+  lines.push("---");
+  lines.push("");
+
+  // Content with page markers
+  for (let i = 0; i < pages.length; i++) {
+    if (i > 0) {
+      lines.push("");
+      lines.push(`<!-- Page ${pages[i].pageNum} -->`);
+      lines.push("");
+    }
+    lines.push(pages[i].text);
+  }
+
+  if (truncated) {
+    lines.push("");
+    lines.push("---");
+    lines.push("");
+    lines.push(`*[Truncated: Only first ${pagesToExtract} of ${pdf.numPages} pages extracted]*`);
+  }
+
+  const content = lines.join("\n");
+
+  // Generate output filename
+  const outputFilename = filename || sanitizeFilename(title) + ".md";
+  const outputPath = join(outputDir, outputFilename);
+
+  // Ensure output directory exists
+  await mkdir(outputDir, { recursive: true });
+
+  // Write file
+  await writeFile(outputPath, content, "utf-8");
+
+  return {
+    title,
+    pages: pdf.numPages,
+    chars: content.length,
+    outputPath,
+  };
+}
+
+/**
+ * Extract a reasonable title from URL
+ */
+function extractTitleFromURL(url: string): string {
+  try {
+    const urlObj = new URL(url);
+    const pathname = urlObj.pathname;
+    
+    // Get filename without extension
+    let filename = basename(pathname, ".pdf");
+    
+    // Handle arxiv URLs: /pdf/1706.03762 → "arxiv-1706.03762"
+    if (urlObj.hostname === "arxiv.org" || urlObj.hostname.endsWith(".arxiv.org")) {
+      const match = pathname.match(/\/(?:pdf|abs)\/(\d+\.\d+)/);
+      if (match) {
+        filename = `arxiv-${match[1]}`;
+      }
+    }
+    
+    // Clean up filename
+    filename = filename
+      .replace(/[_-]+/g, " ")
+      .replace(/\s+/g, " ")
+      .trim();
+    
+    return filename || "document";
+  } catch {
+    return "document";
+  }
+}
+
+/**
+ * Sanitize string for use as filename
+ */
+function sanitizeFilename(name: string): string {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, "")
+    .replace(/\s+/g, "-")
+    .replace(/-+/g, "-")
+    .slice(0, 100)
+    .replace(/^-|-$/g, "")
+    || "document";
+}
+
+/**
+ * Check if URL or content-type indicates a PDF
+ */
+export function isPDF(url: string, contentType?: string): boolean {
+  if (contentType?.includes("application/pdf")) {
+    return true;
+  }
+  try {
+    const urlObj = new URL(url);
+    return urlObj.pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return false;
+  }
+}

package/dist/builtin/web-access/perplexity.ts

@@ -0,0 +1,196 @@
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { CONFIG_DIR_NAME } from "@bastani/atomic";
+import { activityMonitor } from "./activity.js";
+import type { ExtractedContent } from "./extract.js";
+
+const PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions";
+const CONFIG_PATH = join(homedir(), CONFIG_DIR_NAME, "web-search.json");
+
+const RATE_LIMIT = {
+	maxRequests: 10,
+	windowMs: 60 * 1000,
+};
+
+const requestTimestamps: number[] = [];
+
+export interface SearchResult {
+	title: string;
+	url: string;
+	snippet: string;
+}
+
+export interface SearchResponse {
+	answer: string;
+	results: SearchResult[];
+	inlineContent?: ExtractedContent[];
+}
+
+export interface SearchOptions {
+	numResults?: number;
+	recencyFilter?: "day" | "week" | "month" | "year";
+	domainFilter?: string[];
+	signal?: AbortSignal;
+}
+
+interface WebSearchConfig {
+	perplexityApiKey?: unknown;
+}
+
+let cachedConfig: WebSearchConfig | null = null;
+
+function loadConfig(): WebSearchConfig {
+	if (cachedConfig) return cachedConfig;
+	if (!existsSync(CONFIG_PATH)) {
+		cachedConfig = {};
+		return cachedConfig;
+	}
+
+	const content = readFileSync(CONFIG_PATH, "utf-8");
+	try {
+		cachedConfig = JSON.parse(content) as WebSearchConfig;
+		return cachedConfig;
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+	}
+}
+
+function normalizeApiKey(value: unknown): string | null {
+	if (typeof value !== "string") return null;
+	const normalized = value.trim();
+	return normalized.length > 0 ? normalized : null;
+}
+
+function getApiKey(): string {
+	const config = loadConfig();
+	const key = normalizeApiKey(process.env.PERPLEXITY_API_KEY) ?? normalizeApiKey(config.perplexityApiKey);
+	if (!key) {
+		throw new Error(
+			"Perplexity API key not found. Either:\n" +
+			`  1. Create ${CONFIG_PATH} with { "perplexityApiKey": "your-key" }\n` +
+			"  2. Set PERPLEXITY_API_KEY environment variable\n" +
+			"Get a key at https://perplexity.ai/settings/api"
+		);
+	}
+	return key;
+}
+
+function checkRateLimit(): void {
+	const now = Date.now();
+	const windowStart = now - RATE_LIMIT.windowMs;
+
+	while (requestTimestamps.length > 0 && requestTimestamps[0] < windowStart) {
+		requestTimestamps.shift();
+	}
+
+	if (requestTimestamps.length >= RATE_LIMIT.maxRequests) {
+		const waitMs = requestTimestamps[0] + RATE_LIMIT.windowMs - now;
+		throw new Error(`Rate limited. Try again in ${Math.ceil(waitMs / 1000)}s`);
+	}
+
+	requestTimestamps.push(now);
+}
+
+function validateDomainFilter(domains: string[]): string[] {
+	return domains.filter((d) => {
+		const domain = d.startsWith("-") ? d.slice(1) : d;
+		return /^[a-zA-Z0-9][a-zA-Z0-9-_.]*\.[a-zA-Z]{2,}$/.test(domain);
+	});
+}
+
+export function isPerplexityAvailable(): boolean {
+	const config = loadConfig();
+	return !!(normalizeApiKey(process.env.PERPLEXITY_API_KEY) ?? normalizeApiKey(config.perplexityApiKey));
+}
+
+export async function searchWithPerplexity(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
+	checkRateLimit();
+
+	const activityId = activityMonitor.logStart({ type: "api", query });
+
+	activityMonitor.updateRateLimit({
+		used: requestTimestamps.length,
+		max: RATE_LIMIT.maxRequests,
+		oldestTimestamp: requestTimestamps[0] ?? null,
+		windowMs: RATE_LIMIT.windowMs,
+	});
+
+	const apiKey = getApiKey();
+	const numResults = Math.min(options.numResults ?? 5, 20);
+
+	const requestBody: Record<string, unknown> = {
+		model: "sonar",
+		messages: [{ role: "user", content: query }],
+		max_tokens: 1024,
+		return_related_questions: false,
+	};
+
+	if (options.recencyFilter) {
+		requestBody.search_recency_filter = options.recencyFilter;
+	}
+
+	if (options.domainFilter && options.domainFilter.length > 0) {
+		const validated = validateDomainFilter(options.domainFilter);
+		if (validated.length > 0) {
+			requestBody.search_domain_filter = validated;
+		}
+	}
+
+	let response: Response;
+	try {
+		response = await fetch(PERPLEXITY_API_URL, {
+			method: "POST",
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+				"Content-Type": "application/json",
+			},
+			body: JSON.stringify(requestBody),
+			signal: options.signal,
+		});
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		if (message.toLowerCase().includes("abort")) {
+			activityMonitor.logComplete(activityId, 0);
+		} else {
+			activityMonitor.logError(activityId, message);
+		}
+		throw err;
+	}
+
+	if (!response.ok) {
+		activityMonitor.logComplete(activityId, response.status);
+		const errorText = await response.text();
+		throw new Error(`Perplexity API error ${response.status}: ${errorText}`);
+	}
+
+	let data: Record<string, unknown>;
+	try {
+		data = await response.json();
+	} catch (err) {
+		activityMonitor.logComplete(activityId, response.status);
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Perplexity API returned invalid JSON: ${message}`);
+	}
+
+	const answer = (data.choices as Array<{ message?: { content?: string } }>)?.[0]?.message?.content || "";
+	const citations = Array.isArray(data.citations) ? data.citations : [];
+
+	const results: SearchResult[] = [];
+	for (let i = 0; i < Math.min(citations.length, numResults); i++) {
+		const citation = citations[i];
+		if (typeof citation === "string") {
+			results.push({ title: `Source ${i + 1}`, url: citation, snippet: "" });
+		} else if (citation && typeof citation === "object" && typeof citation.url === "string") {
+			results.push({
+				title: citation.title || `Source ${i + 1}`,
+				url: citation.url,
+				snippet: "",
+			});
+		}
+	}
+
+	activityMonitor.logComplete(activityId, response.status);
+	return { answer, results };
+}