@rol1n/pi-kit 0.1.1 → 0.1.2

package/web-reach/page.ts

@@ -1,214 +0,0 @@
-import { DEFAULT_USER_AGENT, envNum, truncate, withTimeout } from "./transport.ts";
-import { extractRSCContent } from "./rsc.ts";
-import { extractTitleFromHtml, htmlToMarkdown, isLikelyJSRendered } from "./html.ts";
-
-const JINA_READER_BASE = "https://r.jina.ai/";
-
-export async function fetchViaJinaReader(
-  url: string,
-  signal?: AbortSignal,
-): Promise<{ text: string; title: string } | null> {
-  const { ac, done } = withTimeout(30_000, "Jina Reader timed out", signal);
-  try {
-    const res = await fetch(`${JINA_READER_BASE}${url}`, {
-      headers: {
-        Accept: "text/markdown",
-        "X-No-Cache": "true",
-        "User-Agent": DEFAULT_USER_AGENT,
-      },
-      signal: ac.signal,
-    });
-    if (!res.ok) return null;
-
-    const raw = await res.text();
-    const contentIndex = raw.indexOf("Markdown Content:");
-    const markdown = contentIndex >= 0 ? raw.slice(contentIndex + 17).trim() : raw.trim();
-
-    if (
-      markdown.length < 80 ||
-      markdown.startsWith("Loading...") ||
-      markdown.startsWith("Please enable JavaScript")
-    ) {
-      return null;
-    }
-
-    const titleMatch = markdown.match(/^#+\s+(.+)$/m);
-    const title = titleMatch?.[1]?.trim() || "";
-
-    return { text: markdown, title };
-  } catch {
-    return null;
-  } finally {
-    done();
-  }
-}
-
-export function isPdfUrl(url: string, contentType?: string): boolean {
-  if (contentType?.includes("application/pdf")) return true;
-  try {
-    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
-  } catch {
-    return false;
-  }
-}
-
-export async function extractPdfBuffer(
-  buffer: ArrayBuffer,
-  url: string,
-): Promise<{ text: string; title: string }> {
-  try {
-    const { getDocumentProxy } = await import("unpdf");
-    const pdf = await getDocumentProxy(new Uint8Array(buffer));
-    const maxPages = Math.min(pdf.numPages, 80);
-    const pages: string[] = [];
-
-    for (let i = 1; i <= maxPages; i++) {
-      const page = await pdf.getPage(i);
-      const textContent = await page.getTextContent();
-      const pageText = textContent.items
-        .map((item: unknown) => (item as { str?: string }).str || "")
-        .join(" ")
-        .replace(/\s+/g, " ")
-        .trim();
-      if (pageText) pages.push(`--- Page ${i} ---\n${pageText}`);
-    }
-
-    const title = new URL(url).pathname.split("/").pop() || "PDF Document";
-    const text = `# ${title}\n\nPages: ${pdf.numPages}\n\n${pages.join("\n\n")}`;
-    return { text, title };
-  } catch {
-    // Fallback plain text extraction
-    const text = Buffer.from(buffer).toString("latin1").replace(/[^\x20-\x7E\n\r\t]/g, " ");
-    return {
-      text: `# PDF Document\n\n${text.slice(0, 8000)}`,
-      title: "PDF Document",
-    };
-  }
-}
-
-const MIN_USEFUL_CONTENT_LENGTH = 300;
-
-export async function fetchUrl(
-  url: string,
-  opts: { mode?: "readable" | "raw" } = {},
-  signal?: AbortSignal,
-): Promise<{ text: string; title: string; contentType: string; status: number; engine: string }> {
-  const mode = opts.mode ?? "readable";
-  const { ac, done } = withTimeout(
-    envNum("SRP_WEB_FETCH_TIMEOUT_MS", 20_000),
-    "Web fetch timed out",
-    signal,
-  );
-
-  try {
-    const res = await fetch(url, {
-      redirect: "follow",
-      headers: {
-        "User-Agent": DEFAULT_USER_AGENT,
-        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/pdf,text/plain,*/*;q=0.8",
-      },
-      signal: ac.signal,
-    });
-
-    if (!res.ok) {
-      throw new Error(`HTTP ${res.status} ${res.statusText}`);
-    }
-
-    const contentType = (res.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase();
-    const maxBytes = envNum("SRP_WEB_FETCH_MAX_BYTES", 4 * 1024 * 1024);
-    const arrayBuf = await res.arrayBuffer();
-
-    if (arrayBuf.byteLength > maxBytes) {
-      throw new Error(`Content exceeds ${Math.round((maxBytes / 1024 / 1024) * 10) / 10} MiB limit.`);
-    }
-
-    // 1. PDF extraction
-    if (isPdfUrl(url, contentType)) {
-      const { text, title } = await extractPdfBuffer(arrayBuf, url);
-      return { text: truncate(text), title, contentType, status: res.status, engine: "unpdf" };
-    }
-
-    const body = Buffer.from(arrayBuf).toString("utf8");
-
-    // 2. Raw mode
-    if (mode === "raw") {
-      return {
-        text: truncate(body),
-        title: extractTitleFromHtml(body) || url,
-        contentType,
-        status: res.status,
-        engine: "raw",
-      };
-    }
-
-    // 3. JSON formatting
-    if (contentType.includes("json")) {
-      try {
-        const text = JSON.stringify(JSON.parse(body), null, 2);
-        return { text: truncate(text), title: "JSON Response", contentType, status: res.status, engine: "json" };
-      } catch {
-        return { text: truncate(body), title: "JSON", contentType, status: res.status, engine: "text" };
-      }
-    }
-
-    // 4. HTML processing
-    if (contentType.includes("html") || contentType.includes("xhtml")) {
-      // 4.1 Attempt Next.js RSC extraction
-      const rsc = extractRSCContent(body);
-      if (rsc && rsc.content.length >= MIN_USEFUL_CONTENT_LENGTH) {
-        return {
-          text: truncate(rsc.content),
-          title: rsc.title || extractTitleFromHtml(body),
-          contentType,
-          status: res.status,
-          engine: "nextjs-rsc",
-        };
-      }
-
-      // 4.2 Semantic HTML to Markdown
-      const title = extractTitleFromHtml(body);
-      const markdown = htmlToMarkdown(body);
-
-      if (markdown.length >= MIN_USEFUL_CONTENT_LENGTH && !isLikelyJSRendered(body)) {
-        return {
-          text: truncate(markdown),
-          title,
-          contentType,
-          status: res.status,
-          engine: "html-markdown",
-        };
-      }
-
-      // 4.3 Dynamic JS page or short content: attempt Jina Reader fallback
-      const jina = await fetchViaJinaReader(url, signal);
-      if (jina && jina.text.length >= 100) {
-        return {
-          text: truncate(jina.text),
-          title: jina.title || title,
-          contentType: "text/markdown",
-          status: res.status,
-          engine: "jina-reader",
-        };
-      }
-
-      return {
-        text: truncate(markdown || body.slice(0, 4000)),
-        title,
-        contentType,
-        status: res.status,
-        engine: "html-fallback",
-      };
-    }
-
-    // 5. Plain text and other types
-    return {
-      text: truncate(body),
-      title: url.split("/").pop() || "Plain Text",
-      contentType,
-      status: res.status,
-      engine: "text",
-    };
-  } finally {
-    done();
-  }
-}

package/web-reach/rsc.ts

@@ -1,161 +0,0 @@
-export function extractRSCContent(html: string): { title: string; content: string } | null {
-  if (!html.includes("self.__next_f.push")) return null;
-
-  const chunkMap = new Map<string, string>();
-  const scriptRegex = /<script>self\.__next_f\.push\(\[1,"([\s\S]*?)"\]\)<\/script>/g;
-
-  for (const match of html.matchAll(scriptRegex)) {
-    let content: string;
-    try {
-      content = JSON.parse('"' + match[1] + '"');
-    } catch {
-      continue;
-    }
-    for (const line of content.split("\n")) {
-      if (!line.trim()) continue;
-      const colonIdx = line.indexOf(":");
-      if (colonIdx <= 0 || colonIdx > 4) continue;
-      const id = line.slice(0, colonIdx);
-      if (!/^[0-9a-f]+$/i.test(id)) continue;
-      const payload = line.slice(colonIdx + 1);
-      if (!payload) continue;
-      const existing = chunkMap.get(id);
-      if (!existing || payload.length > existing.length) {
-        chunkMap.set(id, payload);
-      }
-    }
-  }
-
-  if (chunkMap.size === 0) return null;
-
-  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-  const title = titleMatch?.[1]?.split("|")[0]?.trim() || "";
-
-  const parsedCache = new Map<string, unknown>();
-  function getParsedChunk(id: string): unknown | null {
-    if (parsedCache.has(id)) return parsedCache.get(id);
-    const chunk = chunkMap.get(id);
-    if (!chunk || !chunk.startsWith("[")) {
-      parsedCache.set(id, null);
-      return null;
-    }
-    try {
-      const parsed = JSON.parse(chunk);
-      parsedCache.set(id, parsed);
-      return parsed;
-    } catch {
-      parsedCache.set(id, null);
-      return null;
-    }
-  }
-
-  const visitedRefs = new Set<string>();
-
-  function extractNode(node: unknown, ctx = { inCode: false }): string {
-    if (node === null || node === undefined) return "";
-    if (typeof node === "string") {
-      const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
-      if (refMatch) {
-        const refId = refMatch[1];
-        if (visitedRefs.has(refId)) return "";
-        visitedRefs.add(refId);
-        const refNode = getParsedChunk(refId);
-        const result = refNode ? extractNode(refNode, ctx) : "";
-        visitedRefs.delete(refId);
-        return result;
-      }
-      if (!ctx.inCode && (node === "$undefined" || node === "$" || /^\$[A-Z]/.test(node))) {
-        return "";
-      }
-      return node.trim() ? node : "";
-    }
-    if (typeof node === "number") return String(node);
-    if (typeof node === "boolean") return "";
-    if (!Array.isArray(node)) return "";
-
-    if (node[0] === "$" && typeof node[1] === "string") {
-      const tag = node[1] as string;
-      const props = (node[3] || {}) as Record<string, unknown>;
-      const skipTags = [
-        "script", "style", "svg", "path", "circle", "link", "meta",
-        "template", "button", "input", "nav", "footer", "aside",
-      ];
-      if (skipTags.includes(tag)) return "";
-
-      if (tag.startsWith("$L")) {
-        const refId = tag.slice(2);
-        if (visitedRefs.has(refId)) return "";
-        if (props.baseId && props.children) {
-          return `## ${String(props.children)}\n\n`;
-        }
-        visitedRefs.add(refId);
-        const refNode = getParsedChunk(refId);
-        let result = "";
-        if (refNode) result = extractNode(refNode, ctx);
-        else if (props.children) result = extractNode(props.children, ctx);
-        visitedRefs.delete(refId);
-        return result;
-      }
-
-      const children = props.children;
-      const content = children ? extractNode(children, ctx) : "";
-
-      switch (tag) {
-        case "h1": return `# ${content.trim()}\n\n`;
-        case "h2": return `## ${content.trim()}\n\n`;
-        case "h3": return `### ${content.trim()}\n\n`;
-        case "h4": return `#### ${content.trim()}\n\n`;
-        case "h5": return `##### ${content.trim()}\n\n`;
-        case "h6": return `###### ${content.trim()}\n\n`;
-        case "p": return `${content.trim()}\n\n`;
-        case "code": {
-          const cc = children ? extractNode(children, { inCode: true }) : "";
-          return ctx.inCode ? cc : `\`${cc}\``;
-        }
-        case "pre": {
-          const pc = children ? extractNode(children, { inCode: true }) : "";
-          return "```\n" + pc + "\n```\n\n";
-        }
-        case "strong": case "b": return `**${content}**`;
-        case "em": case "i": return `*${content}*`;
-        case "li": return `- ${content.trim()}\n`;
-        case "ul": case "ol": return content + "\n";
-        case "blockquote": return `> ${content.trim()}\n\n`;
-        case "a": {
-          const href = props.href as string | undefined;
-          return href && !href.startsWith("#") ? `[${content}](${href})` : content;
-        }
-        default: return content;
-      }
-    }
-
-    return (node as unknown[]).map((n) => extractNode(n, ctx)).join("");
-  }
-
-  const contentParts: { order: number; text: string }[] = [];
-  for (const [id] of chunkMap) {
-    const parsed = getParsedChunk(id);
-    if (!parsed) continue;
-    visitedRefs.clear();
-    const text = extractNode(parsed);
-    if (text.trim().length > 50 && !text.includes("404") && !text.includes("not found")) {
-      contentParts.push({ order: parseInt(id, 16) || 0, text: text.trim() });
-    }
-  }
-
-  if (contentParts.length === 0) return null;
-  contentParts.sort((a, b) => a.order - b.order);
-
-  const seen = new Set<string>();
-  const uniqueParts: string[] = [];
-  for (const part of contentParts) {
-    const key = part.text.slice(0, 150);
-    if (!seen.has(key)) {
-      seen.add(key);
-      uniqueParts.push(part.text);
-    }
-  }
-
-  const content = uniqueParts.join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
-  return content.length > 100 ? { title, content } : null;
-}

package/web-reach/search.ts

@@ -1,85 +0,0 @@
-import { envNum, truncate, withTimeout } from "./transport.ts";
-
-const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
-const EXA_SEARCH_TOOL = "web_search_exa";
-
-interface ExaMcpResponse {
-  result?: {
-    content?: Array<{ type?: string; text?: string }>;
-    isError?: boolean;
-  };
-  error?: { code?: number; message?: string };
-}
-
-function parseExaResponse(body: string): ExaMcpResponse | undefined {
-  for (const line of body.split("\n")) {
-    if (!line.startsWith("data:")) continue;
-    const payload = line.slice(5).trim();
-    if (!payload) continue;
-    try {
-      const parsed = JSON.parse(payload) as ExaMcpResponse;
-      if (parsed.result || parsed.error) return parsed;
-    } catch {
-      /* continue to next SSE event */
-    }
-  }
-  try {
-    const parsed = JSON.parse(body) as ExaMcpResponse;
-    return parsed.result || parsed.error ? parsed : undefined;
-  } catch {
-    return undefined;
-  }
-}
-
-export async function searchWeb(
-  query: string,
-  signal?: AbortSignal,
-): Promise<{ text: string; provider: string }> {
-  query = query.trim();
-  if (!query) throw new Error("Search query cannot be empty.");
-  const { ac, done } = withTimeout(
-    envNum("SRP_WEB_SEARCH_TIMEOUT_MS", 60_000),
-    "Search request timed out",
-    signal,
-  );
-  try {
-    const res = await fetch(`${EXA_MCP_URL}?tools=${EXA_SEARCH_TOOL}`, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Accept: "application/json, text/event-stream",
-        "x-exa-source": "web-reach",
-      },
-      body: JSON.stringify({
-        jsonrpc: "2.0",
-        id: 1,
-        method: "tools/call",
-        params: { name: EXA_SEARCH_TOOL, arguments: { query, numResults: 6 } },
-      }),
-      signal: ac.signal,
-    });
-    const body = await res.text();
-    if (!res.ok) {
-      const hint = res.status === 429 ? " (Exa free tier rate limit reached; please try again later)" : "";
-      throw new Error(`Exa MCP HTTP ${res.status}${hint}: ${body.slice(0, 300)}`);
-    }
-
-    const payload = parseExaResponse(body);
-    if (!payload) throw new Error("Exa MCP returned an unparseable response.");
-    if (payload.error) {
-      throw new Error(
-        `Exa MCP error${payload.error.code ? ` ${payload.error.code}` : ""}: ${payload.error.message ?? "Unknown error"}`,
-      );
-    }
-
-    const text = payload.result?.content
-      ?.find((item) => item.type === "text" && item.text?.trim())
-      ?.text?.trim();
-    if (payload.result?.isError || !text) {
-      throw new Error(text || "Exa MCP returned empty content.");
-    }
-    return { text: truncate(text), provider: "exa-mcp" };
-  } finally {
-    done();
-  }
-}

package/web-reach/transport.ts

@@ -1,35 +0,0 @@
-export const DEFAULT_USER_AGENT =
-  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 web-reach/2.0";
-
-const MAX_RETURN_CHARS = 35_000;
-
-export function envNum(name: string, fallback: number): number {
-  const n = Number(process.env[name]);
-  return Number.isFinite(n) && n > 0 ? n : fallback;
-}
-
-// ============================ Timeout & Helpers ============================
-
-export function withTimeout(
-  ms: number,
-  label: string,
-  signal?: AbortSignal,
-): { ac: AbortController; done: () => void } {
-  const ac = new AbortController();
-  const timer = setTimeout(() => ac.abort(new Error(`${label} timed out (${ms / 1000}s)`)), ms);
-  const onAbort = () => ac.abort(signal?.reason);
-  signal?.addEventListener("abort", onAbort, { once: true });
-  if (signal?.aborted) onAbort();
-  return {
-    ac,
-    done: () => {
-      clearTimeout(timer);
-      signal?.removeEventListener("abort", onAbort);
-    },
-  };
-}
-
-export function truncate(text: string, limit = MAX_RETURN_CHARS): string {
-  if (text.length <= limit) return text;
-  return text.slice(0, limit) + `\n\n...(content truncated to ${limit} characters)`;
-}