@rol1n/web-reach 0.1.0 → 0.1.1

package/README.md

@@ -1,6 +1,6 @@
 # web-reach
 
-Web search and page extraction tools for Pi. Registers `web_search` through Exa's public MCP endpoint and `web_fetch` for readable Markdown or raw responses.
+Keyless web search (`web_search` via Exa MCP) and multi-tier page extraction (`web_fetch` supporting Next.js RSC, semantic HTML-to-Markdown, optional `unpdf` PDF extraction, and Jina Reader fallback).
 
 ## Install
 
@@ -8,30 +8,12 @@ Web search and page extraction tools for Pi. Registers `web_search` through Exa'
 pi install npm:@rol1n/web-reach
 ```
 
-Also included in `@rol1n/pi-kit`. Avoid enabling another extension that registers the same tool names unless overriding them is intentional.
-
-## Use
-
-```json
-{"query":"Pi coding agent package resource filtering"}
-```
-
-```json
-{"url":"https://example.com/docs","mode":"readable"}
-```
-
-The first object is for `web_search`; the second is for `web_fetch`. Fetch mode defaults to `readable` and also accepts `raw`. Extraction includes HTML-to-Markdown, JSON formatting, PDF handling, and a Jina Reader fallback for pages that need it. Returned text is limited to 35,000 characters.
+Also included in `@rol1n/pi-kit`.
 
 ## Configuration
 
 | Environment variable | Default |
 |---|---|
-| `SRP_WEB_SEARCH_TIMEOUT_MS` | 60000 |
-| `SRP_WEB_FETCH_TIMEOUT_MS` | 20000 |
-| `SRP_WEB_FETCH_MAX_BYTES` | 4194304 (4 MiB) |
-
-No separate search API key is configured; availability and rate limits depend on the public Exa and Jina services. PDF extraction optionally imports `unpdf` if it is available. Without that package the implementation falls back to a basic text extraction path, with lower PDF text quality.
-
-## Source layout
-
-`index.ts` registers the tools and re-exports `searchWeb` and `fetchUrl`. `search.ts` handles Exa MCP requests and response parsing; `transport.ts` shares timeout, cancellation, environment, and truncation helpers. `page.ts` coordinates page fetching and extraction, using `rsc.ts` for Next.js data, `html.ts` for semantic Markdown, `pdf.ts` for optional PDF extraction, and `reader.ts` for the Jina Reader fallback.
+| `SRP_WEB_SEARCH_TIMEOUT_MS` | `60000` |
+| `SRP_WEB_FETCH_TIMEOUT_MS` | `20000` |
+| `SRP_WEB_FETCH_MAX_BYTES` | `4194304` (4 MiB) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rol1n/web-reach",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "type": "module",
   "description": "web-reach extension from pi-kit",
   "keywords": [
@@ -9,7 +9,7 @@
   "repository": {
     "type": "git",
     "url": "git+https://github.com/RolinShmily/pi-kit.git",
-    "directory": "extensions/web-reach"
+    "directory": "web-reach"
   },
   "homepage": "https://github.com/RolinShmily/pi-kit#readme",
   "bugs": {
@@ -21,10 +21,6 @@
   "publishConfig": {
     "access": "public"
   },
-  "files": [
-    "extensions/web-reach",
-    "extensions/web-reach/README.md"
-  ],
   "peerDependencies": {
     "@earendil-works/pi-ai": "*",
     "@earendil-works/pi-agent-core": "*",
@@ -34,7 +30,7 @@
   },
   "pi": {
     "extensions": [
-      "./extensions/web-reach/index.ts"
+      "./index.ts"
     ],
     "skills": [],
     "prompts": [],

package/{extensions/web-reach/page.ts → page.ts}

@@ -1,8 +1,90 @@
 import { DEFAULT_USER_AGENT, envNum, truncate, withTimeout } from "./transport.ts";
 import { extractRSCContent } from "./rsc.ts";
 import { extractTitleFromHtml, htmlToMarkdown, isLikelyJSRendered } from "./html.ts";
-import { fetchViaJinaReader } from "./reader.ts";
-import { extractPdfBuffer, isPdfUrl } from "./pdf.ts";
+
+const JINA_READER_BASE = "https://r.jina.ai/";
+
+export async function fetchViaJinaReader(
+  url: string,
+  signal?: AbortSignal,
+): Promise<{ text: string; title: string } | null> {
+  const { ac, done } = withTimeout(30_000, "Jina Reader timed out", signal);
+  try {
+    const res = await fetch(`${JINA_READER_BASE}${url}`, {
+      headers: {
+        Accept: "text/markdown",
+        "X-No-Cache": "true",
+        "User-Agent": DEFAULT_USER_AGENT,
+      },
+      signal: ac.signal,
+    });
+    if (!res.ok) return null;
+
+    const raw = await res.text();
+    const contentIndex = raw.indexOf("Markdown Content:");
+    const markdown = contentIndex >= 0 ? raw.slice(contentIndex + 17).trim() : raw.trim();
+
+    if (
+      markdown.length < 80 ||
+      markdown.startsWith("Loading...") ||
+      markdown.startsWith("Please enable JavaScript")
+    ) {
+      return null;
+    }
+
+    const titleMatch = markdown.match(/^#+\s+(.+)$/m);
+    const title = titleMatch?.[1]?.trim() || "";
+
+    return { text: markdown, title };
+  } catch {
+    return null;
+  } finally {
+    done();
+  }
+}
+
+export function isPdfUrl(url: string, contentType?: string): boolean {
+  if (contentType?.includes("application/pdf")) return true;
+  try {
+    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return false;
+  }
+}
+
+export async function extractPdfBuffer(
+  buffer: ArrayBuffer,
+  url: string,
+): Promise<{ text: string; title: string }> {
+  try {
+    const { getDocumentProxy } = await import("unpdf");
+    const pdf = await getDocumentProxy(new Uint8Array(buffer));
+    const maxPages = Math.min(pdf.numPages, 80);
+    const pages: string[] = [];
+
+    for (let i = 1; i <= maxPages; i++) {
+      const page = await pdf.getPage(i);
+      const textContent = await page.getTextContent();
+      const pageText = textContent.items
+        .map((item: unknown) => (item as { str?: string }).str || "")
+        .join(" ")
+        .replace(/\s+/g, " ")
+        .trim();
+      if (pageText) pages.push(`--- Page ${i} ---\n${pageText}`);
+    }
+
+    const title = new URL(url).pathname.split("/").pop() || "PDF Document";
+    const text = `# ${title}\n\nPages: ${pdf.numPages}\n\n${pages.join("\n\n")}`;
+    return { text, title };
+  } catch {
+    // Fallback plain text extraction
+    const text = Buffer.from(buffer).toString("latin1").replace(/[^\x20-\x7E\n\r\t]/g, " ");
+    return {
+      text: `# PDF Document\n\n${text.slice(0, 8000)}`,
+      title: "PDF Document",
+    };
+  }
+}
 
 const MIN_USEFUL_CONTENT_LENGTH = 300;
 

package/extensions/web-reach/README.md

@@ -1,37 +0,0 @@
-# web-reach
-
-Web search and page extraction tools for Pi. Registers `web_search` through Exa's public MCP endpoint and `web_fetch` for readable Markdown or raw responses.
-
-## Install
-
-```sh
-pi install npm:@rol1n/web-reach
-```
-
-Also included in `@rol1n/pi-kit`. Avoid enabling another extension that registers the same tool names unless overriding them is intentional.
-
-## Use
-
-```json
-{"query":"Pi coding agent package resource filtering"}
-```
-
-```json
-{"url":"https://example.com/docs","mode":"readable"}
-```
-
-The first object is for `web_search`; the second is for `web_fetch`. Fetch mode defaults to `readable` and also accepts `raw`. Extraction includes HTML-to-Markdown, JSON formatting, PDF handling, and a Jina Reader fallback for pages that need it. Returned text is limited to 35,000 characters.
-
-## Configuration
-
-| Environment variable | Default |
-|---|---|
-| `SRP_WEB_SEARCH_TIMEOUT_MS` | 60000 |
-| `SRP_WEB_FETCH_TIMEOUT_MS` | 20000 |
-| `SRP_WEB_FETCH_MAX_BYTES` | 4194304 (4 MiB) |
-
-No separate search API key is configured; availability and rate limits depend on the public Exa and Jina services. PDF extraction optionally imports `unpdf` if it is available. Without that package the implementation falls back to a basic text extraction path, with lower PDF text quality.
-
-## Source layout
-
-`index.ts` registers the tools and re-exports `searchWeb` and `fetchUrl`. `search.ts` handles Exa MCP requests and response parsing; `transport.ts` shares timeout, cancellation, environment, and truncation helpers. `page.ts` coordinates page fetching and extraction, using `rsc.ts` for Next.js data, `html.ts` for semantic Markdown, `pdf.ts` for optional PDF extraction, and `reader.ts` for the Jina Reader fallback.

package/extensions/web-reach/pdf.ts

@@ -1,42 +0,0 @@
-export function isPdfUrl(url: string, contentType?: string): boolean {
-  if (contentType?.includes("application/pdf")) return true;
-  try {
-    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
-  } catch {
-    return false;
-  }
-}
-
-export async function extractPdfBuffer(
-  buffer: ArrayBuffer,
-  url: string,
-): Promise<{ text: string; title: string }> {
-  try {
-    const { getDocumentProxy } = await import("unpdf");
-    const pdf = await getDocumentProxy(new Uint8Array(buffer));
-    const maxPages = Math.min(pdf.numPages, 80);
-    const pages: string[] = [];
-
-    for (let i = 1; i <= maxPages; i++) {
-      const page = await pdf.getPage(i);
-      const textContent = await page.getTextContent();
-      const pageText = textContent.items
-        .map((item: unknown) => (item as { str?: string }).str || "")
-        .join(" ")
-        .replace(/\s+/g, " ")
-        .trim();
-      if (pageText) pages.push(`--- Page ${i} ---\n${pageText}`);
-    }
-
-    const title = new URL(url).pathname.split("/").pop() || "PDF Document";
-    const text = `# ${title}\n\nPages: ${pdf.numPages}\n\n${pages.join("\n\n")}`;
-    return { text, title };
-  } catch {
-    // Fallback plain text extraction
-    const text = Buffer.from(buffer).toString("latin1").replace(/[^\x20-\x7E\n\r\t]/g, " ");
-    return {
-      text: `# PDF Document\n\n${text.slice(0, 8000)}`,
-      title: "PDF Document",
-    };
-  }
-}

package/extensions/web-reach/reader.ts

@@ -1,42 +0,0 @@
-import { DEFAULT_USER_AGENT, withTimeout } from "./transport.ts";
-
-const JINA_READER_BASE = "https://r.jina.ai/";
-
-export async function fetchViaJinaReader(
-  url: string,
-  signal?: AbortSignal,
-): Promise<{ text: string; title: string } | null> {
-  const { ac, done } = withTimeout(30_000, "Jina Reader timed out", signal);
-  try {
-    const res = await fetch(`${JINA_READER_BASE}${url}`, {
-      headers: {
-        Accept: "text/markdown",
-        "X-No-Cache": "true",
-        "User-Agent": DEFAULT_USER_AGENT,
-      },
-      signal: ac.signal,
-    });
-    if (!res.ok) return null;
-
-    const raw = await res.text();
-    const contentIndex = raw.indexOf("Markdown Content:");
-    const markdown = contentIndex >= 0 ? raw.slice(contentIndex + 17).trim() : raw.trim();
-
-    if (
-      markdown.length < 80 ||
-      markdown.startsWith("Loading...") ||
-      markdown.startsWith("Please enable JavaScript")
-    ) {
-      return null;
-    }
-
-    const titleMatch = markdown.match(/^#+\s+(.+)$/m);
-    const title = titleMatch?.[1]?.trim() || "";
-
-    return { text: markdown, title };
-  } catch {
-    return null;
-  } finally {
-    done();
-  }
-}