@openlines/opl-pi-sht 0.1.18 → 0.1.20

package/extensions/opl-webaccess/extract.ts

@@ -2,16 +2,45 @@ import { Readability } from "@mozilla/readability";
 import { parseHTML } from "linkedom";
 import TurndownService from "turndown";
 import { extractPdfBuffer } from "./pdf.js";
-import { errorMessage, isAbortError, isPdfUrl, isPdfContentType } from "./utils.js";
+import { errorMessage, isAbortError, isPdfUrl, isPdfContentType, assertHttpUrl } from "./utils.js";
 import type { ExtractedContent } from "./types.js";
 
 const CONCURRENT_LIMIT = 3;
+const MAX_FETCH_BYTES = 10 * 1024 * 1024; // 10 MB response cap
+const FETCH_TIMEOUT_MS = 30_000;
 
 const td = new TurndownService({
   headingStyle: "atx",
   codeBlockStyle: "fenced",
 });
 
+/** Read a response body into bytes, aborting past maxBytes (guards memory against huge payloads). */
+async function readBufferCapped(response: Response, maxBytes: number): Promise<ArrayBuffer> {
+  const declared = Number(response.headers.get("content-length") ?? "0");
+  if (declared > maxBytes) throw new Error(`Response too large (${declared} bytes)`);
+  const reader = response.body?.getReader();
+  if (!reader) return await response.arrayBuffer();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    total += value.byteLength;
+    if (total > maxBytes) {
+      await reader.cancel().catch(() => {});
+      throw new Error(`Response exceeds ${maxBytes} byte limit`);
+    }
+    chunks.push(value);
+  }
+  const merged = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return merged.buffer;
+}
+
 export async function fetchAllContent(
   urls: string[],
   signal?: AbortSignal
@@ -28,8 +57,12 @@ export async function fetchAllContent(
 
 async function fetchOne(url: string, signal?: AbortSignal): Promise<ExtractedContent> {
   try {
-    const response = await fetch(url, {
-      signal,
+    const target = assertHttpUrl(url);
+    const fetchSignal = signal
+      ? AbortSignal.any([signal, AbortSignal.timeout(FETCH_TIMEOUT_MS)])
+      : AbortSignal.timeout(FETCH_TIMEOUT_MS);
+    const response = await fetch(target, {
+      signal: fetchSignal,
       headers: {
         "User-Agent": "Mozilla/5.0 (compatible; pi-web-access/1.0)",
         Accept: "text/html,application/xhtml+xml,application/pdf,*/*",
@@ -43,7 +76,7 @@ async function fetchOne(url: string, signal?: AbortSignal): Promise<ExtractedCon
     const contentType = response.headers.get("content-type") ?? "";
 
     if (isPdfUrl(url) || isPdfContentType(contentType)) {
-      const buffer = await response.arrayBuffer();
+      const buffer = await readBufferCapped(response, MAX_FETCH_BYTES);
       const content = await extractPdfBuffer(buffer);
       return { url, title: url, content, error: null };
     }
@@ -51,7 +84,7 @@ async function fetchOne(url: string, signal?: AbortSignal): Promise<ExtractedCon
     const isHtml =
       contentType.includes("text/html") || contentType.includes("application/xhtml");
 
-    const body = await response.text();
+    const body = new TextDecoder().decode(await readBufferCapped(response, MAX_FETCH_BYTES));
 
     if (!isHtml) {
       // Plain text, markdown, JSON, etc. — return as-is

package/extensions/opl-webaccess/providers/gemini.ts

@@ -27,9 +27,9 @@ export async function searchGemini(
 
   let response: Response;
   try {
-    response = await fetch(`${base}/models/${model}:generateContent?key=${apiKey}`, {
+    response = await fetch(`${base}/models/${model}:generateContent`, {
       method: "POST",
-      headers: { "Content-Type": "application/json" },
+      headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey },
       body: JSON.stringify({
         contents: [{ role: "user", parts: [{ text: query }] }],
         tools: [{ google_search: {} }],

package/extensions/opl-webaccess/utils.ts

@@ -26,3 +26,17 @@ export function isPdfUrl(url: string): boolean {
 export function isPdfContentType(contentType: string): boolean {
   return contentType.toLowerCase().includes("application/pdf");
 }
+
+/** Normalize a URL, allowing only http/https (file:/data:/etc. are rejected). */
+export function assertHttpUrl(url: string): string {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error(`Invalid URL: ${url}`);
+  }
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new Error(`Unsupported protocol "${parsed.protocol}//" — only http/https`);
+  }
+  return parsed.href;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openlines/opl-pi-sht",
-  "version": "0.1.18",
-  "description": "Pi coding-agent extension kit: token-efficient context, safe modes, fingerprinted /init, model benchmarking, web access, and browser automation.",
+  "version": "0.1.20",
+  "description": "opl-pi-sht (shouldHaveThat) Pi coding-agent extension kit: token-efficient context, safe modes, fingerprinted /init, model benchmarking, web access, and browser automation.",
   "repository": {
     "type": "git",
     "url": "https://github.com/linellazatin/opl-pi-sht.git"