lobster-cli 0.2.0 → 0.3.0

package/dist/lib.js

@@ -1,3 +1,211 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/browser/pdf.ts
+var pdf_exports = {};
+__export(pdf_exports, {
+  extractPdf: () => extractPdf,
+  isPdfResponse: () => isPdfResponse,
+  isPdfUrl: () => isPdfUrl,
+  tryExtractPdf: () => tryExtractPdf
+});
+import { readFileSync as readFileSync3 } from "fs";
+async function getPdfParser() {
+  if (!pdfParseFn) {
+    const mod = await import("pdf-parse");
+    const PDFParseClass = mod.PDFParse;
+    if (PDFParseClass && typeof PDFParseClass === "function") {
+      pdfParseFn = async (buffer) => {
+        const parser = new PDFParseClass(buffer);
+        return parser.parse ? await parser.parse() : parser;
+      };
+    } else {
+      pdfParseFn = mod.default || mod;
+    }
+  }
+  return pdfParseFn;
+}
+function isPdfUrl(urlOrPath) {
+  const lower = urlOrPath.toLowerCase();
+  if (lower.endsWith(".pdf")) return true;
+  if (/\/pdf\//.test(lower)) return true;
+  if (/arxiv\.org\/pdf\//.test(lower)) return true;
+  if (/[?&]format=pdf/i.test(lower)) return true;
+  if (/[?&]type=pdf/i.test(lower)) return true;
+  return false;
+}
+function isPdfResponse(contentType) {
+  return contentType.includes("application/pdf");
+}
+async function downloadPdf(url) {
+  const response = await fetch(url, {
+    headers: {
+      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      "Accept": "application/pdf,*/*"
+    },
+    redirect: "follow"
+  });
+  if (!response.ok) {
+    throw new Error(`Failed to download PDF: ${response.status} ${response.statusText}`);
+  }
+  const contentType = response.headers.get("content-type") || "";
+  if (!contentType.includes("pdf") && !isPdfUrl(url)) {
+  }
+  const arrayBuffer = await response.arrayBuffer();
+  return Buffer.from(arrayBuffer);
+}
+function readLocalPdf(filePath) {
+  return readFileSync3(filePath);
+}
+function textToMarkdown(text, metadata) {
+  const lines = text.split("\n");
+  const mdLines = [];
+  if (metadata.title && metadata.title !== "untitled") {
+    mdLines.push(`# ${metadata.title}`);
+    mdLines.push("");
+    if (metadata.author) {
+      mdLines.push(`**Authors:** ${metadata.author}`);
+      mdLines.push("");
+    }
+    mdLines.push("---");
+    mdLines.push("");
+  }
+  let inReferences = false;
+  let prevWasBlank = false;
+  let paragraphBuffer = [];
+  function flushParagraph() {
+    if (paragraphBuffer.length > 0) {
+      mdLines.push(paragraphBuffer.join(" "));
+      mdLines.push("");
+      paragraphBuffer = [];
+    }
+  }
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line) {
+      if (!prevWasBlank) {
+        flushParagraph();
+      }
+      prevWasBlank = true;
+      continue;
+    }
+    prevWasBlank = false;
+    const numberedHeading = line.match(/^(\d+\.?\d*\.?\d*)\s+([A-Z][A-Za-z\s:&-]+)$/);
+    if (numberedHeading && line.length < 80) {
+      flushParagraph();
+      const depth = numberedHeading[1].split(".").filter(Boolean).length;
+      const prefix = depth <= 1 ? "##" : depth === 2 ? "###" : "####";
+      mdLines.push(`${prefix} ${line}`);
+      mdLines.push("");
+      continue;
+    }
+    if (line === line.toUpperCase() && line.length > 3 && line.length < 60 && /^[A-Z\s:&-]+$/.test(line)) {
+      flushParagraph();
+      mdLines.push(`## ${line.charAt(0) + line.slice(1).toLowerCase()}`);
+      mdLines.push("");
+      continue;
+    }
+    const knownHeadings = /^(Abstract|Introduction|Conclusion|Discussion|Results|Methods|Methodology|Background|Related Work|Acknowledgments|Acknowledgements|References|Bibliography|Appendix)/i;
+    if (knownHeadings.test(line) && line.length < 40) {
+      flushParagraph();
+      if (/^(References|Bibliography)/i.test(line)) {
+        inReferences = true;
+      }
+      mdLines.push(`## ${line}`);
+      mdLines.push("");
+      continue;
+    }
+    if (/^[-•∙◦▪]/.test(line)) {
+      flushParagraph();
+      mdLines.push(`- ${line.replace(/^[-•∙◦▪]\s*/, "")}`);
+      continue;
+    }
+    if (/^\(\d+\)|^[a-z]\)/.test(line)) {
+      flushParagraph();
+      mdLines.push(`- ${line}`);
+      continue;
+    }
+    if (inReferences && /^\[?\d+\]?\.?\s/.test(line)) {
+      flushParagraph();
+      mdLines.push(`- ${line}`);
+      continue;
+    }
+    if (line.endsWith("-") && i + 1 < lines.length) {
+      paragraphBuffer.push(line.slice(0, -1));
+    } else {
+      paragraphBuffer.push(line);
+    }
+  }
+  flushParagraph();
+  return mdLines.join("\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+async function extractFromBuffer(buffer) {
+  const parse = await getPdfParser();
+  const result = await parse(buffer);
+  const info = result.info || {};
+  const metadata = {
+    title: info.Title || "untitled",
+    author: info.Author || "",
+    pages: result.numpages,
+    creator: info.Creator || "",
+    producer: info.Producer || "",
+    creationDate: info.CreationDate || ""
+  };
+  const text = result.text || "";
+  const pageTexts = text.split(/\f/).filter(Boolean);
+  const pages = pageTexts.length === result.numpages ? pageTexts : [text];
+  const markdown = textToMarkdown(text, metadata);
+  const words = text.split(/\s+/).filter(Boolean);
+  return {
+    metadata,
+    text,
+    markdown,
+    pages: pages.map((p) => p.trim()),
+    wordCount: words.length,
+    charCount: text.length
+  };
+}
+async function extractPdf(urlOrPath) {
+  let buffer;
+  if (urlOrPath.startsWith("http://") || urlOrPath.startsWith("https://")) {
+    buffer = await downloadPdf(urlOrPath);
+  } else {
+    buffer = readLocalPdf(urlOrPath);
+  }
+  if (buffer[0] !== 37 || buffer[1] !== 80 || buffer[2] !== 68 || buffer[3] !== 70) {
+    throw new Error("Not a valid PDF file (invalid magic bytes)");
+  }
+  return extractFromBuffer(buffer);
+}
+async function tryExtractPdf(url) {
+  if (isPdfUrl(url)) {
+    return extractPdf(url);
+  }
+  try {
+    const head = await fetch(url, { method: "HEAD", redirect: "follow" });
+    const contentType = head.headers.get("content-type") || "";
+    if (isPdfResponse(contentType)) {
+      return extractPdf(url);
+    }
+  } catch {
+  }
+  return null;
+}
+var pdfParseFn;
+var init_pdf = __esm({
+  "src/browser/pdf.ts"() {
+    "use strict";
+    pdfParseFn = null;
+  }
+});
+
 // src/brain/index.ts
 var CLASSIFIER_PROMPT = `You are an intent classifier for a web automation tool. Given a user's question about a webpage, decide what data sources are needed to answer it.
 
@@ -263,6 +471,69 @@ var OpenAIClient = class {
       } : void 0
     };
   }
+  /**
+   * Simple vision call — send a screenshot + text prompt, get text back.
+   * Used by PDF Doctor for targeted issue resolution.
+   */
+  async chatWithVision(prompt, screenshotBase64) {
+    const headers = this.buildHeaders();
+    if (this.config.provider === "anthropic") {
+      const body2 = {
+        model: this.config.model,
+        max_tokens: 1024,
+        temperature: 0.1,
+        messages: [{
+          role: "user",
+          content: [
+            {
+              type: "image",
+              source: {
+                type: "base64",
+                media_type: "image/jpeg",
+                data: screenshotBase64
+              }
+            },
+            { type: "text", text: prompt }
+          ]
+        }]
+      };
+      const resp2 = await fetch(`${this.config.baseURL}/messages`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(body2)
+      });
+      if (!resp2.ok) throw new Error(`Anthropic vision error: ${resp2.status}`);
+      const json2 = await resp2.json();
+      const content = json2.content;
+      return content?.[0]?.text || "";
+    }
+    const body = {
+      model: this.config.model,
+      max_tokens: 1024,
+      temperature: 0.1,
+      messages: [{
+        role: "user",
+        content: [
+          {
+            type: "image_url",
+            image_url: {
+              url: `data:image/jpeg;base64,${screenshotBase64}`
+            }
+          },
+          { type: "text", text: prompt }
+        ]
+      }]
+    };
+    const resp = await fetch(`${this.config.baseURL}/chat/completions`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(body)
+    });
+    if (!resp.ok) throw new Error(`Vision API error: ${resp.status}`);
+    const json = await resp.json();
+    const choice = json.choices?.[0];
+    return choice?.message?.content || "";
+  }
 };
 
 // src/llm/utils.ts
@@ -3274,6 +3545,43 @@ async function lobsterFetch(url, options) {
   const timeout = options?.timeout || 3e4;
   const dump = options?.dump || "markdown";
   const start = Date.now();
+  const { isPdfUrl: isPdfUrl2, isPdfResponse: isPdfResponse2, extractPdf: extractPdf2 } = await Promise.resolve().then(() => (init_pdf(), pdf_exports));
+  if (isPdfUrl2(url)) {
+    const pdfResult = await extractPdf2(url);
+    const duration2 = Date.now() - start;
+    let content2;
+    switch (dump) {
+      case "markdown":
+        content2 = pdfResult.markdown;
+        break;
+      case "text":
+        content2 = pdfResult.text;
+        break;
+      case "html":
+        content2 = `<pre>${pdfResult.text}</pre>`;
+        break;
+      case "snapshot":
+        content2 = `[PDF] ${pdfResult.metadata.title} (${pdfResult.metadata.pages} pages, ${pdfResult.wordCount} words)
+
+${pdfResult.text.slice(0, 5e3)}`;
+        break;
+      case "links":
+        content2 = "";
+        break;
+      default:
+        content2 = pdfResult.markdown;
+    }
+    return {
+      url,
+      finalUrl: url,
+      title: pdfResult.metadata.title,
+      content: content2,
+      links: [],
+      engine: "pdf",
+      duration: duration2,
+      statusCode: 200
+    };
+  }
   const resp = await fetch(url, {
     headers: {
       "User-Agent": "LobsterCLI/0.1 (+https://github.com/iexcalibur/lobster-cli)",
@@ -3287,6 +3595,48 @@ async function lobsterFetch(url, options) {
   if (!resp.ok) {
     throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
   }
+  const contentType = resp.headers.get("content-type") || "";
+  if (isPdfResponse2(contentType)) {
+    const arrayBuffer = await resp.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+    const pdfMod = await import("pdf-parse");
+    const pdfParseFn2 = pdfMod.PDFParse || pdfMod.default || pdfMod;
+    const pdfResult = await pdfParseFn2(buffer);
+    const info = pdfResult.info || {};
+    const metadata = {
+      title: info.Title || "untitled",
+      author: info.Author || "",
+      pages: pdfResult.numpages
+    };
+    const duration2 = Date.now() - start;
+    const text = pdfResult.text || "";
+    let content2;
+    switch (dump) {
+      case "text":
+        content2 = text;
+        break;
+      case "html":
+        content2 = `<pre>${text}</pre>`;
+        break;
+      case "snapshot":
+        content2 = `[PDF] ${metadata.title} (${metadata.pages} pages)
+
+${text.slice(0, 5e3)}`;
+        break;
+      default:
+        content2 = text;
+    }
+    return {
+      url,
+      finalUrl: resp.url || url,
+      title: metadata.title,
+      content: content2,
+      links: [],
+      engine: "pdf",
+      duration: duration2,
+      statusCode: 200
+    };
+  }
   const html = await resp.text();
   const duration = Date.now() - start;
   const finalUrl = resp.url || url;
@@ -4268,7 +4618,7 @@ function makeRoutingDecision(request) {
 }
 
 // src/agent/core.ts
-import { readFileSync as readFileSync3 } from "fs";
+import { readFileSync as readFileSync4 } from "fs";
 import { join as join4, dirname } from "path";
 import { fileURLToPath } from "url";
 
@@ -4661,7 +5011,7 @@ var AgentCore = class {
     const macroTool = packMacroTool(tools);
     let systemPrompt;
     try {
-      systemPrompt = readFileSync3(join4(__dirname, "prompts", "system.md"), "utf-8");
+      systemPrompt = readFileSync4(join4(__dirname, "prompts", "system.md"), "utf-8");
     } catch {
       systemPrompt = "You are an AI web agent that navigates web pages to complete tasks.";
     }