lobster-cli 0.2.0 → 0.3.0

package/dist/browser/index.js

@@ -1,3 +1,211 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/browser/pdf.ts
+var pdf_exports = {};
+__export(pdf_exports, {
+  extractPdf: () => extractPdf,
+  isPdfResponse: () => isPdfResponse,
+  isPdfUrl: () => isPdfUrl,
+  tryExtractPdf: () => tryExtractPdf
+});
+import { readFileSync as readFileSync3 } from "fs";
+async function getPdfParser() {
+  if (!pdfParseFn) {
+    const mod = await import("pdf-parse");
+    const PDFParseClass = mod.PDFParse;
+    if (PDFParseClass && typeof PDFParseClass === "function") {
+      pdfParseFn = async (buffer) => {
+        const parser = new PDFParseClass(buffer);
+        return parser.parse ? await parser.parse() : parser;
+      };
+    } else {
+      pdfParseFn = mod.default || mod;
+    }
+  }
+  return pdfParseFn;
+}
+function isPdfUrl(urlOrPath) {
+  const lower = urlOrPath.toLowerCase();
+  if (lower.endsWith(".pdf")) return true;
+  if (/\/pdf\//.test(lower)) return true;
+  if (/arxiv\.org\/pdf\//.test(lower)) return true;
+  if (/[?&]format=pdf/i.test(lower)) return true;
+  if (/[?&]type=pdf/i.test(lower)) return true;
+  return false;
+}
+function isPdfResponse(contentType) {
+  return contentType.includes("application/pdf");
+}
+async function downloadPdf(url) {
+  const response = await fetch(url, {
+    headers: {
+      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      "Accept": "application/pdf,*/*"
+    },
+    redirect: "follow"
+  });
+  if (!response.ok) {
+    throw new Error(`Failed to download PDF: ${response.status} ${response.statusText}`);
+  }
+  const contentType = response.headers.get("content-type") || "";
+  if (!contentType.includes("pdf") && !isPdfUrl(url)) {
+  }
+  const arrayBuffer = await response.arrayBuffer();
+  return Buffer.from(arrayBuffer);
+}
+function readLocalPdf(filePath) {
+  return readFileSync3(filePath);
+}
+function textToMarkdown(text, metadata) {
+  const lines = text.split("\n");
+  const mdLines = [];
+  if (metadata.title && metadata.title !== "untitled") {
+    mdLines.push(`# ${metadata.title}`);
+    mdLines.push("");
+    if (metadata.author) {
+      mdLines.push(`**Authors:** ${metadata.author}`);
+      mdLines.push("");
+    }
+    mdLines.push("---");
+    mdLines.push("");
+  }
+  let inReferences = false;
+  let prevWasBlank = false;
+  let paragraphBuffer = [];
+  function flushParagraph() {
+    if (paragraphBuffer.length > 0) {
+      mdLines.push(paragraphBuffer.join(" "));
+      mdLines.push("");
+      paragraphBuffer = [];
+    }
+  }
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line) {
+      if (!prevWasBlank) {
+        flushParagraph();
+      }
+      prevWasBlank = true;
+      continue;
+    }
+    prevWasBlank = false;
+    const numberedHeading = line.match(/^(\d+\.?\d*\.?\d*)\s+([A-Z][A-Za-z\s:&-]+)$/);
+    if (numberedHeading && line.length < 80) {
+      flushParagraph();
+      const depth = numberedHeading[1].split(".").filter(Boolean).length;
+      const prefix = depth <= 1 ? "##" : depth === 2 ? "###" : "####";
+      mdLines.push(`${prefix} ${line}`);
+      mdLines.push("");
+      continue;
+    }
+    if (line === line.toUpperCase() && line.length > 3 && line.length < 60 && /^[A-Z\s:&-]+$/.test(line)) {
+      flushParagraph();
+      mdLines.push(`## ${line.charAt(0) + line.slice(1).toLowerCase()}`);
+      mdLines.push("");
+      continue;
+    }
+    const knownHeadings = /^(Abstract|Introduction|Conclusion|Discussion|Results|Methods|Methodology|Background|Related Work|Acknowledgments|Acknowledgements|References|Bibliography|Appendix)/i;
+    if (knownHeadings.test(line) && line.length < 40) {
+      flushParagraph();
+      if (/^(References|Bibliography)/i.test(line)) {
+        inReferences = true;
+      }
+      mdLines.push(`## ${line}`);
+      mdLines.push("");
+      continue;
+    }
+    if (/^[-•∙◦▪]/.test(line)) {
+      flushParagraph();
+      mdLines.push(`- ${line.replace(/^[-•∙◦▪]\s*/, "")}`);
+      continue;
+    }
+    if (/^\(\d+\)|^[a-z]\)/.test(line)) {
+      flushParagraph();
+      mdLines.push(`- ${line}`);
+      continue;
+    }
+    if (inReferences && /^\[?\d+\]?\.?\s/.test(line)) {
+      flushParagraph();
+      mdLines.push(`- ${line}`);
+      continue;
+    }
+    if (line.endsWith("-") && i + 1 < lines.length) {
+      paragraphBuffer.push(line.slice(0, -1));
+    } else {
+      paragraphBuffer.push(line);
+    }
+  }
+  flushParagraph();
+  return mdLines.join("\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+async function extractFromBuffer(buffer) {
+  const parse = await getPdfParser();
+  const result = await parse(buffer);
+  const info = result.info || {};
+  const metadata = {
+    title: info.Title || "untitled",
+    author: info.Author || "",
+    pages: result.numpages,
+    creator: info.Creator || "",
+    producer: info.Producer || "",
+    creationDate: info.CreationDate || ""
+  };
+  const text = result.text || "";
+  const pageTexts = text.split(/\f/).filter(Boolean);
+  const pages = pageTexts.length === result.numpages ? pageTexts : [text];
+  const markdown = textToMarkdown(text, metadata);
+  const words = text.split(/\s+/).filter(Boolean);
+  return {
+    metadata,
+    text,
+    markdown,
+    pages: pages.map((p) => p.trim()),
+    wordCount: words.length,
+    charCount: text.length
+  };
+}
+async function extractPdf(urlOrPath) {
+  let buffer;
+  if (urlOrPath.startsWith("http://") || urlOrPath.startsWith("https://")) {
+    buffer = await downloadPdf(urlOrPath);
+  } else {
+    buffer = readLocalPdf(urlOrPath);
+  }
+  if (buffer[0] !== 37 || buffer[1] !== 80 || buffer[2] !== 68 || buffer[3] !== 70) {
+    throw new Error("Not a valid PDF file (invalid magic bytes)");
+  }
+  return extractFromBuffer(buffer);
+}
+async function tryExtractPdf(url) {
+  if (isPdfUrl(url)) {
+    return extractPdf(url);
+  }
+  try {
+    const head = await fetch(url, { method: "HEAD", redirect: "follow" });
+    const contentType = head.headers.get("content-type") || "";
+    if (isPdfResponse(contentType)) {
+      return extractPdf(url);
+    }
+  } catch {
+  }
+  return null;
+}
+var pdfParseFn;
+var init_pdf = __esm({
+  "src/browser/pdf.ts"() {
+    "use strict";
+    pdfParseFn = null;
+  }
+});
+
 // src/browser/manager.ts
 import puppeteer from "puppeteer-core";
 import { existsSync as existsSync3 } from "fs";
@@ -2782,6 +2990,43 @@ async function lobsterFetch(url, options) {
   const timeout = options?.timeout || 3e4;
   const dump = options?.dump || "markdown";
   const start = Date.now();
+  const { isPdfUrl: isPdfUrl2, isPdfResponse: isPdfResponse2, extractPdf: extractPdf2 } = await Promise.resolve().then(() => (init_pdf(), pdf_exports));
+  if (isPdfUrl2(url)) {
+    const pdfResult = await extractPdf2(url);
+    const duration2 = Date.now() - start;
+    let content2;
+    switch (dump) {
+      case "markdown":
+        content2 = pdfResult.markdown;
+        break;
+      case "text":
+        content2 = pdfResult.text;
+        break;
+      case "html":
+        content2 = `<pre>${pdfResult.text}</pre>`;
+        break;
+      case "snapshot":
+        content2 = `[PDF] ${pdfResult.metadata.title} (${pdfResult.metadata.pages} pages, ${pdfResult.wordCount} words)
+
+${pdfResult.text.slice(0, 5e3)}`;
+        break;
+      case "links":
+        content2 = "";
+        break;
+      default:
+        content2 = pdfResult.markdown;
+    }
+    return {
+      url,
+      finalUrl: url,
+      title: pdfResult.metadata.title,
+      content: content2,
+      links: [],
+      engine: "pdf",
+      duration: duration2,
+      statusCode: 200
+    };
+  }
   const resp = await fetch(url, {
     headers: {
       "User-Agent": "LobsterCLI/0.1 (+https://github.com/iexcalibur/lobster-cli)",
@@ -2795,6 +3040,48 @@ async function lobsterFetch(url, options) {
   if (!resp.ok) {
     throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
   }
+  const contentType = resp.headers.get("content-type") || "";
+  if (isPdfResponse2(contentType)) {
+    const arrayBuffer = await resp.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+    const pdfMod = await import("pdf-parse");
+    const pdfParseFn2 = pdfMod.PDFParse || pdfMod.default || pdfMod;
+    const pdfResult = await pdfParseFn2(buffer);
+    const info = pdfResult.info || {};
+    const metadata = {
+      title: info.Title || "untitled",
+      author: info.Author || "",
+      pages: pdfResult.numpages
+    };
+    const duration2 = Date.now() - start;
+    const text = pdfResult.text || "";
+    let content2;
+    switch (dump) {
+      case "text":
+        content2 = text;
+        break;
+      case "html":
+        content2 = `<pre>${text}</pre>`;
+        break;
+      case "snapshot":
+        content2 = `[PDF] ${metadata.title} (${metadata.pages} pages)
+
+${text.slice(0, 5e3)}`;
+        break;
+      default:
+        content2 = text;
+    }
+    return {
+      url,
+      finalUrl: resp.url || url,
+      title: metadata.title,
+      content: content2,
+      links: [],
+      engine: "pdf",
+      duration: duration2,
+      statusCode: 200
+    };
+  }
   const html = await resp.text();
   const duration = Date.now() - start;
   const finalUrl = resp.url || url;
@@ -2834,6 +3121,9 @@ async function lobsterFetch(url, options) {
   }
   return { url, finalUrl, status: resp.status, title, content, links, duration };
 }
+
+// src/browser/index.ts
+init_pdf();
 export {
   BrowserManager,
   COMPACT_SNAPSHOT_SCRIPT,
@@ -2854,11 +3144,14 @@ export {
   discoverChrome,
   extractLinks,
   extractMarkdown,
+  extractPdf,
   extractSnapshot,
   extractText,
   flatTreeToString,
   getProfileDataDir,
   injectStealth,
+  isPdfResponse,
+  isPdfUrl,
   listProfiles,
   lobsterFetch,
   parseHtml,
@@ -2866,6 +3159,7 @@ export {
   resetProfileCache,
   resolveAttachTarget,
   semanticFind,
+  tryExtractPdf,
   waitForCondition
 };
 //# sourceMappingURL=index.js.map