@graphai/puppeteer_agent 0.0.2 → 0.0.4

package/lib/puppeteer_agent.js

@@ -7,28 +7,70 @@ exports.puppeteerAgent = void 0;
 const puppeteer_1 = __importDefault(require("puppeteer"));
 const readability_1 = require("@mozilla/readability");
 const jsdom_1 = require("jsdom");
-const puppeteerAgent = async ({ namedInputs, params }) => {
-    const { url } = namedInputs;
+const NAV_TIMEOUT = 45_000;
+const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
+const normalize = (s) => s
+    .replace(/\r\n/g, "\n")
+    .replace(/[\n\t]{2,}/g, "\n")
+    .trim();
+/* global document */
+async function waitStable(page, ms = 1200, step = 200) {
+    let last = -1;
+    let stable = 0;
+    while (stable < ms) {
+        const len = await page.evaluate(() => document.body?.innerText?.length || 0);
+        stable = len === last ? stable + step : 0;
+        last = len;
+        await new Promise((r) => setTimeout(r, step));
+    }
+}
+async function fetchArticle(url) {
+    const browser = await puppeteer_1.default.launch({
+        headless: true,
+        args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"],
+    });
+    const page = await browser.newPage();
+    await page.setUserAgent(userAgent);
+    await page.setViewport({ width: 1366, height: 900 });
     try {
-        const browser = await puppeteer_1.default.launch();
-        const page = await browser.newPage();
-        //console.log(1);
-        await page.goto(url, { waitUntil: "networkidle2" });
-        // console.log(2);
+        await page.goto(url, { waitUntil: "networkidle2", timeout: NAV_TIMEOUT });
+        await Promise.race([page.waitForSelector("article, main, [role=main], .article, .post", { timeout: 8000 }), new Promise((r) => setTimeout(r, 8000))]);
+        await waitStable(page, 1200);
         const html = await page.content();
         const dom = new jsdom_1.JSDOM(html, { url: page.url() });
         const reader = new readability_1.Readability(dom.window.document);
-        const article = reader.parse();
-        // console.log(3);
-        await browser.close();
+        const a = reader.parse();
+        const title = a?.title || (await page.title()) || null;
+        const text = normalize(a?.textContent || "");
+        let finalText = text;
+        if (finalText.length < 100) {
+            const raw = await page.evaluate(() => {
+                const el = document.querySelector("article, main, [role=main], .article, .post") || document.body;
+                return el?.textContent || "";
+            });
+            finalText = normalize(raw);
+        }
+        return {
+            url,
+            title,
+            byline: a?.byline || null,
+            excerpt: a?.excerpt || null,
+            length: a?.length ?? (finalText?.length || null),
+            textContent: finalText || null,
+        };
+    }
+    finally {
+        await page.close().catch(() => { });
+        await browser.close().catch(() => { });
+    }
+}
+const puppeteerAgent = async ({ namedInputs, params }) => {
+    const { url } = namedInputs;
+    try {
+        const data = await fetchArticle(url);
         return {
-            title: article?.title,
-            content: (article?.title ?? "") +
-                "\n" +
-                (article?.textContent ?? "")
-                    .replace(/\r\n/g, "\n")
-                    .replace(/(\n[ \t]*){2,}/g, "\n")
-                    .trim(),
+            title: data?.title,
+            content: (data?.title ?? "") + "\n" + (data?.textContent ?? ""),
         };
     }
     catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graphai/puppeteer_agent",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -26,12 +26,12 @@
   },
   "homepage": "https://github.com/receptron/graphai-agents#readme",
   "devDependencies": {
-    "@types/jsdom": "^21.1.7"
+    "@types/jsdom": "^28.0.3"
   },
   "dependencies": {
     "@mozilla/readability": "^0.6.0",
-    "jsdom": "^26.1.0",
-    "puppeteer": "^24.16.1"
+    "jsdom": "^29.1.1",
+    "puppeteer": "^25.2.1"
   },
   "types": "./lib/index.d.ts",
   "directories": {