@purepageio/fetch-engines 0.9.1 → 0.10.1

package/dist/utils/render-detection.js

@@ -0,0 +1,145 @@
+const ROOT_CONTAINER_REGEX = /<div[^>]+id=["']?(?:root|app)\b["']?[^>]*>\s*(?:<!--[\s\S]*?-->\s*)*<\/div>/i;
+const HAS_ROOT_CONTAINER_REGEX = /<(?:div|main|section)[^>]+id=["']?(?:root|app)\b["']?[^>]*>/i;
+const TITLE_REGEX = /<title[^>]*>([\s\S]*?)<\/title>/i;
+const HEADING_REGEX = /<h[1-3][^>]*>/gi;
+const MAIN_LIKE_REGEX = /<(?:main|article)[^>]*>/i;
+const NOSCRIPT_ENABLE_JS_REGEX = /<noscript[\s\S]*?(enable javascript|requires javascript|javascript to run)/i;
+const SCRIPT_TAG_REGEX = /<script\b/gi;
+// Soft-block / challenge page detection
+const SOFT_BLOCK_TITLE_REGEX = /just a moment|attention required|access denied|please wait|one more step|checking your browser|security check|you have been blocked|blocked by|are you a robot/i;
+const SOFT_BLOCK_BODY_REGEX = /checking your browser|verify you.{0,10}(?:are |'re )?(?:not a )?(?:ro)?bot|verify you.{0,10}human|please complete the security check|cf-challenge|captcha-container|hcaptcha|recaptcha|cf-turnstile|enable (?:javascript|cookies) to (?:continue|access|view)|automated (?:access|request)|bot detect|suspicious activity|unusual traffic|too many requests|rate limit exceeded|we need to verify/i;
+function collapseWhitespace(value) {
+    return value.replace(/\s+/g, " ").trim();
+}
+function htmlEntityDecode(value) {
+    return value
+        .replace(/&nbsp;/gi, " ")
+        .replace(/&amp;/gi, "&")
+        .replace(/&lt;/gi, "<")
+        .replace(/&gt;/gi, ">")
+        .replace(/&quot;/gi, '"')
+        .replace(/&#39;/gi, "'");
+}
+function stripHtmlToVisibleText(html) {
+    return collapseWhitespace(htmlEntityDecode(html
+        .replace(/<!--[\s\S]*?-->/g, " ")
+        .replace(/<script[\s\S]*?<\/script>/gi, " ")
+        .replace(/<style[\s\S]*?<\/style>/gi, " ")
+        .replace(/<svg[\s\S]*?<\/svg>/gi, " ")
+        .replace(/<noscript[\s\S]*?<\/noscript>/gi, " ")
+        .replace(/<[^>]+>/g, " ")));
+}
+function stripMarkdownToVisibleText(markdown) {
+    return collapseWhitespace(markdown
+        .replace(/```[\s\S]*?```/g, " ")
+        .replace(/`[^`]*`/g, " ")
+        .replace(/!\[[^\]]*]\([^)]*\)/g, " ")
+        .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
+        .replace(/^#{1,6}\s+/gm, "")
+        .replace(/[*_~>-]/g, " "));
+}
+function scoreTextSignals(textLength, titleLength, hasMainLike, headingCount) {
+    let score = 0;
+    score += Math.min(6, Math.floor(textLength / 120));
+    score += Math.min(2, Math.floor(titleLength / 12));
+    if (hasMainLike)
+        score += 2;
+    if (headingCount > 0)
+        score += 1;
+    return score;
+}
+export function assessHtmlRenderNeed(html) {
+    const htmlLength = html.length;
+    const visibleText = stripHtmlToVisibleText(html);
+    const visibleTextLength = visibleText.length;
+    const titleMatch = html.match(TITLE_REGEX);
+    const titleLength = collapseWhitespace(htmlEntityDecode(titleMatch?.[1] || "")).length;
+    const scriptCount = (html.match(SCRIPT_TAG_REGEX) || []).length;
+    const headingCount = (html.match(HEADING_REGEX) || []).length;
+    const hasMainLike = MAIN_LIKE_REGEX.test(html);
+    const hasRootContainer = HAS_ROOT_CONTAINER_REGEX.test(html);
+    const hasEmptyRootContainer = ROOT_CONTAINER_REGEX.test(html);
+    const hasNoscriptEnableJs = NOSCRIPT_ENABLE_JS_REGEX.test(html);
+    let renderLikelyNeededScore = 0;
+    if (titleLength === 0)
+        renderLikelyNeededScore += 3;
+    if (visibleTextLength < 80)
+        renderLikelyNeededScore += 3;
+    if (hasEmptyRootContainer)
+        renderLikelyNeededScore += 3;
+    if (hasNoscriptEnableJs)
+        renderLikelyNeededScore += 2;
+    if (htmlLength < 2000)
+        renderLikelyNeededScore += 1;
+    if (scriptCount >= 3 && visibleTextLength < 200)
+        renderLikelyNeededScore += 1;
+    if (hasRootContainer && visibleTextLength < 160)
+        renderLikelyNeededScore += 1;
+    if (!hasMainLike && headingCount === 0 && visibleTextLength < 120)
+        renderLikelyNeededScore += 1;
+    let qualityScore = scoreTextSignals(visibleTextLength, titleLength, hasMainLike, headingCount);
+    if (hasEmptyRootContainer)
+        qualityScore -= 3;
+    if (titleLength === 0)
+        qualityScore -= 2;
+    if (visibleTextLength < 80)
+        qualityScore -= 2;
+    return {
+        htmlLength,
+        visibleTextLength,
+        titleLength,
+        scriptCount,
+        headingCount,
+        hasMainLike,
+        hasRootContainer,
+        hasEmptyRootContainer,
+        hasNoscriptEnableJs,
+        qualityScore,
+        renderLikelyNeededScore,
+        renderLikelyNeeded: renderLikelyNeededScore >= 4,
+    };
+}
+/**
+ * Detect if an HTTP response is a soft-block page (Cloudflare challenge, CAPTCHA,
+ * "verify you're human", etc.) that looks like a real HTML document but contains no
+ * actual page content.
+ */
+export function isSoftBlockPage(html) {
+    const visibleText = stripHtmlToVisibleText(html);
+    // Genuine content pages produce substantial text; soft blocks rarely exceed ~1500 visible chars.
+    if (visibleText.length > 1500)
+        return false;
+    const titleMatch = html.match(TITLE_REGEX);
+    const title = titleMatch?.[1] || "";
+    if (SOFT_BLOCK_TITLE_REGEX.test(title))
+        return true;
+    return SOFT_BLOCK_BODY_REGEX.test(html);
+}
+export function assessSerializedContent(content, contentType) {
+    if (contentType === "html") {
+        const assessment = assessHtmlRenderNeed(content);
+        return {
+            textLength: assessment.visibleTextLength,
+            titleLength: assessment.titleLength,
+            qualityScore: assessment.qualityScore,
+        };
+    }
+    const visibleText = stripMarkdownToVisibleText(content);
+    const firstHeadingMatch = content.match(/^#\s+(.+)$/m);
+    const titleLength = collapseWhitespace(firstHeadingMatch?.[1] || "").length;
+    return {
+        textLength: visibleText.length,
+        titleLength,
+        qualityScore: scoreTextSignals(visibleText.length, titleLength, false, firstHeadingMatch ? 1 : 0),
+    };
+}
+export function isRenderedContentMeaningfullyBetter(baseline, candidate) {
+    if (candidate.qualityScore >= baseline.qualityScore + 2)
+        return true;
+    if (candidate.textLength >= Math.max(200, baseline.textLength * 2))
+        return true;
+    if (candidate.titleLength > 0 && baseline.titleLength === 0 && candidate.textLength >= baseline.textLength)
+        return true;
+    return false;
+}
+//# sourceMappingURL=render-detection.js.map

package/dist/utils/render-detection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"render-detection.js","sourceRoot":"","sources":["../../src/utils/render-detection.ts"],"names":[],"mappings":"AAqBA,MAAM,oBAAoB,GAAG,8EAA8E,CAAC;AAC5G,MAAM,wBAAwB,GAAG,8DAA8D,CAAC;AAChG,MAAM,WAAW,GAAG,kCAAkC,CAAC;AACvD,MAAM,aAAa,GAAG,iBAAiB,CAAC;AACxC,MAAM,eAAe,GAAG,0BAA0B,CAAC;AACnD,MAAM,wBAAwB,GAAG,6EAA6E,CAAC;AAC/G,MAAM,gBAAgB,GAAG,aAAa,CAAC;AAEvC,wCAAwC;AACxC,MAAM,sBAAsB,GAC1B,iKAAiK,CAAC;AAEpK,MAAM,qBAAqB,GACzB,oYAAoY,CAAC;AAEvY,SAAS,kBAAkB,CAAC,KAAa;IACvC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAa;IACrC,OAAO,KAAK;SACT,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,kBAAkB,CACvB,gBAAgB,CACd,IAAI;SACD,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC;SAChC,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC;SAC3C,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC;SACzC,OAAO,CAAC,uBAAuB,EAAE,GAAG,CAAC;SACrC,OAAO,CAAC,iCAAiC,EAAE,GAAG,CAAC;SAC/C,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAC5B,CACF,CAAC;AACJ,CAAC;AAED,SAAS,0BAA0B,CAAC,QAAgB;IAClD,OAAO,kBAAkB,CACvB,QAAQ;SACL,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC;SACpC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC;SACvC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAC5B,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,UAAkB,EAAE,WAAmB,EAAE,WAAoB,EAAE,YAAoB;IAC3G,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC;IACnD,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC,CAAC;IACnD,IAAI,WAAW;QAAE,KAAK,IAAI,CAAC,CAAC;IAC5B,IAAI,YAAY,GAAG,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;IAC/B,MAAM,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACjD,MAAM,iBAAiB,GAAG,WAAW,CAAC,MAAM,CAAC;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,kBAAkB,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IACvF,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC9D,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,gBAAgB,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,MAAM,qBAAqB,GAAG,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9D,MAAM,mBAAmB,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEhE,IAAI,uBAAuB,GAAG,CAAC,CAAC;IAChC,IAAI,WAAW,KAAK,CAAC;QAAE,uBAAuB,IAAI,CAAC,CAAC;IACpD,IAAI,iBAAiB,GAAG,EAAE;QAAE,uBAAuB,IAAI,CAAC,CAAC;IACzD,IAAI,qBAAqB;QAAE,uBAAuB,IAAI,CAAC,CAAC;IACxD,IAAI,mBAAmB;QAAE,uBAAuB,IAAI,CAAC,CAAC;IACtD,IAAI,UAAU,GAAG,IAAI;QAAE,uBAAuB,IAAI,CAAC,CAAC;IACpD,IAAI,WAAW,IAAI,CAAC,IAAI,iBAAiB,GAAG,GAAG;QAAE,uBAAuB,IAAI,CAAC,CAAC;IAC9E,IAAI,gBAAgB,IAAI,iBAAiB,GAAG,GAAG;QAAE,uBAAuB,IAAI,CAAC,CAAC;IAC9E,IAAI,CAAC,WAAW,IAAI,YAAY,KAAK,CAAC,IAAI,iBAAiB,GAAG,GAAG;QAAE,uBAAuB,IAAI,CAAC,CAAC;IAEhG,IAAI,YAAY,GAAG,gBAAgB,CAAC,iBAAiB,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;IAC/F,IAAI,qBAAqB;QAAE,YAAY,IAAI,CAAC,CAAC;IAC7C,IAAI,WAAW,KAAK,CAAC;QAAE,YAAY,IAAI,CAAC,CAAC;IACzC,IAAI,iBAAiB,GAAG,EAAE;QAAE,YAAY,IAAI,CAAC,CAAC;IAE9C,OAAO;QACL,UAAU;QACV,iBAAiB;QACjB,WAAW;QACX,WAAW;QACX,YAAY;QACZ,WAAW;QACX,gBAAgB;QAChB,qBAAqB;QACrB,mBAAmB;QACnB,YAAY;QACZ,uBAAuB;QACvB,kBAAkB,EAAE,uBAAuB,IAAI,CAAC;KACjD,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACjD,iGAAiG;IACjG,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI;QAAE,OAAO,KAAK,CAAC;IAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpD,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,OAAe,EACf,WAAgC;IAEhC,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACjD,OAAO;YACL,UAAU,EAAE,UAAU,CAAC,iBAAiB;YACxC,WAAW,EAAE,UAAU,CAAC,WAAW;YACnC,YAAY,EAAE,UAAU,CAAC,YAAY;SACtC,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACvD,MAAM,WAAW,GAAG,kBAAkB,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAE5E,OAAO;QACL,UAAU,EAAE,WAAW,CAAC,MAAM;QAC9B,WAAW;QACX,YAAY,EAAE,gBAAgB,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAClG,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mCAAmC,CACjD,QAAqC,EACrC,SAAsC;IAEtC,IAAI,SAAS,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACrE,IAAI,SAAS,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAChF,IAAI,SAAS,CAAC,WAAW,GAAG,CAAC,IAAI,QAAQ,CAAC,WAAW,KAAK,CAAC,IAAI,SAAS,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU;QACxG,OAAO,IAAI,CAAC;IACd,OAAO,KAAK,CAAC;AACf,CAAC"}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@purepageio/fetch-engines",
-  "version": "0.9.1",
+  "version": "0.10.1",
   "type": "module",
-  "description": "Fetch web pages as clean Markdown or structured data. HTTP-first with automatic Playwright fallback, built for RAG pipelines and content extraction.",
+  "description": "Production-grade web extraction: clean Markdown, Playwright fallback, soft-block detection, and structured data extraction.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [
@@ -75,12 +75,15 @@
   },
   "scripts": {
     "build": "tsc",
+    "typecheck": "tsc --noEmit",
     "lint": "eslint \"src/**/*.ts\" \"examples/**/*.ts\"",
     "format": "prettier --write \"src/**/*.ts\" \"examples/**/*.ts\" \"*.{js,cjs,json,md}\"",
     "test": "vitest run",
     "test:unit": "vitest run",
     "test:live": "LIVE_NETWORK=1 vitest run test/live/*.test.ts",
+    "test:live:auto-render": "LIVE_NETWORK=1 vitest run test/live/AutoRenderHypothesis.test.ts",
     "examples:hybrid-md": "node scripts/hybrid-md-dump.mjs",
+    "eval:auto-render": "pnpm build && node scripts/eval-auto-render.mjs",
     "simple-scraping": "tsx examples/simple-scraping.ts",
     "smart-scraping": "tsx examples/smart-scraping.ts",
     "ai-extraction": "tsx examples/ai-extraction.ts"