@detiq/crawler 1.1.8 → 1.1.9

package/dist/markdown-extract.js

@@ -1,54 +1,67 @@
-import { parseHTML } from "linkedom";
-import { Readability } from "@mozilla/readability";
-import TurndownService from "turndown";
-const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
-turndown.remove(["script", "style", "noscript", "svg", "iframe"]);
+/**
+ * HTML → Markdown extraction for crawled pages — matches crawl4AI's markdown
+ * output capability. Two-tier, mirroring crawl4AI's "markdown" vs "fit markdown":
+ *
+ *  - "fit" markdown: Readability's best guess at the primary content (title +
+ *    article body), stripped of nav/ads/chrome. Ideal for content/marketing
+ *    pages and RAG chunking. null when Readability finds no article (most
+ *    app UI screens — dashboards, forms — legitimately have none).
+ *  - "full" markdown: the whole visible page converted straight to Markdown,
+ *    always present. The fallback for app screens where there's no "article".
+ *
+ * Runs entirely on an already-captured `page.content()` string — no extra
+ * network fetch, no extra page navigation.
+ */
+import { parseHTML } from 'linkedom';
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+turndown.remove(['script', 'style', 'noscript', 'svg', 'iframe']);
 function stripNoiseElements(document) {
-  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
-  noisy.forEach((el) => el.remove());
+    const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
+    noisy.forEach((el) => el.remove());
 }
-function extractMarkdown(html, url) {
-  try {
-    const { document } = parseHTML(html);
-    stripNoiseElements(document);
-    let readabilityResult = null;
+/** Best-effort HTML→Markdown for one crawled page. Never throws. */
+export function extractMarkdown(html, url) {
     try {
-      const { document: readerDoc } = parseHTML(html);
-      stripNoiseElements(readerDoc);
-      readabilityResult = new Readability(readerDoc, { charThreshold: 50 }).parse();
-    } catch {
-      readabilityResult = null;
+        const { document } = parseHTML(html);
+        stripNoiseElements(document);
+        let readabilityResult = null;
+        try {
+            // Readability mutates the DOM it's given — clone via re-parsing so the
+            // full-page fallback below still has an intact document.
+            const { document: readerDoc } = parseHTML(html);
+            stripNoiseElements(readerDoc);
+            readabilityResult = new Readability(readerDoc, { charThreshold: 50 }).parse();
+        }
+        catch {
+            readabilityResult = null;
+        }
+        if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
+            const markdown = turndown.turndown(readabilityResult.content);
+            return {
+                markdown: `# ${readabilityResult.title || document.title || url}\n\n${markdown}`,
+                method: 'readability',
+                title: readabilityResult.title || document.title || null,
+                wordCount: readabilityResult.textContent.trim().split(/\s+/).length,
+            };
+        }
+        const body = document.body;
+        if (!body)
+            return null;
+        const markdown = turndown.turndown(body.innerHTML || '');
+        const text = markdown.trim();
+        if (!text)
+            return null;
+        const title = document.title || null;
+        return {
+            markdown: title ? `# ${title}\n\n${markdown}` : markdown,
+            method: 'full-page',
+            title,
+            wordCount: text.split(/\s+/).length,
+        };
     }
-    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
-      const markdown2 = turndown.turndown(readabilityResult.content);
-      return {
-        markdown: `# ${readabilityResult.title || document.title || url}
-
-${markdown2}`,
-        method: "readability",
-        title: readabilityResult.title || document.title || null,
-        wordCount: readabilityResult.textContent.trim().split(/\s+/).length
-      };
+    catch {
+        return null;
     }
-    const body = document.body;
-    if (!body) return null;
-    const markdown = turndown.turndown(body.innerHTML || "");
-    const text = markdown.trim();
-    if (!text) return null;
-    const title = document.title || null;
-    return {
-      markdown: title ? `# ${title}
-
-${markdown}` : markdown,
-      method: "full-page",
-      title,
-      wordCount: text.split(/\s+/).length
-    };
-  } catch {
-    return null;
-  }
 }
-export {
-  extractMarkdown
-};
-//# sourceMappingURL=markdown-extract.js.map