@detiq/crawler 1.1.9 → 1.1.11

package/dist/markdown-extract.js

@@ -1,67 +1,54 @@
-/**
- * HTML → Markdown extraction for crawled pages — matches crawl4AI's markdown
- * output capability. Two-tier, mirroring crawl4AI's "markdown" vs "fit markdown":
- *
- *  - "fit" markdown: Readability's best guess at the primary content (title +
- *    article body), stripped of nav/ads/chrome. Ideal for content/marketing
- *    pages and RAG chunking. null when Readability finds no article (most
- *    app UI screens — dashboards, forms — legitimately have none).
- *  - "full" markdown: the whole visible page converted straight to Markdown,
- *    always present. The fallback for app screens where there's no "article".
- *
- * Runs entirely on an already-captured `page.content()` string — no extra
- * network fetch, no extra page navigation.
- */
-import { parseHTML } from 'linkedom';
-import { Readability } from '@mozilla/readability';
-import TurndownService from 'turndown';
-const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
-turndown.remove(['script', 'style', 'noscript', 'svg', 'iframe']);
+import { parseHTML } from "linkedom";
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+turndown.remove(["script", "style", "noscript", "svg", "iframe"]);
 function stripNoiseElements(document) {
-    const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
-    noisy.forEach((el) => el.remove());
+  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
+  noisy.forEach((el) => el.remove());
 }
-/** Best-effort HTML→Markdown for one crawled page. Never throws. */
-export function extractMarkdown(html, url) {
+function extractMarkdown(html, url) {
+  try {
+    const { document } = parseHTML(html);
+    stripNoiseElements(document);
+    let readabilityResult = null;
     try {
-        const { document } = parseHTML(html);
-        stripNoiseElements(document);
-        let readabilityResult = null;
-        try {
-            // Readability mutates the DOM it's given — clone via re-parsing so the
-            // full-page fallback below still has an intact document.
-            const { document: readerDoc } = parseHTML(html);
-            stripNoiseElements(readerDoc);
-            readabilityResult = new Readability(readerDoc, { charThreshold: 50 }).parse();
-        }
-        catch {
-            readabilityResult = null;
-        }
-        if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
-            const markdown = turndown.turndown(readabilityResult.content);
-            return {
-                markdown: `# ${readabilityResult.title || document.title || url}\n\n${markdown}`,
-                method: 'readability',
-                title: readabilityResult.title || document.title || null,
-                wordCount: readabilityResult.textContent.trim().split(/\s+/).length,
-            };
-        }
-        const body = document.body;
-        if (!body)
-            return null;
-        const markdown = turndown.turndown(body.innerHTML || '');
-        const text = markdown.trim();
-        if (!text)
-            return null;
-        const title = document.title || null;
-        return {
-            markdown: title ? `# ${title}\n\n${markdown}` : markdown,
-            method: 'full-page',
-            title,
-            wordCount: text.split(/\s+/).length,
-        };
+      const { document: readerDoc } = parseHTML(html);
+      stripNoiseElements(readerDoc);
+      readabilityResult = new Readability(readerDoc, { charThreshold: 50 }).parse();
+    } catch {
+      readabilityResult = null;
     }
-    catch {
-        return null;
+    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
+      const markdown2 = turndown.turndown(readabilityResult.content);
+      return {
+        markdown: `# ${readabilityResult.title || document.title || url}
+
+${markdown2}`,
+        method: "readability",
+        title: readabilityResult.title || document.title || null,
+        wordCount: readabilityResult.textContent.trim().split(/\s+/).length
+      };
     }
+    const body = document.body;
+    if (!body) return null;
+    const markdown = turndown.turndown(body.innerHTML || "");
+    const text = markdown.trim();
+    if (!text) return null;
+    const title = document.title || null;
+    return {
+      markdown: title ? `# ${title}
+
+${markdown}` : markdown,
+      method: "full-page",
+      title,
+      wordCount: text.split(/\s+/).length
+    };
+  } catch {
+    return null;
+  }
 }
+export {
+  extractMarkdown
+};
+//# sourceMappingURL=markdown-extract.js.map