@aria-cli/tools 1.0.19 → 1.0.31

package/dist/extraction/content-extraction.js

@@ -0,0 +1,248 @@
+/**
+ * Content Extraction — HTML to Markdown conversion with article detection
+ *
+ * Three-tier fallback strategy:
+ * 1. Readability.js (Mozilla) + Turndown for article-like content
+ * 2. Turndown raw HTML if Readability fails
+ * 3. Regex-based strip if both fail
+ */
+// Lazy-load jsdom — heavy dependency only needed for HTML parsing
+async function loadJSDOM() {
+    try {
+        const { JSDOM } = await import("jsdom");
+        return JSDOM;
+    }
+    catch {
+        throw new Error("HTML content extraction requires jsdom (optional dependency).\n" +
+            "Install it with: npm install jsdom\n" +
+            "The browse/web_fetch tools will fall back to regex-based extraction.");
+    }
+}
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+/** Maximum content length (50K chars) */
+const MAX_CONTENT_LENGTH = 50_000;
+/** Timeout for Readability parse in milliseconds */
+const PARSE_TIMEOUT_MS = 10_000;
+/** Content types that should be parsed as HTML */
+const HTML_CONTENT_TYPES = ["text/html", "text/xhtml+xml", "application/xhtml+xml"];
+// Simple concurrency semaphore for JSDOM memory protection
+const configuredConcurrentExtractions = Number.parseInt(process.env.ARIA_MAX_CONCURRENT_EXTRACTIONS ?? "3", 10);
+const MAX_CONCURRENT_EXTRACTIONS = Number.isFinite(configuredConcurrentExtractions) && configuredConcurrentExtractions > 0
+    ? configuredConcurrentExtractions
+    : 3;
+let activeExtractions = 0;
+const extractionQueue = [];
+async function acquireExtractionSlot() {
+    if (activeExtractions < MAX_CONCURRENT_EXTRACTIONS) {
+        activeExtractions++;
+        return;
+    }
+    return new Promise((resolve) => {
+        extractionQueue.push(() => {
+            activeExtractions++;
+            resolve();
+        });
+    });
+}
+function releaseExtractionSlot() {
+    activeExtractions--;
+    const next = extractionQueue.shift();
+    if (next)
+        next();
+}
+/**
+ * Checks whether a Content-Type header value represents HTML content
+ * that should be processed through Readability/Turndown extraction.
+ */
+export function isHtmlContentType(contentType) {
+    if (!contentType)
+        return true; // Default to HTML extraction if no Content-Type
+    const [mime] = contentType.toLowerCase().split(";");
+    const lower = (mime ?? "").trim();
+    return HTML_CONTENT_TYPES.includes(lower);
+}
+/**
+ * Extracts content from a response body, checking Content-Type first.
+ * Non-HTML content types (JSON, PDF, images, etc.) are returned as-is
+ * or with a descriptive message rather than being fed through Readability.
+ *
+ * @param body - The response body text
+ * @param url - The source URL
+ * @param contentType - The Content-Type header value (optional)
+ * @returns Extracted content
+ */
+export async function extractFromResponse(body, url, contentType) {
+    if (!isHtmlContentType(contentType)) {
+        // Non-HTML content: return raw text truncated to limit
+        const [rawMimeType] = contentType?.split(";") ?? [];
+        const mimeType = (rawMimeType ?? "unknown").trim() || "unknown";
+        const truncated = body.slice(0, MAX_CONTENT_LENGTH);
+        return {
+            title: "",
+            content: truncated || `[Non-HTML content: ${mimeType}]`,
+            isArticle: false,
+        };
+    }
+    return extractContent(body, url);
+}
+/**
+ * Wraps a synchronous/async operation with a timeout.
+ * Returns the result if completed within the timeout, or rejects.
+ */
+function withTimeout(promise, ms, label) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            reject(new Error(`${label} timed out after ${ms}ms`));
+        }, ms);
+        promise.then((val) => {
+            clearTimeout(timer);
+            resolve(val);
+        }, (err) => {
+            clearTimeout(timer);
+            reject(err);
+        });
+    });
+}
+function extractOversizedHtmlTitle(html) {
+    const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    return match?.[1]?.replace(/\s+/g, " ").trim() ?? "";
+}
+/**
+ * Extracts content from HTML and converts to Markdown.
+ *
+ * Strategy:
+ * 1. Try Readability.js (article detection) + Turndown
+ * 2. Fall back to Turndown on raw HTML
+ * 3. Fall back to regex-based tag stripping
+ *
+ * @param html - The HTML content to extract from
+ * @param url - The source URL (for link resolution)
+ * @returns Extracted content with title and article detection
+ */
+export async function extractContent(html, url) {
+    // Handle empty input
+    if (!html || html.trim().length === 0) {
+        return { title: "", content: "", isArticle: false };
+    }
+    // Oversized pages will be truncated anyway, so skip expensive DOM/readability
+    // work and strip directly from the raw response body.
+    if (html.length > MAX_CONTENT_LENGTH * 2) {
+        return {
+            title: extractOversizedHtmlTitle(html),
+            content: stripHtmlTags(html).slice(0, MAX_CONTENT_LENGTH),
+            isArticle: false,
+        };
+    }
+    await acquireExtractionSlot();
+    try {
+        // Parse HTML with JSDOM
+        const JSDOM = await loadJSDOM();
+        const dom = new JSDOM(html, { url });
+        try {
+            const document = dom.window.document;
+            // Extract title
+            const title = document.querySelector("title")?.textContent?.trim() || "";
+            // Detect semantic article markers before Readability mutates the DOM
+            const hasArticleMarkers = !!document.querySelector("article, [role='article'], [role='main'], [itemtype*='Article']");
+            // Initialize Turndown for Markdown conversion
+            const turndown = new TurndownService({
+                headingStyle: "atx",
+                codeBlockStyle: "fenced",
+            });
+            // Remove noise elements before conversion
+            turndown.remove(["script", "style", "meta", "link", "noscript"]);
+            // Tier 1: Try Readability.js for article extraction (only for article-like pages)
+            if (hasArticleMarkers) {
+                try {
+                    const reader = new Readability(document.cloneNode(true));
+                    // Wrap Readability.parse() with a timeout to prevent hung parsing on malformed HTML
+                    const article = await withTimeout(Promise.resolve(reader.parse()), PARSE_TIMEOUT_MS, "Readability.parse()");
+                    if (article && article.content) {
+                        // Readability succeeded — convert to Markdown, normalize URLs
+                        let markdown = turndown.turndown(article.content);
+                        markdown = normalizeTrailingSlashes(markdown);
+                        const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
+                        return {
+                            title: article.title || title,
+                            content: truncated,
+                            isArticle: true,
+                        };
+                    }
+                }
+                catch {
+                    // Readability failed or timed out, fall through to Tier 2
+                }
+            }
+            // Tier 2: Turndown on raw body HTML (non-article pages or Readability failure)
+            try {
+                const bodyHtml = document.body?.innerHTML || "";
+                if (bodyHtml) {
+                    let markdown = turndown.turndown(bodyHtml);
+                    markdown = normalizeTrailingSlashes(markdown);
+                    const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
+                    return {
+                        title,
+                        content: truncated,
+                        isArticle: false,
+                    };
+                }
+            }
+            catch {
+                // Turndown failed, fall through to Tier 3
+            }
+            // Tier 3: Regex-based fallback (strip tags, extract text)
+            const textContent = stripHtmlTags(html);
+            const truncated = textContent.slice(0, MAX_CONTENT_LENGTH);
+            return {
+                title,
+                content: truncated,
+                isArticle: false,
+            };
+        }
+        finally {
+            dom.window.close();
+        }
+    }
+    catch {
+        // Catastrophic failure — return empty result
+        return { title: "", content: "", isArticle: false };
+    }
+    finally {
+        releaseExtractionSlot();
+    }
+}
+/**
+ * Remove trailing slashes from URLs in markdown links that were added by JSDOM normalization.
+ * JSDOM normalizes `https://example.com` → `https://example.com/`, breaking exact URL matches.
+ */
+function normalizeTrailingSlashes(markdown) {
+    return markdown.replace(/\]\(([^)]+?)\/\)/g, (match, url) => {
+        // Only strip trailing slash from URLs that are just a domain (no path)
+        try {
+            const parsed = new URL(url + "/");
+            if (parsed.pathname === "/") {
+                return `](${url})`;
+            }
+        }
+        catch {
+            // Not a valid URL, leave as-is
+        }
+        return match;
+    });
+}
+/**
+ * Strips HTML tags using regex (last resort fallback)
+ */
+function stripHtmlTags(html) {
+    return (html
+        // Remove script and style tags and their content
+        .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "")
+        .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "")
+        // Remove all other HTML tags
+        .replace(/<[^>]+>/g, " ")
+        // Collapse multiple whitespace
+        .replace(/\s+/g, " ")
+        .trim());
+}
+//# sourceMappingURL=content-extraction.js.map

package/dist/extraction/index.js

@@ -0,0 +1,5 @@
+/**
+ * Content extraction utilities
+ */
+export { extractContent } from "./content-extraction.js";
+//# sourceMappingURL=index.js.map