@aria-cli/tools 1.0.12 → 1.0.14

package/dist-cjs/extraction/content-extraction.js

@@ -1,243 +0,0 @@
-"use strict";
-/**
- * Content Extraction — HTML to Markdown conversion with article detection
- *
- * Three-tier fallback strategy:
- * 1. Readability.js (Mozilla) + Turndown for article-like content
- * 2. Turndown raw HTML if Readability fails
- * 3. Regex-based strip if both fail
- */
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.isHtmlContentType = isHtmlContentType;
-exports.extractFromResponse = extractFromResponse;
-exports.extractContent = extractContent;
-const jsdom_1 = require("jsdom");
-const readability_1 = require("@mozilla/readability");
-const turndown_1 = __importDefault(require("turndown"));
-/** Maximum content length (50K chars) */
-const MAX_CONTENT_LENGTH = 50_000;
-/** Timeout for Readability parse in milliseconds */
-const PARSE_TIMEOUT_MS = 10_000;
-/** Content types that should be parsed as HTML */
-const HTML_CONTENT_TYPES = ["text/html", "text/xhtml+xml", "application/xhtml+xml"];
-// Simple concurrency semaphore for JSDOM memory protection
-const configuredConcurrentExtractions = Number.parseInt(process.env.ARIA_MAX_CONCURRENT_EXTRACTIONS ?? "3", 10);
-const MAX_CONCURRENT_EXTRACTIONS = Number.isFinite(configuredConcurrentExtractions) && configuredConcurrentExtractions > 0
-    ? configuredConcurrentExtractions
-    : 3;
-let activeExtractions = 0;
-const extractionQueue = [];
-async function acquireExtractionSlot() {
-    if (activeExtractions < MAX_CONCURRENT_EXTRACTIONS) {
-        activeExtractions++;
-        return;
-    }
-    return new Promise((resolve) => {
-        extractionQueue.push(() => {
-            activeExtractions++;
-            resolve();
-        });
-    });
-}
-function releaseExtractionSlot() {
-    activeExtractions--;
-    const next = extractionQueue.shift();
-    if (next)
-        next();
-}
-/**
- * Checks whether a Content-Type header value represents HTML content
- * that should be processed through Readability/Turndown extraction.
- */
-function isHtmlContentType(contentType) {
-    if (!contentType)
-        return true; // Default to HTML extraction if no Content-Type
-    const [mime] = contentType.toLowerCase().split(";");
-    const lower = (mime ?? "").trim();
-    return HTML_CONTENT_TYPES.includes(lower);
-}
-/**
- * Extracts content from a response body, checking Content-Type first.
- * Non-HTML content types (JSON, PDF, images, etc.) are returned as-is
- * or with a descriptive message rather than being fed through Readability.
- *
- * @param body - The response body text
- * @param url - The source URL
- * @param contentType - The Content-Type header value (optional)
- * @returns Extracted content
- */
-async function extractFromResponse(body, url, contentType) {
-    if (!isHtmlContentType(contentType)) {
-        // Non-HTML content: return raw text truncated to limit
-        const [rawMimeType] = contentType?.split(";") ?? [];
-        const mimeType = (rawMimeType ?? "unknown").trim() || "unknown";
-        const truncated = body.slice(0, MAX_CONTENT_LENGTH);
-        return {
-            title: "",
-            content: truncated || `[Non-HTML content: ${mimeType}]`,
-            isArticle: false,
-        };
-    }
-    return extractContent(body, url);
-}
-/**
- * Wraps a synchronous/async operation with a timeout.
- * Returns the result if completed within the timeout, or rejects.
- */
-function withTimeout(promise, ms, label) {
-    return new Promise((resolve, reject) => {
-        const timer = setTimeout(() => {
-            reject(new Error(`${label} timed out after ${ms}ms`));
-        }, ms);
-        promise.then((val) => {
-            clearTimeout(timer);
-            resolve(val);
-        }, (err) => {
-            clearTimeout(timer);
-            reject(err);
-        });
-    });
-}
-function extractOversizedHtmlTitle(html) {
-    const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-    return match?.[1]?.replace(/\s+/g, " ").trim() ?? "";
-}
-/**
- * Extracts content from HTML and converts to Markdown.
- *
- * Strategy:
- * 1. Try Readability.js (article detection) + Turndown
- * 2. Fall back to Turndown on raw HTML
- * 3. Fall back to regex-based tag stripping
- *
- * @param html - The HTML content to extract from
- * @param url - The source URL (for link resolution)
- * @returns Extracted content with title and article detection
- */
-async function extractContent(html, url) {
-    // Handle empty input
-    if (!html || html.trim().length === 0) {
-        return { title: "", content: "", isArticle: false };
-    }
-    // Oversized pages will be truncated anyway, so skip expensive DOM/readability
-    // work and strip directly from the raw response body.
-    if (html.length > MAX_CONTENT_LENGTH * 2) {
-        return {
-            title: extractOversizedHtmlTitle(html),
-            content: stripHtmlTags(html).slice(0, MAX_CONTENT_LENGTH),
-            isArticle: false,
-        };
-    }
-    await acquireExtractionSlot();
-    try {
-        // Parse HTML with JSDOM
-        const dom = new jsdom_1.JSDOM(html, { url });
-        try {
-            const document = dom.window.document;
-            // Extract title
-            const title = document.querySelector("title")?.textContent?.trim() || "";
-            // Detect semantic article markers before Readability mutates the DOM
-            const hasArticleMarkers = !!document.querySelector("article, [role='article'], [role='main'], [itemtype*='Article']");
-            // Initialize Turndown for Markdown conversion
-            const turndown = new turndown_1.default({
-                headingStyle: "atx",
-                codeBlockStyle: "fenced",
-            });
-            // Remove noise elements before conversion
-            turndown.remove(["script", "style", "meta", "link", "noscript"]);
-            // Tier 1: Try Readability.js for article extraction (only for article-like pages)
-            if (hasArticleMarkers) {
-                try {
-                    const reader = new readability_1.Readability(document.cloneNode(true));
-                    // Wrap Readability.parse() with a timeout to prevent hung parsing on malformed HTML
-                    const article = await withTimeout(Promise.resolve(reader.parse()), PARSE_TIMEOUT_MS, "Readability.parse()");
-                    if (article && article.content) {
-                        // Readability succeeded — convert to Markdown, normalize URLs
-                        let markdown = turndown.turndown(article.content);
-                        markdown = normalizeTrailingSlashes(markdown);
-                        const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
-                        return {
-                            title: article.title || title,
-                            content: truncated,
-                            isArticle: true,
-                        };
-                    }
-                }
-                catch {
-                    // Readability failed or timed out, fall through to Tier 2
-                }
-            }
-            // Tier 2: Turndown on raw body HTML (non-article pages or Readability failure)
-            try {
-                const bodyHtml = document.body?.innerHTML || "";
-                if (bodyHtml) {
-                    let markdown = turndown.turndown(bodyHtml);
-                    markdown = normalizeTrailingSlashes(markdown);
-                    const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
-                    return {
-                        title,
-                        content: truncated,
-                        isArticle: false,
-                    };
-                }
-            }
-            catch {
-                // Turndown failed, fall through to Tier 3
-            }
-            // Tier 3: Regex-based fallback (strip tags, extract text)
-            const textContent = stripHtmlTags(html);
-            const truncated = textContent.slice(0, MAX_CONTENT_LENGTH);
-            return {
-                title,
-                content: truncated,
-                isArticle: false,
-            };
-        }
-        finally {
-            dom.window.close();
-        }
-    }
-    catch {
-        // Catastrophic failure — return empty result
-        return { title: "", content: "", isArticle: false };
-    }
-    finally {
-        releaseExtractionSlot();
-    }
-}
-/**
- * Remove trailing slashes from URLs in markdown links that were added by JSDOM normalization.
- * JSDOM normalizes `https://example.com` → `https://example.com/`, breaking exact URL matches.
- */
-function normalizeTrailingSlashes(markdown) {
-    return markdown.replace(/\]\(([^)]+?)\/\)/g, (match, url) => {
-        // Only strip trailing slash from URLs that are just a domain (no path)
-        try {
-            const parsed = new URL(url + "/");
-            if (parsed.pathname === "/") {
-                return `](${url})`;
-            }
-        }
-        catch {
-            // Not a valid URL, leave as-is
-        }
-        return match;
-    });
-}
-/**
- * Strips HTML tags using regex (last resort fallback)
- */
-function stripHtmlTags(html) {
-    return (html
-        // Remove script and style tags and their content
-        .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "")
-        .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "")
-        // Remove all other HTML tags
-        .replace(/<[^>]+>/g, " ")
-        // Collapse multiple whitespace
-        .replace(/\s+/g, " ")
-        .trim());
-}

package/dist-cjs/extraction/index.js

@@ -1,8 +0,0 @@
-"use strict";
-/**
- * Content extraction utilities
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractContent = void 0;
-var content_extraction_js_1 = require("./content-extraction.js");
-Object.defineProperty(exports, "extractContent", { enumerable: true, get: function () { return content_extraction_js_1.extractContent; } });