@aria-cli/tools 1.0.0

package/dist/extraction/content-extraction.js

@@ -0,0 +1,236 @@
+/**
+ * Content Extraction — HTML to Markdown conversion with article detection
+ *
+ * Three-tier fallback strategy:
+ * 1. Readability.js (Mozilla) + Turndown for article-like content
+ * 2. Turndown raw HTML if Readability fails
+ * 3. Regex-based strip if both fail
+ */
+import { JSDOM } from "jsdom";
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+/** Maximum content length (50K chars) */
+const MAX_CONTENT_LENGTH = 50_000;
+/** Timeout for Readability parse in milliseconds */
+const PARSE_TIMEOUT_MS = 10_000;
+/** Content types that should be parsed as HTML */
+const HTML_CONTENT_TYPES = ["text/html", "text/xhtml+xml", "application/xhtml+xml"];
+// Simple concurrency semaphore for JSDOM memory protection
+const configuredConcurrentExtractions = Number.parseInt(process.env.ARIA_MAX_CONCURRENT_EXTRACTIONS ?? "3", 10);
+const MAX_CONCURRENT_EXTRACTIONS = Number.isFinite(configuredConcurrentExtractions) && configuredConcurrentExtractions > 0
+    ? configuredConcurrentExtractions
+    : 3;
+let activeExtractions = 0;
+const extractionQueue = [];
+async function acquireExtractionSlot() {
+    if (activeExtractions < MAX_CONCURRENT_EXTRACTIONS) {
+        activeExtractions++;
+        return;
+    }
+    return new Promise((resolve) => {
+        extractionQueue.push(() => {
+            activeExtractions++;
+            resolve();
+        });
+    });
+}
+function releaseExtractionSlot() {
+    activeExtractions--;
+    const next = extractionQueue.shift();
+    if (next)
+        next();
+}
+/**
+ * Checks whether a Content-Type header value represents HTML content
+ * that should be processed through Readability/Turndown extraction.
+ */
+export function isHtmlContentType(contentType) {
+    if (!contentType)
+        return true; // Default to HTML extraction if no Content-Type
+    const [mime] = contentType.toLowerCase().split(";");
+    const lower = (mime ?? "").trim();
+    return HTML_CONTENT_TYPES.includes(lower);
+}
+/**
+ * Extracts content from a response body, checking Content-Type first.
+ * Non-HTML content types (JSON, PDF, images, etc.) are returned as-is
+ * or with a descriptive message rather than being fed through Readability.
+ *
+ * @param body - The response body text
+ * @param url - The source URL
+ * @param contentType - The Content-Type header value (optional)
+ * @returns Extracted content
+ */
+export async function extractFromResponse(body, url, contentType) {
+    if (!isHtmlContentType(contentType)) {
+        // Non-HTML content: return raw text truncated to limit
+        const [rawMimeType] = contentType?.split(";") ?? [];
+        const mimeType = (rawMimeType ?? "unknown").trim() || "unknown";
+        const truncated = body.slice(0, MAX_CONTENT_LENGTH);
+        return {
+            title: "",
+            content: truncated || `[Non-HTML content: ${mimeType}]`,
+            isArticle: false,
+        };
+    }
+    return extractContent(body, url);
+}
+/**
+ * Wraps a synchronous/async operation with a timeout.
+ * Returns the result if completed within the timeout, or rejects.
+ */
+function withTimeout(promise, ms, label) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            reject(new Error(`${label} timed out after ${ms}ms`));
+        }, ms);
+        promise.then((val) => {
+            clearTimeout(timer);
+            resolve(val);
+        }, (err) => {
+            clearTimeout(timer);
+            reject(err);
+        });
+    });
+}
+function extractOversizedHtmlTitle(html) {
+    const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    return match?.[1]?.replace(/\s+/g, " ").trim() ?? "";
+}
+/**
+ * Extracts content from HTML and converts to Markdown.
+ *
+ * Strategy:
+ * 1. Try Readability.js (article detection) + Turndown
+ * 2. Fall back to Turndown on raw HTML
+ * 3. Fall back to regex-based tag stripping
+ *
+ * @param html - The HTML content to extract from
+ * @param url - The source URL (for link resolution)
+ * @returns Extracted content with title and article detection
+ */
+export async function extractContent(html, url) {
+    // Handle empty input
+    if (!html || html.trim().length === 0) {
+        return { title: "", content: "", isArticle: false };
+    }
+    // Oversized pages will be truncated anyway, so skip expensive DOM/readability
+    // work and strip directly from the raw response body.
+    if (html.length > MAX_CONTENT_LENGTH * 2) {
+        return {
+            title: extractOversizedHtmlTitle(html),
+            content: stripHtmlTags(html).slice(0, MAX_CONTENT_LENGTH),
+            isArticle: false,
+        };
+    }
+    await acquireExtractionSlot();
+    try {
+        // Parse HTML with JSDOM
+        const dom = new JSDOM(html, { url });
+        try {
+            const document = dom.window.document;
+            // Extract title
+            const title = document.querySelector("title")?.textContent?.trim() || "";
+            // Detect semantic article markers before Readability mutates the DOM
+            const hasArticleMarkers = !!document.querySelector("article, [role='article'], [role='main'], [itemtype*='Article']");
+            // Initialize Turndown for Markdown conversion
+            const turndown = new TurndownService({
+                headingStyle: "atx",
+                codeBlockStyle: "fenced",
+            });
+            // Remove noise elements before conversion
+            turndown.remove(["script", "style", "meta", "link", "noscript"]);
+            // Tier 1: Try Readability.js for article extraction (only for article-like pages)
+            if (hasArticleMarkers) {
+                try {
+                    const reader = new Readability(document.cloneNode(true));
+                    // Wrap Readability.parse() with a timeout to prevent hung parsing on malformed HTML
+                    const article = await withTimeout(Promise.resolve(reader.parse()), PARSE_TIMEOUT_MS, "Readability.parse()");
+                    if (article && article.content) {
+                        // Readability succeeded — convert to Markdown, normalize URLs
+                        let markdown = turndown.turndown(article.content);
+                        markdown = normalizeTrailingSlashes(markdown);
+                        const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
+                        return {
+                            title: article.title || title,
+                            content: truncated,
+                            isArticle: true,
+                        };
+                    }
+                }
+                catch {
+                    // Readability failed or timed out, fall through to Tier 2
+                }
+            }
+            // Tier 2: Turndown on raw body HTML (non-article pages or Readability failure)
+            try {
+                const bodyHtml = document.body?.innerHTML || "";
+                if (bodyHtml) {
+                    let markdown = turndown.turndown(bodyHtml);
+                    markdown = normalizeTrailingSlashes(markdown);
+                    const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
+                    return {
+                        title,
+                        content: truncated,
+                        isArticle: false,
+                    };
+                }
+            }
+            catch {
+                // Turndown failed, fall through to Tier 3
+            }
+            // Tier 3: Regex-based fallback (strip tags, extract text)
+            const textContent = stripHtmlTags(html);
+            const truncated = textContent.slice(0, MAX_CONTENT_LENGTH);
+            return {
+                title,
+                content: truncated,
+                isArticle: false,
+            };
+        }
+        finally {
+            dom.window.close();
+        }
+    }
+    catch {
+        // Catastrophic failure — return empty result
+        return { title: "", content: "", isArticle: false };
+    }
+    finally {
+        releaseExtractionSlot();
+    }
+}
+/**
+ * Remove trailing slashes from URLs in markdown links that were added by JSDOM normalization.
+ * JSDOM normalizes `https://example.com` → `https://example.com/`, breaking exact URL matches.
+ */
+function normalizeTrailingSlashes(markdown) {
+    return markdown.replace(/\]\(([^)]+?)\/\)/g, (match, url) => {
+        // Only strip trailing slash from URLs that are just a domain (no path)
+        try {
+            const parsed = new URL(url + "/");
+            if (parsed.pathname === "/") {
+                return `](${url})`;
+            }
+        }
+        catch {
+            // Not a valid URL, leave as-is
+        }
+        return match;
+    });
+}
+/**
+ * Strips HTML tags using regex (last resort fallback)
+ */
+function stripHtmlTags(html) {
+    return (html
+        // Remove script and style tags and their content
+        .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "")
+        .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "")
+        // Remove all other HTML tags
+        .replace(/<[^>]+>/g, " ")
+        // Collapse multiple whitespace
+        .replace(/\s+/g, " ")
+        .trim());
+}
+//# sourceMappingURL=content-extraction.js.map

package/dist/extraction/content-extraction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-extraction.js","sourceRoot":"","sources":["../../src/extraction/content-extraction.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,eAAe,MAAM,UAAU,CAAC;AAEvC,yCAAyC;AACzC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,oDAAoD;AACpD,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,kDAAkD;AAClD,MAAM,kBAAkB,GAAG,CAAC,WAAW,EAAE,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;AAEpF,2DAA2D;AAC3D,MAAM,+BAA+B,GAAG,MAAM,CAAC,QAAQ,CACrD,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,GAAG,EAClD,EAAE,CACH,CAAC;AACF,MAAM,0BAA0B,GAC9B,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAC,IAAI,+BAA+B,GAAG,CAAC;IACrF,CAAC,CAAC,+BAA+B;IACjC,CAAC,CAAC,CAAC,CAAC;AACR,IAAI,iBAAiB,GAAG,CAAC,CAAC;AAC1B,MAAM,eAAe,GAAsB,EAAE,CAAC;AAE9C,KAAK,UAAU,qBAAqB;IAClC,IAAI,iBAAiB,GAAG,0BAA0B,EAAE,CAAC;QACnD,iBAAiB,EAAE,CAAC;QACpB,OAAO;IACT,CAAC;IACD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACnC,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE;YACxB,iBAAiB,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB;IAC5B,iBAAiB,EAAE,CAAC;IACpB,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,EAAE,CAAC;IACrC,IAAI,IAAI;QAAE,IAAI,EAAE,CAAC;AACnB,CAAC;AAcD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,WAAsC;IACtE,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAC,CAAC,gDAAgD;IAC/E,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClC,OAAO,kBAAkB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,GAAW,EACX,WAA2B;IAE3B,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAE,CAAC;QACpC,uDAAuD;QACvD,MAAM,CAAC,WAAW,CAAC,GAAG,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpD,MAAM,QAAQ,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;QACpD,OAAO;YACL,KAAK,EAAE,EAAE;YACT,OAAO,EAAE,SAAS,IAAI,sBAAsB,QAAQ,GAAG;YACvD,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IACD,OAAO,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAI,OAAmB,EAAE,EAAU,EAAE,KAAa;IACpE,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,KAAK,oBAAoB,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,OAAO,CAAC,IAAI,CACV,CAAC,GAAG,EAAE,EAAE;YACN,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC,EACD,CAAC,GAAG,EAAE,EAAE;YACN,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,yBAAyB,CAAC,IAAY;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC7D,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;AACvD,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IAC5D,qBAAqB;IACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAED,8EAA8E;IAC9E,sDAAsD;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,GAAG,CAAC,EAAE,CAAC;QACzC,OAAO;YACL,KAAK,EAAE,yBAAyB,CAAC,IAAI,CAAC;YACtC,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC;YACzD,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IAED,MAAM,qBAAqB,EAAE,CAAC;IAC9B,IAAI,CAAC;QACH,wBAAwB;QACxB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;YAErC,gBAAgB;YAChB,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAEzE,qEAAqE;YACrE,MAAM,iBAAiB,GAAG,CAAC,CAAC,QAAQ,CAAC,aAAa,CAChD,iEAAiE,CAClE,CAAC;YAEF,8CAA8C;YAC9C,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;gBACnC,YAAY,EAAE,KAAK;gBACnB,cAAc,EAAE,QAAQ;aACzB,CAAC,CAAC;YACH,0CAA0C;YAC1C,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;YAEjE,kFAAkF;YAClF,IAAI,iBAAiB,EAAE,CAAC;gBACtB,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAa,CAAC,CAAC;oBACrE,oFAAoF;oBACpF,MAAM,OAAO,GAAG,MAAM,WAAW,CAC/B,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAC/B,gBAAgB,EAChB,qBAAqB,CACtB,CAAC;oBAEF,IAAI,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;wBAC/B,8DAA8D;wBAC9D,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;wBAClD,QAAQ,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;wBAC9C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;wBACxD,OAAO;4BACL,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK;4BAC7B,OAAO,EAAE,SAAS;4BAClB,SAAS,EAAE,IAAI;yBAChB,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,0DAA0D;gBAC5D,CAAC;YACH,CAAC;YAED,+EAA+E;YAC/E,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC;gBAChD,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBAC3C,QAAQ,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;oBAC9C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;oBACxD,OAAO;wBACL,KAAK;wBACL,OAAO,EAAE,SAAS;wBAClB,SAAS,EAAE,KAAK;qBACjB,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0CAA0C;YAC5C,CAAC;YAED,0DAA0D;YAC1D,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAC3D,OAAO;gBACL,KAAK;gBACL,OAAO,EAAE,SAAS;gBAClB,SAAS,EAAE,KAAK;aACjB,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;QAC7C,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;YAAS,CAAC;QACT,qBAAqB,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,wBAAwB,CAAC,QAAgB;IAChD,OAAO,QAAQ,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC1D,uEAAuE;QACvE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;YAClC,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO,KAAK,GAAG,GAAG,CAAC;YACrB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,CACL,IAAI;QACF,iDAAiD;SAChD,OAAO,CAAC,qDAAqD,EAAE,EAAE,CAAC;SAClE,OAAO,CAAC,kDAAkD,EAAE,EAAE,CAAC;QAChE,6BAA6B;SAC5B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;QACzB,+BAA+B;SAC9B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CACV,CAAC;AACJ,CAAC"}

package/dist/extraction/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Content extraction utilities
+ */
+export { extractContent, type ExtractedContent } from "./content-extraction.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/extraction/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/extraction/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,EAAE,MAAM,yBAAyB,CAAC"}

package/dist/extraction/index.js

@@ -0,0 +1,5 @@
+/**
+ * Content extraction utilities
+ */
+export { extractContent } from "./content-extraction.js";
+//# sourceMappingURL=index.js.map

package/dist/extraction/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/extraction/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,cAAc,EAAyB,MAAM,yBAAyB,CAAC"}