@aria-cli/tools 1.0.11 → 1.0.13

package/dist-cjs/extraction/content-extraction.js

@@ -1,244 +0,0 @@
-"use strict";
-/**
- * Content Extraction — HTML to Markdown conversion with article detection
- *
- * Three-tier fallback strategy:
- * 1. Readability.js (Mozilla) + Turndown for article-like content
- * 2. Turndown raw HTML if Readability fails
- * 3. Regex-based strip if both fail
- */
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.isHtmlContentType = isHtmlContentType;
-exports.extractFromResponse = extractFromResponse;
-exports.extractContent = extractContent;
-const jsdom_1 = require("jsdom");
-const readability_1 = require("@mozilla/readability");
-const turndown_1 = __importDefault(require("turndown"));
-/** Maximum content length (50K chars) */
-const MAX_CONTENT_LENGTH = 50_000;
-/** Timeout for Readability parse in milliseconds */
-const PARSE_TIMEOUT_MS = 10_000;
-/** Content types that should be parsed as HTML */
-const HTML_CONTENT_TYPES = ["text/html", "text/xhtml+xml", "application/xhtml+xml"];
-// Simple concurrency semaphore for JSDOM memory protection
-const configuredConcurrentExtractions = Number.parseInt(process.env.ARIA_MAX_CONCURRENT_EXTRACTIONS ?? "3", 10);
-const MAX_CONCURRENT_EXTRACTIONS = Number.isFinite(configuredConcurrentExtractions) && configuredConcurrentExtractions > 0
-    ? configuredConcurrentExtractions
-    : 3;
-let activeExtractions = 0;
-const extractionQueue = [];
-async function acquireExtractionSlot() {
-    if (activeExtractions < MAX_CONCURRENT_EXTRACTIONS) {
-        activeExtractions++;
-        return;
-    }
-    return new Promise((resolve) => {
-        extractionQueue.push(() => {
-            activeExtractions++;
-            resolve();
-        });
-    });
-}
-function releaseExtractionSlot() {
-    activeExtractions--;
-    const next = extractionQueue.shift();
-    if (next)
-        next();
-}
-/**
- * Checks whether a Content-Type header value represents HTML content
- * that should be processed through Readability/Turndown extraction.
- */
-function isHtmlContentType(contentType) {
-    if (!contentType)
-        return true; // Default to HTML extraction if no Content-Type
-    const [mime] = contentType.toLowerCase().split(";");
-    const lower = (mime ?? "").trim();
-    return HTML_CONTENT_TYPES.includes(lower);
-}
-/**
- * Extracts content from a response body, checking Content-Type first.
- * Non-HTML content types (JSON, PDF, images, etc.) are returned as-is
- * or with a descriptive message rather than being fed through Readability.
- *
- * @param body - The response body text
- * @param url - The source URL
- * @param contentType - The Content-Type header value (optional)
- * @returns Extracted content
- */
-async function extractFromResponse(body, url, contentType) {
-    if (!isHtmlContentType(contentType)) {
-        // Non-HTML content: return raw text truncated to limit
-        const [rawMimeType] = contentType?.split(";") ?? [];
-        const mimeType = (rawMimeType ?? "unknown").trim() || "unknown";
-        const truncated = body.slice(0, MAX_CONTENT_LENGTH);
-        return {
-            title: "",
-            content: truncated || `[Non-HTML content: ${mimeType}]`,
-            isArticle: false,
-        };
-    }
-    return extractContent(body, url);
-}
-/**
- * Wraps a synchronous/async operation with a timeout.
- * Returns the result if completed within the timeout, or rejects.
- */
-function withTimeout(promise, ms, label) {
-    return new Promise((resolve, reject) => {
-        const timer = setTimeout(() => {
-            reject(new Error(`${label} timed out after ${ms}ms`));
-        }, ms);
-        promise.then((val) => {
-            clearTimeout(timer);
-            resolve(val);
-        }, (err) => {
-            clearTimeout(timer);
-            reject(err);
-        });
-    });
-}
-function extractOversizedHtmlTitle(html) {
-    const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-    return match?.[1]?.replace(/\s+/g, " ").trim() ?? "";
-}
-/**
- * Extracts content from HTML and converts to Markdown.
- *
- * Strategy:
- * 1. Try Readability.js (article detection) + Turndown
- * 2. Fall back to Turndown on raw HTML
- * 3. Fall back to regex-based tag stripping
- *
- * @param html - The HTML content to extract from
- * @param url - The source URL (for link resolution)
- * @returns Extracted content with title and article detection
- */
-async function extractContent(html, url) {
-    // Handle empty input
-    if (!html || html.trim().length === 0) {
-        return { title: "", content: "", isArticle: false };
-    }
-    // Oversized pages will be truncated anyway, so skip expensive DOM/readability
-    // work and strip directly from the raw response body.
-    if (html.length > MAX_CONTENT_LENGTH * 2) {
-        return {
-            title: extractOversizedHtmlTitle(html),
-            content: stripHtmlTags(html).slice(0, MAX_CONTENT_LENGTH),
-            isArticle: false,
-        };
-    }
-    await acquireExtractionSlot();
-    try {
-        // Parse HTML with JSDOM
-        const dom = new jsdom_1.JSDOM(html, { url });
-        try {
-            const document = dom.window.document;
-            // Extract title
-            const title = document.querySelector("title")?.textContent?.trim() || "";
-            // Detect semantic article markers before Readability mutates the DOM
-            const hasArticleMarkers = !!document.querySelector("article, [role='article'], [role='main'], [itemtype*='Article']");
-            // Initialize Turndown for Markdown conversion
-            const turndown = new turndown_1.default({
-                headingStyle: "atx",
-                codeBlockStyle: "fenced",
-            });
-            // Remove noise elements before conversion
-            turndown.remove(["script", "style", "meta", "link", "noscript"]);
-            // Tier 1: Try Readability.js for article extraction (only for article-like pages)
-            if (hasArticleMarkers) {
-                try {
-                    const reader = new readability_1.Readability(document.cloneNode(true));
-                    // Wrap Readability.parse() with a timeout to prevent hung parsing on malformed HTML
-                    const article = await withTimeout(Promise.resolve(reader.parse()), PARSE_TIMEOUT_MS, "Readability.parse()");
-                    if (article && article.content) {
-                        // Readability succeeded — convert to Markdown, normalize URLs
-                        let markdown = turndown.turndown(article.content);
-                        markdown = normalizeTrailingSlashes(markdown);
-                        const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
-                        return {
-                            title: article.title || title,
-                            content: truncated,
-                            isArticle: true,
-                        };
-                    }
-                }
-                catch {
-                    // Readability failed or timed out, fall through to Tier 2
-                }
-            }
-            // Tier 2: Turndown on raw body HTML (non-article pages or Readability failure)
-            try {
-                const bodyHtml = document.body?.innerHTML || "";
-                if (bodyHtml) {
-                    let markdown = turndown.turndown(bodyHtml);
-                    markdown = normalizeTrailingSlashes(markdown);
-                    const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
-                    return {
-                        title,
-                        content: truncated,
-                        isArticle: false,
-                    };
-                }
-            }
-            catch {
-                // Turndown failed, fall through to Tier 3
-            }
-            // Tier 3: Regex-based fallback (strip tags, extract text)
-            const textContent = stripHtmlTags(html);
-            const truncated = textContent.slice(0, MAX_CONTENT_LENGTH);
-            return {
-                title,
-                content: truncated,
-                isArticle: false,
-            };
-        }
-        finally {
-            dom.window.close();
-        }
-    }
-    catch {
-        // Catastrophic failure — return empty result
-        return { title: "", content: "", isArticle: false };
-    }
-    finally {
-        releaseExtractionSlot();
-    }
-}
-/**
- * Remove trailing slashes from URLs in markdown links that were added by JSDOM normalization.
- * JSDOM normalizes `https://example.com` → `https://example.com/`, breaking exact URL matches.
- */
-function normalizeTrailingSlashes(markdown) {
-    return markdown.replace(/\]\(([^)]+?)\/\)/g, (match, url) => {
-        // Only strip trailing slash from URLs that are just a domain (no path)
-        try {
-            const parsed = new URL(url + "/");
-            if (parsed.pathname === "/") {
-                return `](${url})`;
-            }
-        }
-        catch {
-            // Not a valid URL, leave as-is
-        }
-        return match;
-    });
-}
-/**
- * Strips HTML tags using regex (last resort fallback)
- */
-function stripHtmlTags(html) {
-    return (html
-        // Remove script and style tags and their content
-        .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "")
-        .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "")
-        // Remove all other HTML tags
-        .replace(/<[^>]+>/g, " ")
-        // Collapse multiple whitespace
-        .replace(/\s+/g, " ")
-        .trim());
-}
-//# sourceMappingURL=content-extraction.js.map

package/dist-cjs/extraction/content-extraction.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"content-extraction.js","sourceRoot":"","sources":["../../src/extraction/content-extraction.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;;;AA8DH,8CAKC;AAYD,kDAiBC;AAyCD,wCAoGC;AA3OD,iCAA8B;AAC9B,sDAAmD;AACnD,wDAAuC;AAEvC,yCAAyC;AACzC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,oDAAoD;AACpD,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,kDAAkD;AAClD,MAAM,kBAAkB,GAAG,CAAC,WAAW,EAAE,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;AAEpF,2DAA2D;AAC3D,MAAM,+BAA+B,GAAG,MAAM,CAAC,QAAQ,CACrD,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,GAAG,EAClD,EAAE,CACH,CAAC;AACF,MAAM,0BAA0B,GAC9B,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAC,IAAI,+BAA+B,GAAG,CAAC;IACrF,CAAC,CAAC,+BAA+B;IACjC,CAAC,CAAC,CAAC,CAAC;AACR,IAAI,iBAAiB,GAAG,CAAC,CAAC;AAC1B,MAAM,eAAe,GAAsB,EAAE,CAAC;AAE9C,KAAK,UAAU,qBAAqB;IAClC,IAAI,iBAAiB,GAAG,0BAA0B,EAAE,CAAC;QACnD,iBAAiB,EAAE,CAAC;QACpB,OAAO;IACT,CAAC;IACD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACnC,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE;YACxB,iBAAiB,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB;IAC5B,iBAAiB,EAAE,CAAC;IACpB,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,EAAE,CAAC;IACrC,IAAI,IAAI;QAAE,IAAI,EAAE,CAAC;AACnB,CAAC;AAcD;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,WAAsC;IACtE,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAC,CAAC,gDAAgD;IAC/E,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClC,OAAO,kBAAkB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,GAAW,EACX,WAA2B;IAE3B,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAE,CAAC;QACpC,uDAAuD;QACvD,MAAM,CAAC,WAAW,CAAC,GAAG,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpD,MAAM,QAAQ,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;QACpD,OAAO;YACL,KAAK,EAAE,EAAE;YACT,OAAO,EAAE,SAAS,IAAI,sBAAsB,QAAQ,GAAG;YACvD,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IACD,OAAO,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAI,OAAmB,EAAE,EAAU,EAAE,KAAa;IACpE,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,KAAK,oBAAoB,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,OAAO,CAAC,IAAI,CACV,CAAC,GAAG,EAAE,EAAE;YACN,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC,EACD,CAAC,GAAG,EAAE,EAAE;YACN,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,yBAAyB,CAAC,IAAY;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC7D,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;AACvD,CAAC;AAED;;;;;;;;;;;GAWG;AACI,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IAC5D,qBAAqB;IACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAED,8EAA8E;IAC9E,sDAAsD;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,GAAG,CAAC,EAAE,CAAC;QACzC,OAAO;YACL,KAAK,EAAE,yBAAyB,CAAC,IAAI,CAAC;YACtC,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC;YACzD,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IAED,MAAM,qBAAqB,EAAE,CAAC;IAC9B,IAAI,CAAC;QACH,wBAAwB;QACxB,MAAM,GAAG,GAAG,IAAI,aAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;YAErC,gBAAgB;YAChB,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAEzE,qEAAqE;YACrE,MAAM,iBAAiB,GAAG,CAAC,CAAC,QAAQ,CAAC,aAAa,CAChD,iEAAiE,CAClE,CAAC;YAEF,8CAA8C;YAC9C,MAAM,QAAQ,GAAG,IAAI,kBAAe,CAAC;gBACnC,YAAY,EAAE,KAAK;gBACnB,cAAc,EAAE,QAAQ;aACzB,CAAC,CAAC;YACH,0CAA0C;YAC1C,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;YAEjE,kFAAkF;YAClF,IAAI,iBAAiB,EAAE,CAAC;gBACtB,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,yBAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAa,CAAC,CAAC;oBACrE,oFAAoF;oBACpF,MAAM,OAAO,GAAG,MAAM,WAAW,CAC/B,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAC/B,gBAAgB,EAChB,qBAAqB,CACtB,CAAC;oBAEF,IAAI,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;wBAC/B,8DAA8D;wBAC9D,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;wBAClD,QAAQ,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;wBAC9C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;wBACxD,OAAO;4BACL,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK;4BAC7B,OAAO,EAAE,SAAS;4BAClB,SAAS,EAAE,IAAI;yBAChB,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,0DAA0D;gBAC5D,CAAC;YACH,CAAC;YAED,+EAA+E;YAC/E,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC;gBAChD,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBAC3C,QAAQ,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;oBAC9C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;oBACxD,OAAO;wBACL,KAAK;wBACL,OAAO,EAAE,SAAS;wBAClB,SAAS,EAAE,KAAK;qBACjB,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0CAA0C;YAC5C,CAAC;YAED,0DAA0D;YAC1D,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAC3D,OAAO;gBACL,KAAK;gBACL,OAAO,EAAE,SAAS;gBAClB,SAAS,EAAE,KAAK;aACjB,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;QAC7C,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;YAAS,CAAC;QACT,qBAAqB,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,wBAAwB,CAAC,QAAgB;IAChD,OAAO,QAAQ,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC1D,uEAAuE;QACvE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;YAClC,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO,KAAK,GAAG,GAAG,CAAC;YACrB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,CACL,IAAI;QACF,iDAAiD;SAChD,OAAO,CAAC,qDAAqD,EAAE,EAAE,CAAC;SAClE,OAAO,CAAC,kDAAkD,EAAE,EAAE,CAAC;QAChE,6BAA6B;SAC5B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;QACzB,+BAA+B;SAC9B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CACV,CAAC;AACJ,CAAC"}

package/dist-cjs/extraction/index.js

@@ -1,9 +0,0 @@
-"use strict";
-/**
- * Content extraction utilities
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractContent = void 0;
-var content_extraction_js_1 = require("./content-extraction.js");
-Object.defineProperty(exports, "extractContent", { enumerable: true, get: function () { return content_extraction_js_1.extractContent; } });
-//# sourceMappingURL=index.js.map

package/dist-cjs/extraction/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/extraction/index.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,iEAAgF;AAAvE,uHAAA,cAAc,OAAA"}