@aria-cli/tools 1.0.9 → 1.0.11

package/src/extraction/content-extraction.ts

@@ -1,281 +0,0 @@
-/**
- * Content Extraction — HTML to Markdown conversion with article detection
- *
- * Three-tier fallback strategy:
- * 1. Readability.js (Mozilla) + Turndown for article-like content
- * 2. Turndown raw HTML if Readability fails
- * 3. Regex-based strip if both fail
- */
-
-import { JSDOM } from "jsdom";
-import { Readability } from "@mozilla/readability";
-import TurndownService from "turndown";
-
-/** Maximum content length (50K chars) */
-const MAX_CONTENT_LENGTH = 50_000;
-
-/** Timeout for Readability parse in milliseconds */
-const PARSE_TIMEOUT_MS = 10_000;
-
-/** Content types that should be parsed as HTML */
-const HTML_CONTENT_TYPES = ["text/html", "text/xhtml+xml", "application/xhtml+xml"];
-
-// Simple concurrency semaphore for JSDOM memory protection
-const configuredConcurrentExtractions = Number.parseInt(
-  process.env.ARIA_MAX_CONCURRENT_EXTRACTIONS ?? "3",
-  10,
-);
-const MAX_CONCURRENT_EXTRACTIONS =
-  Number.isFinite(configuredConcurrentExtractions) && configuredConcurrentExtractions > 0
-    ? configuredConcurrentExtractions
-    : 3;
-let activeExtractions = 0;
-const extractionQueue: Array<() => void> = [];
-
-async function acquireExtractionSlot(): Promise<void> {
-  if (activeExtractions < MAX_CONCURRENT_EXTRACTIONS) {
-    activeExtractions++;
-    return;
-  }
-  return new Promise<void>((resolve) => {
-    extractionQueue.push(() => {
-      activeExtractions++;
-      resolve();
-    });
-  });
-}
-
-function releaseExtractionSlot(): void {
-  activeExtractions--;
-  const next = extractionQueue.shift();
-  if (next) next();
-}
-
-/**
- * Result of content extraction
- */
-export interface ExtractedContent {
-  /** Page title */
-  title: string;
-  /** Extracted content as Markdown */
-  content: string;
-  /** Whether Readability detected article-like content */
-  isArticle: boolean;
-}
-
-/**
- * Checks whether a Content-Type header value represents HTML content
- * that should be processed through Readability/Turndown extraction.
- */
-export function isHtmlContentType(contentType: string | undefined | null): boolean {
-  if (!contentType) return true; // Default to HTML extraction if no Content-Type
-  const [mime] = contentType.toLowerCase().split(";");
-  const lower = (mime ?? "").trim();
-  return HTML_CONTENT_TYPES.includes(lower);
-}
-
-/**
- * Extracts content from a response body, checking Content-Type first.
- * Non-HTML content types (JSON, PDF, images, etc.) are returned as-is
- * or with a descriptive message rather than being fed through Readability.
- *
- * @param body - The response body text
- * @param url - The source URL
- * @param contentType - The Content-Type header value (optional)
- * @returns Extracted content
- */
-export async function extractFromResponse(
-  body: string,
-  url: string,
-  contentType?: string | null,
-): Promise<ExtractedContent> {
-  if (!isHtmlContentType(contentType)) {
-    // Non-HTML content: return raw text truncated to limit
-    const [rawMimeType] = contentType?.split(";") ?? [];
-    const mimeType = (rawMimeType ?? "unknown").trim() || "unknown";
-    const truncated = body.slice(0, MAX_CONTENT_LENGTH);
-    return {
-      title: "",
-      content: truncated || `[Non-HTML content: ${mimeType}]`,
-      isArticle: false,
-    };
-  }
-  return extractContent(body, url);
-}
-
-/**
- * Wraps a synchronous/async operation with a timeout.
- * Returns the result if completed within the timeout, or rejects.
- */
-function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
-  return new Promise<T>((resolve, reject) => {
-    const timer = setTimeout(() => {
-      reject(new Error(`${label} timed out after ${ms}ms`));
-    }, ms);
-    promise.then(
-      (val) => {
-        clearTimeout(timer);
-        resolve(val);
-      },
-      (err) => {
-        clearTimeout(timer);
-        reject(err);
-      },
-    );
-  });
-}
-
-function extractOversizedHtmlTitle(html: string): string {
-  const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-  return match?.[1]?.replace(/\s+/g, " ").trim() ?? "";
-}
-
-/**
- * Extracts content from HTML and converts to Markdown.
- *
- * Strategy:
- * 1. Try Readability.js (article detection) + Turndown
- * 2. Fall back to Turndown on raw HTML
- * 3. Fall back to regex-based tag stripping
- *
- * @param html - The HTML content to extract from
- * @param url - The source URL (for link resolution)
- * @returns Extracted content with title and article detection
- */
-export async function extractContent(html: string, url: string): Promise<ExtractedContent> {
-  // Handle empty input
-  if (!html || html.trim().length === 0) {
-    return { title: "", content: "", isArticle: false };
-  }
-
-  // Oversized pages will be truncated anyway, so skip expensive DOM/readability
-  // work and strip directly from the raw response body.
-  if (html.length > MAX_CONTENT_LENGTH * 2) {
-    return {
-      title: extractOversizedHtmlTitle(html),
-      content: stripHtmlTags(html).slice(0, MAX_CONTENT_LENGTH),
-      isArticle: false,
-    };
-  }
-
-  await acquireExtractionSlot();
-  try {
-    // Parse HTML with JSDOM
-    const dom = new JSDOM(html, { url });
-    try {
-      const document = dom.window.document;
-
-      // Extract title
-      const title = document.querySelector("title")?.textContent?.trim() || "";
-
-      // Detect semantic article markers before Readability mutates the DOM
-      const hasArticleMarkers = !!document.querySelector(
-        "article, [role='article'], [role='main'], [itemtype*='Article']",
-      );
-
-      // Initialize Turndown for Markdown conversion
-      const turndown = new TurndownService({
-        headingStyle: "atx",
-        codeBlockStyle: "fenced",
-      });
-      // Remove noise elements before conversion
-      turndown.remove(["script", "style", "meta", "link", "noscript"]);
-
-      // Tier 1: Try Readability.js for article extraction (only for article-like pages)
-      if (hasArticleMarkers) {
-        try {
-          const reader = new Readability(document.cloneNode(true) as Document);
-          // Wrap Readability.parse() with a timeout to prevent hung parsing on malformed HTML
-          const article = await withTimeout(
-            Promise.resolve(reader.parse()),
-            PARSE_TIMEOUT_MS,
-            "Readability.parse()",
-          );
-
-          if (article && article.content) {
-            // Readability succeeded — convert to Markdown, normalize URLs
-            let markdown = turndown.turndown(article.content);
-            markdown = normalizeTrailingSlashes(markdown);
-            const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
-            return {
-              title: article.title || title,
-              content: truncated,
-              isArticle: true,
-            };
-          }
-        } catch {
-          // Readability failed or timed out, fall through to Tier 2
-        }
-      }
-
-      // Tier 2: Turndown on raw body HTML (non-article pages or Readability failure)
-      try {
-        const bodyHtml = document.body?.innerHTML || "";
-        if (bodyHtml) {
-          let markdown = turndown.turndown(bodyHtml);
-          markdown = normalizeTrailingSlashes(markdown);
-          const truncated = markdown.slice(0, MAX_CONTENT_LENGTH);
-          return {
-            title,
-            content: truncated,
-            isArticle: false,
-          };
-        }
-      } catch {
-        // Turndown failed, fall through to Tier 3
-      }
-
-      // Tier 3: Regex-based fallback (strip tags, extract text)
-      const textContent = stripHtmlTags(html);
-      const truncated = textContent.slice(0, MAX_CONTENT_LENGTH);
-      return {
-        title,
-        content: truncated,
-        isArticle: false,
-      };
-    } finally {
-      dom.window.close();
-    }
-  } catch {
-    // Catastrophic failure — return empty result
-    return { title: "", content: "", isArticle: false };
-  } finally {
-    releaseExtractionSlot();
-  }
-}
-
-/**
- * Remove trailing slashes from URLs in markdown links that were added by JSDOM normalization.
- * JSDOM normalizes `https://example.com` → `https://example.com/`, breaking exact URL matches.
- */
-function normalizeTrailingSlashes(markdown: string): string {
-  return markdown.replace(/\]\(([^)]+?)\/\)/g, (match, url) => {
-    // Only strip trailing slash from URLs that are just a domain (no path)
-    try {
-      const parsed = new URL(url + "/");
-      if (parsed.pathname === "/") {
-        return `](${url})`;
-      }
-    } catch {
-      // Not a valid URL, leave as-is
-    }
-    return match;
-  });
-}
-
-/**
- * Strips HTML tags using regex (last resort fallback)
- */
-function stripHtmlTags(html: string): string {
-  return (
-    html
-      // Remove script and style tags and their content
-      .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "")
-      .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "")
-      // Remove all other HTML tags
-      .replace(/<[^>]+>/g, " ")
-      // Collapse multiple whitespace
-      .replace(/\s+/g, " ")
-      .trim()
-  );
-}

package/src/extraction/index.ts

@@ -1,5 +0,0 @@
-/**
- * Content extraction utilities
- */
-
-export { extractContent, type ExtractedContent } from "./content-extraction.js";