@robot-resources/scraper 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,373 @@
+/**
+ * Types for scraper
+ * Context compression for AI agents
+ */
+/**
+ * Fetch tier selection for scrape()
+ *
+ * - 'fast': Tier 1 — plain HTTP fetch (default Node.js fetch)
+ * - 'stealth': Tier 2 — TLS fingerprint impersonation (bypasses anti-bot)
+ * - 'render': Tier 3 — Playwright headless browser (JS-rendered pages)
+ * - 'auto': Try fast first, fall back to stealth on 403/challenge detection
+ */
+type FetchMode = 'fast' | 'stealth' | 'render' | 'auto';
+/**
+ * Options for the scrape() function
+ */
+interface ScrapeOptions {
+    /** Fetch tier to use (default: 'auto') */
+    mode?: FetchMode;
+    /** Request timeout in milliseconds (default: 10000) */
+    timeout?: number;
+    /** Maximum retry attempts for failed requests (default: 3) */
+    maxRetries?: number;
+    /** Custom user agent string */
+    userAgent?: string;
+    /** Check robots.txt before fetching (default: false) */
+    respectRobots?: boolean;
+}
+/**
+ * Result from the scrape() function
+ */
+interface ScrapeResult {
+    /** Compressed markdown content */
+    markdown: string;
+    /** Estimated token count */
+    tokenCount: number;
+    /** Page title if extracted */
+    title?: string;
+    /** Author if extracted */
+    author?: string;
+    /** Site name if extracted */
+    siteName?: string;
+    /** Published date if extracted */
+    publishedAt?: string;
+    /** Final URL after redirects */
+    url: string;
+}
+/**
+ * Result from fetch layer
+ */
+interface FetchResult {
+    /** Raw HTML content */
+    html: string;
+    /** Final URL after redirects */
+    url: string;
+    /** HTTP status code */
+    statusCode: number;
+    /** Response headers */
+    headers: Record<string, string>;
+}
+/**
+ * Result from extract layer
+ */
+interface ExtractResult {
+    /** Extracted HTML content (article body) */
+    content: string;
+    /** Page title */
+    title?: string;
+    /** Author name */
+    author?: string;
+    /** Published timestamp */
+    publishedAt?: string;
+    /** Site name */
+    siteName?: string;
+}
+/**
+ * Result from convert layer
+ */
+interface ConvertResult {
+    /** Markdown content */
+    markdown: string;
+    /** Estimated token count */
+    tokenCount: number;
+}
+/**
+ * Options for the crawl() function
+ */
+interface CrawlOptions {
+    /** Starting URL to crawl */
+    url: string;
+    /** Max crawl depth — 0 means only starting URL (default: 2) */
+    depth?: number;
+    /** Max pages to crawl (default: 50) */
+    limit?: number;
+    /** Fetch mode per page (default: 'auto') */
+    mode?: FetchMode;
+    /** URL patterns to include — glob-style (if set, URL must match at least one) */
+    include?: string[];
+    /** URL patterns to exclude — glob-style (URL must not match any) */
+    exclude?: string[];
+    /** Per-page timeout in milliseconds (default: 10000) */
+    timeout?: number;
+    /** Parallel page fetches (default: 3) */
+    concurrency?: number;
+    /** Check robots.txt before crawling (default: true) */
+    respectRobots?: boolean;
+}
+/**
+ * Result from a single crawled page — extends ScrapeResult with crawl depth
+ */
+interface CrawlPageResult extends ScrapeResult {
+    /** Crawl depth where this page was discovered */
+    depth: number;
+}
+/**
+ * Error for a single URL during crawl
+ */
+interface CrawlError {
+    /** URL that failed */
+    url: string;
+    /** Error message */
+    error: string;
+    /** Crawl depth where error occurred */
+    depth: number;
+}
+/**
+ * Aggregate result from crawl()
+ */
+interface CrawlResult {
+    /** Successfully scraped pages */
+    pages: CrawlPageResult[];
+    /** Total URLs discovered (including skipped/errored) */
+    totalDiscovered: number;
+    /** Total URLs successfully scraped */
+    totalCrawled: number;
+    /** Total URLs skipped (robots, filter, limit) */
+    totalSkipped: number;
+    /** Per-URL errors */
+    errors: CrawlError[];
+    /** Total crawl duration in milliseconds */
+    duration: number;
+}
+
+/**
+ * Layer 1: Fetch
+ * HTTP fetching with smart headers and retries
+ */
+
+interface FetchOptions {
+    timeout?: number;
+    maxRetries?: number;
+    userAgent?: string;
+}
+/**
+ * Error class for fetch-related errors
+ */
+declare class FetchError extends Error {
+    readonly statusCode?: number | undefined;
+    readonly retryable: boolean;
+    constructor(message: string, statusCode?: number | undefined, retryable?: boolean);
+}
+/**
+ * Fetch URL content with smart headers and retry logic
+ */
+declare function fetchUrl(url: string, options?: FetchOptions): Promise<FetchResult>;
+
+/**
+ * Layer 1b: Stealth Fetch
+ * TLS fingerprint impersonation via impit (optional peer dependency)
+ *
+ * Uses Rust-based browser TLS fingerprinting to bypass anti-bot systems
+ * (Cloudflare, Akamai, PerimeterX) without a full browser.
+ *
+ * Requires: npm install impit (Node >= 20)
+ */
+
+/**
+ * Fetch URL with browser TLS fingerprint impersonation.
+ *
+ * Uses impit (Apify) to produce Chrome-like JA3/JA4 fingerprints at the
+ * TLS handshake level. This bypasses anti-bot systems that reject default
+ * Node.js TLS signatures.
+ *
+ * impit is an optional peer dependency — if not installed, throws a clear
+ * error message with install instructions.
+ */
+declare function fetchStealth(url: string, options?: FetchOptions): Promise<FetchResult>;
+
+/**
+ * Layer 1c: Render Fetch
+ * Playwright headless browser for JS-rendered pages (optional peer dependency)
+ *
+ * Uses Chromium to fully render SPAs (React, Next.js, Vue) that return
+ * empty/partial HTML to tiers 1 and 2. Extracts the fully rendered DOM.
+ *
+ * Requires: npm install playwright
+ */
+
+/**
+ * Fetch URL using a headless Chromium browser to render JavaScript.
+ *
+ * Launches a fresh browser per call (no shared state), navigates to the URL,
+ * waits for network idle, then extracts the fully rendered HTML.
+ *
+ * Playwright is an optional peer dependency — if not installed, throws a clear
+ * error message with install instructions.
+ */
+declare function fetchRender(url: string, options?: FetchOptions): Promise<FetchResult>;
+
+/**
+ * Layer 2: Extract
+ * Content extraction using Readability
+ */
+
+/**
+ * Error class for extraction-related errors
+ */
+declare class ExtractionError extends Error {
+    readonly code: string;
+    constructor(message: string, code: string);
+}
+/**
+ * Extract main content from HTML using Readability
+ */
+declare function extractContent(fetchResult: FetchResult): Promise<ExtractResult>;
+
+/**
+ * Layer 3: Convert
+ * HTML to Markdown conversion
+ */
+
+/**
+ * Convert extracted HTML to clean Markdown
+ */
+declare function convertToMarkdown(extractResult: ExtractResult): Promise<ConvertResult>;
+/**
+ * Content-aware token estimator.
+ *
+ * Segments text by content type and applies calibrated character-per-token
+ * ratios derived from cl100k_base (GPT-4) empirical measurements.
+ *
+ * Ratios:
+ *   Code blocks  — 3.2 chars/token (operators, camelCase split into subwords)
+ *   Inline code  — 3.5 chars/token (variable names, short expressions)
+ *   URLs         — 5.0 chars/token (path segments tokenize efficiently)
+ *   Prose        — 4.3 chars/token (words, punctuation, markdown formatting)
+ *
+ * Accuracy: within ±15% of actual BPE tokenization for English content.
+ */
+declare function estimateTokens(text: string): number;
+
+/**
+ * robots.txt compliance layer
+ *
+ * Opt-in for single-page scraping (respectRobots option),
+ * foundation for FTR-ORG-019 crawl mode where it becomes default.
+ */
+/**
+ * Check if a URL is allowed by robots.txt.
+ * Returns true if allowed or if robots.txt cannot be fetched (fail-open).
+ */
+declare function isAllowedByRobots(url: string, timeout?: number): Promise<boolean>;
+/**
+ * Extract Sitemap: URLs from robots.txt.
+ * Returns empty array if no Sitemap directives or robots.txt unreachable (fail-open).
+ * Reuses cached robots.txt parser.
+ */
+declare function getSitemapUrls(url: string, timeout?: number): Promise<string[]>;
+/**
+ * Extract Crawl-delay value from robots.txt for ScraperBot user agent.
+ * Returns delay in seconds, or null if not specified or robots.txt unreachable (fail-open).
+ * Reuses cached robots.txt parser.
+ */
+declare function getCrawlDelay(url: string, timeout?: number): Promise<number | null>;
+/**
+ * Clear the robots.txt cache. Exported for testing.
+ */
+declare function clearRobotsCache(): void;
+
+/**
+ * Sitemap parser
+ *
+ * Fetches and parses sitemap.xml for crawl mode seed URLs.
+ * Regex-based XML parsing — no XML parser dependency.
+ * Handles sitemap index files with recursion limit.
+ * Mirrors robots.ts fetch+cache+fail-open pattern.
+ */
+/**
+ * A single entry from a sitemap
+ */
+interface SitemapEntry {
+    /** URL from <loc> tag */
+    loc: string;
+    /** Last modification date from <lastmod> tag */
+    lastmod?: string;
+    /** Priority from <priority> tag (0.0 to 1.0) */
+    priority?: number;
+}
+/**
+ * Parse a sitemap.xml and return URL entries.
+ *
+ * - Fetches the sitemap from the given URL
+ * - Extracts <loc> URLs via regex (no XML parser dependency)
+ * - Handles sitemap index files (recursive, max depth 2)
+ * - Caches results per URL with 1-hour TTL
+ * - Fail-open: returns empty array if sitemap is unreachable or invalid
+ * - Filters to same-origin URLs only
+ *
+ * @param url - Full URL of the sitemap.xml
+ * @param timeout - Fetch timeout in ms (default: 10000)
+ * @returns Array of SitemapEntry objects
+ */
+declare function parseSitemap(url: string, timeout?: number): Promise<SitemapEntry[]>;
+/**
+ * Clear the sitemap cache. Exported for testing.
+ */
+declare function clearSitemapCache(): void;
+
+/**
+ * Crawl: BFS multi-page orchestrator
+ * TKT-SCRAPER-079: Crawl multiple pages from a starting URL
+ *
+ * Composes: sitemap seeding, robots.txt, link extraction, URL normalization,
+ * depth/limit/filter constraints, concurrency, and scrape pipeline per page.
+ */
+
+declare function normalizeUrl(url: string): string;
+declare function extractLinks(html: string, baseUrl: string): string[];
+declare function crawl(options: CrawlOptions): Promise<CrawlResult>;
+
+/**
+ * Mode-aware fetch routing with challenge detection.
+ *
+ * Shared between scrape() (index.ts) and crawl() (crawl.ts)
+ * to avoid duplicating mode routing and challenge detection logic.
+ */
+
+/**
+ * Detect anti-bot challenge pages that return HTTP 200 but contain
+ * challenge/verification HTML instead of real content.
+ */
+declare function isChallengeResponse(fetchResult: FetchResult): boolean;
+declare function fetchWithMode(url: string, mode: FetchMode, options: FetchOptions): Promise<FetchResult>;
+
+/**
+ * scraper
+ * Context compression for AI agents
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Compress web content for AI agents
+ *
+ * Pipeline: Fetch -> Extract -> Convert
+ * No LLM dependency. Reduces tokens by 70-80%.
+ *
+ * @example
+ * ```typescript
+ * import { scrape } from '@robot-resources/scraper';
+ *
+ * const result = await scrape('https://example.com/article');
+ * console.log(result.markdown);
+ * console.log(result.tokenCount);
+ * ```
+ *
+ * @param url - URL to fetch and compress
+ * @param options - Optional configuration
+ * @returns Compressed content with metadata
+ */
+declare function scrape(url: string, options?: ScrapeOptions): Promise<ScrapeResult>;
+
+export { type ConvertResult, type CrawlError, type CrawlOptions, type CrawlPageResult, type CrawlResult, type ExtractResult, ExtractionError, FetchError, type FetchMode, type FetchResult, type ScrapeOptions, type ScrapeResult, type SitemapEntry, clearRobotsCache, clearSitemapCache, convertToMarkdown, crawl, estimateTokens, extractContent, extractLinks, fetchRender, fetchStealth, fetchUrl, fetchWithMode, getCrawlDelay, getSitemapUrls, isAllowedByRobots, isChallengeResponse, normalizeUrl, parseSitemap, scrape };