@wenathlan/saddle 1.8.17 → 2.0.2

package/dist/webscrape.js

@@ -0,0 +1,2697 @@
+/**
+ * webscrape.ts — the legacy WebScrape toolkit generation.
+ *
+ * This file carries the complete legacy scrape sub-project of the former
+ * repository (thirty sections: types, errors, utils, port, tokens,
+ * chunking, formats, serialize, headers, session, pool, proxy, proxypool,
+ * rate-limiter, retry, middleware, events, renderer, sitemap, agent,
+ * llms-txt, fetch, batch, browser, scrape, crawler, server, cli and
+ * dev-server) consolidated into the single root-level file of the grand
+ * merge. The optional npm runtimes of the toolkit (emittery, p-limit,
+ * p-retry, cheerio, playwright, hono, commander) load lazily so nothing is
+ * required at module load that the package does not ship; the CLI,
+ * dev-server and hono API bootstraps are exported factories.
+ */
+import { z } from "zod";
+import { JSDOM } from "jsdom";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { spawn } from "node:child_process";
+import { createServer as createnodeserver } from "node:http";
+import { dirname, join, resolve } from "node:path";
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 12: scrape/types.ts */
+/* ════════════════════════════════════════════════════════════════════ */
+export const ScrapeFormatSchema = z.enum(['markdown', 'html', 'text', 'json', 'xml', 'redis']);
+export const SerializeTargetSchema = z.enum(['markdown', 'xml', 'json', 'redis', 'text']);
+export const ScrapeOptionsSchema = z.strictObject({
+    url: z.string().url().optional(),
+    html: z.string().optional(),
+    mode: z.enum(['auto', 'fetch', 'browser']).prefault('auto').optional(),
+    waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle', 'commit']).prefault('networkidle').optional(),
+    timeout: z.number().positive().prefault(30000).optional(),
+    scroll: z.boolean().prefault(false).optional(),
+    scrollDelay: z.number().nonnegative().prefault(300).optional(),
+    maxScrolls: z.number().positive().prefault(50).optional(),
+    removeSelectors: z.array(z.string()).prefault([]).optional(),
+    extractImages: z.boolean().prefault(true).optional(),
+    extractLinks: z.boolean().prefault(true).optional(),
+    extractTables: z.boolean().prefault(true).optional(),
+    maxContentLength: z.number().positive().optional(),
+    format: ScrapeFormatSchema.prefault('markdown').optional(),
+    language: z.string().optional(),
+    proxy: z.string().optional(),
+    retries: z.number().nonnegative().prefault(3).optional(),
+    retryDelay: z.number().positive().prefault(1000).optional(),
+    userAgent: z.string().optional(),
+    headers: z.record(z.string(), z.string()).prefault({}).optional(),
+    cache: z.boolean().prefault(false).optional(),
+    cacheTtlMs: z.number().positive().prefault(300000).optional(),
+    readable: z.boolean().optional(),
+});
+export const BrowserAgentConfigSchema = z.strictObject({
+    headless: z.boolean().prefault(true).optional(),
+    userAgent: z.string().optional(),
+    viewport: z
+        .strictObject({
+        width: z.number().positive().prefault(1280),
+        height: z.number().positive().prefault(720),
+    })
+        .prefault({ width: 1280, height: 720 })
+        .optional(),
+    locale: z.string().prefault('en-US').optional(),
+    geolocation: z
+        .strictObject({
+        latitude: z.number(),
+        longitude: z.number(),
+    })
+        .optional(),
+    timeout: z.number().positive().prefault(30000).optional(),
+    recordVideo: z.boolean().prefault(false).optional(),
+    proxy: z.string().prefault('').optional(),
+    storageState: z.string().prefault('').optional(),
+    blockAds: z.boolean().prefault(true).optional(),
+    stealth: z.boolean().prefault(true).optional(),
+});
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 13: scrape/errors.ts */
+/* ════════════════════════════════════════════════════════════════════ */
+export const ErrorCode = {
+    VALIDATION_FAILED: 'VALIDATION_FAILED',
+    INVALID_URL: 'INVALID_URL',
+    TIMEOUT: 'TIMEOUT',
+    BLOCKED: 'BLOCKED',
+    RATE_LIMITED: 'RATE_LIMITED',
+    PROXY_ERROR: 'PROXY_ERROR',
+    PARSE_ERROR: 'PARSE_ERROR',
+    AUTH_REQUIRED: 'AUTH_REQUIRED',
+    NETWORK_ERROR: 'NETWORK_ERROR',
+    BROWSER_NOT_AVAILABLE: 'BROWSER_NOT_AVAILABLE',
+    CRAWL_DEPTH_EXCEEDED: 'CRAWL_DEPTH_EXCEEDED',
+    MAX_RETRIES_EXCEEDED: 'MAX_RETRIES_EXCEEDED',
+};
+export class WebScrapeError extends Error {
+    code;
+    statusCode;
+    isRetryable;
+    timestamp;
+    details;
+    cause;
+    constructor(message, code, statusCode, isRetryable, details, cause) {
+        super(message);
+        this.name = 'WebScrapeError';
+        this.code = code;
+        this.statusCode = statusCode;
+        this.isRetryable = isRetryable;
+        this.timestamp = new Date().toISOString();
+        this.details = details;
+        this.cause = cause;
+        Object.setPrototypeOf(this, new.target.prototype);
+        Error.captureStackTrace?.(this, this.constructor);
+    }
+    toJSON() {
+        return {
+            error: {
+                name: this.name,
+                code: this.code,
+                message: this.message,
+                statusCode: this.statusCode,
+                isRetryable: this.isRetryable,
+                timestamp: this.timestamp,
+                details: this.details,
+            },
+        };
+    }
+}
+export class ValidationError extends WebScrapeError {
+    constructor(message, details, cause) {
+        super(message, ErrorCode.VALIDATION_FAILED, 400, false, details, cause);
+        this.name = 'ValidationError';
+    }
+}
+export class TimeoutError extends WebScrapeError {
+    constructor(message, details, cause) {
+        super(message, ErrorCode.TIMEOUT, 504, true, details, cause);
+        this.name = 'TimeoutError';
+    }
+}
+export class BlockedError extends WebScrapeError {
+    constructor(message, details, cause) {
+        super(message, ErrorCode.BLOCKED, 403, false, details, cause);
+        this.name = 'BlockedError';
+    }
+}
+export class RateLimitError extends WebScrapeError {
+    retryAfterMs;
+    constructor(message, retryAfterMs, details, cause) {
+        super(message, ErrorCode.RATE_LIMITED, 429, true, { ...details, retryAfterMs }, cause);
+        this.name = 'RateLimitError';
+        this.retryAfterMs = retryAfterMs;
+    }
+}
+export class ProxyError extends WebScrapeError {
+    constructor(message, details, cause) {
+        super(message, ErrorCode.PROXY_ERROR, 502, true, details, cause);
+        this.name = 'ProxyError';
+    }
+}
+export class ParseError extends WebScrapeError {
+    constructor(message, details, cause) {
+        super(message, ErrorCode.PARSE_ERROR, 422, false, details, cause);
+        this.name = 'ParseError';
+    }
+}
+export class AuthError extends WebScrapeError {
+    constructor(message, details, cause) {
+        super(message, ErrorCode.AUTH_REQUIRED, 401, false, details, cause);
+        this.name = 'AuthError';
+    }
+}
+export class NetworkError extends WebScrapeError {
+    constructor(message, details, cause) {
+        super(message, ErrorCode.NETWORK_ERROR, 503, true, details, cause);
+        this.name = 'NetworkError';
+    }
+}
+export class BrowserNotAvailableError extends WebScrapeError {
+    constructor(message = 'Playwright is not installed. Install it with: npm install playwright', details) {
+        super(message, ErrorCode.BROWSER_NOT_AVAILABLE, 500, false, details);
+        this.name = 'BrowserNotAvailableError';
+    }
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 14: scrape/utils.ts */
+/* ════════════════════════════════════════════════════════════════════ */
+export function ensureDir(path) {
+    const dir = dirname(path);
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+}
+export function writeOutput(path, content) {
+    ensureDir(path);
+    writeFileSync(path, content, 'utf-8');
+}
+export function slugify(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\w\s-]/g, '')
+        .replace(/[\s_]+/g, '-')
+        .replace(/^-+|-+$/g, '');
+}
+export function truncate(text, max, suffix = '...') {
+    if (text.length <= max)
+        return text;
+    return text.slice(0, max - suffix.length) + suffix;
+}
+export function legacyestimatetokens(text) {
+    return Math.ceil(text.length / 4);
+}
+export function legacychunktext(text, chunkSize) {
+    const chunks = [];
+    let i = 0;
+    while (i < text.length) {
+        let end = i + chunkSize;
+        if (end < text.length) {
+            const breakIdx = text.lastIndexOf('\n', end);
+            if (breakIdx > i)
+                end = breakIdx + 1;
+        }
+        chunks.push(text.slice(i, end).trim());
+        i = end;
+    }
+    return chunks;
+}
+export function delay(ms) {
+    return new Promise(r => setTimeout(r, ms));
+}
+export function isValidUrl(url) {
+    try {
+        new URL(url);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export function normalizeUrl(url, base) {
+    try {
+        return new URL(url, base).href;
+    }
+    catch {
+        return url;
+    }
+}
+export function isInternalUrl(href, base) {
+    try {
+        const baseUrl = new URL(base);
+        const target = new URL(href, base);
+        return target.hostname === baseUrl.hostname;
+    }
+    catch {
+        return false;
+    }
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 15: scrape/port.ts — Generate a random port number in the ephemeral port range (1024-65535). */
+/* ════════════════════════════════════════════════════════════════════ */
+/**
+ * Generate a random port number in the ephemeral port range (1024-65535).
+ * The port is calculated once and then fixed for the lifetime of the process.
+ */
+let _cachedPort = null;
+export function randomPort() {
+    if (_cachedPort !== null)
+        return _cachedPort;
+    _cachedPort = Math.floor(Math.random() * 64511) + 1024;
+    return _cachedPort;
+}
+/**
+ * Reset the cached port (for testing).
+ */
+export function resetPort() {
+    _cachedPort = null;
+}
+const MODEL_TOKEN_RATIOS = {
+    'gpt-4o': 3.5,
+    'gpt-4': 3.5,
+    'gpt-3.5-turbo': 4,
+    'claude': 3.2,
+    'gemini': 3.5,
+    'default': 4,
+};
+export function estimateTokens(text, model = 'default') {
+    const ratio = MODEL_TOKEN_RATIOS[model] || 4;
+    return Math.ceil(text.length / ratio);
+}
+export function countTokens(text, model = 'default') {
+    return estimateTokens(text, model);
+}
+export function fitsInContext(text, maxTokens, model = 'default') {
+    return estimateTokens(text, model) <= maxTokens;
+}
+export function truncateToTokens(text, maxTokens, model = 'default') {
+    const estimatedTokens = estimateTokens(text, model);
+    if (estimatedTokens <= maxTokens)
+        return text;
+    const charLimit = Math.floor(maxTokens * (MODEL_TOKEN_RATIOS[model] || 4));
+    return text.slice(0, charLimit) + '...';
+}
+export function tokenCost(text, model, pricePer1kTokens) {
+    const tokens = estimateTokens(text, model);
+    return (tokens / 1000) * pricePer1kTokens;
+}
+export const MODEL_PRICES = {
+    'gpt-4o': { input: 0.0025, output: 0.01 },
+    'gpt-4o-mini': { input: 0.00015, output: 0.0006 },
+    'gpt-4-turbo': { input: 0.01, output: 0.03 },
+    'claude-3-5-sonnet': { input: 0.003, output: 0.015 },
+    'claude-3-haiku': { input: 0.00025, output: 0.00125 },
+    'gemini-1.5-pro': { input: 0.00125, output: 0.005 },
+    'gemini-1.5-flash': { input: 0.000075, output: 0.0003 },
+};
+const DEFAULT_CHUNK_OPTIONS = {
+    maxTokens: 512,
+    overlapTokens: 50,
+    preserveCodeBlocks: true,
+    includeHeadingPath: true,
+};
+function splitByHeaders(markdown) {
+    const sections = [];
+    const lines = markdown.split('\n');
+    let currentHeading = '';
+    let currentLevel = 0;
+    let currentContent = [];
+    for (const line of lines) {
+        const headerMatch = line.match(/^(#{1,6})\s+(.+)/);
+        if (headerMatch) {
+            if (currentContent.length > 0 || currentHeading) {
+                sections.push({
+                    heading: currentHeading,
+                    level: currentLevel,
+                    content: currentContent.join('\n').trim(),
+                });
+            }
+            currentLevel = headerMatch[1].length;
+            currentHeading = headerMatch[2];
+            currentContent = [];
+        }
+        else {
+            currentContent.push(line);
+        }
+    }
+    if (currentContent.length > 0 || currentHeading) {
+        sections.push({
+            heading: currentHeading,
+            level: currentLevel,
+            content: currentContent.join('\n').trim(),
+        });
+    }
+    return sections;
+}
+function splitByParagraphs(text) {
+    return text.split(/\n\n+/).filter(p => p.trim().length > 0);
+}
+function splitBySentences(text) {
+    return text.match(/[^.!?]+[.!?]+/g) || [text];
+}
+export function chunkMarkdown(markdown, options = {}) {
+    const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
+    const chunks = [];
+    const sections = splitByHeaders(markdown);
+    const headingPath = [];
+    let headingDepth = 0;
+    for (const section of sections) {
+        while (headingDepth > 0 && headingDepth >= section.level) {
+            headingDepth--;
+            headingPath.pop();
+        }
+        if (section.heading) {
+            headingDepth = section.level;
+            headingPath.push(section.heading);
+        }
+        const sectionText = section.heading ? `#${'#'.repeat(section.level - 1)} ${section.heading}\n\n${section.content}` : section.content;
+        if (estimateTokens(sectionText) <= opts.maxTokens) {
+            chunks.push({
+                content: sectionText,
+                headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+                tokenCount: estimateTokens(sectionText),
+                index: chunks.length,
+            });
+        }
+        else {
+            const paragraphs = splitByParagraphs(sectionText);
+            let currentChunk = '';
+            for (const paragraph of paragraphs) {
+                const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
+                if (estimateTokens(testChunk) <= opts.maxTokens) {
+                    currentChunk = testChunk;
+                }
+                else {
+                    if (currentChunk) {
+                        chunks.push({
+                            content: currentChunk,
+                            headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+                            tokenCount: estimateTokens(currentChunk),
+                            index: chunks.length,
+                        });
+                    }
+                    if (estimateTokens(paragraph) > opts.maxTokens) {
+                        const sentences = splitBySentences(paragraph);
+                        let sentenceChunk = '';
+                        for (const sentence of sentences) {
+                            const testSentence = sentenceChunk ? `${sentenceChunk} ${sentence}` : sentence;
+                            if (estimateTokens(testSentence) <= opts.maxTokens) {
+                                sentenceChunk = testSentence;
+                            }
+                            else {
+                                if (sentenceChunk) {
+                                    chunks.push({
+                                        content: sentenceChunk,
+                                        headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+                                        tokenCount: estimateTokens(sentenceChunk),
+                                        index: chunks.length,
+                                    });
+                                }
+                                sentenceChunk = sentence;
+                            }
+                        }
+                        currentChunk = sentenceChunk;
+                    }
+                    else {
+                        currentChunk = paragraph;
+                    }
+                }
+            }
+            if (currentChunk) {
+                chunks.push({
+                    content: currentChunk,
+                    headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+                    tokenCount: estimateTokens(currentChunk),
+                    index: chunks.length,
+                });
+            }
+        }
+    }
+    return chunks;
+}
+export function chunkText(text, options = {}) {
+    const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
+    const chunks = [];
+    const paragraphs = splitByParagraphs(text);
+    let currentChunk = '';
+    for (const paragraph of paragraphs) {
+        const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
+        if (estimateTokens(testChunk) <= opts.maxTokens) {
+            currentChunk = testChunk;
+        }
+        else {
+            if (currentChunk) {
+                chunks.push({
+                    content: currentChunk,
+                    headingPath: [],
+                    tokenCount: estimateTokens(currentChunk),
+                    index: chunks.length,
+                });
+            }
+            currentChunk = paragraph;
+        }
+    }
+    if (currentChunk) {
+        chunks.push({
+            content: currentChunk,
+            headingPath: [],
+            tokenCount: estimateTokens(currentChunk),
+            index: chunks.length,
+        });
+    }
+    return chunks;
+}
+export function formatChunksForRAG(chunks) {
+    return chunks.map(chunk => {
+        const heading = chunk.headingPath.length > 0 ? `## ${chunk.headingPath.join(' > ')}\n\n` : '';
+        return `${heading}${chunk.content}`;
+    }).join('\n\n---\n\n');
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 18: scrape/formats.ts */
+/* ════════════════════════════════════════════════════════════════════ */
+const FORMAT_ALIASES = {
+    md: 'markdown',
+    txt: 'text',
+    plain: 'text',
+    xml: 'xml',
+    json: 'json',
+    redis: 'redis',
+    html: 'html',
+    h: 'html',
+};
+const FORMAT_EXTENSIONS = {
+    '.md': 'markdown',
+    '.markdown': 'markdown',
+    '.html': 'html',
+    '.htm': 'html',
+    '.txt': 'text',
+    '.json': 'json',
+    '.xml': 'xml',
+    '.redis': 'redis',
+};
+export function resolveFormat(format) {
+    const lower = format.toLowerCase().trim();
+    return FORMAT_ALIASES[lower] || lower;
+}
+export function formatFromExtension(ext) {
+    return FORMAT_EXTENSIONS[ext.toLowerCase()] ?? null;
+}
+export function extensionForFormat(format) {
+    const map = {
+        markdown: '.md',
+        html: '.html',
+        text: '.txt',
+        json: '.json',
+        xml: '.xml',
+        redis: '.redis',
+    };
+    return map[format] || '.md';
+}
+export function detectContentType(html) {
+    if (html.includes('<article') || html.includes('entry-content') || html.includes('class="post-content"') || html.includes('class="article-content"') || html.includes('role="main"')) {
+        return 'article';
+    }
+    const listCount = (html.match(/<li>/gi) || []).length;
+    if (listCount > 20)
+        return 'list';
+    if (html.includes('<article') || html.includes('entry-content'))
+        return 'article';
+    return 'page';
+}
+export function buildSerializeOptions(format, pretty = true) {
+    const targetMap = {
+        markdown: 'markdown',
+        html: 'markdown',
+        text: 'text',
+        json: 'json',
+        xml: 'xml',
+        redis: 'redis',
+    };
+    return {
+        format: targetMap[format] || 'markdown',
+        pretty,
+        includeMetadata: true,
+    };
+}
+export async function convertResult(result, targetFormat) {
+    if (result.format === targetFormat)
+        return result;
+    const opts = buildSerializeOptions(targetFormat);
+    const serialized = sr(result, opts);
+    return {
+        ...result,
+        content: serialized.content,
+        format: targetFormat,
+        size: serialized.size,
+    };
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 19: scrape/serialize.ts */
+/* ════════════════════════════════════════════════════════════════════ */
+function htmlToMarkdown(html) {
+    const dom = new JSDOM(html);
+    const doc = dom.window.document;
+    let md = '';
+    function processNode(node, depth = 0) {
+        if (node.nodeType === 3) {
+            const text = node.textContent?.trim();
+            if (text)
+                md += text + ' ';
+            return;
+        }
+        if (node.nodeType !== 1)
+            return;
+        const el = node;
+        const tag = el.tagName.toLowerCase();
+        switch (tag) {
+            case 'h1':
+            case 'h2':
+            case 'h3':
+            case 'h4':
+            case 'h5':
+            case 'h6': {
+                const level = parseInt(tag[1]);
+                md += '\n' + '#'.repeat(level) + ' ' + el.textContent?.trim() + '\n\n';
+                break;
+            }
+            case 'p':
+                md += '\n' + el.textContent?.trim() + '\n\n';
+                break;
+            case 'br':
+                md += '\n';
+                break;
+            case 'hr':
+                md += '\n---\n\n';
+                break;
+            case 'ul':
+            case 'ol': {
+                el.childNodes.forEach(li => {
+                    if (li.nodeType === 1) {
+                        const liEl = li;
+                        if (liEl.tagName.toLowerCase() === 'li') {
+                            md += '  '.repeat(depth) + '- ' + liEl.textContent?.trim() + '\n';
+                        }
+                    }
+                });
+                md += '\n';
+                break;
+            }
+            case 'a': {
+                const href = el.getAttribute('href') || '';
+                const text = el.textContent?.trim() || href;
+                if (href && href !== '#')
+                    md += `[${text}](${href}) `;
+                else
+                    md += text + ' ';
+                break;
+            }
+            case 'img': {
+                const src = el.getAttribute('src') || '';
+                const alt = el.getAttribute('alt') || '';
+                if (src)
+                    md += `![${alt}](${src}) `;
+                break;
+            }
+            case 'strong':
+            case 'b':
+                md += '**' + el.textContent?.trim() + '** ';
+                break;
+            case 'em':
+            case 'i':
+                md += '*' + el.textContent?.trim() + '* ';
+                break;
+            case 'code':
+                md += '`' + el.textContent?.trim() + '` ';
+                break;
+            case 'pre': {
+                const code = el.querySelector('code');
+                const lang = code?.getAttribute('class')?.replace(/^language-/, '') || '';
+                md += '\n```' + lang + '\n' + (code || el).textContent + '\n```\n\n';
+                break;
+            }
+            case 'blockquote':
+                md += '\n> ' + el.textContent?.trim().replace(/\n/g, '\n> ') + '\n\n';
+                break;
+            case 'table': {
+                const rows = el.querySelectorAll('tr');
+                rows.forEach((row, i) => {
+                    const cells = row.querySelectorAll('td, th');
+                    md += '| ' + Array.from(cells).map(c => c.textContent?.trim()).join(' | ') + ' |\n';
+                    if (i === 0 && row.parentElement?.tagName === 'THEAD') {
+                        md += '|' + Array.from(cells).map(() => ' --- ').join('|') + '|\n';
+                    }
+                });
+                md += '\n';
+                break;
+            }
+            default:
+                el.childNodes.forEach(child => processNode(child, depth));
+        }
+    }
+    doc.body?.childNodes.forEach((child) => processNode(child));
+    return md.replace(/\n{3,}/g, '\n\n').trim();
+}
+function htmlToXml(html, root = 'document') {
+    const dom = new JSDOM(html);
+    const doc = dom.window.document;
+    function serialize(el, depth = 0) {
+        const indent = '  '.repeat(depth);
+        const tag = el.tagName.toLowerCase();
+        if (el.childNodes.length === 0) {
+            return `${indent}<${tag} />\n`;
+        }
+        const text = el.textContent?.trim();
+        if (el.childNodes.length === 1 && el.firstChild?.nodeType === 3 && text) {
+            return `${indent}<${tag}>${escapeXml(text)}</${tag}>\n`;
+        }
+        let xml = `${indent}<${tag}>\n`;
+        el.childNodes.forEach(child => {
+            if (child.nodeType === 1)
+                xml += serialize(child, depth + 1);
+            else if (child.nodeType === 3 && child.textContent?.trim()) {
+                xml += `${indent}  ${escapeXml(child.textContent.trim())}\n`;
+            }
+        });
+        xml += `${indent}</${tag}>\n`;
+        return xml;
+    }
+    function escapeXml(s) {
+        return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
+    }
+    let xml = `<?xml version="1.0" encoding="UTF-8"?>\n<${root}>\n`;
+    doc.body?.childNodes.forEach((child) => {
+        if (child.nodeType === 1)
+            xml += serialize(child, 1);
+    });
+    xml += `</${root}>\n`;
+    return xml;
+}
+function htmlToRedisJson(html, key = 'page') {
+    const dom = new JSDOM(html);
+    const doc = dom.window.document;
+    function extract(el) {
+        const tag = el.tagName.toLowerCase();
+        const children = Array.from(el.children);
+        if (children.length === 0)
+            return { _tag: tag, _text: el.textContent?.trim() || '' };
+        const result = { _tag: tag };
+        if (el.getAttribute('href'))
+            result.href = el.getAttribute('href');
+        if (el.getAttribute('src'))
+            result.src = el.getAttribute('src');
+        if (el.getAttribute('alt'))
+            result.alt = el.getAttribute('alt');
+        if (el.getAttribute('class'))
+            result.class = el.getAttribute('class');
+        children.forEach(child => {
+            const childTag = child.tagName.toLowerCase();
+            if (result[childTag]) {
+                if (!Array.isArray(result[childTag]))
+                    result[childTag] = [result[childTag]];
+                result[childTag].push(extract(child));
+            }
+            else {
+                result[childTag] = extract(child);
+            }
+        });
+        return result;
+    }
+    const title = doc.title;
+    const bodyArr = [];
+    doc.body?.childNodes.forEach((child) => {
+        if (child.nodeType === 1)
+            bodyArr.push(extract(child));
+    });
+    const json = {
+        title,
+        body: bodyArr,
+        links: Array.from(doc.querySelectorAll('a[href]')).map((a) => ({
+            href: a.getAttribute('href'),
+            text: a.textContent?.trim(),
+        })),
+        images: Array.from(doc.querySelectorAll('img[src]')).map((img) => ({
+            src: img.getAttribute('src'),
+            alt: img.getAttribute('alt'),
+        })),
+    };
+    return `JSON.SET ${key} $ ${JSON.stringify(json, null, 2)}`;
+}
+/** Single-pass HTML entity decode map: the replace callback resolves each entity exactly once so a double-escaped `&amp;lt;` decodes to the literal text `&lt;` and never again to `<`. */
+const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', '#39': "'" };
+/** Linear single-pass script/style block removal: case-insensitive, tolerates `</script >` closing whitespace, and drops any unterminated block so no `<script` fragment can survive. One scan, no regex backtracking. */
+function stripblocks(html, ...tags) {
+    const open = new RegExp(`<(${tags.join('|')})\\b[^<>]*>`, 'i');
+    let out = '';
+    let cursor = 0;
+    let match;
+    while ((match = open.exec(html.slice(cursor))) !== null) {
+        const start = cursor + match.index;
+        out += html.slice(cursor, start);
+        const closeTag = `</${match[1].toLowerCase()}`;
+        const close = html.toLowerCase().indexOf(closeTag, start + match[0].length);
+        if (close < 0)
+            return out; /* unterminated block: the rest is dropped */
+        const gt = html.indexOf('>', close);
+        cursor = gt < 0 ? html.length : gt + 1;
+    }
+    return out + html.slice(cursor);
+}
+/** Converts sanitized HTML to plain text: script/style blocks drop in one linear pass, remaining tags strip once and entities decode once. */
+function plainText(html) {
+    return stripblocks(html, 'script', 'style')
+        .replace(/<br\s*\/?>/gi, '\n')
+        .replace(/<\/p>/gi, '\n\n')
+        .replace(/<\/h[1-6]>/gi, '\n')
+        .replace(/<\/li>/gi, '\n')
+        .replace(/<\/tr>/gi, '\n')
+        .replace(/<\/td>/gi, ' | ')
+        .replace(/<\/th>/gi, ' | ')
+        .replace(/<[^<>]+>/g, '')
+        .replace(/<\/?(?:script|style)\b[^<>]*$/i, '')
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&(amp|lt|gt|quot|#39);/g, (whole, entity) => ENTITIES[entity] ?? whole)
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+function tableToMarkdown(headers, rows, caption) {
+    let md = caption ? `### ${caption}\n\n` : '';
+    md += '| ' + headers.join(' | ') + ' |\n';
+    md += '|' + headers.map(() => ' --- ').join('|') + '|\n';
+    for (const row of rows) {
+        md += '| ' + row.join(' | ') + ' |\n';
+    }
+    return md + '\n';
+}
+export function serializeResult(result, options) {
+    let content = '';
+    if (options.includeMetadata !== false) {
+        content += `# ${result.title}\n\n`;
+        content += `- **URL:** ${result.url}\n`;
+        content += `- **Extracted:** ${result.extractedAt}\n`;
+        if (result.metadata.author)
+            content += `- **Author:** ${result.metadata.author}\n`;
+        if (result.metadata.publishedDate)
+            content += `- **Published:** ${result.metadata.publishedDate}\n`;
+        content += `- **Size:** ${result.size} bytes\n\n---\n\n`;
+    }
+    switch (options.format) {
+        case 'markdown':
+            content += result.content;
+            if (result.tables.length) {
+                content += '\n\n## Tables\n\n';
+                for (const t of result.tables) {
+                    content += tableToMarkdown(t.headers, t.rows, t.caption);
+                }
+            }
+            break;
+        case 'text':
+            content += result.text;
+            break;
+        case 'xml':
+            content = htmlToXml(`<html><head><title>${result.title}</title></head><body>${result.content}</body></html>`, options.xmlRoot || 'document');
+            break;
+        case 'json':
+            content = JSON.stringify({
+                title: result.title,
+                url: result.url,
+                metadata: result.metadata,
+                content: result.content,
+                links: result.links,
+                images: result.images,
+                tables: result.tables,
+                extractedAt: result.extractedAt,
+                duration: result.duration,
+            }, null, options.pretty ? 2 : undefined);
+            break;
+        case 'redis':
+            content = htmlToRedisJson(`<html><head><title>${result.title}</title></head><body>${result.content}</body></html>`, options.redisKey || `page:${encodeURIComponent(result.url)}`);
+            break;
+    }
+    const chunks = options.maxChunkSize ? legacychunktext(content, options.maxChunkSize) : [];
+    const output = {
+        format: options.format,
+        content,
+        chunks: chunks.length ? chunks : undefined,
+        size: content.length,
+    };
+    if (options.includeMetadata) {
+        output.metadata = {
+            title: result.title,
+            url: result.url,
+            format: result.format,
+            extractedAt: result.extractedAt,
+            duration: result.duration,
+            linkCount: result.links.length,
+            imageCount: result.images.length,
+            tableCount: result.tables.length,
+        };
+    }
+    return output;
+}
+export function serializeHtml(html, options) {
+    const content = (() => {
+        switch (options.format) {
+            case 'markdown': return htmlToMarkdown(html);
+            case 'xml': return htmlToXml(html, options.xmlRoot || 'document');
+            case 'json': return JSON.stringify({ html }, null, options.pretty ? 2 : undefined);
+            case 'redis': return htmlToRedisJson(html, options.redisKey || 'page:1');
+            case 'text': return plainText(html);
+            default: return plainText(html);
+        }
+    })();
+    const chunks = options.maxChunkSize ? legacychunktext(content, options.maxChunkSize) : [];
+    return {
+        format: options.format,
+        content,
+        chunks: chunks.length ? chunks : undefined,
+        size: content.length,
+    };
+}
+const CHROME_PROFILES = [
+    {
+        userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+        accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+        acceptLanguage: 'en-US,en;q=0.9',
+        secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
+        secChUaPlatform: '"Windows"',
+    },
+    {
+        userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+        accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+        acceptLanguage: 'en-US,en;q=0.9',
+        secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
+        secChUaPlatform: '"macOS"',
+    },
+    {
+        userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
+        accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+        acceptLanguage: 'en-US,en;q=0.5',
+        secChUa: '',
+        secChUaPlatform: '"Windows"',
+    },
+    {
+        userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
+        accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        acceptLanguage: 'en-US,en;q=0.9',
+        secChUa: '',
+        secChUaPlatform: '"macOS"',
+    },
+];
+export function getRandomProfile() {
+    return CHROME_PROFILES[Math.floor(Math.random() * CHROME_PROFILES.length)];
+}
+export function getHeaders(profile) {
+    const p = profile || getRandomProfile();
+    const headers = {
+        'User-Agent': p.userAgent,
+        'Accept': p.accept,
+        'Accept-Language': p.acceptLanguage,
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-User': '?1',
+    };
+    if (p.secChUa) {
+        headers['sec-ch-ua'] = p.secChUa;
+        headers['sec-ch-ua-mobile'] = '?0';
+        headers['sec-ch-ua-platform'] = p.secChUaPlatform;
+    }
+    return headers;
+}
+export function mergeHeaders(base, override) {
+    return { ...base, ...override };
+}
+export class CookieJar {
+    cookies = new Map();
+    set(cookie) {
+        const key = `${cookie.domain || ''}:${cookie.path || '/'}:${cookie.name}`;
+        this.cookies.set(key, cookie);
+    }
+    get(name, domain, _path) {
+        for (const [key, cookie] of this.cookies) {
+            if (cookie.name !== name)
+                continue;
+            if (domain && cookie.domain && !domain.includes(cookie.domain))
+                continue;
+            if (cookie.expires && cookie.expires < Date.now()) {
+                this.cookies.delete(key);
+                continue;
+            }
+            return cookie;
+        }
+        return undefined;
+    }
+    getAll(domain) {
+        const result = [];
+        const now = Date.now();
+        for (const [key, cookie] of this.cookies) {
+            if (cookie.expires && cookie.expires < now) {
+                this.cookies.delete(key);
+                continue;
+            }
+            if (domain && cookie.domain && !domain.includes(cookie.domain))
+                continue;
+            result.push(cookie);
+        }
+        return result;
+    }
+    toString(domain) {
+        return this.getAll(domain)
+            .map(c => `${c.name}=${c.value}`)
+            .join('; ');
+    }
+    parseSetCookie(header, defaultDomain) {
+        const parts = header.split(';').map(s => s.trim());
+        const [nameValue, ...attrs] = parts;
+        const [name, ...valueParts] = nameValue.split('=');
+        const value = valueParts.join('=');
+        const cookie = {
+            name: name.trim(),
+            value: value.trim(),
+            domain: defaultDomain,
+            path: '/',
+        };
+        for (const attr of attrs) {
+            const [key, val] = attr.split('=').map(s => s.trim());
+            const lower = key.toLowerCase();
+            if (lower === 'domain')
+                cookie.domain = val;
+            else if (lower === 'path')
+                cookie.path = val;
+            else if (lower === 'expires')
+                cookie.expires = new Date(val).getTime();
+            else if (lower === 'httponly')
+                cookie.httpOnly = true;
+            else if (lower === 'secure')
+                cookie.secure = true;
+        }
+        this.set(cookie);
+    }
+    clear() {
+        this.cookies.clear();
+    }
+}
+export class ScrapingSession {
+    cookieJar;
+    id;
+    data = new Map();
+    constructor(id) {
+        this.id = id || crypto.randomUUID();
+        this.cookieJar = new CookieJar();
+    }
+    get(key) {
+        return this.data.get(key);
+    }
+    set(key, value) {
+        this.data.set(key, value);
+    }
+    clear() {
+        this.cookieJar.clear();
+        this.data.clear();
+    }
+}
+export function createSession(id) {
+    return new ScrapingSession(id);
+}
+export class BrowserPool {
+    pool = [];
+    waitQueue = [];
+    config;
+    browserConfig;
+    constructor(config = {}, browserConfig = {}) {
+        this.config = {
+            maxSize: config.maxSize || 3,
+            acquireTimeoutMs: config.acquireTimeoutMs || 30000,
+        };
+        this.browserConfig = browserConfig;
+    }
+    async acquire() {
+        const idle = this.pool.find(e => !e.inUse && e.browser.isConnected());
+        if (idle) {
+            idle.inUse = true;
+            return idle.browser;
+        }
+        if (this.pool.length < this.config.maxSize) {
+            const browser = new AgentBrowser(this.browserConfig);
+            await browser.launch();
+            const entry = { browser, inUse: true, createdAt: Date.now() };
+            this.pool.push(entry);
+            return browser;
+        }
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                const idx = this.waitQueue.indexOf(waiter);
+                if (idx !== -1)
+                    this.waitQueue.splice(idx, 1);
+                reject(new Error('Timeout waiting for browser'));
+            }, this.config.acquireTimeoutMs);
+            const waiter = (browser) => {
+                clearTimeout(timeout);
+                resolve(browser);
+            };
+            this.waitQueue.push(waiter);
+        });
+    }
+    release(browser) {
+        const entry = this.pool.find(e => e.browser === browser);
+        if (entry) {
+            entry.inUse = false;
+            if (this.waitQueue.length > 0) {
+                const waiter = this.waitQueue.shift();
+                entry.inUse = true;
+                waiter(browser);
+            }
+        }
+    }
+    async destroy() {
+        for (const entry of this.pool) {
+            await entry.browser.close().catch(() => { });
+        }
+        this.pool = [];
+        for (const waiter of this.waitQueue) {
+            waiter(new AgentBrowser());
+        }
+        this.waitQueue = [];
+    }
+    getStatus() {
+        return {
+            total: this.pool.length,
+            idle: this.pool.filter(e => !e.inUse).length,
+            inUse: this.pool.filter(e => e.inUse).length,
+            waiting: this.waitQueue.length,
+        };
+    }
+}
+export function createPool(config, browserConfig) {
+    return new BrowserPool(config, browserConfig);
+}
+const DEFAULT_POOL_CONFIG = {
+    maxFailures: 3,
+    reviveAfterMs: 30 * 60 * 1000,
+    strategy: 'round-robin',
+};
+export class ProxyPool {
+    proxies;
+    stats = new Map();
+    currentIndex = 0;
+    config;
+    constructor(proxies, config = {}) {
+        this.proxies = proxies;
+        this.config = { ...DEFAULT_POOL_CONFIG, ...config };
+        for (const p of proxies) {
+            this.stats.set(p.url, { failures: 0, successes: 0, consecutiveFailures: 0, disabled: false });
+        }
+    }
+    getActiveProxies() {
+        const now = Date.now();
+        return this.proxies.filter(p => {
+            const s = this.stats.get(p.url);
+            if (!s)
+                return true;
+            if (s.disabled && s.disabledAt && (now - s.disabledAt) >= this.config.reviveAfterMs) {
+                s.disabled = false;
+                s.failures = 0;
+                s.consecutiveFailures = 0;
+                return true;
+            }
+            return !s.disabled;
+        });
+    }
+    next() {
+        const active = this.getActiveProxies();
+        if (active.length === 0)
+            return null;
+        if (this.config.strategy === 'random') {
+            return active[Math.floor(Math.random() * active.length)];
+        }
+        const proxy = active[this.currentIndex % active.length];
+        this.currentIndex = (this.currentIndex + 1) % active.length;
+        return proxy;
+    }
+    reportSuccess(proxy) {
+        const s = this.stats.get(proxy.url);
+        if (s) {
+            s.successes++;
+            s.consecutiveFailures = 0;
+            s.lastSuccess = Date.now();
+        }
+    }
+    reportFailure(proxy) {
+        const s = this.stats.get(proxy.url);
+        if (s) {
+            s.failures++;
+            s.consecutiveFailures++;
+            s.lastFailure = Date.now();
+            if (s.consecutiveFailures >= this.config.maxFailures) {
+                s.disabled = true;
+                s.disabledAt = Date.now();
+            }
+        }
+    }
+    getStatus() {
+        const active = this.getActiveProxies();
+        return {
+            total: this.proxies.length,
+            active: active.length,
+            disabled: this.proxies.length - active.length,
+        };
+    }
+}
+export function createProxyPool(proxies, config) {
+    return new ProxyPool(proxies, config);
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 24: scrape/proxypool.ts — proxy pool chooses healthy least used entries and never rotates blindly. */
+/* ════════════════════════════════════════════════════════════════════ */
+/**
+ * proxy pool chooses healthy least used entries and never rotates blindly.
+ */
+export function proxypool(options = {}) {
+    const entries = (options.proxies ?? []).map((proxy) => ({ ...proxy, failures: 0, uses: 0, status: "active", lastused: 0 }));
+    const threshold = options.failurethreshold ?? 3;
+    const recovery = options.recoverytime ?? 300000;
+    function revive() { const now = Date.now(); for (const entry of entries)
+        if (entry.status === "graveyard" && now - entry.failedat >= recovery) {
+            entry.status = "active";
+            entry.failures = 0;
+        } }
+    function choose() { revive(); const available = entries.filter((entry) => entry.status === "active"); if (!available.length)
+        throw new Error("proxy pool has no healthy entries"); const selected = [...available].sort((left, right) => left.uses - right.uses || left.lastused - right.lastused)[0]; selected.uses += 1; selected.lastused = Date.now(); return { ...selected }; }
+    function report(id, result = {}) { const entry = entries.find((value) => value.id === id); if (!entry)
+        return null; if (result.ok) {
+        entry.failures = 0;
+        entry.status = "active";
+    }
+    else {
+        entry.failures += 1;
+        entry.status = entry.failures >= threshold ? "graveyard" : "active";
+        entry.failedat = Date.now();
+    } return { ...entry }; }
+    return { choose, report, list() { revive(); return entries.map((entry) => ({ ...entry })); } };
+}
+const DEFAULT_CONFIG = {
+    maxTokensPerInterval: 10,
+    intervalMs: 1000,
+    maxConcurrent: 5,
+};
+export class RateLimiter {
+    buckets = new Map();
+    active = new Map();
+    config;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+    }
+    getBucket(key) {
+        if (!this.buckets.has(key)) {
+            this.buckets.set(key, {
+                tokens: this.config.maxTokensPerInterval,
+                lastRefill: Date.now(),
+                maxTokens: this.config.maxTokensPerInterval,
+                refillRate: this.config.maxTokensPerInterval / (this.config.intervalMs / 1000),
+            });
+        }
+        return this.buckets.get(key);
+    }
+    refill(bucket) {
+        const now = Date.now();
+        const elapsed = (now - bucket.lastRefill) / 1000;
+        bucket.tokens = Math.min(bucket.maxTokens, bucket.tokens + elapsed * bucket.refillRate);
+        bucket.lastRefill = now;
+    }
+    async acquire(key) {
+        const bucket = this.getBucket(key);
+        const currentActive = this.active.get(key) || 0;
+        if (currentActive >= this.config.maxConcurrent) {
+            await new Promise((resolve) => setTimeout(resolve, 100));
+            return this.acquire(key);
+        }
+        this.refill(bucket);
+        if (bucket.tokens < 1) {
+            const waitMs = ((1 - bucket.tokens) / bucket.refillRate) * 1000;
+            await new Promise((resolve) => setTimeout(resolve, Math.ceil(waitMs)));
+            this.refill(bucket);
+        }
+        bucket.tokens -= 1;
+        this.active.set(key, currentActive + 1);
+    }
+    release(key) {
+        const current = this.active.get(key) || 0;
+        if (current <= 1) {
+            this.active.delete(key);
+        }
+        else {
+            this.active.set(key, current - 1);
+        }
+    }
+    getStats(key) {
+        const bucket = this.buckets.get(key);
+        if (bucket)
+            this.refill(bucket);
+        return {
+            tokens: bucket?.tokens ?? this.config.maxTokensPerInterval,
+            active: this.active.get(key) || 0,
+        };
+    }
+    clear() {
+        this.buckets.clear();
+        this.active.clear();
+    }
+}
+export function createRateLimiter(config) {
+    return new RateLimiter(config);
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 26: scrape/retry.ts */
+/* ════════════════════════════════════════════════════════════════════ */
+export const DEFAULT_RETRY_CONFIG = {
+    retries: 3,
+    delay: 1000,
+    backoff: "exponential",
+    maxDelay: 30000,
+    statusCodes: [408, 429, 500, 502, 503, 504],
+};
+const RETRYABLE_NETWORK_CODES = ["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "EPIPE", "ENOTFOUND"];
+export function isRetryableError(error) {
+    const code = error.code;
+    if (code && RETRYABLE_NETWORK_CODES.includes(code))
+        return true;
+    const status = error.statusCode || error.status;
+    if (status && [408, 429, 500, 502, 503, 504].includes(status))
+        return true;
+    if (error.name === "AbortError")
+        return false;
+    return false;
+}
+function calculateDelay(attempt, config) {
+    switch (config.backoff) {
+        case "exponential":
+            return Math.min(config.delay * 2 ** attempt, config.maxDelay);
+        case "linear":
+            return Math.min(config.delay * (attempt + 1), config.maxDelay);
+        case "constant":
+            return config.delay;
+    }
+}
+export async function withRetry(fn, config = {}, onRetry) {
+    const cfg = { ...DEFAULT_RETRY_CONFIG, ...config };
+    const { default: pRetry } = await import("p-retry");
+    return pRetry(async (attemptNumber) => {
+        try {
+            return await fn(attemptNumber);
+        }
+        catch (error) {
+            if (!isRetryableError(error)) {
+                throw new AbortError(error.message);
+            }
+            throw error;
+        }
+    }, {
+        retries: cfg.retries,
+        minTimeout: calculateDelay(0, cfg),
+        maxTimeout: cfg.maxDelay,
+        randomize: cfg.backoff === "exponential",
+        onFailedAttempt: (context) => {
+            onRetry?.(context.error, context.attemptNumber);
+        },
+    });
+}
+/**
+ * Loads the p-retry AbortError class lazily (the class was a value re-export
+ * of the former scrape/retry.ts module; the grand merge keeps it reachable
+ * without requiring p-retry at module load time).
+ */
+export async function loadaborterror() {
+    const { AbortError } = await import("p-retry");
+    return AbortError;
+}
+export class MiddlewarePipeline {
+    middlewares = [];
+    use(middleware) {
+        this.middlewares.push(middleware);
+        return this;
+    }
+    async execute(initialData) {
+        const ctx = {
+            data: initialData,
+            state: {},
+            startTime: Date.now(),
+            aborted: false,
+            abort() { this.aborted = true; },
+        };
+        let index = 0;
+        const next = async () => {
+            if (ctx.aborted)
+                return;
+            if (index < this.middlewares.length) {
+                const middleware = this.middlewares[index++];
+                await middleware(ctx, next);
+            }
+        };
+        await next();
+        return ctx;
+    }
+    clear() {
+        this.middlewares = [];
+    }
+}
+export function loggingMiddleware(logger) {
+    return async (_ctx, next) => {
+        const start = Date.now();
+        await next();
+        const duration = Date.now() - start;
+        logger?.info(`Request completed in ${duration}ms`);
+    };
+}
+export function timeoutMiddleware(timeoutMs) {
+    return async (ctx, next) => {
+        const timer = setTimeout(() => ctx.abort(), timeoutMs);
+        try {
+            await next();
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    };
+}
+export function retryMiddleware(maxRetries, delayMs) {
+    return async (ctx, next) => {
+        let attempt = 0;
+        while (attempt <= maxRetries) {
+            ctx.state.attempt = attempt;
+            try {
+                await next();
+                return;
+            }
+            catch (error) {
+                if (attempt === maxRetries)
+                    throw error;
+                await new Promise(r => setTimeout(r, delayMs * Math.pow(2, attempt)));
+                attempt++;
+            }
+        }
+    };
+}
+export function createPipeline() {
+    return new MiddlewarePipeline();
+}
+/**
+ * Creates a fresh emittery event bus. Emittery joined the lazy-loading
+ * contract of the grand merge: the optional runtime dependency is imported
+ * on first use, so loading this module never requires it to be installed.
+ */
+export async function createEventEmitter() {
+    const { default: EmitteryCtor } = await import("emittery");
+    return new EmitteryCtor();
+}
+/**
+ * Resolves the process-wide emitter on first call. The former eager
+ * `globalEmitter` value of scrape/events.ts became this lazy memoized
+ * factory in the grand merge: emittery is an optional runtime and nothing
+ * may request it as an import side effect of loading the module.
+ */
+let globalemitterpromise = null;
+export function globalEmitter() {
+    globalemitterpromise ??= createEventEmitter();
+    return globalemitterpromise;
+}
+const DEFAULT_PYGAME_CONFIG = {
+    pythonPath: 'python',
+    width: 1280,
+    height: 720,
+    fps: 30,
+    fullscreen: false,
+};
+function generatePygameScript(imagePath, config, interactive = false) {
+    const lines = [
+        'import sys, json',
+        'import pygame',
+        'pygame.init()',
+        `screen = pygame.display.set_mode((${config.width}, ${config.height})${config.fullscreen ? ', pygame.FULLSCREEN' : ''})`,
+        'pygame.display.set_caption("DevThink WebScrape Renderer")',
+        `clock = pygame.time.Clock()`,
+        '',
+        'try:',
+        `  img = pygame.image.load(${JSON.stringify(imagePath)})`,
+        '  img_rect = img.get_rect(center=(screen.get_width()//2, screen.get_height()//2))',
+        'except:',
+        '  img = None',
+        '',
+        'font = pygame.font.Font(None, 36)',
+        '',
+        'running = True',
+        'while running:',
+        '  for event in pygame.event.get():',
+        '    if event.type == pygame.QUIT:',
+        '      running = False',
+        '    elif event.type == pygame.KEYDOWN:',
+        '      if event.key == pygame.K_ESCAPE:',
+        '        running = False',
+        ...(interactive ? [
+            '      elif event.key == pygame.K_SPACE:',
+            '        print(json.dumps({"action": "next"}))',
+            '        sys.stdout.flush()',
+        ] : []),
+        '',
+        '  screen.fill((30, 30, 40))',
+        '  if img:',
+        '    screen.blit(img, img_rect)',
+        '  else:',
+        '    text = font.render("No screenshot available", True, (200, 200, 200))',
+        '    screen.blit(text, (config.width//2 - text.get_width()//2, config.height//2))',
+        '',
+        '  pygame.display.flip()',
+        '  clock.tick(30)',
+        '',
+        'pygame.quit()',
+    ];
+    return lines.join('\n');
+}
+function generateDoomScript(config) {
+    const lines = [
+        'import sys, json, asyncio, subprocess',
+        'import pygame',
+        'pygame.init()',
+        `screen = pygame.display.set_mode((${config.width}, ${config.height})${config.fullscreen ? ', pygame.FULLSCREEN' : ''})`,
+        'pygame.display.set_caption("DevThink WebScrape - DOOM")',
+        '',
+        'font = pygame.font.Font(None, 24)',
+        'small_font = pygame.font.Font(None, 18)',
+        '',
+        'frames = []',
+        'current_frame = 0',
+        'running = True',
+        '',
+        'def load_frames(path):',
+        '  import os',
+        '  frames_dir = os.path.expanduser(path)',
+        '  if os.path.isdir(frames_dir):',
+        '    files = sorted(os.listdir(frames_dir))',
+        '    for f in files:',
+        '      if f.endswith((".png", ".jpg", ".jpeg")):',
+        '        try:',
+        '          img = pygame.image.load(os.path.join(frames_dir, f))',
+        '          frames.append(img)',
+        '        except: pass',
+        '',
+        'load_frames("~/devthink/webscrape/doom_frames")',
+        '',
+        'while running:',
+        '  for event in pygame.event.get():',
+        '    if event.type == pygame.QUIT:',
+        '      running = False',
+        '    elif event.type == pygame.KEYDOWN:',
+        '      if event.key == pygame.K_ESCAPE: running = False',
+        '      elif event.key == pygame.K_LEFT: current_frame = max(0, current_frame - 1)',
+        '      elif event.key == pygame.K_RIGHT: current_frame = min(len(frames)-1, current_frame + 1)',
+        '      elif event.key == pygame.K_SPACE:',
+        '        out = {"action": "frame", "index": current_frame, "total": len(frames)}',
+        '        print(json.dumps(out))',
+        '        sys.stdout.flush()',
+        '',
+        '  screen.fill((20, 20, 30))',
+        '  if frames and current_frame < len(frames):',
+        '    img = pygame.transform.scale(frames[current_frame], (config.width, config.height - 60))',
+        '    screen.blit(img, (0, 0))',
+        '  else:',
+        '    text = font.render("No DOOM frames loaded", True, (200, 80, 80))',
+        '    screen.blit(text, (config.width//2 - 150, config.height//2))',
+        '',
+        '  info = f"Frame {current_frame+1}/{len(frames)} | ESC: quit | <- ->: navigate | SPACE: capture"',
+        '  info_surf = small_font.render(info, True, (180, 180, 180))',
+        '  screen.blit(info_surf, (10, config.height - 40))',
+        '',
+        '  pygame.display.flip()',
+        '  clock.tick(30)',
+        '',
+        'pygame.quit()',
+    ];
+    return lines.join('\n');
+}
+export class PygameRenderer {
+    config;
+    process = null;
+    tempDir;
+    screenDir;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_PYGAME_CONFIG, ...config };
+        this.tempDir = join(process.cwd(), '.webscrape');
+        this.screenDir = join(this.tempDir, 'screenshots');
+        if (!existsSync(this.tempDir))
+            mkdirSync(this.tempDir, { recursive: true });
+        if (!existsSync(this.screenDir))
+            mkdirSync(this.screenDir, { recursive: true });
+    }
+    async renderScreenshot(imageBuffer) {
+        const imagePath = join(this.screenDir, 'current.png');
+        writeFileSync(imagePath, imageBuffer);
+        const script = generatePygameScript(imagePath, this.config);
+        const scriptPath = join(this.tempDir, 'render_screenshot.py');
+        writeFileSync(scriptPath, script);
+        await this.runScript(scriptPath);
+    }
+    async renderDoom() {
+        const script = generateDoomScript(this.config);
+        const scriptPath = join(this.tempDir, 'render_doom.py');
+        writeFileSync(scriptPath, script);
+        await this.runScript(scriptPath);
+    }
+    runScript(scriptPath) {
+        return new Promise((resolve, reject) => {
+            this.process = spawn(this.config.pythonPath, [scriptPath], {
+                stdio: ['inherit', 'inherit', 'pipe'],
+            });
+            this.process.on('close', code => {
+                this.process = null;
+                if (code === 0)
+                    resolve();
+                else
+                    reject(new Error(`Pygame exited with code ${code}`));
+            });
+            this.process.on('error', err => reject(err));
+        });
+    }
+    close() {
+        if (this.process && !this.process.killed) {
+            this.process.kill();
+            this.process = null;
+        }
+    }
+}
+export function createRenderer(config) {
+    return new PygameRenderer(config);
+}
+function parseSitemapXml(xml) {
+    const urls = [];
+    const urlMatches = xml.match(/<url[^>]*>([\s\S]*?)<\/url>/gi) || [];
+    for (const urlBlock of urlMatches) {
+        const loc = urlBlock.match(/<loc>([\s\S]*?)<\/loc>/i)?.[1]?.trim();
+        if (!loc)
+            continue;
+        const lastmod = urlBlock.match(/<lastmod>([\s\S]*?)<\/lastmod>/i)?.[1]?.trim();
+        const changefreq = urlBlock.match(/<changefreq>([\s\S]*?)<\/changefreq>/i)?.[1]?.trim();
+        const priorityStr = urlBlock.match(/<priority>([\s\S]*?)<\/priority>/i)?.[1]?.trim();
+        const priority = priorityStr ? parseFloat(priorityStr) : undefined;
+        urls.push({ loc, lastmod, changefreq, priority });
+    }
+    return urls;
+}
+function extractSitemapUrls(xml) {
+    const urls = [];
+    const sitemapMatches = xml.match(/<sitemap[^>]*>[\s\S]*?<loc>([\s\S]*?)<\/loc>[\s\S]*?<\/sitemap>/gi) || [];
+    for (const match of sitemapMatches) {
+        const loc = match.match(/<loc>([\s\S]*?)<\/loc>/i)?.[1]?.trim();
+        if (loc)
+            urls.push(loc);
+    }
+    return urls;
+}
+export async function fetchSitemap(url, options = {}) {
+    const startTime = Date.now();
+    const fetcher = options.fetcher ?? fetch;
+    try {
+        const response = await fetcher(url, {
+            headers: {
+                'User-Agent': 'DevThink-WebScrape/2.0 (sitemap parser)',
+                ...options.headers,
+            },
+            signal: AbortSignal.timeout(options.timeout || 10000),
+        });
+        if (!response.ok) {
+            throw new WebScrapeError(`Failed to fetch sitemap: ${response.status}`, ErrorCode.NETWORK_ERROR, response.status, false, { url });
+        }
+        const text = await response.text();
+        const isIndex = text.includes('<sitemapindex');
+        const urls = isIndex ? [] : parseSitemapXml(text);
+        const sitemaps = isIndex ? extractSitemapUrls(text) : [];
+        return {
+            urls,
+            sitemaps,
+            duration: Date.now() - startTime,
+        };
+    }
+    catch (error) {
+        if (error instanceof WebScrapeError)
+            throw error;
+        throw new WebScrapeError(`Failed to parse sitemap: ${error.message}`, ErrorCode.PARSE_ERROR, 422, false, { url }, error);
+    }
+}
+export async function discoverSitemaps(siteUrl) {
+    const candidates = [
+        '/sitemap.xml',
+        '/sitemap_index.xml',
+        '/sitemap-index.xml',
+        '/sitemaps.xml',
+        '/robots.txt',
+    ];
+    const found = [];
+    const base = siteUrl.replace(/\/$/, '');
+    for (const path of candidates) {
+        try {
+            const url = path === '/robots.txt' ? `${base}/robots.txt` : `${base}${path}`;
+            const response = await fetch(url, {
+                headers: { 'User-Agent': 'DevThink-WebScrape/2.0' },
+                signal: AbortSignal.timeout(5000),
+            });
+            if (response.ok) {
+                const text = await response.text();
+                if (path === '/robots.txt') {
+                    const sitemapUrls = text.match(/Sitemap:\s*(https?:\/\/[^\s]+)/gi);
+                    if (sitemapUrls) {
+                        for (const s of sitemapUrls) {
+                            found.push(s.replace(/Sitemap:\s*/i, '').trim());
+                        }
+                    }
+                }
+                else {
+                    found.push(url);
+                }
+            }
+        }
+        catch { }
+    }
+    return [...new Set(found)];
+}
+export async function parseSitemap(url, options = {}) {
+    const maxUrls = Math.max(0, Number(options.maxUrls ?? 10000));
+    const maxDepth = Math.max(0, Number(options.maxDepth ?? 8));
+    const allUrls = [];
+    const seenSitemaps = new Set();
+    const seenUrls = new Set();
+    async function visit(current, depth) {
+        const identity = sitemapidentity(current);
+        if (seenSitemaps.has(identity) || depth > maxDepth || allUrls.length >= maxUrls)
+            return;
+        seenSitemaps.add(identity);
+        const result = await fetchSitemap(current, options);
+        for (const item of result.urls) {
+            const itemidentity = sitemapidentity(item.loc);
+            if (!seenUrls.has(itemidentity)) {
+                seenUrls.add(itemidentity);
+                allUrls.push(item);
+                if (allUrls.length >= maxUrls)
+                    return;
+            }
+        }
+        if (options.followIndexes === false || depth >= maxDepth)
+            return;
+        for (const child of result.sitemaps) {
+            if (allUrls.length >= maxUrls)
+                return;
+            await visit(child, depth + 1);
+        }
+    }
+    await visit(url, 0);
+    return allUrls;
+}
+function sitemapidentity(value) { try {
+    const parsed = new URL(value);
+    parsed.hash = "";
+    return parsed.href;
+}
+catch {
+    return String(value).trim();
+} }
+const DEFAULT_OPTIONS = {
+    includeSummary: true,
+    includeKeyPoints: true,
+    includeUrls: true,
+    includeRaw: true,
+    maxTokens: 8000,
+    chunkSize: 512,
+    model: 'default',
+};
+function generateSummary(result) {
+    const wordCount = result.text.split(/\s+/).length;
+    return `This page titled "${result.title}" contains approximately ${wordCount} words with ${result.links.length} links and ${result.images.length} images.`;
+}
+function generateKeyPoints(result) {
+    const points = [];
+    const text = result.text.slice(0, 5000);
+    if (result.metadata.author)
+        points.push(`Author: ${result.metadata.author}`);
+    if (result.metadata.publishedDate)
+        points.push(`Published: ${result.metadata.publishedDate}`);
+    if (result.metadata.description)
+        points.push(`Description: ${result.metadata.description.slice(0, 200)}`);
+    const sentences = text.match(/[^.!?]+[.!?]+/g) || [];
+    for (const s of sentences.slice(0, 5)) {
+        const trimmed = s.trim();
+        if (trimmed.length > 30)
+            points.push(trimmed);
+    }
+    return points.slice(0, 8);
+}
+function buildSystemPrompt(result) {
+    let prompt = `# Web Page Content\n\n`;
+    prompt += `## Title\n${result.title}\n\n`;
+    prompt += `## URL\n${result.url}\n\n`;
+    if (result.metadata.description) {
+        prompt += `## Description\n${result.metadata.description}\n\n`;
+    }
+    if (result.tables.length > 0) {
+        prompt += `## Tables (${result.tables.length} found)\n`;
+        for (const t of result.tables.slice(0, 3)) {
+            prompt += `- ${t.caption || 'Unnamed table'}: ${t.headers.join(', ')}\n`;
+        }
+        prompt += '\n';
+    }
+    prompt += `## Content\n${result.content}\n\n`;
+    return prompt;
+}
+export function formatForAgent(result, options) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    let content = '';
+    if (opts.includeSummary) {
+        content += `## Summary\n${generateSummary(result)}\n\n`;
+    }
+    if (opts.includeKeyPoints) {
+        const points = generateKeyPoints(result);
+        content += `## Key Points\n${points.map(p => `- ${p}`).join('\n')}\n\n`;
+    }
+    if (opts.includeUrls) {
+        const externalUrls = result.links.filter(l => l.isExternal).slice(0, 15);
+        if (externalUrls.length) {
+            content += `## External Links\n${externalUrls.map(l => `- [${l.text || l.href}](${l.href})`).join('\n')}\n\n`;
+        }
+        const internalUrls = result.links.filter(l => l.isInternal).slice(0, 10);
+        if (internalUrls.length) {
+            content += `## Internal Links\n${internalUrls.map(l => `- [${l.text || l.href}](${l.href})`).join('\n')}\n\n`;
+        }
+    }
+    if (opts.includeRaw) {
+        content += `## Full Content\n${result.content}\n\n`;
+    }
+    const totalTokens = estimateTokens(content, opts.model);
+    if (totalTokens > opts.maxTokens) {
+        const charLimit = Math.floor(opts.maxTokens * 4);
+        content = content.slice(0, charLimit) + '\n\n... (truncated)';
+    }
+    const chunks = chunkMarkdown(content, { maxTokens: opts.chunkSize });
+    const output = {
+        summary: generateSummary(result),
+        content,
+        keyPoints: generateKeyPoints(result),
+        relevantUrls: result.links.filter(l => l.isExternal).map(l => l.href).slice(0, 10),
+        tokens: estimateTokens(content, opts.model),
+    };
+    if (chunks.length > 1) {
+        output.chunks = chunks.map(c => c.content);
+    }
+    return output;
+}
+export function buildContext(result) {
+    return buildSystemPrompt(result);
+}
+export function generateLlmsTxt(results, options = {}) {
+    const siteName = options.siteName || results[0]?.metadata?.title || 'Site';
+    const description = options.description || results[0]?.metadata?.description || '';
+    const lines = [];
+    lines.push(`# ${siteName}`);
+    lines.push('');
+    if (description) {
+        lines.push(`> ${description}`);
+        lines.push('');
+    }
+    const pages = [];
+    const docs = [];
+    for (const result of results) {
+        const url = result.url.toLowerCase();
+        if (url.includes('doc') || url.includes('guide') || url.includes('api')) {
+            docs.push(result);
+        }
+        else {
+            pages.push(result);
+        }
+    }
+    if (pages.length > 0) {
+        lines.push('## Pages');
+        lines.push('');
+        for (const page of pages.slice(0, 20)) {
+            const title = page.metadata.title || page.title;
+            const desc = page.metadata.description || '';
+            lines.push(`- ${title}: ${desc || page.url}`);
+        }
+        lines.push('');
+    }
+    if (docs.length > 0) {
+        lines.push('## Documentation');
+        lines.push('');
+        for (const doc of docs.slice(0, 20)) {
+            const title = doc.metadata.title || doc.title;
+            const desc = doc.metadata.description || '';
+            lines.push(`- ${title}: ${desc || doc.url}`);
+        }
+        lines.push('');
+    }
+    if (options.includeOptional && results.length > 5) {
+        lines.push('## Optional');
+        lines.push('');
+        lines.push(`- Full content: ${results.length} pages scraped`);
+        lines.push(`- Last updated: ${new Date().toISOString().split('T')[0]}`);
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+export function generateLlmsFullTxt(results) {
+    const parts = [];
+    for (const result of results) {
+        parts.push(`# ${result.title || result.metadata.title || 'Untitled'}`);
+        parts.push('');
+        parts.push(`Source: ${result.url}`);
+        parts.push('');
+        parts.push(result.content);
+        parts.push('');
+        parts.push('---');
+        parts.push('');
+    }
+    return parts.join('\n');
+}
+export async function fetchHtml(url, options = {}) {
+    const startTime = Date.now();
+    const profile = getRandomProfile();
+    const baseHeaders = getHeaders(profile);
+    const headers = { ...baseHeaders, ...options.headers };
+    if (options.userAgent)
+        headers['User-Agent'] = options.userAgent;
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), options.timeout || 30000);
+    const signal = options.signal ? AbortSignal.any([controller.signal, options.signal]) : controller.signal;
+    try {
+        const response = await fetch(url, {
+            method: 'GET',
+            headers,
+            signal,
+            redirect: 'follow',
+        });
+        clearTimeout(timeoutId);
+        if (!response.ok) {
+            throw new WebScrapeError(`HTTP ${response.status}: ${response.statusText}`, response.status === 403 ? ErrorCode.BLOCKED :
+                response.status === 429 ? ErrorCode.RATE_LIMITED :
+                    response.status >= 500 ? ErrorCode.NETWORK_ERROR :
+                        ErrorCode.NETWORK_ERROR, response.status, [408, 429, 500, 502, 503, 504].includes(response.status), { url, status: response.status });
+        }
+        const html = await response.text();
+        const responseHeaders = {};
+        response.headers.forEach((value, key) => { responseHeaders[key] = value; });
+        return {
+            html,
+            url: response.url || url,
+            status: response.status,
+            headers: responseHeaders,
+            duration: Date.now() - startTime,
+        };
+    }
+    catch (error) {
+        clearTimeout(timeoutId);
+        if (error instanceof WebScrapeError)
+            throw error;
+        if (error.name === 'AbortError') {
+            throw new WebScrapeError(`Request timeout after ${options.timeout || 30000}ms`, ErrorCode.TIMEOUT, 504, true, { url, timeout: options.timeout || 30000 });
+        }
+        throw new WebScrapeError(`Network error: ${error.message}`, ErrorCode.NETWORK_ERROR, 503, true, { url }, error);
+    }
+}
+export async function fetchAndParse(url, options = {}) {
+    return fetchHtml(url, {
+        timeout: options.timeout,
+        headers: options.headers,
+        userAgent: options.userAgent,
+        proxy: options.proxy,
+    });
+}
+export async function detectRenderingMode(html) {
+    if (/<div id=["'](?:root|app|__next|__nuxt)["']>\s*<\/div>/.test(html))
+        return 'spa';
+    if (html.includes('__NEXT_DATA__') || html.includes('__NUXT__') || html.includes('__VUE_SSR_DATA__'))
+        return 'hydrated';
+    const cheeriomodule = await import("cheerio");
+    const $ = cheeriomodule.load(html);
+    if ($('script[type="application/ld+json"]').length > 0)
+        return 'static';
+    const textContent = $.text().replace(/\s+/g, ' ').trim();
+    if (textContent.length > 500)
+        return 'static';
+    return 'spa';
+}
+export async function batchScrape(options, scrapeOpts = {}) {
+    const startTime = Date.now();
+    const { default: pLimit } = await import("p-limit");
+    const limit = pLimit(options.concurrency || 5);
+    const results = [];
+    const errors = [];
+    const tasks = options.urls.map((url) => limit(async () => {
+        try {
+            const result = await scrapeUrl(url, { ...scrapeOpts, mode: scrapeOpts.mode || 'auto' });
+            results.push(result);
+            options.onProgress?.(results.length + errors.length, options.urls.length, url);
+        }
+        catch (error) {
+            errors.push({ url, error: error });
+            options.onError?.(url, error);
+            options.onProgress?.(results.length + errors.length, options.urls.length, url);
+        }
+    }));
+    await Promise.allSettled(tasks);
+    return {
+        results,
+        errors,
+        successful: results.length,
+        failed: errors.length,
+        duration: Date.now() - startTime,
+    };
+}
+export async function batchScrapeSequential(urls, options = {}, callbacks) {
+    const startTime = Date.now();
+    const results = [];
+    const errors = [];
+    for (let i = 0; i < urls.length; i++) {
+        try {
+            const result = await scrapeUrl(urls[i], options);
+            results.push(result);
+            callbacks?.onResult?.(result, i);
+        }
+        catch (error) {
+            errors.push({ url: urls[i], error: error });
+            callbacks?.onError?.(urls[i], error, i);
+        }
+        if (callbacks?.delayMs && i < urls.length - 1) {
+            await new Promise(r => setTimeout(r, callbacks.delayMs));
+        }
+    }
+    return {
+        results,
+        errors,
+        successful: results.length,
+        failed: errors.length,
+        duration: Date.now() - startTime,
+    };
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 35: scrape/browser.ts */
+/* ════════════════════════════════════════════════════════════════════ */
+export class AgentBrowser {
+    browser = null;
+    context = null;
+    page = null;
+    config;
+    static DEFAULTS = {
+        headless: true,
+        userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+        viewport: { width: 1280, height: 720 },
+        locale: 'en-US',
+        timeout: 30000,
+        recordVideo: false,
+        proxy: '',
+        storageState: '',
+        blockAds: true,
+        stealth: true,
+    };
+    constructor(config = {}) {
+        this.config = { ...AgentBrowser.DEFAULTS, ...config };
+    }
+    async launch(browserType = 'chromium') {
+        try {
+            const playwrightmodule = await import("playwright");
+            const launcher = { chromium: playwrightmodule.chromium, firefox: playwrightmodule.firefox, webkit: playwrightmodule.webkit }[browserType];
+            const launchOpts = {
+                headless: this.config.headless,
+                args: [
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-dev-shm-usage',
+                    '--disable-web-security',
+                    '--disable-features=IsolateOrigins,site-per-process',
+                ],
+            };
+            if (this.config.proxy) {
+                launchOpts.proxy = { server: this.config.proxy };
+            }
+            this.browser = await launcher.launch(launchOpts);
+            const ctxOpts = {
+                viewport: this.config.viewport,
+                locale: this.config.locale,
+                userAgent: this.config.userAgent,
+            };
+            if (this.config.recordVideo) {
+                ctxOpts.recordVideo = { dir: './videos' };
+            }
+            if (this.config.geolocation) {
+                ctxOpts.geolocation = this.config.geolocation;
+                ctxOpts.permissions = ['geolocation'];
+            }
+            if (this.config.storageState) {
+                ctxOpts.storageState = this.config.storageState;
+            }
+            this.context = await this.browser.newContext(ctxOpts);
+            if (this.config.blockAds) {
+                await this.context.route('**/*.{png,jpg,jpeg,gif,svg,ico,css,woff,woff2,mp4,webm}', r => r.abort());
+            }
+            this.page = await this.context.newPage();
+            this.page.setDefaultTimeout(this.config.timeout);
+        }
+        catch (error) {
+            if (error.message?.includes('playwright') || error.message?.includes('chromium')) {
+                throw new BrowserNotAvailableError();
+            }
+            throw error;
+        }
+    }
+    async navigate(url, waitUntil = 'networkidle') {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        await this.page.goto(url, { waitUntil, timeout: this.config.timeout });
+    }
+    async screenshot(opts = {}) {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        return this.page.screenshot({
+            fullPage: opts.fullPage ?? false,
+            type: opts.type ?? 'png',
+            quality: opts.quality ?? 80,
+        });
+    }
+    async evaluate(fn, args) {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        if (typeof fn === 'string') {
+            return this.page.evaluate(new Function(fn));
+        }
+        return this.page.evaluate(fn, args);
+    }
+    async click(selector) {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        await this.page.click(selector);
+    }
+    async type(selector, text, delayMs = 0) {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        await this.page.fill(selector, '');
+        await this.page.type(selector, text, { delay: delayMs });
+    }
+    async scrollToBottom(step = 800, delayMs = 300, maxScrolls = 50) {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        let scrolls = 0;
+        let prevHeight = 0;
+        while (scrolls < maxScrolls) {
+            const height = await this.page.evaluate(() => document.body.scrollHeight);
+            if (height === prevHeight && scrolls > 2)
+                break;
+            prevHeight = height;
+            await this.page.evaluate(y => window.scrollBy(0, y), step);
+            await new Promise(r => setTimeout(r, delayMs));
+            scrolls++;
+        }
+        return scrolls;
+    }
+    async scrollTo(x, y) {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        await this.page.evaluate(([px, py]) => window.scrollTo(px, py), [x, y]);
+    }
+    async waitFor(selector, timeout) {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        await this.page.waitForSelector(selector, { timeout: timeout ?? this.config.timeout });
+    }
+    async waitForLoad() {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        await this.page.waitForLoadState('networkidle');
+    }
+    async html() {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        return this.page.content();
+    }
+    async text() {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        return this.page.evaluate(() => document.body.innerText);
+    }
+    async title() {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        return this.page.title();
+    }
+    async url() {
+        if (!this.page)
+            throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+        return this.page.url();
+    }
+    async executeCommands(commands) {
+        const results = [];
+        for (const cmd of commands) {
+            switch (cmd.action) {
+                case 'goto':
+                    await this.navigate(cmd.args?.url);
+                    results.push({ ok: true });
+                    break;
+                case 'click':
+                    await this.click(cmd.args?.selector);
+                    results.push({ ok: true });
+                    break;
+                case 'type':
+                    await this.type(cmd.args?.selector, cmd.args?.text);
+                    results.push({ ok: true });
+                    break;
+                case 'screenshot':
+                    results.push(await this.screenshot(cmd.args));
+                    break;
+                case 'evaluate':
+                    results.push(await this.evaluate(cmd.args?.fn));
+                    break;
+                case 'wait':
+                    await this.waitFor(cmd.args?.selector);
+                    results.push({ ok: true });
+                    break;
+                case 'scroll':
+                    if (cmd.args?.to) {
+                        await this.scrollTo(0, cmd.args.to);
+                    }
+                    else {
+                        await this.scrollToBottom();
+                    }
+                    results.push({ ok: true });
+                    break;
+                case 'extract':
+                    results.push({
+                        html: await this.html(),
+                        text: await this.text(),
+                        title: await this.title(),
+                    });
+                    break;
+            }
+        }
+        return results;
+    }
+    async close() {
+        if (this.page)
+            await this.page.close().catch(() => { });
+        if (this.context)
+            await this.context.close().catch(() => { });
+        if (this.browser)
+            await this.browser.close().catch(() => { });
+        this.page = null;
+        this.context = null;
+        this.browser = null;
+    }
+    isConnected() {
+        return this.browser?.isConnected() ?? false;
+    }
+}
+export function createBrowser(config) {
+    return new AgentBrowser(config);
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 36: scrape/scrape.ts */
+/* ════════════════════════════════════════════════════════════════════ */
+export async function scrapeUrl(url, options = {}) {
+    if (!url || !url.startsWith('http')) {
+        throw new WebScrapeError(`Invalid URL: ${url}`, ErrorCode.VALIDATION_FAILED, 400, false);
+    }
+    const startTime = Date.now();
+    const mode = options.mode || 'auto';
+    if (mode === 'fetch' || mode === 'auto') {
+        try {
+            const fetchResult = await fetchHtml(url, {
+                timeout: options.timeout,
+                headers: options.headers,
+                userAgent: options.userAgent,
+            });
+            if (mode === 'auto') {
+                const renderMode = await detectRenderingMode(fetchResult.html);
+                if (renderMode === 'static') {
+                    return buildResult(fetchResult.html, fetchResult.url, options, startTime);
+                }
+            }
+            else {
+                return buildResult(fetchResult.html, fetchResult.url, options, startTime);
+            }
+        }
+        catch (error) {
+            if (mode === 'fetch')
+                throw error;
+            if (!(error instanceof WebScrapeError) || error.isRetryable) {
+                // Fall through to browser mode
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    const browser = new AgentBrowser({
+        headless: true,
+        timeout: options.timeout ?? 30000,
+        proxy: options.proxy ?? '',
+        userAgent: options.userAgent,
+    });
+    try {
+        await browser.launch();
+        await browser.navigate(url, options.waitUntil ?? 'networkidle');
+        if (options.scroll) {
+            await browser.scrollToBottom(800, options.scrollDelay ?? 300, options.maxScrolls ?? 50);
+        }
+        const html = await browser.html();
+        const pageTitle = await browser.title();
+        const pageUrl = await browser.url();
+        return buildResult(html, pageUrl, options, startTime, pageTitle);
+    }
+    finally {
+        await browser.close();
+    }
+}
+export async function scrapeHtml(html, options = {}) {
+    const startTime = Date.now();
+    return buildResult(html, 'about:blank', options, startTime);
+}
+export async function scrapeWithBrowser(browser, url, options = {}) {
+    const startTime = Date.now();
+    await browser.navigate(url, options.waitUntil ?? 'networkidle');
+    if (options.scroll) {
+        await browser.scrollToBottom(800, options.scrollDelay ?? 300, options.maxScrolls ?? 50);
+    }
+    const html = await browser.html();
+    const pageTitle = await browser.title();
+    const pageUrl = await browser.url();
+    return buildResult(html, pageUrl, options, startTime, pageTitle);
+}
+async function buildResult(html, url, options, startTime, title) {
+    const extracted = await extractContent(html, {
+        readable: true,
+        preserveLinks: options.extractLinks ?? true,
+        preserveImages: options.extractImages ?? true,
+        preserveTables: options.extractTables ?? true,
+        maxLength: options.maxContentLength,
+        removeSelectors: options.removeSelectors,
+    });
+    const duration = Date.now() - startTime;
+    return {
+        url,
+        title: title || extracted.metadata.title || '',
+        content: extracted.content,
+        format: options.format ?? 'markdown',
+        text: extracted.text,
+        links: extracted.links,
+        images: extracted.images,
+        tables: extracted.tables,
+        metadata: extracted.metadata,
+        extractedAt: new Date().toISOString(),
+        duration,
+        size: extracted.content.length + extracted.text.length,
+    };
+}
+function isSameDomain(url1, url2) {
+    try {
+        return new URL(url1).hostname === new URL(url2).hostname;
+    }
+    catch {
+        return false;
+    }
+}
+function normalizeForDedup(url) {
+    try {
+        const u = new URL(url);
+        u.hash = '';
+        u.search = '';
+        return u.href.replace(/\/+$/, '') || u.href;
+    }
+    catch {
+        return url;
+    }
+}
+export async function legacycrawl(startUrl, options = {}) {
+    const { maxDepth = 2, maxPages = 50, maxConcurrent = 3, sameDomain = true, delayMs = 1000, onDiscover, onResult, onError, ...scrapeOpts } = options;
+    const startTime = Date.now();
+    const visited = new Set();
+    const queue = [{ url: startUrl, depth: 0 }];
+    const results = [];
+    let failed = 0;
+    visited.add(normalizeForDedup(startUrl));
+    while (queue.length > 0 && results.length < maxPages) {
+        const batch = queue.splice(0, maxConcurrent);
+        const promises = batch.map(async (entry) => {
+            if (entry.depth > maxDepth)
+                return;
+            try {
+                const result = await scrapeUrl(entry.url, scrapeOpts);
+                results.push(result);
+                onResult?.(result);
+                // Extract links for further crawling
+                if (entry.depth < maxDepth) {
+                    for (const link of result.links) {
+                        const normalized = normalizeForDedup(link.href);
+                        if (visited.has(normalized))
+                            continue;
+                        if (sameDomain && !isSameDomain(startUrl, link.href))
+                            continue;
+                        try {
+                            new URL(link.href);
+                        }
+                        catch {
+                            continue;
+                        }
+                        visited.add(normalized);
+                        queue.push({ url: link.href, depth: entry.depth + 1 });
+                        onDiscover?.(link.href, entry.depth + 1);
+                    }
+                }
+            }
+            catch (error) {
+                failed++;
+                onError?.(entry.url, error);
+            }
+        });
+        await Promise.all(promises);
+        if (delayMs > 0 && queue.length > 0) {
+            await new Promise(r => setTimeout(r, delayMs));
+        }
+    }
+    const duration = Date.now() - startTime;
+    return {
+        results,
+        stats: {
+            totalUrls: visited.size,
+            successful: results.length,
+            failed,
+            skipped: visited.size - results.length - failed,
+            duration,
+            avgResponseTime: results.length > 0 ? results.reduce((sum, r) => sum + r.duration, 0) / results.length : 0,
+        },
+    };
+}
+/**
+ * Builds the hono application lazily — hono is an optional runtime the merged
+ * module never requires at load time (the grand-merge load-safety adaptation
+ * of the former scrape/server.ts).
+ */
+async function buildscrapeserver() {
+    const { Hono } = await import("hono");
+    const { cors } = await import("hono/cors");
+    const app = new Hono();
+    app.use('*', cors());
+    const api = new Hono();
+    api.post('/scrape', async (c) => {
+        const body = await c.req.json();
+        const { url, options = {} } = body;
+        if (!url) {
+            return c.json({ error: { code: 'MISSING_URL', message: 'URL is required' } }, 400);
+        }
+        try {
+            const result = await scrapeUrl(url, options);
+            return c.json({ status: 'ok', data: result });
+        }
+        catch (error) {
+            return c.json({
+                error: {
+                    code: error.code || 'SCRAPE_ERROR',
+                    message: error.message,
+                }
+            }, error.statusCode || 500);
+        }
+    });
+    api.post('/scrape/agent', async (c) => {
+        const body = await c.req.json();
+        const { url, options = {} } = body;
+        if (!url) {
+            return c.json({ error: { code: 'MISSING_URL', message: 'URL is required' } }, 400);
+        }
+        try {
+            const result = await scrapeUrl(url, options);
+            const agentOutput = formatForAgent(result);
+            return c.json({ status: 'ok', data: agentOutput });
+        }
+        catch (error) {
+            return c.json({
+                error: {
+                    code: error.code || 'SCRAPE_ERROR',
+                    message: error.message,
+                }
+            }, error.statusCode || 500);
+        }
+    });
+    api.post('/batch', async (c) => {
+        const body = await c.req.json();
+        const { urls, options = {}, scrapeOptions = {} } = body;
+        if (!urls || !Array.isArray(urls) || urls.length === 0) {
+            return c.json({ error: { code: 'MISSING_URLS', message: 'urls array is required' } }, 400);
+        }
+        try {
+            const result = await batchScrape({ urls, ...options }, scrapeOptions);
+            return c.json({
+                status: 'ok',
+                data: {
+                    successful: result.successful,
+                    failed: result.failed,
+                    duration: result.duration,
+                    results: result.results,
+                    errors: result.errors.map(e => ({ url: e.url, error: e.error.message })),
+                },
+            });
+        }
+        catch (error) {
+            return c.json({
+                error: {
+                    code: 'BATCH_ERROR',
+                    message: error.message,
+                }
+            }, 500);
+        }
+    });
+    api.post('/crawl', async (c) => {
+        const body = await c.req.json();
+        const { url, options = {} } = body;
+        if (!url) {
+            return c.json({ error: { code: 'MISSING_URL', message: 'URL is required' } }, 400);
+        }
+        try {
+            const result = await legacycrawl(url, options);
+            return c.json({
+                status: 'ok',
+                data: {
+                    stats: result.stats,
+                    results: result.results.slice(0, 100),
+                },
+            });
+        }
+        catch (error) {
+            return c.json({
+                error: {
+                    code: 'CRAWL_ERROR',
+                    message: error.message,
+                }
+            }, 500);
+        }
+    });
+    api.get('/health', (c) => {
+        return c.json({ status: 'ok', version: '2.0.0', uptime: process.uptime() });
+    });
+    app.route('/v1', api);
+    return app;
+}
+/**
+ * Creates the self-hosted scrape API surface. The factory became async with
+ * the grand merge: the hono application is built on first call and the port
+ * draw keeps using the process-wide random port helper.
+ */
+export async function createServer(config = {}) {
+    const app = await buildscrapeserver();
+    return {
+        fetch: app.fetch,
+        port: config.port || randomPort(),
+        host: config.host || '0.0.0.0',
+    };
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 39: scrape/cli.ts — the webscrape command-line interface (grand-merge adaptation: commander became a lazy runtime import and the top-level program construction became an exported factory). */
+/* ════════════════════════════════════════════════════════════════════ */
+/**
+ * Runs the webscrape CLI. The former top-level commander construction of
+ * scrape/cli.ts became this exported factory in the grand merge: commander
+ * is imported on first call and loading the module never requires it.
+ */
+export async function runscrapecli() {
+    const { Command } = await import("commander");
+    const program = new Command();
+    program
+        .name('webscrape')
+        .description('DevThink WebScrape — Universal AI-powered web scraping toolkit')
+        .version('2.0.0');
+    program
+        .argument('[url]', 'URL to scrape')
+        .option('-f, --format <type>', 'Output format: markdown, html, text, json, xml, redis', 'markdown')
+        .option('-o, --output <file>', 'Write output to file')
+        .option('--file <path>', 'Scrape HTML from local file')
+        .option('--mode <mode>', 'Scraping mode: auto, fetch, browser', 'auto')
+        .option('--no-headless', 'Show browser window')
+        .option('--timeout <ms>', 'Navigation timeout', '30000')
+        .option('--scroll', 'Scroll page to load dynamic content')
+        .option('--readable', 'Extract readable content only')
+        .option('--agent', 'Format output for AI agent consumption')
+        .option('--pretty', 'Pretty-print JSON output')
+        .option('--proxy <url>', 'Proxy server URL')
+        .option('--retries <n>', 'Number of retries', '3')
+        .option('--user-agent <ua>', 'Custom User-Agent string')
+        .action(async (url, options) => {
+        if (!url && !options.file) {
+            program.help();
+        }
+        const format = resolveFormat(options.format);
+        const scrapeOpts = {
+            format,
+            timeout: parseInt(options.timeout),
+            scroll: options.scroll,
+            readable: options.readable,
+            mode: options.mode,
+            proxy: options.proxy,
+            retries: parseInt(options.retries),
+            userAgent: options.userAgent,
+            extractLinks: true,
+            extractImages: true,
+            extractTables: true,
+        };
+        if (options.readable) {
+            scrapeOpts.removeSelectors = [
+                'script', 'style', 'nav', 'footer', 'header',
+                '.sidebar', '.advertisement', '.ads', '.menu',
+                '.comments', '.comment', '#comments',
+            ];
+        }
+        console.error(`Scraping: ${url || options.file}`);
+        let result;
+        if (options.file) {
+            const filePath = resolve(options.file);
+            if (!existsSync(filePath)) {
+                console.error(`File not found: ${filePath}`);
+                process.exit(1);
+            }
+            const html = readFileSync(filePath, 'utf-8');
+            result = await scrapeHtml(html, scrapeOpts);
+        }
+        else {
+            result = await scrapeUrl(url, scrapeOpts);
+        }
+        console.error(`Done in ${result.duration}ms (${(result.size / 1024).toFixed(1)} KB)`);
+        if (options.agent) {
+            const agentOutput = formatForAgent(result);
+            const output = JSON.stringify(agentOutput, null, 2);
+            if (options.output) {
+                writeOutput(options.output, output);
+                console.error(`Saved to ${options.output}`);
+            }
+            else {
+                console.log(output);
+            }
+        }
+        else {
+            const serializeOpts = buildSerializeOptions(format, options.pretty);
+            const serialized = serializeResult(result, serializeOpts);
+            if (options.output) {
+                const ext = extensionForFormat(format);
+                const finalPath = options.output.endsWith(ext) ? options.output : options.output + ext;
+                writeOutput(finalPath, serialized.content);
+                console.error(`Saved to ${finalPath}`);
+            }
+            else {
+                console.log(serialized.content);
+            }
+        }
+    });
+    program.parse();
+}
+/* ════════════════════════════════════════════════════════════════════ */
+/* Section 40: scrape/dev-server.ts — the zero-dependency local API server (grand-merge adaptation: the top-level listener became an exported factory so importing the module never binds a port). */
+/* ════════════════════════════════════════════════════════════════════ */
+/**
+ * Starts the local scrape API dev server. The former top-level bootstrap
+ * of scrape/dev-server.ts became this exported factory in the grand merge
+ * (a merged module must never bind a port as an import side effect).
+ */
+export function startdevserver() {
+    const PORT = randomPort();
+    const server = createnodeserver(async (req, res) => {
+        res.setHeader('Access-Control-Allow-Origin', '*');
+        res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
+        res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+        if (req.method === 'OPTIONS') {
+            res.writeHead(200);
+            res.end();
+            return;
+        }
+        if (req.method !== 'POST' || req.url !== '/api/scrape') {
+            res.writeHead(404);
+            res.end(JSON.stringify({ error: 'Not found' }));
+            return;
+        }
+        let body = '';
+        req.on('data', (chunk) => { body += chunk.toString(); });
+        req.on('end', async () => {
+            try {
+                const { url, format: fmt, options = {} } = JSON.parse(body);
+                if (!url)
+                    throw new Error('URL is required');
+                const format = resolveFormat(fmt || 'markdown');
+                const result = await scrapeUrl(url, {
+                    timeout: 30000,
+                    scroll: options.scroll,
+                    extractLinks: options.extractLinks ?? true,
+                    extractImages: options.extractImages ?? true,
+                    extractTables: options.extractTables ?? true,
+                });
+                let response;
+                if (options.agent) {
+                    const agentOutput = formatForAgent(result);
+                    response = {
+                        success: true,
+                        title: result.title,
+                        url: result.url,
+                        duration: result.duration,
+                        size: result.size,
+                        agentOutput,
+                    };
+                }
+                else {
+                    const serializeOpts = buildSerializeOptions(format);
+                    const serialized = serializeResult(result, serializeOpts);
+                    response = {
+                        success: true,
+                        title: result.title,
+                        url: result.url,
+                        format,
+                        content: serialized.content,
+                        duration: result.duration,
+                        size: serialized.size,
+                        metadata: result.metadata,
+                        links: result.links,
+                        images: result.images,
+                        tables: result.tables,
+                        result,
+                    };
+                }
+                res.writeHead(200, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify(response));
+            }
+            catch (err) {
+                const error = err;
+                res.writeHead(500, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify({ error: error.message }));
+            }
+        });
+    });
+    server.listen(PORT, () => {
+        console.log(`API server running at http://localhost:${PORT}`);
+        console.log(`POST /api/scrape with { url, format?, options? }`);
+    });
+    return server;
+}
+//# sourceMappingURL=webscrape.js.map