@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/chunking.js

@@ -0,0 +1,181 @@
+/**
+ * Smart content chunking for LLM processing.
+ *
+ * Splits content into manageable pieces with configurable overlap and strategy.
+ * Zero external dependencies; target <1ms for typical page content.
+ */
+/** Estimate token count using chars/4 heuristic. Accurate within ±10%. */
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Split content into chunks suitable for LLM processing.
+ */
+export function chunkContent(content, options) {
+    const chunkSize = options?.chunkSize ?? 4000;
+    const overlap = options?.overlap ?? 200;
+    const strategy = options?.strategy ?? 'semantic';
+    const originalTokens = estimateTokens(content);
+    if (content.length === 0) {
+        return { chunks: [], totalChunks: 0, totalTokens: 0, originalTokens: 0 };
+    }
+    let rawChunks;
+    switch (strategy) {
+        case 'fixed':
+            rawChunks = chunkFixed(content, chunkSize, overlap);
+            break;
+        case 'paragraph':
+            rawChunks = chunkParagraph(content, chunkSize);
+            break;
+        case 'semantic':
+        default:
+            rawChunks = chunkSemantic(content, chunkSize, overlap);
+            break;
+    }
+    const chunks = rawChunks.map((raw, i) => ({
+        index: i,
+        content: raw.content,
+        tokens: estimateTokens(raw.content),
+        startOffset: raw.startOffset,
+        isLast: i === rawChunks.length - 1,
+    }));
+    const totalTokens = chunks.reduce((sum, c) => sum + c.tokens, 0);
+    return {
+        chunks,
+        totalChunks: chunks.length,
+        totalTokens,
+        originalTokens,
+    };
+}
+// ---------------------------------------------------------------------------
+// Fixed strategy
+// ---------------------------------------------------------------------------
+function chunkFixed(content, chunkSize, overlap) {
+    const chunkChars = chunkSize * 4;
+    const overlapChars = overlap * 4;
+    const result = [];
+    let start = 0;
+    while (start < content.length) {
+        const end = Math.min(start + chunkChars, content.length);
+        result.push({ content: content.slice(start, end), startOffset: start });
+        if (end >= content.length)
+            break;
+        start = end - overlapChars;
+    }
+    return result;
+}
+// ---------------------------------------------------------------------------
+// Semantic strategy
+// ---------------------------------------------------------------------------
+function findNaturalBreak(content, target, tolerance) {
+    const min = target - tolerance;
+    const max = Math.min(target + tolerance, content.length);
+    // 1. Heading break: newline immediately before ##
+    const headingRe = /\n(?=#{1,6} )/g;
+    let best = null;
+    headingRe.lastIndex = min;
+    let m = headingRe.exec(content);
+    while (m && m.index <= max) {
+        const dist = Math.abs(m.index - target);
+        if (!best || best.priority < 4 || dist < Math.abs(best.pos - target)) {
+            best = { pos: m.index, priority: 4 };
+        }
+        m = headingRe.exec(content);
+    }
+    // 2. Paragraph break: \n\n
+    const paraRe = /\n\n/g;
+    paraRe.lastIndex = min;
+    m = paraRe.exec(content);
+    while (m && m.index <= max) {
+        const dist = Math.abs(m.index - target);
+        if (!best || best.priority < 3 || (best.priority === 3 && dist < Math.abs(best.pos - target))) {
+            best = { pos: m.index + 2, priority: 3 };
+        }
+        m = paraRe.exec(content);
+    }
+    // 3. Sentence end: '. ', '! ', '? ' followed by capital or newline
+    const sentRe = /[.!?](?:\s+(?=[A-Z\n])|(?=\n))/g;
+    sentRe.lastIndex = min;
+    m = sentRe.exec(content);
+    while (m && m.index <= max) {
+        const pos = m.index + m[0].length;
+        const dist = Math.abs(pos - target);
+        if (!best || best.priority < 2 || (best.priority === 2 && dist < Math.abs(best.pos - target))) {
+            best = { pos, priority: 2 };
+        }
+        m = sentRe.exec(content);
+    }
+    // 4. Word boundary (space)
+    if (!best || best.priority < 1) {
+        const spaceRe = / /g;
+        spaceRe.lastIndex = min;
+        m = spaceRe.exec(content);
+        while (m && m.index <= max) {
+            const pos = m.index + 1;
+            const dist = Math.abs(pos - target);
+            if (!best || (best.priority < 2 && dist < Math.abs(best.pos - target))) {
+                best = { pos, priority: 1 };
+            }
+            m = spaceRe.exec(content);
+        }
+    }
+    return best ? best.pos : Math.min(target, content.length);
+}
+function chunkSemantic(content, chunkSize, overlap) {
+    const chunkChars = chunkSize * 4;
+    const overlapChars = overlap * 4;
+    const tolerance = Math.floor(chunkChars * 0.2);
+    const result = [];
+    let start = 0;
+    while (start < content.length) {
+        const remaining = content.length - start;
+        // If the rest fits, take it all
+        if (remaining <= chunkChars + tolerance) {
+            result.push({ content: content.slice(start), startOffset: start });
+            break;
+        }
+        const breakPos = findNaturalBreak(content, start + chunkChars, tolerance);
+        const end = Math.max(breakPos, start + 1); // always advance
+        result.push({ content: content.slice(start, end), startOffset: start });
+        // Next chunk starts with overlap from end of this chunk
+        const nextStart = Math.max(start + 1, end - overlapChars);
+        start = nextStart;
+    }
+    return result;
+}
+// ---------------------------------------------------------------------------
+// Paragraph strategy
+// ---------------------------------------------------------------------------
+function chunkParagraph(content, chunkSize) {
+    const chunkChars = chunkSize * 4;
+    const paragraphs = content.split(/\n\n/);
+    const result = [];
+    let currentParts = [];
+    let currentLen = 0;
+    let currentOffset = 0;
+    let offsetTracker = 0;
+    for (let i = 0; i < paragraphs.length; i++) {
+        const para = paragraphs[i];
+        // +2 for the \n\n separator (except for first paragraph)
+        const paraLen = para.length + (currentParts.length > 0 ? 2 : 0);
+        if (currentParts.length > 0 && currentLen + paraLen > chunkChars) {
+            // Flush current group
+            result.push({ content: currentParts.join('\n\n'), startOffset: currentOffset });
+            currentOffset = offsetTracker;
+            currentParts = [para];
+            currentLen = para.length;
+        }
+        else {
+            if (currentParts.length === 0) {
+                currentOffset = offsetTracker;
+            }
+            currentParts.push(para);
+            currentLen += paraLen;
+        }
+        offsetTracker += para.length + 2; // account for \n\n
+    }
+    if (currentParts.length > 0) {
+        result.push({ content: currentParts.join('\n\n'), startOffset: currentOffset });
+    }
+    return result;
+}

package/dist/core/circuit-breaker.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Circuit breaker for browser/Chromium operations.
+ *
+ * States:
+ * - CLOSED (normal): requests pass through
+ * - OPEN (tripped): requests immediately fail, no browser launch attempted
+ * - HALF_OPEN (testing): allow 1 request through to test if browser works again
+ *
+ * Transitions:
+ * - CLOSED → OPEN: after `failureThreshold` consecutive failures (default: 3)
+ * - OPEN → HALF_OPEN: after `resetTimeoutMs` (default: 60s)
+ * - HALF_OPEN → CLOSED: if test request succeeds
+ * - HALF_OPEN → OPEN: if test request fails (reset timer)
+ */
+export type CircuitState = 'closed' | 'open' | 'half_open';
+export interface CircuitBreakerOptions {
+    failureThreshold?: number;
+    resetTimeoutMs?: number;
+    name?: string;
+}
+export declare class CircuitBreaker {
+    private state;
+    private failureCount;
+    private lastFailureTime;
+    private readonly failureThreshold;
+    private readonly resetTimeoutMs;
+    private readonly name;
+    constructor(options?: CircuitBreakerOptions);
+    /** Check if the circuit allows a request through */
+    canExecute(): boolean;
+    /** Record a successful operation */
+    recordSuccess(): void;
+    /** Record a failed operation */
+    recordFailure(error?: Error): void;
+    /** Get current state for health checks */
+    getState(): {
+        state: CircuitState;
+        failureCount: number;
+        lastFailureTime: number;
+    };
+    /** Force reset (e.g., on manual intervention) */
+    reset(): void;
+}
+export declare const browserCircuitBreaker: CircuitBreaker;

package/dist/core/circuit-breaker.js

@@ -0,0 +1,85 @@
+/**
+ * Circuit breaker for browser/Chromium operations.
+ *
+ * States:
+ * - CLOSED (normal): requests pass through
+ * - OPEN (tripped): requests immediately fail, no browser launch attempted
+ * - HALF_OPEN (testing): allow 1 request through to test if browser works again
+ *
+ * Transitions:
+ * - CLOSED → OPEN: after `failureThreshold` consecutive failures (default: 3)
+ * - OPEN → HALF_OPEN: after `resetTimeoutMs` (default: 60s)
+ * - HALF_OPEN → CLOSED: if test request succeeds
+ * - HALF_OPEN → OPEN: if test request fails (reset timer)
+ */
+export class CircuitBreaker {
+    state = 'closed';
+    failureCount = 0;
+    lastFailureTime = 0;
+    failureThreshold;
+    resetTimeoutMs;
+    name;
+    constructor(options = {}) {
+        this.failureThreshold = options.failureThreshold ?? 3;
+        this.resetTimeoutMs = options.resetTimeoutMs ?? 60000;
+        this.name = options.name ?? 'browser';
+    }
+    /** Check if the circuit allows a request through */
+    canExecute() {
+        if (this.state === 'closed')
+            return true;
+        if (this.state === 'open') {
+            // Check if enough time has passed to try again
+            if (Date.now() - this.lastFailureTime >= this.resetTimeoutMs) {
+                this.state = 'half_open';
+                console.log(`[circuit-breaker:${this.name}] HALF_OPEN — testing browser availability`);
+                return true;
+            }
+            return false;
+        }
+        // half_open: allow one request
+        return true;
+    }
+    /** Record a successful operation */
+    recordSuccess() {
+        if (this.state === 'half_open') {
+            console.log(`[circuit-breaker:${this.name}] CLOSED — browser recovered`);
+        }
+        this.failureCount = 0;
+        this.state = 'closed';
+    }
+    /** Record a failed operation */
+    recordFailure(error) {
+        this.failureCount++;
+        this.lastFailureTime = Date.now();
+        if (this.state === 'half_open') {
+            this.state = 'open';
+            console.error(`[circuit-breaker:${this.name}] OPEN — test request failed, waiting ${this.resetTimeoutMs / 1000}s`);
+            return;
+        }
+        if (this.failureCount >= this.failureThreshold) {
+            this.state = 'open';
+            console.error(`[circuit-breaker:${this.name}] OPEN — ${this.failureCount} consecutive failures (${error?.message ?? 'unknown'}). Falling back to HTTP-only for ${this.resetTimeoutMs / 1000}s`);
+        }
+    }
+    /** Get current state for health checks */
+    getState() {
+        return {
+            state: this.state,
+            failureCount: this.failureCount,
+            lastFailureTime: this.lastFailureTime,
+        };
+    }
+    /** Force reset (e.g., on manual intervention) */
+    reset() {
+        this.state = 'closed';
+        this.failureCount = 0;
+        console.log(`[circuit-breaker:${this.name}] RESET — manually closed`);
+    }
+}
+// Singleton browser circuit breaker
+export const browserCircuitBreaker = new CircuitBreaker({
+    name: 'browser',
+    failureThreshold: 3,
+    resetTimeoutMs: 60000, // 1 minute cooldown
+});

package/dist/core/content-pruner.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Content Density Pruner
+ *
+ * Two-pass pruning to reduce HTML before markdown conversion:
+ *
+ *   Pass 1 — Semantic removal: strip elements whose tag or class/id clearly
+ *            mark them as page chrome (nav, footer, sidebar, cookie banners, ads).
+ *
+ *   Pass 2 — Density scoring: score remaining block elements by text density,
+ *            link density, tag importance, and word count. Remove low-scorers.
+ *
+ * Inspired by Crawl4AI's PruningContentFilter — targets 40-60% token savings.
+ */
+export interface PruneOptions {
+    /** Score threshold (0-1). Blocks below this are removed. Default: 0.3 */
+    threshold?: number;
+    /** Minimum word count for a block to be considered. Default: 3 */
+    minWords?: number;
+    /** Whether threshold adapts to content distribution. Default: true */
+    dynamic?: boolean;
+}
+export interface PruneResult {
+    /** Pruned HTML */
+    html: string;
+    /** Number of nodes removed */
+    nodesRemoved: number;
+    /** Percentage of content removed (by character count) */
+    reductionPercent: number;
+}
+/**
+ * Prune low-value HTML blocks using two-pass approach:
+ *   1. Semantic tag/class removal
+ *   2. Density scoring of remaining blocks
+ *
+ * @param html - Raw HTML to prune
+ * @param options - Pruning configuration
+ * @returns Pruned HTML with stats
+ */
+export declare function pruneContent(html: string, options?: PruneOptions): PruneResult;
+/**
+ * Post-process markdown output to remove UI elements that leak through
+ * from content scrapers (buttons, empty images, consecutive hr separators).
+ *
+ * @param markdown - Raw markdown string
+ * @returns Cleaned markdown string
+ */
+export declare function pruneMarkdown(markdown: string): string;