@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/synonyms.js

@@ -0,0 +1,184 @@
+/**
+ * Synonym expansion for query broadening.
+ *
+ * Provides stemmed synonym groups and a function to expand a set of stemmed
+ * query tokens with related synonyms (at a lower weight).
+ *
+ * Usage:
+ *   const queryTerms = tokenizeQuestion(question); // already stemmed
+ *   const expanded = expandWithSynonyms(queryTerms);
+ *   // expanded includes originals (weight=1.0) + synonyms (weight=0.5)
+ */
+import { stem } from './stemmer.js';
+// ---------------------------------------------------------------------------
+// Synonym groups (raw, unstemmed)
+// ---------------------------------------------------------------------------
+/**
+ * Raw synonym groups. Each group is a set of words with equivalent or near-
+ * equivalent meaning in the context of software/web documentation queries.
+ *
+ * These are stored in unstemmed form for readability; the build process stems
+ * them into STEMMED_SYNONYM_GROUPS and builds an index.
+ */
+export const SYNONYM_GROUPS = [
+    // Price/cost
+    ['price', 'cost', 'fee', 'charge', 'rate', 'pricing', 'subscription', 'plan'],
+    // Create/build
+    ['create', 'build', 'make', 'develop', 'construct', 'design', 'author', 'write', 'conceive'],
+    // Fast/quick
+    ['fast', 'quick', 'rapid', 'speedy', 'swift', 'performant', 'efficient'],
+    // Big/large
+    ['big', 'large', 'huge', 'enormous', 'massive', 'significant', 'substantial'],
+    // Small/tiny
+    ['small', 'tiny', 'little', 'minor', 'minimal', 'compact', 'lightweight'],
+    // Error/bug
+    ['error', 'bug', 'issue', 'problem', 'fault', 'defect', 'failure', 'crash'],
+    // Start/begin
+    ['start', 'begin', 'launch', 'initiate', 'commence', 'found', 'establish', 'release', 'introduce'],
+    // Stop/end
+    ['stop', 'end', 'finish', 'terminate', 'halt', 'cease', 'conclude'],
+    // Use/utilize
+    ['use', 'utilize', 'employ', 'leverage', 'apply', 'adopt'],
+    // Help/assist
+    ['help', 'assist', 'support', 'aid', 'guide', 'facilitate'],
+    // Show/display
+    ['show', 'display', 'render', 'present', 'exhibit', 'demonstrate'],
+    // Get/obtain
+    ['get', 'obtain', 'acquire', 'retrieve', 'fetch', 'receive', 'gain'],
+    // Send/transmit
+    ['send', 'transmit', 'deliver', 'dispatch', 'forward', 'emit'],
+    // Change/modify
+    ['change', 'modify', 'alter', 'update', 'revise', 'adjust', 'edit'],
+    // Delete/remove
+    ['delete', 'remove', 'eliminate', 'erase', 'clear', 'purge', 'drop'],
+    // Allow/permit
+    ['allow', 'permit', 'enable', 'authorize', 'grant', 'let'],
+    // Prevent/block
+    ['prevent', 'block', 'prohibit', 'restrict', 'deny', 'forbid', 'disable'],
+    // Location/place
+    ['location', 'place', 'position', 'site', 'area', 'region', 'spot'],
+    // Person/individual
+    ['person', 'individual', 'user', 'member', 'participant', 'developer'],
+    // Feature/capability
+    ['feature', 'capability', 'functionality', 'ability', 'function', 'capacity'],
+    // Limit/restrict
+    ['limit', 'restrict', 'constrain', 'cap', 'bound', 'throttle'],
+    // Install/setup
+    ['install', 'setup', 'configure', 'deploy', 'provision'],
+    // Compare/contrast
+    ['compare', 'contrast', 'versus', 'differ', 'distinction', 'difference'],
+    // Require/need
+    ['require', 'need', 'demand', 'necessitate', 'depend'],
+    // Advantage/benefit/feature
+    ['advantage', 'benefit', 'pro', 'strength', 'upside', 'merit', 'feature', 'perk'],
+    // Disadvantage/drawback
+    ['disadvantage', 'drawback', 'con', 'weakness', 'downside', 'limitation'],
+    // Learn/study
+    ['learn', 'study', 'understand', 'explore', 'discover', 'research'],
+    // Test/verify
+    ['test', 'verify', 'validate', 'check', 'confirm', 'inspect'],
+    // Connect/integrate
+    ['connect', 'integrate', 'link', 'combine', 'join', 'merge', 'attach'],
+    // Document/describe
+    ['document', 'describe', 'explain', 'detail', 'outline', 'specify'],
+    // Improve/optimize
+    ['improve', 'optimize', 'enhance', 'upgrade', 'refine', 'boost'],
+    // Location state (where questions)
+    ['base', 'headquarter', 'locate', 'situate'],
+    // Software referents (helps with coreference)
+    ['platform', 'service', 'product', 'tool', 'application', 'system', 'software'],
+    // Movement/direction
+    ['move', 'transfer', 'migrate', 'shift', 'relocate', 'transition'],
+    // Exist/available
+    ['exist', 'available', 'present', 'accessible', 'offered'],
+    // Medical/health
+    ['symptom', 'sign', 'indication', 'manifestation'],
+    ['treatment', 'therapy', 'cure', 'remedy', 'medication', 'medicine'],
+    ['diagnosis', 'assessment', 'evaluation', 'examination'],
+    ['disease', 'illness', 'condition', 'disorder', 'ailment', 'sickness'],
+    // Financial/business
+    ['revenue', 'income', 'earnings', 'sales', 'turnover'],
+    ['expense', 'cost', 'spending', 'expenditure', 'outlay'],
+    ['profit', 'gain', 'return', 'margin', 'surplus'],
+    ['invest', 'fund', 'finance', 'capitalize', 'back'],
+    // Importance/significance
+    ['important', 'significant', 'crucial', 'critical', 'vital', 'essential', 'key'],
+    ['minor', 'trivial', 'negligible', 'insignificant', 'marginal'],
+    // Communication
+    ['say', 'state', 'declare', 'announce', 'claim', 'assert', 'mention'],
+    ['ask', 'question', 'inquire', 'query', 'request'],
+    ['answer', 'reply', 'respond', 'response'],
+    // Think/believe
+    ['think', 'believe', 'consider', 'regard', 'view', 'deem'],
+    ['decide', 'determine', 'conclude', 'resolve', 'settle'],
+    // Outcome/result
+    ['result', 'outcome', 'consequence', 'effect', 'impact', 'aftermath'],
+    ['cause', 'reason', 'factor', 'trigger', 'source', 'origin'],
+    // Amount/quantity
+    ['many', 'numerous', 'several', 'multiple', 'various', 'countless'],
+    ['few', 'scarce', 'rare', 'limited', 'sparse'],
+    // Time
+    ['before', 'prior', 'previous', 'preceding', 'earlier', 'former'],
+    ['after', 'subsequent', 'following', 'later', 'next', 'succeeding'],
+    ['recent', 'latest', 'newest', 'current', 'contemporary', 'modern'],
+    ['old', 'ancient', 'historical', 'legacy', 'outdated', 'obsolete'],
+];
+// ---------------------------------------------------------------------------
+// Build stemmed synonym index
+// ---------------------------------------------------------------------------
+/**
+ * Stemmed synonym groups.
+ * Each word in each group has been run through the Porter stemmer.
+ * Duplicate stems within a group are deduplicated.
+ */
+export const STEMMED_SYNONYM_GROUPS = SYNONYM_GROUPS.map(group => {
+    const stemmed = group.map(w => stem(w));
+    // Deduplicate (multiple words may stem to the same root)
+    return [...new Set(stemmed)];
+});
+/**
+ * Index: stemmed_word → set of other stemmed words in the same group.
+ * A word may appear in only one group (first match wins if duplicates exist).
+ */
+const synonymIndex = new Map();
+for (const group of STEMMED_SYNONYM_GROUPS) {
+    for (const word of group) {
+        if (!synonymIndex.has(word)) {
+            // Store all OTHER words in the group as synonyms
+            const others = new Set(group.filter(w => w !== word));
+            synonymIndex.set(word, others);
+        }
+    }
+}
+/**
+ * Expand a list of stemmed query tokens with their synonyms.
+ *
+ * @param terms - Already-stemmed tokens from the query
+ * @returns Array of ExpandedTerm objects. Original terms have weight=1.0,
+ *          synonym expansions have weight=0.5.
+ *          The returned array preserves originals first, then synonyms.
+ */
+export function expandWithSynonyms(terms) {
+    const result = [];
+    const seen = new Set();
+    // Add all original terms first
+    for (const term of terms) {
+        if (!seen.has(term)) {
+            seen.add(term);
+            result.push({ term, weight: 1.0, isOriginal: true });
+        }
+    }
+    // Add synonyms for each original term
+    for (const term of terms) {
+        const synonyms = synonymIndex.get(term);
+        if (!synonyms)
+            continue;
+        for (const syn of synonyms) {
+            if (!seen.has(syn)) {
+                seen.add(syn);
+                result.push({ term: syn, weight: 0.5, isOriginal: false });
+            }
+        }
+    }
+    return result;
+}

package/dist/core/system-monitor.d.ts

@@ -0,0 +1,61 @@
+/**
+ * System resource monitor — cgroup-aware for K8s containers.
+ *
+ * In K8s, os.freemem() gives WRONG numbers because it reads host memory,
+ * not the container's cgroup limit. We read /sys/fs/cgroup directly.
+ *
+ * Three-tier memory management (from Crawl4AI):
+ * - Normal: < threshold (85%) — accept all jobs
+ * - Pressure: >= threshold (85%) — log warning, reject new render jobs
+ * - Critical: >= critical (95%) — reject ALL new jobs
+ *
+ * Also tracks: CPU usage, connection count, uptime.
+ */
+export interface SystemHealth {
+    memory: {
+        usedBytes: number;
+        totalBytes: number;
+        usedPercent: number;
+        tier: 'normal' | 'pressure' | 'critical';
+    };
+    uptime: number;
+    isK8s: boolean;
+    canAcceptJob: boolean;
+    canAcceptRenderJob: boolean;
+}
+export interface SystemMonitorOptions {
+    memoryThresholdPercent?: number;
+    criticalThresholdPercent?: number;
+    recoveryThresholdPercent?: number;
+}
+declare class SystemMonitor {
+    private readonly IS_K8S;
+    private readonly MEM_CURRENT;
+    private readonly MEM_MAX;
+    private readonly MEM_CURRENT_V1;
+    private readonly MEM_MAX_V1;
+    private memoryThreshold;
+    private criticalThreshold;
+    private recoveryThreshold;
+    private inPressureMode;
+    private readonly startTime;
+    private consecutiveRejects;
+    private readonly MAX_CONSECUTIVE_REJECTS;
+    constructor(options?: SystemMonitorOptions);
+    /** Read memory from cgroups (K8s) or OS */
+    getMemoryUsage(): {
+        usedBytes: number;
+        totalBytes: number;
+        usedPercent: number;
+    };
+    private readCgroupMemory;
+    /** Get full health snapshot */
+    getHealth(): SystemHealth;
+    /** Quick check: can we accept a new job? */
+    canAccept(requireRender?: boolean): boolean;
+    /** Check if the worker appears stalled */
+    isStalled(): boolean;
+}
+/** Singleton system monitor */
+export declare const systemMonitor: SystemMonitor;
+export {};

package/dist/core/system-monitor.js

@@ -0,0 +1,133 @@
+/**
+ * System resource monitor — cgroup-aware for K8s containers.
+ *
+ * In K8s, os.freemem() gives WRONG numbers because it reads host memory,
+ * not the container's cgroup limit. We read /sys/fs/cgroup directly.
+ *
+ * Three-tier memory management (from Crawl4AI):
+ * - Normal: < threshold (85%) — accept all jobs
+ * - Pressure: >= threshold (85%) — log warning, reject new render jobs
+ * - Critical: >= critical (95%) — reject ALL new jobs
+ *
+ * Also tracks: CPU usage, connection count, uptime.
+ */
+import { readFileSync, existsSync } from 'fs';
+import os from 'os';
+import { createLogger } from './logger.js';
+const log = createLogger('monitor');
+class SystemMonitor {
+    IS_K8S;
+    MEM_CURRENT = '/sys/fs/cgroup/memory.current';
+    MEM_MAX = '/sys/fs/cgroup/memory.max';
+    // cgroup v1 fallback paths
+    MEM_CURRENT_V1 = '/sys/fs/cgroup/memory/memory.usage_in_bytes';
+    MEM_MAX_V1 = '/sys/fs/cgroup/memory/memory.limit_in_bytes';
+    memoryThreshold;
+    criticalThreshold;
+    recoveryThreshold;
+    inPressureMode = false;
+    startTime = Date.now();
+    consecutiveRejects = 0;
+    MAX_CONSECUTIVE_REJECTS = 25; // Firecrawl-style stall detection
+    constructor(options = {}) {
+        this.memoryThreshold = (options.memoryThresholdPercent ?? 85) / 100;
+        this.criticalThreshold = (options.criticalThresholdPercent ?? 95) / 100;
+        this.recoveryThreshold = (options.recoveryThresholdPercent ?? 80) / 100;
+        // Detect K8s by checking for cgroup files
+        this.IS_K8S = existsSync(this.MEM_CURRENT) || existsSync(this.MEM_CURRENT_V1);
+        if (this.IS_K8S) {
+            log.info('K8s cgroup memory monitoring active');
+        }
+        else {
+            log.info('Using OS-level memory monitoring (non-K8s)');
+        }
+    }
+    /** Read memory from cgroups (K8s) or OS */
+    getMemoryUsage() {
+        if (this.IS_K8S) {
+            return this.readCgroupMemory();
+        }
+        const total = os.totalmem();
+        const free = os.freemem();
+        const used = total - free;
+        return { usedBytes: used, totalBytes: total, usedPercent: used / total };
+    }
+    readCgroupMemory() {
+        try {
+            // Try cgroup v2 first, then v1
+            let currentPath = this.MEM_CURRENT;
+            let maxPath = this.MEM_MAX;
+            if (!existsSync(currentPath)) {
+                currentPath = this.MEM_CURRENT_V1;
+                maxPath = this.MEM_MAX_V1;
+            }
+            const current = parseInt(readFileSync(currentPath, 'utf8').trim(), 10);
+            const maxStr = readFileSync(maxPath, 'utf8').trim();
+            // 'max' in cgroup v2 means no limit set
+            const max = maxStr === 'max' || parseInt(maxStr, 10) > os.totalmem() * 10
+                ? os.totalmem()
+                : parseInt(maxStr, 10);
+            return { usedBytes: current, totalBytes: max, usedPercent: current / max };
+        }
+        catch (err) {
+            // Fallback to OS-level if cgroup read fails
+            log.warn('Failed to read cgroup memory, falling back to OS-level');
+            const total = os.totalmem();
+            const free = os.freemem();
+            const used = total - free;
+            return { usedBytes: used, totalBytes: total, usedPercent: used / total };
+        }
+    }
+    /** Get full health snapshot */
+    getHealth() {
+        const mem = this.getMemoryUsage();
+        // Three-tier memory management with hysteresis
+        let tier;
+        if (mem.usedPercent >= this.criticalThreshold) {
+            tier = 'critical';
+            this.inPressureMode = true;
+        }
+        else if (mem.usedPercent >= this.memoryThreshold) {
+            tier = 'pressure';
+            this.inPressureMode = true;
+        }
+        else if (this.inPressureMode && mem.usedPercent > this.recoveryThreshold) {
+            // Hysteresis: stay in pressure until we drop below recovery threshold
+            tier = 'pressure';
+        }
+        else {
+            tier = 'normal';
+            this.inPressureMode = false;
+        }
+        const canAcceptJob = tier !== 'critical';
+        const canAcceptRenderJob = tier === 'normal'; // render jobs use more memory
+        // Track consecutive rejects for stall detection
+        if (!canAcceptJob) {
+            this.consecutiveRejects++;
+            if (this.consecutiveRejects >= this.MAX_CONSECUTIVE_REJECTS) {
+                log.error(`WORKER STALLED — ${this.consecutiveRejects} consecutive job rejections. Memory: ${(mem.usedPercent * 100).toFixed(1)}%`);
+            }
+        }
+        else {
+            this.consecutiveRejects = 0;
+        }
+        return {
+            memory: { ...mem, tier },
+            uptime: (Date.now() - this.startTime) / 1000,
+            isK8s: this.IS_K8S,
+            canAcceptJob,
+            canAcceptRenderJob,
+        };
+    }
+    /** Quick check: can we accept a new job? */
+    canAccept(requireRender = false) {
+        const health = this.getHealth();
+        return requireRender ? health.canAcceptRenderJob : health.canAcceptJob;
+    }
+    /** Check if the worker appears stalled */
+    isStalled() {
+        return this.consecutiveRejects >= this.MAX_CONSECUTIVE_REJECTS;
+    }
+}
+/** Singleton system monitor */
+export const systemMonitor = new SystemMonitor();

package/dist/core/table-format.d.ts

@@ -0,0 +1,30 @@
+/**
+ * ASCII table renderer using Unicode box-drawing characters.
+ *
+ * Renders an array of objects (e.g. {@link ListingItem}) as a neatly formatted
+ * table with auto-sized columns.
+ *
+ * @module table-format
+ */
+/**
+ * Render an array of row objects as a Unicode box-drawing table.
+ *
+ * Column order follows the insertion order of keys in the first row.
+ * Columns with exclusively `undefined`/empty values are omitted.
+ *
+ * @param rows    - Array of plain objects.
+ * @param columns - Optional explicit column keys to include (in order).
+ * @returns       Multi-line string ready for `console.log`.
+ *
+ * @example
+ * ```typescript
+ * import { formatTable } from './table-format.js';
+ *
+ * const table = formatTable([
+ *   { title: 'Widget A', price: '$10' },
+ *   { title: 'Widget B', price: '$20' },
+ * ]);
+ * console.log(table);
+ * ```
+ */
+export declare function formatTable(rows: Record<string, string | undefined>[], columns?: string[]): string;

package/dist/core/table-format.js

@@ -0,0 +1,146 @@
+/**
+ * ASCII table renderer using Unicode box-drawing characters.
+ *
+ * Renders an array of objects (e.g. {@link ListingItem}) as a neatly formatted
+ * table with auto-sized columns.
+ *
+ * @module table-format
+ */
+/* ------------------------------------------------------------------ */
+/*  Box-drawing character set                                         */
+/* ------------------------------------------------------------------ */
+const BOX = {
+    topLeft: '┌',
+    topRight: '┐',
+    bottomLeft: '└',
+    bottomRight: '┘',
+    horizontal: '─',
+    vertical: '│',
+    teeDown: '┬',
+    teeUp: '┴',
+    teeRight: '├',
+    teeLeft: '┤',
+    cross: '┼',
+};
+/* ------------------------------------------------------------------ */
+/*  Helpers                                                           */
+/* ------------------------------------------------------------------ */
+const MAX_COL_WIDTH = 40;
+const MIN_COL_WIDTH = 3;
+/**
+ * Truncate a string to `max` visible characters. Adds `…` if truncated.
+ */
+function truncate(s, max) {
+    if (s.length <= max)
+        return s;
+    return s.slice(0, max - 1) + '…';
+}
+/**
+ * Pad (right) a string to `width` characters.
+ */
+function padRight(s, width) {
+    if (s.length >= width)
+        return s;
+    return s + ' '.repeat(width - s.length);
+}
+/**
+ * Capitalise the first letter of a string.
+ */
+function capitalise(s) {
+    if (!s)
+        return s;
+    return s.charAt(0).toUpperCase() + s.slice(1);
+}
+/* ------------------------------------------------------------------ */
+/*  Public API                                                        */
+/* ------------------------------------------------------------------ */
+/**
+ * Render an array of row objects as a Unicode box-drawing table.
+ *
+ * Column order follows the insertion order of keys in the first row.
+ * Columns with exclusively `undefined`/empty values are omitted.
+ *
+ * @param rows    - Array of plain objects.
+ * @param columns - Optional explicit column keys to include (in order).
+ * @returns       Multi-line string ready for `console.log`.
+ *
+ * @example
+ * ```typescript
+ * import { formatTable } from './table-format.js';
+ *
+ * const table = formatTable([
+ *   { title: 'Widget A', price: '$10' },
+ *   { title: 'Widget B', price: '$20' },
+ * ]);
+ * console.log(table);
+ * ```
+ */
+export function formatTable(rows, columns) {
+    if (rows.length === 0)
+        return '';
+    // Determine columns: explicit list or derive from all rows
+    const cols = columns ?? deriveColumns(rows);
+    if (cols.length === 0)
+        return '';
+    // Build header labels
+    const headers = cols.map(c => capitalise(c));
+    // Compute column widths (bounded by MIN/MAX)
+    const widths = cols.map((col, i) => {
+        let max = headers[i].length;
+        for (const row of rows) {
+            const val = row[col] ?? '';
+            if (val.length > max)
+                max = val.length;
+        }
+        return Math.max(MIN_COL_WIDTH, Math.min(max, MAX_COL_WIDTH));
+    });
+    // Build lines
+    const lines = [];
+    // Top border
+    lines.push(BOX.topLeft +
+        widths.map(w => BOX.horizontal.repeat(w + 2)).join(BOX.teeDown) +
+        BOX.topRight);
+    // Header row
+    lines.push(BOX.vertical +
+        headers.map((h, i) => ' ' + padRight(truncate(h, widths[i]), widths[i]) + ' ').join(BOX.vertical) +
+        BOX.vertical);
+    // Separator
+    lines.push(BOX.teeRight +
+        widths.map(w => BOX.horizontal.repeat(w + 2)).join(BOX.cross) +
+        BOX.teeLeft);
+    // Data rows
+    for (const row of rows) {
+        lines.push(BOX.vertical +
+            cols.map((col, i) => {
+                const val = row[col] ?? '';
+                return ' ' + padRight(truncate(val, widths[i]), widths[i]) + ' ';
+            }).join(BOX.vertical) +
+            BOX.vertical);
+    }
+    // Bottom border
+    lines.push(BOX.bottomLeft +
+        widths.map(w => BOX.horizontal.repeat(w + 2)).join(BOX.teeUp) +
+        BOX.bottomRight);
+    return lines.join('\n');
+}
+/* ------------------------------------------------------------------ */
+/*  Internals                                                         */
+/* ------------------------------------------------------------------ */
+/**
+ * Derive the set of columns that have at least one non-empty value across
+ * all rows. Preserves insertion order from the first row that provides a key.
+ */
+function deriveColumns(rows) {
+    const seen = new Set();
+    const order = [];
+    for (const row of rows) {
+        for (const key of Object.keys(row)) {
+            if (!seen.has(key)) {
+                seen.add(key);
+                order.push(key);
+            }
+        }
+    }
+    // Filter out columns that are entirely empty
+    return order.filter(col => rows.some(r => r[col] && r[col].trim().length > 0));
+}

package/dist/core/threat-feeds.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Community threat intelligence feeds — cached in-memory, refreshed every 6 hours.
+ *
+ * Sources (all free, no API key):
+ * - URLhaus (abuse.ch): ~150K active malware distribution URLs
+ *   API: https://urlhaus-api.abuse.ch/v1/url/ (POST with url=<url>)
+ * - PhishTank: community-verified phishing URLs
+ *   API: https://checkurl.phishtank.com/checkurl/ (POST with url=<url>&format=json)
+ * - OpenPhish: curated phishing feed
+ *   Feed: https://openphish.com/feed.txt (plain text, one URL per line, ~5K URLs)
+ *
+ * Strategy:
+ * - On startup, fetch OpenPhish feed into a Set (small, fast lookup)
+ * - For URLhaus and PhishTank, do real-time API lookups with 2s timeout
+ * - Cache results for 1 hour to avoid hammering APIs
+ */
+export interface ThreatFeedResult {
+    safe: boolean;
+    threats: string[];
+    source: 'urlhaus' | 'phishtank' | 'openphish' | 'none';
+    details?: string;
+}
+export declare function checkThreatFeeds(url: string): Promise<ThreatFeedResult>;