@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/rate-governor.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Rate Governor — controls application rate to stay within safe limits.
+ *
+ * Prevents over-application and bot-detection triggers by:
+ * - Capping daily application count
+ * - Enforcing minimum/maximum delays between applications
+ * - Restricting activity to configured active hours
+ * - Supporting optional weekday-only mode
+ * - Entering cooldown after a CAPTCHA / warning signal is detected
+ *
+ * State is persisted to ~/.webpeel/rate-state.json.
+ */
+export interface RateConfig {
+    /** Max applications per day. Default: 5 */
+    maxPerDay: number;
+    /** Min delay between applications in ms. Default: 900000 (15 min) */
+    minDelayMs: number;
+    /** Max delay between applications in ms. Default: 2700000 (45 min) */
+    maxDelayMs: number;
+    /** Active hours range [start, end] in 24h format. Default: [9, 18] */
+    activeHours: [number, number];
+    /** Only apply on weekdays. Default: true */
+    weekdaysOnly: boolean;
+    /** Cooldown period after CAPTCHA/warning detection in ms. Default: 172800000 (48h) */
+    warningCooldownMs: number;
+}
+export interface RateState {
+    /** Applications submitted today (resets at midnight) */
+    todayCount: number;
+    /** Date string (YYYY-MM-DD) for today tracking */
+    todayDate: string;
+    /** Timestamp of last application */
+    lastApplyTimestamp: number;
+    /** Timestamp when cooldown expires (0 if no cooldown) */
+    cooldownUntil: number;
+    /** Total applications all time */
+    totalApplications: number;
+}
+export interface CanApplyResult {
+    allowed: boolean;
+    reason?: string;
+    /** Milliseconds to wait before trying again */
+    waitMs?: number;
+}
+export declare class RateGovernor {
+    private config;
+    private state;
+    constructor(config?: Partial<RateConfig>);
+    /**
+     * Check whether we are allowed to submit an application right now.
+     * Returns `{ allowed: true }` if all checks pass, otherwise returns
+     * `{ allowed: false, reason, waitMs }`.
+     */
+    canApply(): CanApplyResult;
+    /** Record a successful application submission. */
+    recordApplication(): void;
+    /**
+     * Record a warning signal (CAPTCHA detected, rate-limit notice, etc.).
+     * Activates a cooldown period to avoid further detection.
+     */
+    recordWarning(): void;
+    /** Get a snapshot of the current rate state. */
+    getState(): RateState;
+    /** Manually clear the active cooldown. */
+    resetCooldown(): void;
+    /**
+     * Calculate a randomised delay for the next application submission.
+     * Returns a value uniformly distributed between minDelayMs and maxDelayMs.
+     */
+    getNextDelay(): number;
+    /** Expose resolved config (useful for CLI display). */
+    getConfig(): RateConfig;
+    /** Reset today's counter if the calendar date has changed. */
+    private maybeResetDay;
+    private loadState;
+    private freshState;
+    private saveState;
+}
+/** Human-readable duration string from milliseconds. */
+export declare function formatDuration(ms: number): string;

package/dist/core/rate-governor.js

@@ -0,0 +1,238 @@
+/**
+ * Rate Governor — controls application rate to stay within safe limits.
+ *
+ * Prevents over-application and bot-detection triggers by:
+ * - Capping daily application count
+ * - Enforcing minimum/maximum delays between applications
+ * - Restricting activity to configured active hours
+ * - Supporting optional weekday-only mode
+ * - Entering cooldown after a CAPTCHA / warning signal is detected
+ *
+ * State is persisted to ~/.webpeel/rate-state.json.
+ */
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+// ── Defaults ───────────────────────────────────────────────────────────
+const DEFAULT_CONFIG = {
+    maxPerDay: 5,
+    minDelayMs: 900_000, // 15 minutes
+    maxDelayMs: 2_700_000, // 45 minutes
+    activeHours: [9, 18],
+    weekdaysOnly: true,
+    warningCooldownMs: 172_800_000, // 48 hours
+};
+const WEBPEEL_DIR = join(homedir(), '.webpeel');
+const RATE_STATE_PATH = join(WEBPEEL_DIR, 'rate-state.json');
+// ── Helpers ────────────────────────────────────────────────────────────
+/** Format a Date as YYYY-MM-DD in local time */
+function toDateString(d) {
+    const y = d.getFullYear();
+    const m = String(d.getMonth() + 1).padStart(2, '0');
+    const day = String(d.getDate()).padStart(2, '0');
+    return `${y}-${m}-${day}`;
+}
+/** Milliseconds until the next occurrence of a given hour (local time) */
+function msUntilHour(hour) {
+    const now = new Date();
+    const target = new Date(now);
+    target.setHours(hour, 0, 0, 0);
+    if (target <= now) {
+        target.setDate(target.getDate() + 1);
+    }
+    return target.getTime() - now.getTime();
+}
+/** Milliseconds until next weekday morning (Monday 9am) */
+function msUntilNextWeekdayMorning(startHour) {
+    const now = new Date();
+    const target = new Date(now);
+    // Step forward day by day until we hit a weekday
+    for (let i = 1; i <= 7; i++) {
+        target.setDate(now.getDate() + i);
+        target.setHours(startHour, 0, 0, 0);
+        const dow = target.getDay(); // 0=Sun, 6=Sat
+        if (dow >= 1 && dow <= 5)
+            break;
+    }
+    return target.getTime() - now.getTime();
+}
+// ── RateGovernor class ─────────────────────────────────────────────────
+export class RateGovernor {
+    config;
+    state;
+    constructor(config) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.state = this.loadState();
+        this.maybeResetDay();
+    }
+    // ── Public API ───────────────────────────────────────────────────────
+    /**
+     * Check whether we are allowed to submit an application right now.
+     * Returns `{ allowed: true }` if all checks pass, otherwise returns
+     * `{ allowed: false, reason, waitMs }`.
+     */
+    canApply() {
+        const now = Date.now();
+        const nowDate = new Date();
+        // 1. Active cooldown check
+        if (this.state.cooldownUntil > 0 && now < this.state.cooldownUntil) {
+            const waitMs = this.state.cooldownUntil - now;
+            return {
+                allowed: false,
+                reason: `Warning cooldown active — resumes in ${formatDuration(waitMs)}`,
+                waitMs,
+            };
+        }
+        // 2. Weekday-only check
+        if (this.config.weekdaysOnly) {
+            const dow = nowDate.getDay(); // 0=Sun, 6=Sat
+            if (dow === 0 || dow === 6) {
+                const waitMs = msUntilNextWeekdayMorning(this.config.activeHours[0]);
+                return {
+                    allowed: false,
+                    reason: 'Weekday-only mode — today is a weekend',
+                    waitMs,
+                };
+            }
+        }
+        // 3. Active hours check
+        const [startHour, endHour] = this.config.activeHours;
+        const currentHour = nowDate.getHours();
+        if (currentHour < startHour) {
+            const waitMs = msUntilHour(startHour);
+            return {
+                allowed: false,
+                reason: `Outside active hours — active between ${startHour}:00 and ${endHour}:00`,
+                waitMs,
+            };
+        }
+        if (currentHour >= endHour) {
+            // Past today's window; next window is tomorrow morning
+            const waitMs = msUntilHour(startHour) + 0; // already points to tomorrow
+            return {
+                allowed: false,
+                reason: `Outside active hours — active between ${startHour}:00 and ${endHour}:00`,
+                waitMs,
+            };
+        }
+        // 4. Daily cap check
+        if (this.state.todayCount >= this.config.maxPerDay) {
+            const waitMs = msUntilHour(startHour); // Try again tomorrow
+            return {
+                allowed: false,
+                reason: `Daily limit reached (${this.config.maxPerDay} applications today)`,
+                waitMs,
+            };
+        }
+        // 5. Minimum delay between applications
+        if (this.state.lastApplyTimestamp > 0) {
+            const elapsed = now - this.state.lastApplyTimestamp;
+            if (elapsed < this.config.minDelayMs) {
+                const waitMs = this.config.minDelayMs - elapsed;
+                return {
+                    allowed: false,
+                    reason: `Minimum delay not elapsed — please wait ${formatDuration(waitMs)}`,
+                    waitMs,
+                };
+            }
+        }
+        return { allowed: true };
+    }
+    /** Record a successful application submission. */
+    recordApplication() {
+        this.maybeResetDay();
+        this.state.todayCount += 1;
+        this.state.lastApplyTimestamp = Date.now();
+        this.state.totalApplications += 1;
+        this.saveState();
+    }
+    /**
+     * Record a warning signal (CAPTCHA detected, rate-limit notice, etc.).
+     * Activates a cooldown period to avoid further detection.
+     */
+    recordWarning() {
+        this.state.cooldownUntil = Date.now() + this.config.warningCooldownMs;
+        this.saveState();
+    }
+    /** Get a snapshot of the current rate state. */
+    getState() {
+        this.maybeResetDay();
+        return { ...this.state };
+    }
+    /** Manually clear the active cooldown. */
+    resetCooldown() {
+        this.state.cooldownUntil = 0;
+        this.saveState();
+    }
+    /**
+     * Calculate a randomised delay for the next application submission.
+     * Returns a value uniformly distributed between minDelayMs and maxDelayMs.
+     */
+    getNextDelay() {
+        const range = this.config.maxDelayMs - this.config.minDelayMs;
+        return Math.floor(this.config.minDelayMs + Math.random() * range);
+    }
+    /** Expose resolved config (useful for CLI display). */
+    getConfig() {
+        return { ...this.config };
+    }
+    // ── Private helpers ──────────────────────────────────────────────────
+    /** Reset today's counter if the calendar date has changed. */
+    maybeResetDay() {
+        const today = toDateString(new Date());
+        if (this.state.todayDate !== today) {
+            this.state.todayCount = 0;
+            this.state.todayDate = today;
+            this.saveState();
+        }
+    }
+    loadState() {
+        try {
+            if (existsSync(RATE_STATE_PATH)) {
+                const raw = readFileSync(RATE_STATE_PATH, 'utf-8');
+                return JSON.parse(raw);
+            }
+        }
+        catch (e) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'rate state load failed:', e instanceof Error ? e.message : e);
+        }
+        return this.freshState();
+    }
+    freshState() {
+        return {
+            todayCount: 0,
+            todayDate: toDateString(new Date()),
+            lastApplyTimestamp: 0,
+            cooldownUntil: 0,
+            totalApplications: 0,
+        };
+    }
+    saveState() {
+        try {
+            mkdirSync(WEBPEEL_DIR, { recursive: true });
+            writeFileSync(RATE_STATE_PATH, JSON.stringify(this.state, null, 2), 'utf-8');
+        }
+        catch (err) {
+            // Non-fatal — state will be recomputed on next run
+            console.error('[rate-governor] Failed to save state:', err);
+        }
+    }
+}
+// ── Utility ────────────────────────────────────────────────────────────
+/** Human-readable duration string from milliseconds. */
+export function formatDuration(ms) {
+    if (ms <= 0)
+        return '0s';
+    const seconds = Math.floor(ms / 1000);
+    if (seconds < 60)
+        return `${seconds}s`;
+    const minutes = Math.floor(seconds / 60);
+    if (minutes < 60)
+        return `${minutes}m ${seconds % 60}s`;
+    const hours = Math.floor(minutes / 60);
+    if (hours < 24)
+        return `${hours}h ${minutes % 60}m`;
+    const days = Math.floor(hours / 24);
+    return `${days}d ${hours % 24}h`;
+}

package/dist/core/readability.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Readability Engine
+ *
+ * Extracts the core article content from a web page — like Pocket, Instapaper,
+ * or Safari Reader Mode but deterministic, fast, and purpose-built for AI agents.
+ *
+ * Algorithm:
+ *   1. Noise removal — strip nav, footer, aside, ads, cookie banners, etc.
+ *   2. Candidate scoring — score block elements by text density, link density,
+ *      paragraph count, and structural signals.
+ *   3. Best candidate selection — prefer <article> > <main> > highest-scoring div.
+ *   4. Post-selection cleaning — remove inline noise (share buttons, etc.).
+ *   5. Metadata extraction — title, author, date, site name from meta tags / bylines.
+ *   6. Markdown output — via existing htmlToMarkdown().
+ */
+export interface ReadabilityOptions {
+    /** Keep image references in output (default: true) */
+    includeImages?: boolean;
+    /** Keep hyperlinks in output (default: true) */
+    includeLinks?: boolean;
+    /** Keep code blocks in output (default: true) */
+    includeCode?: boolean;
+    /** Keep tables in output (default: true) */
+    includeTables?: boolean;
+    /** Maximum characters to return (applied after conversion) */
+    maxLength?: number;
+}
+export interface ReadabilityResult {
+    /** Article title */
+    title: string;
+    /** Author name or null */
+    author: string | null;
+    /** Publication date string or null */
+    date: string | null;
+    /** Site name or null */
+    siteName: string | null;
+    /** Clean article content as markdown */
+    content: string;
+    /** First 2 complete sentences as excerpt */
+    excerpt: string;
+    /** Estimated word count */
+    wordCount: number;
+    /** Human-readable reading time, e.g. "5 min read" */
+    readingTime: string;
+    /** Language code from <html lang> or null */
+    language: string | null;
+}
+/**
+ * Extract clean, readable article content from raw HTML.
+ *
+ * Mimics browser Reader Mode but deterministic and purpose-built for AI agents.
+ *
+ * @param html  - Raw HTML of the page
+ * @param url   - Source URL (used for resolving relative links in metadata)
+ * @param options - Extraction options
+ */
+export declare function extractReadableContent(html: string, _url: string, options?: ReadabilityOptions): ReadabilityResult;