@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/vertical-search.js

@@ -0,0 +1,158 @@
+/**
+ * Vertical search — specialized endpoints for shopping, news, images, videos.
+ * Primary: SearXNG (reliable, structured JSON, no CAPTCHA issues from datacenter).
+ * Fallback: Google scraping via peel() (for shopping only).
+ */
+import { fetch as undiciFetch } from 'undici';
+import { load } from 'cheerio';
+// ── SearXNG helper ─────────────────────────────────────────────────────────
+async function searxngSearch(query, category, count = 20, language) {
+    const baseUrl = process.env.SEARXNG_URL;
+    if (!baseUrl)
+        return [];
+    const params = new URLSearchParams({
+        q: query,
+        format: 'json',
+        categories: category,
+        safesearch: '0',
+    });
+    if (language)
+        params.set('language', language);
+    try {
+        const url = `${baseUrl.replace(/\/$/, '')}/search?${params}`;
+        const res = await undiciFetch(url, {
+            signal: AbortSignal.timeout(10000),
+            headers: { 'Accept': 'application/json', 'User-Agent': 'WebPeel/1.0' },
+        });
+        if (!res.ok)
+            return [];
+        const data = await res.json();
+        return (data.results || []).slice(0, count);
+    }
+    catch {
+        return [];
+    }
+}
+// ── Shopping Search ────────────────────────────────────────────────────────
+export async function searchShopping(opts) {
+    const { query, count = 10, language } = opts;
+    // Strategy 1: SearXNG general search with price-aware filtering
+    // (SearXNG doesn't have a shopping category that works, but general results
+    // from shopping sites contain price data)
+    const results = await searxngSearch(`${query} price buy`, 'general', count * 3, language);
+    const items = [];
+    for (const r of results) {
+        // Look for results from shopping domains or with price content
+        const url = r.url || '';
+        const content = r.content || '';
+        const title = r.title || '';
+        const isShoppingSite = /amazon|ebay|walmart|bestbuy|target|etsy|aliexpress|newegg|shopify/i.test(url);
+        const hasPrice = /\$[\d,]+(\.\d{2})?|\d+\.\d{2}\s*(USD|EUR|GBP)/.test(content + title);
+        if (isShoppingSite || hasPrice) {
+            // Extract price from content
+            const priceMatch = (content + ' ' + title).match(/\$[\d,]+(?:\.\d{2})?/);
+            items.push({
+                title: title,
+                price: priceMatch ? priceMatch[0] : undefined,
+                store: extractStoreName(url),
+                url: url,
+                imageUrl: r.img_src || r.thumbnail || undefined,
+            });
+        }
+        if (items.length >= count)
+            break;
+    }
+    // Fallback: if SearXNG returned nothing, try Google Shopping via peel()
+    if (items.length === 0) {
+        try {
+            const { peel } = await import('../index.js');
+            const params = new URLSearchParams({ q: query, tbm: 'shop', num: '20' });
+            const result = await peel(`https://www.google.com/search?${params}`, {
+                render: true, stealth: true, format: 'html', wait: 3000, timeout: 15000,
+            });
+            const html = result.content || '';
+            if (html && html.length > 500) {
+                const $ = load(html);
+                $('.sh-dgr__content, .mnr-c .pla-unit, [data-docid], .KZmu8e').each((_, elem) => {
+                    const el = $(elem);
+                    const t = el.find('.tAxDx, .pymv4e, h3, .Xjkr3b').first().text().trim();
+                    const p = el.find('.a8Pemb, .e10twf, .HRLxBb, .kHxwFf').first().text().trim();
+                    const s = el.find('.aULzUe, .LbUacb, .dD8iuc, .IuHnof').first().text().trim();
+                    if (t)
+                        items.push({ title: t, price: p || undefined, store: s || 'Unknown', url: el.find('a[href]').first().attr('href') || '' });
+                });
+            }
+        }
+        catch { /* fallback failed */ }
+    }
+    return items.slice(0, count);
+}
+function extractStoreName(url) {
+    try {
+        const host = new URL(url).hostname.replace('www.', '');
+        const nameMap = {
+            'amazon.com': 'Amazon', 'ebay.com': 'eBay', 'walmart.com': 'Walmart',
+            'bestbuy.com': 'Best Buy', 'target.com': 'Target', 'etsy.com': 'Etsy',
+            'newegg.com': 'Newegg', 'aliexpress.com': 'AliExpress',
+        };
+        return nameMap[host] || host.split('.')[0].charAt(0).toUpperCase() + host.split('.')[0].slice(1);
+    }
+    catch {
+        return 'Unknown';
+    }
+}
+// ── News Search ────────────────────────────────────────────────────────────
+export async function searchNews(opts) {
+    const { query, count = 10, language, freshness } = opts;
+    // Primary: SearXNG news category (tested: 44 results)
+    let searchQuery = query;
+    if (freshness === 'day')
+        searchQuery += ' today';
+    else if (freshness === 'week')
+        searchQuery += ' this week';
+    const results = await searxngSearch(searchQuery, 'news', count * 2, language);
+    const items = results.map((r) => ({
+        title: r.title || '',
+        url: r.url || '',
+        source: r.engine || extractStoreName(r.url || ''),
+        date: r.publishedDate || undefined,
+        snippet: r.content || undefined,
+        imageUrl: r.img_src || r.thumbnail || undefined,
+    })).filter((r) => r.title && r.url);
+    return items.slice(0, count);
+}
+// ── Image Search ────────────────────────────────────────────────────────────
+export async function searchImages(opts) {
+    const { query, count = 20, language } = opts;
+    // Primary: SearXNG images category (tested: 566 results)
+    const results = await searxngSearch(query, 'images', count * 2, language);
+    const items = results
+        .filter((r) => r.img_src || r.thumbnail)
+        .map((r) => ({
+        title: r.title || '',
+        url: r.url || '',
+        imageUrl: r.img_src || r.thumbnail || '',
+        width: r.img_format ? parseInt(r.img_format.split('x')[0]) || undefined : undefined,
+        height: r.img_format ? parseInt(r.img_format.split('x')[1]) || undefined : undefined,
+        source: r.engine || undefined,
+    }));
+    return items.slice(0, count);
+}
+// ── Video Search ────────────────────────────────────────────────────────────
+export async function searchVideos(opts) {
+    const { query, count = 10, language } = opts;
+    // Primary: SearXNG videos category (tested: 120 results)
+    const results = await searxngSearch(query, 'videos', count * 2, language);
+    const items = results
+        .filter((r) => r.url)
+        .map((r) => ({
+        title: r.title || '',
+        url: r.url || '',
+        platform: r.url?.includes('youtube') ? 'YouTube' : r.url?.includes('vimeo') ? 'Vimeo' : r.engine || 'Web',
+        duration: r.length || undefined,
+        date: r.publishedDate || undefined,
+        thumbnailUrl: r.thumbnail || r.img_src || undefined,
+        channel: r.author || undefined,
+    }));
+    return items.slice(0, count);
+}

package/dist/core/watch-manager.d.ts

@@ -0,0 +1,175 @@
+/**
+ * WebPeel WatchManager — Database-backed persistent URL monitoring
+ *
+ * Stores watch entries in PostgreSQL, periodically fetches watched URLs,
+ * compares content fingerprints to detect changes, and fires webhook
+ * notifications when a page is updated.
+ *
+ * This module is complementary to the in-process `watch.ts` poller:
+ *  - `watch.ts`        → ephemeral, CLI/in-process, no DB
+ *  - `watch-manager.ts` → persistent, server-side, PostgreSQL-backed
+ */
+import pg from 'pg';
+export interface WatchEntry {
+    id: string;
+    accountId: string;
+    url: string;
+    webhookUrl?: string;
+    /** HMAC-SHA256 signing secret for webhook deliveries. Stored hashed; never returned in API responses. */
+    webhookSecret?: string;
+    checkIntervalMinutes: number;
+    selector?: string;
+    lastFingerprint?: string;
+    lastCheckedAt?: Date;
+    lastChangedAt?: Date;
+    changeCount: number;
+    status: 'active' | 'paused' | 'error';
+    errorMessage?: string;
+    createdAt: Date;
+    updatedAt: Date;
+}
+export interface CreateWatchOptions {
+    webhookUrl?: string;
+    /** Optional HMAC-SHA256 secret used to sign webhook deliveries (header: X-WebPeel-Signature). */
+    webhookSecret?: string;
+    checkIntervalMinutes?: number;
+    selector?: string;
+}
+export interface WatchDiff {
+    /** Whether content changed since the last recorded fingerprint. */
+    changed: boolean;
+    /** SHA-256 fingerprint of the previous content (empty string if none). */
+    previousFingerprint: string;
+    /** SHA-256 fingerprint of the current content. */
+    currentFingerprint: string;
+    /** Human-readable description of the change. */
+    summary: string;
+    /** Paragraph-level text blocks present in new content but not in old. */
+    addedText: string[];
+    /** Paragraph-level text blocks present in old content but not in new. */
+    removedText: string[];
+    /** Current fetched page content (markdown). */
+    content: string;
+    /** Previous content snapshot, if available. */
+    previousContent: string;
+}
+export interface LineDiff {
+    /** Whether any lines changed between old and new content. */
+    changed: boolean;
+    /** Lines present in new content but not in old. */
+    added: string[];
+    /** Lines present in old content but not in new. */
+    removed: string[];
+    /** One-line human-readable description of the change. */
+    summary: string;
+    /** Percentage of lines that changed (0–100). */
+    changePercent: number;
+}
+/**
+ * Compute a simple Set-based line diff between two content strings.
+ *
+ * Splits both versions by newline, filters blank lines, then computes
+ * symmetric differences to find added/removed lines.  Change percentage
+ * is relative to the larger of the two line-count sets.
+ *
+ * @example
+ * ```typescript
+ * const d = computeLineDiff('line1\nline2', 'line1\nline2\nline3');
+ * d.added   // ['line3']
+ * d.removed // []
+ * d.changed // true
+ * ```
+ */
+export declare function computeLineDiff(oldContent: string, newContent: string): LineDiff;
+/**
+ * Compute a stable SHA-256 fingerprint of page content.
+ * Normalises whitespace so cosmetic-only reformatting doesn't trigger alerts.
+ */
+export declare function computeFingerprint(content: string): string;
+/**
+ * Paragraph-level diff — splits both versions of a page into paragraph blocks
+ * (separated by blank lines), then finds paragraphs that appear exclusively in
+ * each version.  Only blocks longer than 10 characters are considered to avoid
+ * noise from short punctuation-only lines.
+ */
+export declare function computeParagraphDiff(oldContent: string, newContent: string): {
+    addedText: string[];
+    removedText: string[];
+};
+/**
+ * Database-backed URL watch manager.
+ *
+ * Stores watch entries in a PostgreSQL `watches` table (see
+ * `migrations/007_watch.sql`) and handles periodic checks, change detection,
+ * and webhook delivery.
+ *
+ * @example
+ * ```typescript
+ * const manager = new WatchManager(pool);
+ * const watch = await manager.create('acct-uuid', 'https://example.com/pricing', {
+ *   webhookUrl: 'https://hooks.example.com/alert',
+ *   checkIntervalMinutes: 30,
+ * });
+ * const diff = await manager.check(watch.id);
+ * console.log(diff.summary);
+ * ```
+ */
+export declare class WatchManager {
+    private readonly db;
+    constructor(db: pg.Pool);
+    /**
+     * Create a new watch entry for the given URL.
+     * The watch is immediately active; the first check will establish the baseline.
+     */
+    create(accountId: string, url: string, options?: CreateWatchOptions): Promise<WatchEntry>;
+    /** List all watches owned by the given account, most recent first. */
+    list(accountId: string): Promise<WatchEntry[]>;
+    /** Get a single watch by ID, or null if not found. */
+    get(watchId: string): Promise<WatchEntry | null>;
+    /** Pause a watch — it will not be included in {@link checkDue} runs. */
+    pause(watchId: string): Promise<void>;
+    /** Resume a previously paused (or errored) watch. */
+    resume(watchId: string): Promise<void>;
+    /** Permanently delete a watch. */
+    delete(watchId: string): Promise<void>;
+    /**
+     * Update mutable properties of a watch.
+     * Only the fields present in `updates` are changed.
+     */
+    update(watchId: string, updates: Partial<Pick<WatchEntry, 'webhookUrl' | 'webhookSecret' | 'checkIntervalMinutes' | 'selector' | 'status'>>): Promise<WatchEntry | null>;
+    /**
+     * Perform an immediate content check for the given watch ID.
+     *
+     * Steps:
+     * 1. Load watch entry and previous content snapshot from disk (if any).
+     * 2. Fetch the current page via `peel()`.
+     * 3. Compute a SHA-256 fingerprint of the normalised content.
+     * 4. If the fingerprint changed, compute a paragraph-level diff and fire the webhook.
+     * 5. Persist `last_fingerprint`, `last_checked_at`, and `change_count` to the DB.
+     * 6. Return a {@link WatchDiff} describing what changed.
+     */
+    check(watchId: string): Promise<WatchDiff>;
+    /**
+     * Scan the database for watches that are due for a check and run them.
+     *
+     * A watch is "due" when:
+     *  - `status = 'active'`
+     *  - `last_checked_at` is NULL (never checked) OR older than `check_interval_minutes`
+     *
+     * Processes up to 50 watches per invocation to avoid long-running cycles.
+     */
+    checkDue(): Promise<void>;
+}
+/**
+ * Start a background interval that calls {@link WatchManager.checkDue} every
+ * minute.  Wire this up in `app.ts` after the server starts.
+ *
+ * @returns The interval handle (pass to `clearInterval` for clean shutdown).
+ *
+ * @example
+ * ```typescript
+ * const handle = startWatchChecker(pool);
+ * process.on('SIGTERM', () => clearInterval(handle));
+ * ```
+ */
+export declare function startWatchChecker(db: pg.Pool): ReturnType<typeof setInterval>;