@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/watch-manager.js

@@ -0,0 +1,416 @@
+/**
+ * WebPeel WatchManager — Database-backed persistent URL monitoring
+ *
+ * Stores watch entries in PostgreSQL, periodically fetches watched URLs,
+ * compares content fingerprints to detect changes, and fires webhook
+ * notifications when a page is updated.
+ *
+ * This module is complementary to the in-process `watch.ts` poller:
+ *  - `watch.ts`        → ephemeral, CLI/in-process, no DB
+ *  - `watch-manager.ts` → persistent, server-side, PostgreSQL-backed
+ */
+import { createHash, createHmac } from 'crypto';
+import { fetch as undiciFetch } from 'undici';
+/**
+ * Compute a simple Set-based line diff between two content strings.
+ *
+ * Splits both versions by newline, filters blank lines, then computes
+ * symmetric differences to find added/removed lines.  Change percentage
+ * is relative to the larger of the two line-count sets.
+ *
+ * @example
+ * ```typescript
+ * const d = computeLineDiff('line1\nline2', 'line1\nline2\nline3');
+ * d.added   // ['line3']
+ * d.removed // []
+ * d.changed // true
+ * ```
+ */
+export function computeLineDiff(oldContent, newContent) {
+    const oldLines = new Set(oldContent.split('\n').filter(l => l.trim()));
+    const newLines = new Set(newContent.split('\n').filter(l => l.trim()));
+    const added = [...newLines].filter(l => !oldLines.has(l));
+    const removed = [...oldLines].filter(l => !newLines.has(l));
+    const maxLen = Math.max(oldLines.size, newLines.size);
+    const changePercent = maxLen === 0 ? 0 : Math.min(100, Math.round(((added.length + removed.length) / maxLen) * 100));
+    const changed = added.length > 0 || removed.length > 0;
+    let summary;
+    if (!changed) {
+        summary = 'No changes detected.';
+    }
+    else if (changePercent < 5) {
+        summary = `Minor wording changes (${changePercent}% changed)`;
+    }
+    else {
+        const parts = [];
+        if (added.length > 0)
+            parts.push(`${added.length} line${added.length !== 1 ? 's' : ''} added`);
+        if (removed.length > 0)
+            parts.push(`${removed.length} line${removed.length !== 1 ? 's' : ''} removed`);
+        summary = parts.join(', ');
+    }
+    return { changed, added, removed, summary, changePercent };
+}
+// ─── Internal helpers ──────────────────────────────────────────────────────────
+/**
+ * Compute a stable SHA-256 fingerprint of page content.
+ * Normalises whitespace so cosmetic-only reformatting doesn't trigger alerts.
+ */
+export function computeFingerprint(content) {
+    const normalized = content.trim().replace(/\s+/g, ' ');
+    return createHash('sha256').update(normalized).digest('hex');
+}
+/**
+ * Paragraph-level diff — splits both versions of a page into paragraph blocks
+ * (separated by blank lines), then finds paragraphs that appear exclusively in
+ * each version.  Only blocks longer than 10 characters are considered to avoid
+ * noise from short punctuation-only lines.
+ */
+export function computeParagraphDiff(oldContent, newContent) {
+    const toSet = (text) => new Set(text
+        .split(/\n{2,}/)
+        .map(p => p.trim())
+        .filter(p => p.length > 10));
+    const oldSet = toSet(oldContent);
+    const newSet = toSet(newContent);
+    const addedText = [];
+    const removedText = [];
+    for (const p of newSet) {
+        if (!oldSet.has(p))
+            addedText.push(p.slice(0, 500));
+    }
+    for (const p of oldSet) {
+        if (!newSet.has(p))
+            removedText.push(p.slice(0, 500));
+    }
+    return { addedText, removedText };
+}
+/**
+ * Sign a webhook payload body with HMAC-SHA256.
+ *
+ * @param body    - The raw JSON string that will be sent as the request body.
+ * @param secret  - The signing secret shared between WebPeel and the recipient.
+ * @returns       - Hex digest of the HMAC-SHA256 signature.
+ *
+ * Recipients verify delivery authenticity like this:
+ *   const expected = crypto.createHmac('sha256', secret).update(rawBody).digest('hex');
+ *   if (receivedSignature !== `sha256=${expected}`) reject(); // tampered or wrong secret
+ */
+function signWebhookBody(body, secret) {
+    return createHmac('sha256', secret).update(body).digest('hex');
+}
+/** Post a JSON payload to a webhook URL, silently swallowing delivery errors. */
+async function sendWatchWebhook(webhookUrl, payload, webhookSecret) {
+    try {
+        const bodyStr = JSON.stringify(payload);
+        const headers = {
+            'Content-Type': 'application/json',
+            'User-Agent': 'WebPeel-Watch/1.0 (+https://webpeel.dev)',
+        };
+        // Sign the payload when a secret is available (per-watch secret or global fallback).
+        const secret = webhookSecret || process.env.WEBHOOK_SIGNING_SECRET;
+        if (secret) {
+            headers['X-WebPeel-Signature'] = `sha256=${signWebhookBody(bodyStr, secret)}`;
+            headers['X-WebPeel-Timestamp'] = String(Date.now());
+        }
+        await undiciFetch(webhookUrl, {
+            method: 'POST',
+            headers,
+            body: bodyStr,
+            signal: AbortSignal.timeout(10_000),
+        });
+    }
+    catch (err) {
+        process.stderr.write(`[watch-manager] Webhook delivery failed to ${webhookUrl}: ${err instanceof Error ? err.message : String(err)}\n`);
+    }
+}
+/** Map a raw database row to a typed {@link WatchEntry}. */
+function rowToEntry(row) {
+    return {
+        id: row.id,
+        accountId: row.account_id,
+        url: row.url,
+        webhookUrl: row.webhook_url ?? undefined,
+        webhookSecret: row.webhook_secret ?? undefined,
+        checkIntervalMinutes: row.check_interval_minutes ?? 60,
+        selector: row.selector ?? undefined,
+        lastFingerprint: row.last_fingerprint ?? undefined,
+        lastCheckedAt: row.last_checked_at ? new Date(row.last_checked_at) : undefined,
+        lastChangedAt: row.last_changed_at ? new Date(row.last_changed_at) : undefined,
+        changeCount: row.change_count ?? 0,
+        status: row.status ?? 'active',
+        errorMessage: row.error_message ?? undefined,
+        createdAt: new Date(row.created_at),
+        updatedAt: new Date(row.updated_at),
+    };
+}
+// ─── WatchManager ──────────────────────────────────────────────────────────────
+/**
+ * Database-backed URL watch manager.
+ *
+ * Stores watch entries in a PostgreSQL `watches` table (see
+ * `migrations/007_watch.sql`) and handles periodic checks, change detection,
+ * and webhook delivery.
+ *
+ * @example
+ * ```typescript
+ * const manager = new WatchManager(pool);
+ * const watch = await manager.create('acct-uuid', 'https://example.com/pricing', {
+ *   webhookUrl: 'https://hooks.example.com/alert',
+ *   checkIntervalMinutes: 30,
+ * });
+ * const diff = await manager.check(watch.id);
+ * console.log(diff.summary);
+ * ```
+ */
+export class WatchManager {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    // ─── CRUD ────────────────────────────────────────────────────────────────────
+    /**
+     * Create a new watch entry for the given URL.
+     * The watch is immediately active; the first check will establish the baseline.
+     */
+    async create(accountId, url, options = {}) {
+        const { webhookUrl, webhookSecret, checkIntervalMinutes = 60, selector } = options;
+        const result = await this.db.query(`INSERT INTO watches (account_id, url, webhook_url, webhook_secret, check_interval_minutes, selector)
+       VALUES ($1, $2, $3, $4, $5, $6)
+       RETURNING *`, [accountId, url, webhookUrl ?? null, webhookSecret ?? null, checkIntervalMinutes, selector ?? null]);
+        return rowToEntry(result.rows[0]);
+    }
+    /** List all watches owned by the given account, most recent first. */
+    async list(accountId) {
+        const result = await this.db.query(`SELECT * FROM watches WHERE account_id = $1 ORDER BY created_at DESC`, [accountId]);
+        return result.rows.map(rowToEntry);
+    }
+    /** Get a single watch by ID, or null if not found. */
+    async get(watchId) {
+        const result = await this.db.query(`SELECT * FROM watches WHERE id = $1`, [watchId]);
+        if (result.rows.length === 0)
+            return null;
+        return rowToEntry(result.rows[0]);
+    }
+    /** Pause a watch — it will not be included in {@link checkDue} runs. */
+    async pause(watchId) {
+        await this.db.query(`UPDATE watches SET status = 'paused', updated_at = NOW() WHERE id = $1`, [watchId]);
+    }
+    /** Resume a previously paused (or errored) watch. */
+    async resume(watchId) {
+        await this.db.query(`UPDATE watches
+       SET status = 'active', error_message = NULL, updated_at = NOW()
+       WHERE id = $1`, [watchId]);
+    }
+    /** Permanently delete a watch. */
+    async delete(watchId) {
+        await this.db.query(`DELETE FROM watches WHERE id = $1`, [watchId]);
+    }
+    /**
+     * Update mutable properties of a watch.
+     * Only the fields present in `updates` are changed.
+     */
+    async update(watchId, updates) {
+        const setClauses = ['updated_at = NOW()'];
+        const values = [];
+        let idx = 1;
+        if ('webhookUrl' in updates) {
+            setClauses.push(`webhook_url = $${idx++}`);
+            values.push(updates.webhookUrl ?? null);
+        }
+        if ('webhookSecret' in updates) {
+            setClauses.push(`webhook_secret = $${idx++}`);
+            values.push(updates.webhookSecret ?? null);
+        }
+        if ('checkIntervalMinutes' in updates) {
+            setClauses.push(`check_interval_minutes = $${idx++}`);
+            values.push(updates.checkIntervalMinutes);
+        }
+        if ('selector' in updates) {
+            setClauses.push(`selector = $${idx++}`);
+            values.push(updates.selector ?? null);
+        }
+        if ('status' in updates) {
+            setClauses.push(`status = $${idx++}`);
+            values.push(updates.status);
+        }
+        values.push(watchId);
+        const result = await this.db.query(`UPDATE watches SET ${setClauses.join(', ')} WHERE id = $${idx} RETURNING *`, values);
+        if (result.rows.length === 0)
+            return null;
+        return rowToEntry(result.rows[0]);
+    }
+    // ─── Checking ────────────────────────────────────────────────────────────────
+    /**
+     * Perform an immediate content check for the given watch ID.
+     *
+     * Steps:
+     * 1. Load watch entry and previous content snapshot from disk (if any).
+     * 2. Fetch the current page via `peel()`.
+     * 3. Compute a SHA-256 fingerprint of the normalised content.
+     * 4. If the fingerprint changed, compute a paragraph-level diff and fire the webhook.
+     * 5. Persist `last_fingerprint`, `last_checked_at`, and `change_count` to the DB.
+     * 6. Return a {@link WatchDiff} describing what changed.
+     */
+    async check(watchId) {
+        const watch = await this.get(watchId);
+        if (!watch)
+            throw new Error(`Watch not found: ${watchId}`);
+        const now = new Date();
+        try {
+            // Load previous content snapshot for text-diff computation.
+            const { getSnapshot } = await import('./change-tracking.js');
+            const prevSnapshot = await getSnapshot(watch.url);
+            // Fetch current content.
+            const { peel } = await import('../index.js');
+            const peelResult = await peel(watch.url, {
+                format: 'markdown',
+                selector: watch.selector,
+                timeout: 30_000,
+                // Enable change tracking so the snapshot is persisted for future diffs.
+                changeTracking: true,
+            });
+            const currentContent = peelResult.content;
+            const currentFingerprint = computeFingerprint(currentContent);
+            const previousFingerprint = watch.lastFingerprint ?? '';
+            // Determine whether content actually changed relative to our DB record.
+            const isFirstCheck = !previousFingerprint;
+            const changed = !isFirstCheck && currentFingerprint !== previousFingerprint;
+            // Compute text diff when changed and we have old content to compare against.
+            let addedText = [];
+            let removedText = [];
+            let summary;
+            if (isFirstCheck) {
+                summary = 'Baseline fingerprint established — monitoring active.';
+            }
+            else if (changed) {
+                const oldContent = prevSnapshot?.content ?? '';
+                if (oldContent) {
+                    const diff = computeParagraphDiff(oldContent, currentContent);
+                    addedText = diff.addedText;
+                    removedText = diff.removedText;
+                    summary =
+                        addedText.length > 0 || removedText.length > 0
+                            ? `Page updated: ${addedText.length} block${addedText.length !== 1 ? 's' : ''} added, ` +
+                                `${removedText.length} block${removedText.length !== 1 ? 's' : ''} removed.`
+                            : 'Page content changed (fingerprint mismatch — no paragraph-level diff available).';
+                }
+                else {
+                    summary = 'Page content changed (no previous snapshot available for text diff).';
+                }
+            }
+            else {
+                summary = 'No changes detected.';
+            }
+            // Update DB.
+            if (changed) {
+                await this.db.query(`UPDATE watches
+           SET last_fingerprint    = $1,
+               last_checked_at     = $2,
+               last_changed_at     = $2,
+               change_count        = change_count + 1,
+               status              = 'active',
+               error_message       = NULL,
+               updated_at          = $2
+           WHERE id = $3`, [currentFingerprint, now, watchId]);
+                // Reload to get the latest change_count for the webhook payload.
+                const updated = await this.get(watchId);
+                // Fire webhook.
+                if (watch.webhookUrl && updated) {
+                    const payload = {
+                        event: 'watch.changed',
+                        watchId: watch.id,
+                        url: watch.url,
+                        changedAt: now.toISOString(),
+                        changeCount: updated.changeCount,
+                        diff: { addedText, removedText, summary },
+                    };
+                    await sendWatchWebhook(watch.webhookUrl, payload, watch.webhookSecret);
+                }
+            }
+            else {
+                await this.db.query(`UPDATE watches
+           SET last_fingerprint = COALESCE($1, last_fingerprint),
+               last_checked_at  = $2,
+               status           = 'active',
+               error_message    = NULL,
+               updated_at       = $2
+           WHERE id = $3`, [currentFingerprint || null, now, watchId]);
+            }
+            return {
+                changed,
+                previousFingerprint,
+                currentFingerprint,
+                summary,
+                addedText,
+                removedText,
+                content: currentContent,
+                previousContent: prevSnapshot?.content ?? '',
+            };
+        }
+        catch (error) {
+            const errMsg = (error instanceof Error ? error.message : String(error)).slice(0, 500);
+            // Mark the watch as errored so operators can investigate.
+            await this.db.query(`UPDATE watches
+         SET status        = 'error',
+             error_message = $1,
+             last_checked_at = $2,
+             updated_at    = $2
+         WHERE id = $3`, [errMsg, now, watchId]);
+            throw error;
+        }
+    }
+    /**
+     * Scan the database for watches that are due for a check and run them.
+     *
+     * A watch is "due" when:
+     *  - `status = 'active'`
+     *  - `last_checked_at` is NULL (never checked) OR older than `check_interval_minutes`
+     *
+     * Processes up to 50 watches per invocation to avoid long-running cycles.
+     */
+    async checkDue() {
+        const result = await this.db.query(`SELECT * FROM watches
+       WHERE status = 'active'
+         AND (
+           last_checked_at IS NULL
+           OR last_checked_at < NOW() - (check_interval_minutes * INTERVAL '1 minute')
+         )
+       ORDER BY last_checked_at ASC NULLS FIRST
+       LIMIT 50`);
+        for (const row of result.rows) {
+            const watch = rowToEntry(row);
+            try {
+                await this.check(watch.id);
+            }
+            catch (err) {
+                process.stderr.write(`[watch-manager] Error checking watch ${watch.id} (${watch.url}): ` +
+                    `${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        }
+    }
+}
+// ─── Background checker ────────────────────────────────────────────────────────
+/**
+ * Start a background interval that calls {@link WatchManager.checkDue} every
+ * minute.  Wire this up in `app.ts` after the server starts.
+ *
+ * @returns The interval handle (pass to `clearInterval` for clean shutdown).
+ *
+ * @example
+ * ```typescript
+ * const handle = startWatchChecker(pool);
+ * process.on('SIGTERM', () => clearInterval(handle));
+ * ```
+ */
+export function startWatchChecker(db) {
+    const manager = new WatchManager(db);
+    return setInterval(async () => {
+        try {
+            await manager.checkDue();
+        }
+        catch (err) {
+            process.stderr.write(`[watch-manager] Background checker error: ${err instanceof Error ? err.message : String(err)}\n`);
+        }
+    }, 60_000); // Every 1 minute
+}

package/dist/core/watch.d.ts

@@ -0,0 +1,101 @@
+/**
+ * WebPeel Watch - Lightweight URL monitoring with assertions
+ *
+ * Polls a URL on a configurable interval, evaluates assertions against the
+ * response, detects content changes, and optionally fires a webhook on failure.
+ */
+export interface WatchOptions {
+    url: string;
+    /** Polling interval in milliseconds */
+    intervalMs: number;
+    /** Assertions to evaluate on every check */
+    assertions?: Assertion[];
+    /** POST this URL when an assertion fails or content changes */
+    webhookUrl?: string;
+    /** HMAC-SHA256 secret for signing webhook deliveries (header: X-WebPeel-Signature). */
+    webhookSecret?: string;
+    /** Per-request timeout in milliseconds (default: 10 000) */
+    timeout?: number;
+    /** Stop after this many checks (default: unlimited) */
+    maxChecks?: number;
+    /** Use browser rendering instead of simple HTTP fetch */
+    render?: boolean;
+    /** Output each result as NDJSON to stdout */
+    json?: boolean;
+    /** Suppress output unless a failure or change is detected */
+    silent?: boolean;
+    /** Optional callback invoked after every check */
+    onCheck?: (result: WatchCheckResult) => void;
+}
+export interface Assertion {
+    /** Field to evaluate: "status" (HTTP status), "body", "header.<name>",
+     *  or a dot-notation path into the JSON response body (e.g. "data.health") */
+    field: string;
+    operator: '=' | '!=' | '<' | '>' | 'contains';
+    value: string;
+}
+export interface WatchCheckResult {
+    timestamp: string;
+    url: string;
+    /** HTTP status code (0 on network error, 408 on timeout) */
+    status: number;
+    /** Request round-trip time in milliseconds */
+    elapsed: number;
+    /** Per-assertion evaluation results */
+    assertions: AssertionResult[];
+    /** true when every assertion passed (or there are no assertions) */
+    allPassed: boolean;
+    /** true when the response body changed since the previous check */
+    changed: boolean;
+    /** Present only on network/fetch errors */
+    error?: string;
+}
+export interface AssertionResult {
+    field: string;
+    expected: string;
+    actual: string;
+    passed: boolean;
+}
+/**
+ * Parse a human-readable duration string into milliseconds.
+ *
+ * @example
+ * parseDuration("30s")  // → 30 000
+ * parseDuration("5m")   // → 300 000
+ * parseDuration("1h")   // → 3 600 000
+ * parseDuration("500ms")// → 500
+ */
+export declare function parseDuration(duration: string): number;
+/**
+ * Parse an assertion expression into an {@link Assertion} object.
+ *
+ * Supported formats:
+ *   field=value        — equality
+ *   field!=value       — inequality
+ *   field>value        — greater-than (numeric)
+ *   field<value        — less-than (numeric)
+ *   field contains str — substring match
+ *
+ * @example
+ * parseAssertion("status=200")            // HTTP status
+ * parseAssertion("body.status=healthy")   // JSON field
+ * parseAssertion("version!=0.0.0")
+ */
+export declare function parseAssertion(expr: string): Assertion;
+/**
+ * Monitor a URL on a recurring interval.
+ *
+ * Resolves when {@link WatchOptions.maxChecks} is reached (or never, until the
+ * process receives SIGINT/SIGTERM).
+ *
+ * @example
+ * ```typescript
+ * await watch({
+ *   url: 'https://api.example.com/health',
+ *   intervalMs: 60_000,
+ *   assertions: [{ field: 'status', operator: '=', value: 'healthy' }],
+ *   webhookUrl: 'https://hooks.example.com/alert',
+ * });
+ * ```
+ */
+export declare function watch(options: WatchOptions): Promise<void>;