@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/answer.js

@@ -0,0 +1,395 @@
+/**
+ * /answer core implementation
+ *
+ * Flow:
+ * - search the web
+ * - fetch top sources via WebPeel
+ * - call an LLM (BYOK) to generate a cited answer
+ */
+import { peel } from '../index.js';
+import { getSearchProvider } from './search-provider.js';
+function defaultModelForProvider(provider) {
+    switch (provider) {
+        case 'openai':
+            return 'gpt-4o-mini';
+        case 'anthropic':
+            return 'claude-3-5-sonnet-latest';
+        case 'google':
+            return 'gemini-1.5-flash';
+    }
+}
+function clamp(n, min, max) {
+    return Math.min(Math.max(n, min), max);
+}
+function truncateChars(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    return text.slice(0, maxChars) + '\n\n[Truncated]';
+}
+async function fetchSources(results, maxSources, signal) {
+    const chosen = results.slice(0, maxSources);
+    // Fetch in parallel, but keep it modest to avoid resource spikes.
+    const concurrency = 3;
+    const out = [];
+    for (let i = 0; i < chosen.length; i += concurrency) {
+        const batch = chosen.slice(i, i + concurrency);
+        const settled = await Promise.allSettled(batch.map(async (r) => {
+            const timeoutMs = 30_000;
+            if (signal?.aborted) {
+                throw new Error('Aborted');
+            }
+            const pr = await peel(r.url, {
+                format: 'markdown',
+                maxTokens: 1800,
+                timeout: timeoutMs,
+                // Keep it cheap/fast by default; users can always refetch with render=true.
+                render: false,
+            });
+            return { result: r, content: pr.content };
+        }));
+        for (let j = 0; j < settled.length; j++) {
+            const s = settled[j];
+            if (s.status === 'fulfilled') {
+                out.push({
+                    result: s.value.result,
+                    content: s.value.content,
+                });
+            }
+            else {
+                out.push({
+                    result: batch[j],
+                    content: `[Failed to fetch: ${s.reason instanceof Error ? s.reason.message : 'Unknown error'}]`,
+                });
+            }
+        }
+    }
+    return out;
+}
+function buildCitedContext(sources) {
+    const parts = [];
+    sources.forEach((s, i) => {
+        const n = i + 1;
+        const title = s.result.title || '(untitled)';
+        const url = s.result.url;
+        const snippet = s.result.snippet || '';
+        // Sanitize untrusted web content before passing to LLM
+        const rawContent = truncateChars(s.content || '', 20_000);
+        const sanitized = sanitizeForLLM(rawContent);
+        if (sanitized.injectionDetected) {
+            console.log(`[webpeel] [prompt-guard] Injection patterns detected in source [${n}] (${url}): ${sanitized.detectedPatterns.join(', ')}`);
+        }
+        parts.push(`SOURCE [${n}]\nTitle: ${title}\nURL: ${url}\nSnippet: ${truncateChars(snippet, 800)}\n\nContent (markdown):\n${sanitized.content}`);
+    });
+    return parts.join('\n\n---\n\n');
+}
+async function readTextStream(body, onText, signal) {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    try {
+        while (true) {
+            if (signal?.aborted)
+                throw new Error('Aborted');
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            buffer += decoder.decode(value, { stream: true });
+            // Process complete SSE events separated by blank lines
+            while (true) {
+                const idx = buffer.indexOf('\n\n');
+                if (idx === -1)
+                    break;
+                const event = buffer.slice(0, idx);
+                buffer = buffer.slice(idx + 2);
+                const lines = event.split('\n');
+                for (const line of lines) {
+                    const trimmed = line.trim();
+                    if (!trimmed.startsWith('data:'))
+                        continue;
+                    const data = trimmed.slice(5).trim();
+                    onText(data);
+                }
+            }
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+async function callOpenAI(params) {
+    const { apiKey, model, messages, stream, onChunk, signal } = params;
+    const resp = await fetch('https://api.openai.com/v1/chat/completions', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({
+            model,
+            messages,
+            temperature: 0.2,
+            stream,
+        }),
+        signal,
+    });
+    if (!resp.ok) {
+        const text = await resp.text().catch(() => '');
+        throw new Error(`OpenAI API error: HTTP ${resp.status}${text ? ` - ${text}` : ''}`);
+    }
+    if (!stream) {
+        const json = await resp.json();
+        const text = String(json?.choices?.[0]?.message?.content || '').trim();
+        const usage = {
+            input: Number(json?.usage?.prompt_tokens || 0),
+            output: Number(json?.usage?.completion_tokens || 0),
+        };
+        return { text, usage };
+    }
+    if (!resp.body)
+        throw new Error('OpenAI stream error: missing body');
+    let out = '';
+    let usage = { input: 0, output: 0 };
+    await readTextStream(resp.body, (data) => {
+        if (data === '[DONE]')
+            return;
+        let obj;
+        try {
+            obj = JSON.parse(data);
+        }
+        catch {
+            return;
+        }
+        const delta = obj?.choices?.[0]?.delta?.content;
+        if (typeof delta === 'string' && delta.length > 0) {
+            out += delta;
+            onChunk?.(delta);
+        }
+        if (obj?.usage) {
+            usage = {
+                input: Number(obj.usage.prompt_tokens || usage.input),
+                output: Number(obj.usage.completion_tokens || usage.output),
+            };
+        }
+    }, signal);
+    return { text: out.trim(), usage };
+}
+async function callAnthropic(params) {
+    const { apiKey, model, system, user, stream, onChunk, signal } = params;
+    const resp = await fetch('https://api.anthropic.com/v1/messages', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'x-api-key': apiKey,
+            'anthropic-version': '2023-06-01',
+            ...(stream ? { 'Accept': 'text/event-stream' } : {}),
+        },
+        body: JSON.stringify({
+            model,
+            system,
+            messages: [{ role: 'user', content: user }],
+            max_tokens: 4096,
+            temperature: 0.2,
+            stream,
+        }),
+        signal,
+    });
+    if (!resp.ok) {
+        const text = await resp.text().catch(() => '');
+        throw new Error(`Anthropic API error: HTTP ${resp.status}${text ? ` - ${text}` : ''}`);
+    }
+    if (!stream) {
+        const json = await resp.json();
+        const blocks = Array.isArray(json?.content) ? json.content : [];
+        const text = blocks.map((b) => (typeof b?.text === 'string' ? b.text : '')).join('').trim();
+        const usage = {
+            input: Number(json?.usage?.input_tokens || 0),
+            output: Number(json?.usage?.output_tokens || 0),
+        };
+        return { text, usage };
+    }
+    if (!resp.body)
+        throw new Error('Anthropic stream error: missing body');
+    let out = '';
+    let usage = { input: 0, output: 0 };
+    await readTextStream(resp.body, (data) => {
+        let obj;
+        try {
+            obj = JSON.parse(data);
+        }
+        catch {
+            return;
+        }
+        // Streaming event types: content_block_delta, message_delta, message_stop, etc.
+        if (obj?.type === 'content_block_delta') {
+            const delta = obj?.delta?.text;
+            if (typeof delta === 'string' && delta.length > 0) {
+                out += delta;
+                onChunk?.(delta);
+            }
+        }
+        if (obj?.type === 'message_delta' && obj?.usage) {
+            usage = {
+                input: Number(obj.usage.input_tokens || usage.input),
+                output: Number(obj.usage.output_tokens || usage.output),
+            };
+        }
+        if (obj?.type === 'message_stop' && obj?.message?.usage) {
+            usage = {
+                input: Number(obj.message.usage.input_tokens || usage.input),
+                output: Number(obj.message.usage.output_tokens || usage.output),
+            };
+        }
+    }, signal);
+    return { text: out.trim(), usage };
+}
+async function callGoogle(params) {
+    const { apiKey, model, system, user, signal } = params;
+    const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}:generateContent?key=${encodeURIComponent(apiKey)}`;
+    const resp = await fetch(url, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+            contents: [
+                {
+                    role: 'user',
+                    parts: [{ text: `${system}\n\n${user}` }],
+                },
+            ],
+            generationConfig: {
+                temperature: 0.2,
+            },
+        }),
+        signal,
+    });
+    if (!resp.ok) {
+        const text = await resp.text().catch(() => '');
+        throw new Error(`Google API error: HTTP ${resp.status}${text ? ` - ${text}` : ''}`);
+    }
+    const json = await resp.json();
+    const parts = json?.candidates?.[0]?.content?.parts;
+    const text = Array.isArray(parts)
+        ? parts.map((p) => (typeof p?.text === 'string' ? p.text : '')).join('')
+        : '';
+    const usage = {
+        input: Number(json?.usageMetadata?.promptTokenCount || 0),
+        output: Number(json?.usageMetadata?.candidatesTokenCount || 0),
+    };
+    return { text: String(text || '').trim(), usage };
+}
+import { sanitizeForLLM, hardenSystemPrompt, validateOutput } from './prompt-guard.js';
+const BASE_SYSTEM_PROMPT = [
+    'You are a helpful assistant that answers questions using ONLY the provided sources.',
+    'You must cite sources using bracketed numbers like [1], [2], etc. corresponding to the sources list.',
+    'If the sources do not contain the answer, say you do not know.',
+    'Do not fabricate URLs or citations.',
+].join('\n');
+function systemPrompt() {
+    return hardenSystemPrompt(BASE_SYSTEM_PROMPT);
+}
+export async function answerQuestion(req) {
+    const question = (req.question || '').trim();
+    if (!question)
+        throw new Error('Missing or invalid "question"');
+    if (question.length > 2000)
+        throw new Error('Question too long (max 2000 characters)');
+    const searchProvider = req.searchProvider || 'duckduckgo';
+    const maxSources = clamp(req.maxSources ?? 5, 1, 10);
+    const llmProvider = req.llmProvider;
+    const llmApiKey = (req.llmApiKey || '').trim();
+    if (!llmApiKey)
+        throw new Error('Missing or invalid "llmApiKey" (BYOK required)');
+    const llmModel = (req.llmModel || '').trim() || defaultModelForProvider(llmProvider);
+    // 1) Search
+    const provider = getSearchProvider(searchProvider);
+    const searchResults = await provider.searchWeb(question, {
+        count: maxSources,
+        apiKey: req.searchApiKey,
+        signal: req.signal,
+    });
+    const citations = searchResults.slice(0, maxSources).map((r) => ({
+        title: r.title,
+        url: r.url,
+        snippet: r.snippet,
+    }));
+    // 2) Fetch sources
+    const fetched = await fetchSources(searchResults, maxSources, req.signal);
+    const context = buildCitedContext(fetched);
+    // 3) LLM
+    const sys = systemPrompt();
+    const user = `Question:\n${question}\n\nSources:\n\n${context}\n\nWrite the best possible answer. Remember to cite sources like [1], [2].`;
+    const stream = req.stream === true;
+    let answer = '';
+    let tokensUsed = { input: 0, output: 0 };
+    if (llmProvider === 'openai') {
+        const messages = [
+            { role: 'system', content: sys },
+            { role: 'user', content: user },
+        ];
+        const r = await callOpenAI({
+            apiKey: llmApiKey,
+            model: llmModel,
+            messages,
+            stream,
+            onChunk: req.onChunk,
+            signal: req.signal,
+        });
+        answer = r.text;
+        tokensUsed = r.usage;
+    }
+    else if (llmProvider === 'anthropic') {
+        const r = await callAnthropic({
+            apiKey: llmApiKey,
+            model: llmModel,
+            system: sys,
+            user,
+            stream,
+            onChunk: req.onChunk,
+            signal: req.signal,
+        });
+        answer = r.text;
+        tokensUsed = r.usage;
+    }
+    else if (llmProvider === 'google') {
+        // Google streaming is not implemented here; when stream=true, the caller can
+        // still stream the final text in chunks.
+        const r = await callGoogle({
+            apiKey: llmApiKey,
+            model: llmModel,
+            system: sys,
+            user,
+            signal: req.signal,
+        });
+        answer = r.text;
+        tokensUsed = r.usage;
+        if (stream && req.onChunk) {
+            // Emit reasonable chunks so SSE clients can start rendering.
+            const chunkSize = 120;
+            for (let i = 0; i < answer.length; i += chunkSize) {
+                req.onChunk(answer.slice(i, i + chunkSize));
+            }
+        }
+    }
+    else {
+        throw new Error(`Unsupported llmProvider: ${llmProvider}`);
+    }
+    // Validate output for signs of successful injection
+    const outputCheck = validateOutput(answer, [
+        'cite sources using bracketed',
+        'do not fabricate urls',
+        'security rules',
+    ]);
+    if (!outputCheck.clean) {
+        console.log(`[webpeel] [prompt-guard] Output validation issues: ${outputCheck.issues.join(', ')}`);
+        // Don't block the response — log for monitoring. In future, could redact or retry.
+    }
+    return {
+        answer,
+        citations,
+        searchProvider,
+        llmProvider,
+        llmModel,
+        tokensUsed,
+    };
+}

package/dist/core/application-tracker.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Application Tracker — persists job application records to disk.
+ *
+ * Stored at: ~/.webpeel/applications.json
+ *
+ * Features:
+ * - Add / update application records
+ * - Duplicate-URL detection
+ * - Filter by platform, status, or since-date
+ * - Quick stats summary (total, by-platform, by-status, this-week, today)
+ */
+export type ApplicationStatus = 'applied' | 'saved' | 'failed' | 'skipped' | 'interview' | 'rejected' | 'offer';
+export interface ApplicationRecord {
+    /** Unique ID (UUID v4) */
+    id: string;
+    /** Job URL (canonical, used for duplicate detection) */
+    url: string;
+    /** Job title */
+    title: string;
+    /** Company name */
+    company: string;
+    /** Location string */
+    location?: string;
+    /** Salary / compensation info */
+    salary?: string;
+    /** Platform identifier: linkedin, indeed, upwork, etc. */
+    platform: string;
+    /** Current application status */
+    status: ApplicationStatus;
+    /** ISO-8601 timestamp of the initial application */
+    appliedAt: string;
+    /** Cover letter or proposal text used */
+    coverLetter?: string;
+    /** Answers given to screening questions */
+    screeningAnswers?: Record<string, string>;
+    /** Error message if the application failed */
+    error?: string;
+    /** Free-form notes */
+    notes?: string;
+}
+export interface ApplicationFilter {
+    platform?: string;
+    status?: string;
+    /** ISO date string — only return records on or after this date */
+    since?: string;
+}
+export interface ApplicationStats {
+    total: number;
+    byPlatform: Record<string, number>;
+    byStatus: Record<string, number>;
+    thisWeek: number;
+    today: number;
+}
+export declare class ApplicationTracker {
+    private records;
+    constructor();
+    /**
+     * Add a new application record.
+     * Assigns a random UUID and returns the completed record.
+     */
+    add(record: Omit<ApplicationRecord, 'id'>): ApplicationRecord;
+    /**
+     * Update the status (and optionally notes) of an existing record.
+     * Throws if the ID is not found.
+     */
+    updateStatus(id: string, status: ApplicationStatus, notes?: string): void;
+    /**
+     * Return true if there is already an 'applied' record for this URL.
+     * Normalises the URL by stripping trailing slashes and query strings that
+     * are tracking parameters (utm_*, ref, etc.) before comparison.
+     */
+    hasApplied(url: string): boolean;
+    /**
+     * Return records, optionally filtered by platform, status, and/or since-date.
+     * Results are sorted newest-first.
+     */
+    list(filter?: ApplicationFilter): ApplicationRecord[];
+    /**
+     * Return an aggregate stats summary.
+     */
+    stats(): ApplicationStats;
+    private loadRecords;
+    private save;
+}

package/dist/core/application-tracker.js

@@ -0,0 +1,184 @@
+/**
+ * Application Tracker — persists job application records to disk.
+ *
+ * Stored at: ~/.webpeel/applications.json
+ *
+ * Features:
+ * - Add / update application records
+ * - Duplicate-URL detection
+ * - Filter by platform, status, or since-date
+ * - Quick stats summary (total, by-platform, by-status, this-week, today)
+ */
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+import { randomUUID } from 'crypto';
+// ── Storage ────────────────────────────────────────────────────────────
+const WEBPEEL_DIR = join(homedir(), '.webpeel');
+const APPLICATIONS_PATH = join(WEBPEEL_DIR, 'applications.json');
+// ── ApplicationTracker class ───────────────────────────────────────────
+export class ApplicationTracker {
+    records;
+    constructor() {
+        this.records = this.loadRecords();
+    }
+    // ── Public API ───────────────────────────────────────────────────────
+    /**
+     * Add a new application record.
+     * Assigns a random UUID and returns the completed record.
+     */
+    add(record) {
+        const full = {
+            id: randomUUID(),
+            ...record,
+        };
+        this.records.push(full);
+        this.save();
+        return full;
+    }
+    /**
+     * Update the status (and optionally notes) of an existing record.
+     * Throws if the ID is not found.
+     */
+    updateStatus(id, status, notes) {
+        const record = this.records.find((r) => r.id === id);
+        if (!record) {
+            throw new Error(`Application record not found: ${id}`);
+        }
+        record.status = status;
+        if (notes !== undefined) {
+            record.notes = notes;
+        }
+        this.save();
+    }
+    /**
+     * Return true if there is already an 'applied' record for this URL.
+     * Normalises the URL by stripping trailing slashes and query strings that
+     * are tracking parameters (utm_*, ref, etc.) before comparison.
+     */
+    hasApplied(url) {
+        const norm = normaliseUrl(url);
+        return this.records.some((r) => normaliseUrl(r.url) === norm && r.status === 'applied');
+    }
+    /**
+     * Return records, optionally filtered by platform, status, and/or since-date.
+     * Results are sorted newest-first.
+     */
+    list(filter) {
+        let results = [...this.records];
+        if (filter?.platform) {
+            const p = filter.platform.toLowerCase();
+            results = results.filter((r) => r.platform.toLowerCase() === p);
+        }
+        if (filter?.status) {
+            const s = filter.status.toLowerCase();
+            results = results.filter((r) => r.status.toLowerCase() === s);
+        }
+        if (filter?.since) {
+            const sinceTs = Date.parse(filter.since);
+            if (!isNaN(sinceTs)) {
+                results = results.filter((r) => Date.parse(r.appliedAt) >= sinceTs);
+            }
+        }
+        // Newest first
+        results.sort((a, b) => Date.parse(b.appliedAt) - Date.parse(a.appliedAt));
+        return results;
+    }
+    /**
+     * Return an aggregate stats summary.
+     */
+    stats() {
+        const now = new Date();
+        const todayStr = toDateString(now);
+        const weekAgo = new Date(now);
+        weekAgo.setDate(weekAgo.getDate() - 7);
+        const weekAgoTs = weekAgo.getTime();
+        const byPlatform = {};
+        const byStatus = {};
+        let thisWeek = 0;
+        let today = 0;
+        for (const r of this.records) {
+            // Platform counts
+            byPlatform[r.platform] = (byPlatform[r.platform] ?? 0) + 1;
+            // Status counts
+            byStatus[r.status] = (byStatus[r.status] ?? 0) + 1;
+            // Time-based counts
+            const appliedTs = Date.parse(r.appliedAt);
+            if (!isNaN(appliedTs)) {
+                if (appliedTs >= weekAgoTs)
+                    thisWeek++;
+                if (toDateString(new Date(appliedTs)) === todayStr)
+                    today++;
+            }
+        }
+        return {
+            total: this.records.length,
+            byPlatform,
+            byStatus,
+            thisWeek,
+            today,
+        };
+    }
+    // ── Private helpers ──────────────────────────────────────────────────
+    loadRecords() {
+        try {
+            if (existsSync(APPLICATIONS_PATH)) {
+                const raw = readFileSync(APPLICATIONS_PATH, 'utf-8');
+                const parsed = JSON.parse(raw);
+                if (Array.isArray(parsed))
+                    return parsed;
+            }
+        }
+        catch (e) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'records load failed:', e instanceof Error ? e.message : e);
+        }
+        return [];
+    }
+    save() {
+        try {
+            mkdirSync(WEBPEEL_DIR, { recursive: true });
+            writeFileSync(APPLICATIONS_PATH, JSON.stringify(this.records, null, 2), 'utf-8');
+        }
+        catch (err) {
+            console.error('[application-tracker] Failed to save records:', err);
+        }
+    }
+}
+// ── Utility ────────────────────────────────────────────────────────────
+/** Format a Date as YYYY-MM-DD in local time. */
+function toDateString(d) {
+    const y = d.getFullYear();
+    const m = String(d.getMonth() + 1).padStart(2, '0');
+    const day = String(d.getDate()).padStart(2, '0');
+    return `${y}-${m}-${day}`;
+}
+/** Tracking parameters to strip before URL comparison. */
+const TRACKING_PARAMS = new Set([
+    'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+    'ref', 'referer', 'referrer', 'source', 'trk', 'trackingId',
+]);
+/**
+ * Strip trailing slashes, fragments, and common tracking query parameters
+ * from a URL for stable duplicate detection.
+ */
+function normaliseUrl(rawUrl) {
+    try {
+        const u = new URL(rawUrl);
+        u.hash = '';
+        for (const param of TRACKING_PARAMS) {
+            u.searchParams.delete(param);
+        }
+        // Sort remaining params for canonical ordering
+        u.searchParams.sort();
+        let result = u.toString();
+        // Strip trailing slash
+        if (result.endsWith('/'))
+            result = result.slice(0, -1);
+        return result.toLowerCase();
+    }
+    catch {
+        // Not a valid URL — return as-is lowercased
+        return rawUrl.toLowerCase().replace(/\/$/, '');
+    }
+}