@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/server/routes/answer.d.ts

@@ -0,0 +1,5 @@
+/**
+ * /v1/answer — search + fetch + LLM-generated answer with citations (BYOK)
+ */
+import { Router } from 'express';
+export declare function createAnswerRouter(): Router;

package/dist/server/routes/answer.js

@@ -0,0 +1,125 @@
+/**
+ * /v1/answer — search + fetch + LLM-generated answer with citations (BYOK)
+ */
+import { Router } from 'express';
+import { answerQuestion, } from '../../core/answer.js';
+const VALID_LLM_PROVIDERS = ['openai', 'anthropic', 'google'];
+const VALID_SEARCH_PROVIDERS = ['duckduckgo', 'brave'];
+export function createAnswerRouter() {
+    const router = Router();
+    router.post('/v1/answer', async (req, res) => {
+        // Deprecation notice — prefer /v1/fetch?question=... which is LLM-free
+        res.setHeader('X-Deprecated', 'true');
+        res.setHeader('X-Deprecated-Use', '/v1/fetch?question=...');
+        // AUTH: require authentication (global middleware sets req.auth)
+        const ansAuthId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!ansAuthId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required. Get one at https://app.webpeel.dev/keys', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/errors#authentication_required' }, requestId: req.requestId });
+            return;
+        }
+        try {
+            const { question, searchProvider, searchApiKey, llmProvider, llmApiKey, llmModel, maxSources, stream, } = req.body;
+            // --- Validation -----------------------------------------------------------
+            if (!question || typeof question !== 'string' || question.trim().length === 0) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing or invalid "question" parameter', hint: 'Include a "question" string in the request body', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+                return;
+            }
+            if (question.length > 2000) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: '"question" too long (max 2000 characters)', hint: 'Keep the question under 2000 characters', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+                return;
+            }
+            if (!llmProvider || !VALID_LLM_PROVIDERS.includes(llmProvider)) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: `"llmProvider" is required and must be one of: ${VALID_LLM_PROVIDERS.join(', ')}`, hint: `Supported providers: ${VALID_LLM_PROVIDERS.join(', ')}`, docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+                return;
+            }
+            if (!llmApiKey || typeof llmApiKey !== 'string' || llmApiKey.trim().length === 0) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing or invalid "llmApiKey" (BYOK required)', hint: 'Provide your own LLM API key in the "llmApiKey" field', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+                return;
+            }
+            const resolvedSearchProvider = searchProvider && VALID_SEARCH_PROVIDERS.includes(searchProvider)
+                ? searchProvider
+                : 'duckduckgo';
+            // Accept search API key from body or header
+            const resolvedSearchApiKey = searchApiKey || req.headers['x-search-api-key'] || undefined;
+            const resolvedMaxSources = typeof maxSources === 'number'
+                ? Math.min(Math.max(maxSources, 1), 10)
+                : 5;
+            const shouldStream = stream === true;
+            // --- Streaming response (SSE) -------------------------------------------
+            if (shouldStream) {
+                res.setHeader('Content-Type', 'text/event-stream');
+                res.setHeader('Cache-Control', 'no-cache');
+                res.setHeader('Connection', 'keep-alive');
+                res.setHeader('X-Accel-Buffering', 'no'); // nginx
+                res.flushHeaders();
+                const answerReq = {
+                    question: question.trim(),
+                    searchProvider: resolvedSearchProvider,
+                    searchApiKey: resolvedSearchApiKey,
+                    llmProvider: llmProvider,
+                    llmApiKey: llmApiKey.trim(),
+                    llmModel,
+                    maxSources: resolvedMaxSources,
+                    stream: true,
+                    onChunk: (text) => {
+                        const payload = JSON.stringify({ type: 'chunk', text });
+                        res.write(`data: ${payload}\n\n`);
+                    },
+                };
+                try {
+                    const result = await answerQuestion(answerReq);
+                    const donePayload = JSON.stringify({
+                        type: 'done',
+                        citations: result.citations,
+                        searchProvider: result.searchProvider,
+                        llmProvider: result.llmProvider,
+                        llmModel: result.llmModel,
+                        tokensUsed: result.tokensUsed,
+                    });
+                    res.write(`data: ${donePayload}\n\n`);
+                }
+                catch (err) {
+                    const errMsg = err instanceof Error ? err.message : 'Unknown error';
+                    const errPayload = JSON.stringify({ type: 'error', message: errMsg });
+                    res.write(`data: ${errPayload}\n\n`);
+                }
+                res.end();
+                return;
+            }
+            // --- Non-streaming response ---------------------------------------------
+            const answerReq = {
+                question: question.trim(),
+                searchProvider: resolvedSearchProvider,
+                searchApiKey: resolvedSearchApiKey,
+                llmProvider: llmProvider,
+                llmApiKey: llmApiKey.trim(),
+                llmModel,
+                maxSources: resolvedMaxSources,
+                stream: false,
+            };
+            const result = await answerQuestion(answerReq);
+            res.json({
+                answer: result.answer,
+                citations: result.citations,
+                searchProvider: result.searchProvider,
+                llmProvider: result.llmProvider,
+                llmModel: result.llmModel,
+                tokensUsed: result.tokensUsed,
+            });
+        }
+        catch (error) {
+            const err = error;
+            console.error('Answer error:', err);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'answer_failed',
+                    message: 'Failed to generate answer. Please try again.',
+                    docs: 'https://webpeel.dev/docs/errors#answer_failed',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/ask.d.ts

@@ -0,0 +1,28 @@
+/**
+ * GET /v1/ask?q=<question>&sources=3
+ * POST /v1/ask { "question": "...", "sources": 3 }
+ *
+ * LLM-free web Q&A: search → fetch top pages → BM25 → best answer
+ *
+ * Returns:
+ * {
+ *   question: string,
+ *   answer: string,           // best passage from top sources
+ *   confidence: number,       // 0-1
+ *   sources: [{url, title, snippet, confidence}],
+ *   method: "bm25",
+ *   elapsed: number           // ms
+ * }
+ *
+ * No LLM key required — 100% deterministic BM25 ranking.
+ * Competitors: Tavily charges $50/mo and requires an API key.
+ * We do this with zero LLM cost, included in every plan.
+ *
+ * Performance targets:
+ * - Source pages fetched in parallel with 5s timeout (no browser escalation)
+ * - Early termination when high-confidence answer found (>=0.85)
+ * - 10s hard timeout on the entire flow
+ * - 5-minute in-memory cache for repeated questions
+ */
+import { Router } from 'express';
+export declare function createAskRouter(): Router;

package/dist/server/routes/ask.js

@@ -0,0 +1,295 @@
+/**
+ * GET /v1/ask?q=<question>&sources=3
+ * POST /v1/ask { "question": "...", "sources": 3 }
+ *
+ * LLM-free web Q&A: search → fetch top pages → BM25 → best answer
+ *
+ * Returns:
+ * {
+ *   question: string,
+ *   answer: string,           // best passage from top sources
+ *   confidence: number,       // 0-1
+ *   sources: [{url, title, snippet, confidence}],
+ *   method: "bm25",
+ *   elapsed: number           // ms
+ * }
+ *
+ * No LLM key required — 100% deterministic BM25 ranking.
+ * Competitors: Tavily charges $50/mo and requires an API key.
+ * We do this with zero LLM cost, included in every plan.
+ *
+ * Performance targets:
+ * - Source pages fetched in parallel with 5s timeout (no browser escalation)
+ * - Early termination when high-confidence answer found (>=0.85)
+ * - 10s hard timeout on the entire flow
+ * - 5-minute in-memory cache for repeated questions
+ */
+import { Router } from 'express';
+import { peel } from '../../index.js';
+import { quickAnswer } from '../../core/quick-answer.js';
+import { getBestSearchProvider } from '../../core/search-provider.js';
+import { rankSearchResults, scoreFetchedSources, } from '../../core/source-scoring.js';
+import { selectEvidence, isUnusableEvidenceContent, } from '../../core/selective-evidence.js';
+const resultCache = new Map();
+const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
+function getCacheKey(question, numSources) {
+    return `${question.trim().toLowerCase()}|${numSources}`;
+}
+function getFromCache(key) {
+    const entry = resultCache.get(key);
+    if (!entry)
+        return null;
+    if (Date.now() > entry.expiresAt) {
+        resultCache.delete(key);
+        return null;
+    }
+    return entry.result;
+}
+function setInCache(key, result) {
+    // Evict stale entries periodically (simple GC — keep max 500 entries)
+    if (resultCache.size >= 500) {
+        const now = Date.now();
+        for (const [k, v] of resultCache) {
+            if (v.expiresAt < now)
+                resultCache.delete(k);
+        }
+    }
+    resultCache.set(key, { result, expiresAt: Date.now() + CACHE_TTL_MS });
+}
+// ---------------------------------------------------------------------------
+// Route factory
+// ---------------------------------------------------------------------------
+export function createAskRouter() {
+    const router = Router();
+    async function handleAsk(question, numSources, req, res) {
+        const startMs = Date.now();
+        const elapsed = () => Date.now() - startMs;
+        if (!question?.trim()) {
+            res.status(400).json({ success: false, error: { type: 'missing_question', message: 'Provide q= or question= parameter', hint: 'GET /v1/ask?q=your+question or POST {"question": "your question"}', docs: 'https://webpeel.dev/docs/errors#missing_question' }, requestId: req.requestId });
+            return;
+        }
+        // Auth check — global middleware sets req.auth
+        const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!authId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required. Get one at https://app.webpeel.dev/keys', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/errors#authentication_required' }, requestId: req.requestId });
+            return;
+        }
+        const clampedSources = Math.min(Math.max(numSources, 1), 5);
+        // Cache check — return cached result immediately for repeated questions
+        const cacheKey = getCacheKey(question, clampedSources);
+        const cached = getFromCache(cacheKey);
+        if (cached) {
+            if (process.env.DEBUG)
+                console.debug('[ask] cache hit in', elapsed(), 'ms');
+            res.json({ ...cached, elapsed: elapsed() });
+            return;
+        }
+        // -----------------------------------------------------------------------
+        // Total flow timeout — 10s hard cap.
+        // -----------------------------------------------------------------------
+        const TOTAL_TIMEOUT_MS = 10000;
+        let timedOut = false;
+        const totalTimer = setTimeout(() => { timedOut = true; }, TOTAL_TIMEOUT_MS);
+        try {
+            // Step 1: Search — fetch more results so we can intelligently rank/dedup
+            const searchStart = Date.now();
+            const { provider, apiKey } = getBestSearchProvider();
+            let rawSearchResults;
+            try {
+                // Fetch up to 2x more results than needed so ranking has candidates to work with
+                rawSearchResults = await provider.searchWeb(question.trim(), {
+                    count: Math.min(clampedSources * 2, 10),
+                    apiKey,
+                });
+            }
+            catch {
+                rawSearchResults = [];
+            }
+            if (process.env.DEBUG)
+                console.debug(`[ask] search ${Date.now() - searchStart}ms, ${rawSearchResults.length} results`);
+            if (!rawSearchResults.length) {
+                clearTimeout(totalTimer);
+                res.json({
+                    question,
+                    answer: null,
+                    confidence: 0,
+                    sources: [],
+                    method: 'bm25',
+                    elapsed: elapsed(),
+                });
+                return;
+            }
+            // -----------------------------------------------------------------------
+            // Step 2: Rank search results by authority + primary source (pre-fetch)
+            // This prioritizes official/high-authority sources and deduplicates domains
+            // before we spend time fetching pages.
+            // -----------------------------------------------------------------------
+            const rankedResults = rankSearchResults(rawSearchResults, question.trim(), {
+                maxPerDomain: 2,
+            });
+            // Take top N candidates after ranking
+            const sourceUrls = rankedResults.slice(0, clampedSources);
+            // -----------------------------------------------------------------------
+            // Step 3: Fetch top sources in parallel
+            // - noEscalate: true → skip browser escalation (simple HTTP only)
+            // - render: false    → don't start headless browser
+            // - timeout: 5000    → 5s per source max
+            // - budget: 3000     → keep content manageable
+            // -----------------------------------------------------------------------
+            const PER_SOURCE_TIMEOUT_MS = 5000;
+            const fetchStart = Date.now();
+            const fetchPromises = sourceUrls.map((r) => Promise.race([
+                peel(r.url, {
+                    render: false,
+                    noEscalate: true,
+                    format: 'markdown',
+                    timeout: PER_SOURCE_TIMEOUT_MS,
+                    budget: 3000,
+                }).then((result) => ({ result, searchResult: r })),
+                new Promise((_, reject) => setTimeout(() => reject(new Error('per-source timeout')), PER_SOURCE_TIMEOUT_MS)),
+            ]));
+            const fetched = await Promise.allSettled(fetchPromises);
+            if (process.env.DEBUG) {
+                const ok = fetched.filter(f => f.status === 'fulfilled').length;
+                console.debug(`[ask] fetch ${Date.now() - fetchStart}ms, ${ok}/${sourceUrls.length} ok`);
+            }
+            // -----------------------------------------------------------------------
+            // Step 4: BM25 score each fetched page with quickAnswer
+            // Early termination: if any source yields >=0.85 confidence, use it now
+            // -----------------------------------------------------------------------
+            const HIGH_CONFIDENCE_THRESHOLD = 0.85;
+            const answers = [];
+            for (const f of fetched) {
+                if (timedOut)
+                    break;
+                if (f.status !== 'fulfilled')
+                    continue;
+                const { result, searchResult } = f.value;
+                const qa = quickAnswer({
+                    question,
+                    content: result.content,
+                    url: result.url,
+                    maxPassages: 2,
+                });
+                const unusableFetchedContent = isUnusableEvidenceContent(result.content);
+                const effectiveAnswer = unusableFetchedContent
+                    ? (searchResult.snippet || '')
+                    : qa.answer;
+                const effectiveConfidence = unusableFetchedContent
+                    ? Math.min(qa.confidence, 0.15)
+                    : qa.confidence;
+                answers.push({
+                    answer: effectiveAnswer,
+                    bm25Score: effectiveConfidence,
+                    searchResult,
+                    fetchedUrl: result.url,
+                    fetchedTitle: result.title || searchResult.title,
+                    metadata: result.metadata,
+                    freshnessData: result.freshness,
+                });
+                // Early termination on high confidence
+                if (effectiveConfidence >= HIGH_CONFIDENCE_THRESHOLD) {
+                    if (process.env.DEBUG)
+                        console.debug(`[ask] early exit confidence=${effectiveConfidence.toFixed(2)} at ${elapsed()}ms`);
+                    break;
+                }
+            }
+            // -----------------------------------------------------------------------
+            // Step 5: Final combined scoring — BM25 + authority + freshness + primary
+            // -----------------------------------------------------------------------
+            const scoredSources = scoreFetchedSources(answers.map(a => ({
+                searchResult: {
+                    url: a.fetchedUrl,
+                    title: a.fetchedTitle,
+                    snippet: a.searchResult.snippet,
+                },
+                bm25Score: a.bm25Score,
+                metadata: a.metadata,
+                freshnessData: a.freshnessData,
+            })), question.trim(), { maxPerDomain: 2 });
+            // Sort by final score — best answer is the highest-scored source
+            scoredSources.sort((a, b) => b.finalScore - a.finalScore);
+            // -----------------------------------------------------------------------
+            // Step 5b: Selective evidence aggregation (AttnRes-inspired)
+            // Re-rank evidence blocks across all fetched sources using query-aware
+            // scoring, credibility weighting, and domain diversity caps.
+            // -----------------------------------------------------------------------
+            const evidenceSources = answers.map(a => ({
+                url: a.fetchedUrl,
+                title: a.fetchedTitle,
+                content: a.answer || '',
+                snippet: a.searchResult.snippet,
+                metadata: a.metadata,
+            }));
+            const evidenceSelection = selectEvidence({
+                query: question.trim(),
+                sources: evidenceSources,
+                maxBlocks: 5,
+                maxChars: 3000,
+            });
+            // Map back to answer text — prefer the top evidence block when available,
+            // fall back to BM25 answer from the highest-scored source
+            const answerMap = new Map(answers.map(a => [a.fetchedUrl, a.answer]));
+            const bestSource = scoredSources[0];
+            const topEvidenceBlock = evidenceSelection.blocks[0];
+            const bestAnswer = topEvidenceBlock?.text
+                || (bestSource ? answerMap.get(bestSource.url) : undefined);
+            clearTimeout(totalTimer);
+            // Build enriched sources array for the response
+            const enrichedSources = scoredSources.map(s => ({
+                url: s.url,
+                title: s.title,
+                snippet: s.snippet,
+                confidence: s.confidence,
+                authority: s.authority,
+                freshness: s.freshness,
+                isPrimarySource: s.isPrimarySource,
+            }));
+            const response = {
+                question,
+                answer: bestAnswer || null,
+                confidence: bestSource?.confidence || 0,
+                sources: enrichedSources,
+                method: 'bm25',
+                elapsed: elapsed(),
+                evidencePolicy: evidenceSelection.policy.type,
+                evidenceSourcesUsed: evidenceSelection.sourcesUsed,
+            };
+            if (timedOut) {
+                response.warning = 'Partial result — 10s timeout reached';
+            }
+            // Cache successful results (only when we have an answer)
+            if (bestAnswer && !timedOut) {
+                setInCache(cacheKey, response);
+            }
+            if (process.env.DEBUG)
+                console.debug(`[ask] done ${elapsed()}ms confidence=${bestSource?.confidence?.toFixed(2) ?? 0}`);
+            res.json(response);
+        }
+        catch (err) {
+            clearTimeout(totalTimer);
+            if (process.env.DEBUG)
+                console.debug('[ask] error:', err);
+            res.json({
+                question,
+                answer: null,
+                confidence: 0,
+                sources: [],
+                method: 'bm25',
+                elapsed: elapsed(),
+                ...(timedOut ? { warning: 'Request timed out after 10s' } : {}),
+            });
+        }
+    }
+    router.get('/v1/ask', async (req, res) => {
+        const question = req.query.q || req.query.question || '';
+        const sources = Math.min(parseInt(req.query.sources || '3', 10) || 3, 5);
+        await handleAsk(question, sources, req, res);
+    });
+    router.post('/v1/ask', async (req, res) => {
+        const question = req.body?.question || req.body?.q || '';
+        const sources = Math.min(parseInt(req.body?.sources ?? 3, 10) || 3, 5);
+        await handleAsk(question, sources, req, res);
+    });
+    return router;
+}

package/dist/server/routes/batch.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Batch scrape API - process multiple URLs concurrently
+ */
+import { Router } from 'express';
+import type { IJobQueue } from '../job-queue.js';
+export declare function createBatchRouter(jobQueue: IJobQueue): Router;