@finchagentic/mcp 4.7.1 → 4.7.3

package/dist/tools/deep-research-constants.js

@@ -0,0 +1,33 @@
+"use strict";
+// Shared constants for the deep_research pipeline (deep-research-*.ts).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.STOPWORDS = exports.FRESH_TRIGGER_RE = exports.NEWS_DOMAIN_BOOST_RE = exports.DOMAIN_TIER_BONUS = exports.EXCERPT_TOP_CHUNKS = exports.EXCERPT_CHARS_PER_CHUNK = exports.MAX_PER_DOMAIN = exports.FC_BASE = void 0;
+exports.FC_BASE = "https://api.firecrawl.dev/v1";
+exports.MAX_PER_DOMAIN = 2; // source diversity - cap hits per domain
+exports.EXCERPT_CHARS_PER_CHUNK = 600;
+exports.EXCERPT_TOP_CHUNKS = 4; // pick top N relevant chunks per source
+// Quality scoring - higher = more trustworthy primary source
+exports.DOMAIN_TIER_BONUS = [
+    [/\.gov(\b|\/|$)/i, 4],
+    [/\.edu(\b|\/|$)/i, 3],
+    [/(?:nature|science|nih|arxiv|acm|ieee|sciencedirect)\.(?:org|com)/i, 3],
+    // Crypto primary-source boost - these are the authoritative data sources
+    // for protocol TVL, yields, prices, and on-chain analytics. Rank above
+    // generic news for crypto queries.
+    [/(?:defillama|tokenterminal|coingecko|coinmarketcap|dune|messari|artemis)\.(?:com|fi)/i, 3],
+    [/(?:etherscan|basescan|arbiscan|solscan|polygonscan|optimistic\.etherscan)\.(?:io|com)/i, 2],
+    [/(?:reuters|apnews|bbc|economist|ft|wsj|bloomberg)\.com/i, 2],
+    [/(?:coindesk|theblock|cointelegraph|decrypt|theinformation)\.(?:co|com|io)/i, 1],
+    [/(?:wikipedia|github|stackoverflow)\.(?:org|com)/i, 1],
+    [/(?:medium|substack|reddit|twitter|x)\.com/i, -1],
+];
+// News-domain bonus - applied additionally in fresh mode
+exports.NEWS_DOMAIN_BOOST_RE = /(?:reuters|apnews|bbc|economist|ft|wsj|bloomberg|cnbc|theverge|techcrunch|axios|coindesk|theinformation|nytimes|guardian|aljazeera)\.com/i;
+// Keywords that signal a time-sensitive query - trigger fresh mode auto.
+exports.FRESH_TRIGGER_RE = /\b(today|tonight|tomorrow|yesterday|this week|last week|past week|latest|breaking|just now|recent|currently|now|live|happening|this month|last month|past month|past \d+ days?|last \d+ days?|q[1-4]|h[12]|202[6-9])\b/i;
+exports.STOPWORDS = new Set([
+    "the", "and", "for", "with", "that", "this", "what", "when", "where", "how", "why",
+    "have", "has", "had", "are", "was", "were", "will", "would", "could", "should", "does",
+    "did", "being", "been", "from", "into", "over", "under", "about", "into", "than", "then",
+    "your", "yours", "their", "they", "them", "there", "here", "just", "also", "more", "most",
+]);

package/dist/tools/deep-research-firecrawl.js

@@ -0,0 +1,88 @@
+"use strict";
+// Firecrawl search/scrape - BYOK direct call with a Finch backend-proxy
+// fallback for signed-in users without their own key.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeSearchHits = normalizeSearchHits;
+exports.fcSearch = fcSearch;
+exports.fcScrape = fcScrape;
+const convex_js_1 = require("../convex.js");
+const deep_research_constants_js_1 = require("./deep-research-constants.js");
+/**
+ * Firecrawl returns either a flat array or a keyed object of result groups
+ * (`{ web: [...], news: [...] }`). The proxy path assumed the flat shape and
+ * handed an object to `.forEach`, which threw — invisible until now because
+ * without an API key the run always died at the planner first.
+ */
+function normalizeSearchHits(raw) {
+    if (Array.isArray(raw))
+        return raw;
+    if (raw && typeof raw === "object") {
+        const out = [];
+        for (const group of Object.values(raw)) {
+            if (Array.isArray(group))
+                out.push(...group);
+        }
+        return out;
+    }
+    return [];
+}
+async function fcSearch(query, limit) {
+    // BYOK path - direct call to Firecrawl with user's key. Fastest, no proxy hop.
+    const key = process.env.FIRECRAWL_API_KEY;
+    if (key) {
+        try {
+            const res = await fetch(`${deep_research_constants_js_1.FC_BASE}/search`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` },
+                body: JSON.stringify({ query, limit }),
+                signal: AbortSignal.timeout(15000),
+            });
+            if (res.ok) {
+                const data = (await res.json());
+                return normalizeSearchHits(data.data);
+            }
+        }
+        catch { /* fall through */ }
+    }
+    // Backend-proxy path - session-authed; Finch covers Firecrawl cost.
+    try {
+        const data = await (0, convex_js_1.callConvex)("/research/firecrawl-search", "POST", { query, limit }, "web_search", 20000);
+        return normalizeSearchHits(data?.results);
+    }
+    catch {
+        return [];
+    }
+}
+async function fcScrape(url) {
+    const key = process.env.FIRECRAWL_API_KEY;
+    if (key) {
+        try {
+            const res = await fetch(`${deep_research_constants_js_1.FC_BASE}/scrape`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` },
+                body: JSON.stringify({ url, formats: ["markdown"], onlyMainContent: true }),
+                signal: AbortSignal.timeout(20000),
+            });
+            if (res.ok) {
+                const data = (await res.json());
+                const md = data.data?.markdown;
+                if (md) {
+                    const meta = data.data?.metadata;
+                    const publishedAt = meta?.publishedAt ?? meta?.ogPublishedTime ?? meta?.["article:published_time"];
+                    return { markdown: md, publishedAt };
+                }
+            }
+        }
+        catch { /* fall through */ }
+    }
+    // Backend-proxy path. Returns markdown only - no metadata extraction yet,
+    // which means continueFrom can't auto-date proxied scrapes. Acceptable
+    // tradeoff for now; markdown is the primary signal.
+    try {
+        const data = await (0, convex_js_1.callConvex)("/research/firecrawl-scrape", "POST", { url }, "web_scrape", 25000);
+        if (data?.markdown)
+            return { markdown: data.markdown };
+    }
+    catch { /* swallow */ }
+    return null;
+}

package/dist/tools/deep-research-planning.js

@@ -0,0 +1,249 @@
+"use strict";
+// Query planning: sub-query derivation, LLM planners (flat + multi-angle),
+// the deep-mode adversarial critic, and gap-finding reflection.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.deriveSubQueries = deriveSubQueries;
+exports.planQueries = planQueries;
+exports.planAngles = planAngles;
+exports.runCritic = runCritic;
+exports.reflectAndExtend = reflectAndExtend;
+const llm_js_1 = require("../llm.js");
+const deep_research_text_js_1 = require("./deep-research-text.js");
+/**
+ * Sub-queries without an LLM.
+ *
+ * The caller should normally pass `queries` — it is a model and plans better
+ * than any heuristic. This exists so the evidence-pack path still works with
+ * zero configuration, rather than failing when no API key is set.
+ */
+function deriveSubQueries(query, n, focus, freshMode) {
+    const base = query.trim().replace(/\?+$/, "");
+    const recency = freshMode ? ` ${new Date().getFullYear()} latest` : "";
+    const lenses = focus
+        ? [focus, `${focus} risks`, `${focus} data`]
+        : ["overview", "latest developments", "risks and criticism", "data and numbers", "expert analysis"];
+    const out = [`${base}${recency}`];
+    for (const lens of lenses) {
+        if (out.length >= Math.max(2, n))
+            break;
+        out.push(`${base} ${lens}${recency}`);
+    }
+    return out;
+}
+async function planQueries(query, n, focus, priorContext, freshMode) {
+    const focusNote = focus ? ` Focus angle: ${focus}.` : "";
+    const sys = "You are a research planner. Output strict JSON only - no preamble, no markdown.";
+    const freshNote = freshMode
+        ? `
+
+⏱ FRESH MODE - last ${freshMode.days} days:
+- Add a recency hint to most sub-questions: "in ${(0, deep_research_text_js_1.currentYearMonth)()}", "last ${freshMode.days} days", "this week", "as of ${(0, deep_research_text_js_1.todayISO)()}", etc.
+- Prefer queries that surface news / press releases / X posts over evergreen background.
+- Skip generic background - the user wants what's CURRENT, not historical.
+- At least 70% of sub-questions must include a date or recency token.`
+        : "";
+    const continuationNote = priorContext
+        ? `
+
+⚠️ CONTINUATION MODE - there is a PRIOR research report on this topic:
+
+"""
+${priorContext.slice(0, 2500)}
+"""
+
+Your sub-questions must focus on:
+1. UPDATES - what has changed since the prior report (new releases, news, data revisions)
+2. GAPS - angles the prior report explicitly listed as open questions or follow-ups
+3. NEW developments - entities/events the prior report doesn't mention
+4. VERIFICATION - claims the prior report flagged as low-confidence or single-source
+
+DO NOT re-tread material already well-covered in the prior report. The user already has those answers.`
+        : "";
+    const user = `Decompose this research question into ${n} sub-questions that together cover the topic from different angles.${focusNote}${continuationNote}${freshNote}
+
+Rules:
+- Each sub-question must be a standalone web search query, under 90 chars.
+- Cover different facets: definition, current state, key actors, comparisons, counterarguments, recent news, forward outlook.
+- No duplicates, no near-paraphrases.
+
+ENTITY-HUNTING - at least HALF of your sub-questions must target queries likely to surface:
+- Specific company / product / framework names (e.g., "LangGraph adoption stats", "Manus orchestration funding")
+- Dollar amounts (acquisitions, funding rounds, revenue, ARR, market size)
+- Benchmark numbers (% adoption, latency ms, accuracy scores, MMLU/HumanEval/SWE-bench results)
+- Specific dates and timeline events (when X launched, when Y reached scale)
+- Named studies / surveys / reports (e.g., "Anthropic Economic Index 2026", "a16z AI infrastructure report")
+
+Bad: "what is X" → too generic, returns Wikipedia
+Good: "X adoption rate enterprise 2026 survey" → returns concrete stats
+
+Question: "${query}"
+
+Return: {"queries": ["...", "..."]} - exactly ${n} items.`;
+    let raw = "";
+    try {
+        raw = await (0, llm_js_1.callLLM)(sys, user, 500, [], 30000);
+    }
+    catch {
+        return [query];
+    }
+    const parsed = (0, deep_research_text_js_1.safeParseJson)(raw, {});
+    if (!parsed.queries || !Array.isArray(parsed.queries))
+        return [query];
+    const queries = parsed.queries
+        .filter((s) => typeof s === "string")
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0 && s.length <= 200)
+        .slice(0, n);
+    return queries.length > 0 ? queries : [query];
+}
+async function planAngles(query, n, focus, priorContext, freshMode) {
+    const focusNote = focus ? ` Focus angle: ${focus}.` : "";
+    const freshNote = freshMode
+        ? `\n\n⏱ FRESH MODE - last ${freshMode.days} days: bias each angle's queries toward news/recent press/dated reports.`
+        : "";
+    const continuationNote = priorContext
+        ? `\n\n⚠️ CONTINUATION - prior report exists. Angles must focus on UPDATES, GAPS, NEW developments since:\n"""\n${priorContext.slice(0, 1500)}\n"""`
+        : "";
+    const sys = "You are a research planner that decomposes topics into specialist angles. Output strict JSON only.";
+    const user = `Break this research topic into ${n} DIFFERENT specialist angles. Each angle gets its own 2-3 search queries.${focusNote}${continuationNote}${freshNote}
+
+Topic: "${query}"
+
+Pick angles that are GENUINELY different - not paraphrases of the same question. Good angle diversity examples:
+- data / quantitative metrics
+- competitive landscape
+- team / governance / actors
+- recent news / catalysts
+- counterarguments / risks / criticism
+- forward outlook / projections
+- historical context / origins
+- regulatory / policy angle
+- technical / mechanism
+- ecosystem partners
+
+Pick the ${n} angles that most fit THIS specific topic. Each angle should have a label (3-6 words) and rationale (1 sentence). Queries must be standalone, ≤90 chars, entity-rich (specific names, dollar amounts, dates, benchmark numbers - NOT generic background).
+
+Return strict JSON:
+{
+  "angles": [
+    {
+      "label": "...",
+      "rationale": "...",
+      "queries": ["...", "..."]
+    }
+  ]
+}
+
+Exactly ${n} angles. 2-3 queries per angle. No duplicate queries across angles.`;
+    let raw = "";
+    try {
+        raw = await (0, llm_js_1.callLLM)(sys, user, 1200, [], 30000);
+    }
+    catch {
+        return [];
+    }
+    const parsed = (0, deep_research_text_js_1.safeParseJson)(raw, {});
+    if (!parsed.angles || !Array.isArray(parsed.angles))
+        return [];
+    const angles = [];
+    for (const a of parsed.angles) {
+        if (typeof a !== "object" || a === null)
+            continue;
+        const o = a;
+        if (typeof o.label !== "string" || typeof o.rationale !== "string")
+            continue;
+        if (!Array.isArray(o.queries))
+            continue;
+        const queries = o.queries
+            .filter((q) => typeof q === "string")
+            .map((q) => q.trim())
+            .filter((q) => q.length > 0 && q.length <= 200)
+            .slice(0, 3);
+        if (queries.length === 0)
+            continue;
+        angles.push({
+            label: o.label.trim().slice(0, 80),
+            rationale: o.rationale.trim().slice(0, 200),
+            queries,
+        });
+        if (angles.length >= n)
+            break;
+    }
+    return angles;
+}
+// Adversarial critic - only runs for depth=deep. Reads the draft + sources
+// already gathered and produces a structured challenge block: single-source
+// claims, contradictions across angles, speculation framed as fact. The
+// final synthesizer is told to incorporate or refute these challenges
+// explicitly, lifting the quality floor.
+async function runCritic(query, draft, angles, sourceCount) {
+    const sys = "You are an adversarial research critic. Be terse, specific, and unsparing. Output markdown.";
+    const angleLabels = angles.map((a, i) => `${i + 1}. ${a.label}`).join("\n");
+    const user = `Original question: "${query}"
+
+Specialist angles investigated:
+${angleLabels}
+
+Total sources gathered: ${sourceCount}
+
+Draft report:
+"""
+${draft.slice(0, 5000)}
+"""
+
+Audit the draft for quality problems. Be specific and quote spans where possible.
+
+Identify:
+1. **Single-source claims** - major assertions resting on one [N] citation that aren't widely corroborated
+2. **Contradictions** - places where different sources disagree but the draft doesn't surface the disagreement
+3. **Speculation framed as fact** - confident statements about future/intent/cause without evidence
+4. **Coverage gaps** - angles from the list above that got shallow treatment
+5. **Stale or weak sources** - dated material treated as current, blog posts cited as primary data
+
+Output format:
+## Critic notes
+- **[type]**: specific issue + relevant quote or claim. (1-2 sentences each)
+- Skip categories with no findings - don't pad.
+
+End with one line: "Net recommendation: [accept|revise|reject]"
+
+If the draft is solid, say so plainly - don't manufacture issues.`;
+    try {
+        const notes = await (0, llm_js_1.callLLM)(sys, user, 1500, [], 60000);
+        return notes.trim();
+    }
+    catch {
+        return ""; // critic failure shouldn't block final synth
+    }
+}
+async function reflectAndExtend(query, draft, existingQueries, n) {
+    const sys = "You are a research auditor. Find gaps in a draft report and propose follow-up search queries. Output strict JSON.";
+    const user = `Original question: "${query}"
+
+Queries already run:
+${existingQueries.map((q, i) => `${i + 1}. ${q}`).join("\n")}
+
+Draft report:
+"""
+${draft.slice(0, 4000)}
+"""
+
+Identify ${n} GAPS in the draft - angles missing, claims that need verification, counter-perspectives not represented, or recent developments not covered. For each gap, give ONE web search query (≤90 chars) that would fill it.
+
+Return: {"gap_queries": ["...", "..."]} - exactly ${n} items, no duplicates of existing queries.`;
+    let raw = "";
+    try {
+        raw = await (0, llm_js_1.callLLM)(sys, user, 500, [], 30000);
+    }
+    catch {
+        return [];
+    }
+    const parsed = (0, deep_research_text_js_1.safeParseJson)(raw, {});
+    if (!parsed.gap_queries || !Array.isArray(parsed.gap_queries))
+        return [];
+    return parsed.gap_queries
+        .filter((s) => typeof s === "string")
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0 && s.length <= 200)
+        .slice(0, n);
+}