@jayarrowz/mcp-arsr 1.0.0

package/dist/src/services/llm.d.ts

@@ -0,0 +1,56 @@
+import type { ARSRConfig, Claim, ScoredClaim, ClaimEvidence } from "../types.js";
+/**
+ * Generate an initial draft response to a query.
+ * Uses web search to ground the draft in real data — this prevents
+ * the "I don't know" refusal problem where the inner model punts
+ * on questions outside its training data.
+ *
+ * Returns the draft text and a structured is_refusal flag classified
+ * by the inner LLM (replacing brittle string matching).
+ */
+export declare function generateDraft(query: string, context?: string, config?: ARSRConfig): Promise<{
+    draft: string;
+    is_refusal: boolean;
+}>;
+/**
+ * Decompose a draft into individually verifiable atomic claims.
+ * If the draft is a refusal/non-answer, extracts claims from the
+ * original query context instead so the loop can still retrieve evidence.
+ */
+export declare function decomposeClaims(draft: string, originalQuery?: string, isRefusal?: boolean, config?: ARSRConfig): Promise<Claim[]>;
+/**
+ * Score claims by generating multiple rephrasings and measuring agreement.
+ * Uses semantic entropy: low agreement across rephrasings = high uncertainty.
+ */
+export declare function scoreClaims(claims: Claim[], config?: ARSRConfig): Promise<ScoredClaim[]>;
+/**
+ * For low-confidence claims, generate adversarial search queries and retrieve evidence.
+ * Uses the inner LLM + web search to find supporting/contradicting sources.
+ */
+export declare function retrieveEvidence(claims: ScoredClaim[], strategy?: string, config?: ARSRConfig): Promise<ClaimEvidence[]>;
+/**
+ * Revise the draft based on evidence, returning the new text + change log.
+ * If the original draft was a refusal/non-answer, writes a completely new
+ * response from the evidence instead of trying to edit the refusal.
+ */
+export declare function reviseDraft(draft: string, evidence: ClaimEvidence[], scored: ScoredClaim[], originalQuery?: string, isRefusal?: boolean, config?: ARSRConfig): Promise<{
+    revised: string;
+    changes: Array<{
+        claim_id: string;
+        action: string;
+        original: string;
+        revised: string;
+        reason: string;
+    }>;
+    conflicts: Array<{
+        claim_id: string;
+        description: string;
+    }>;
+}>;
+/**
+ * Decide whether to continue the refinement loop.
+ */
+export declare function shouldContinue(iteration: number, scored: ScoredClaim[], maxIterations: number, confidenceThreshold: number, previousAvgConfidence: number | null): Promise<{
+    decision: "continue" | "stop";
+    reason: string;
+}>;

package/dist/src/services/llm.js

@@ -0,0 +1,361 @@
+import Anthropic from "@anthropic-ai/sdk";
+import { DEFAULT_CONFIG } from "../types.js";
+let client = null;
+function getClient() {
+    if (!client) {
+        client = new Anthropic(); // Uses ANTHROPIC_API_KEY env var
+    }
+    return client;
+}
+async function askInner(system, user, config = DEFAULT_CONFIG) {
+    const api = getClient();
+    const response = await api.messages.create({
+        model: config.inner_model,
+        max_tokens: 4096,
+        system,
+        messages: [{ role: "user", content: user }],
+    });
+    const textBlock = response.content.find((b) => b.type === "text");
+    return textBlock ? textBlock.text : "";
+}
+async function askInnerWithSearch(system, user, config = DEFAULT_CONFIG) {
+    const api = getClient();
+    const response = await api.messages.create({
+        model: config.inner_model,
+        max_tokens: 4096,
+        system,
+        messages: [{ role: "user", content: user }],
+        tools: [{ type: "web_search_20250305", name: "web_search" }],
+    });
+    // Extract text and citations from the response
+    let text = "";
+    const citations = [];
+    for (const block of response.content) {
+        if (block.type === "text") {
+            text += block.text;
+            // Extract any inline citations
+            if ("citations" in block && Array.isArray(block.citations)) {
+                for (const cite of block.citations) {
+                    if ("url" in cite && "title" in cite) {
+                        citations.push({
+                            url: cite.url,
+                            title: cite.title,
+                        });
+                    }
+                }
+            }
+        }
+    }
+    return { text, citations };
+}
+function extractJSON(raw) {
+    // Strip markdown fences if present
+    const cleaned = raw
+        .replace(/```json\s*/gi, "")
+        .replace(/```\s*/g, "")
+        .trim();
+    // Try to find JSON object or array
+    const jsonMatch = cleaned.match(/[\[{][\s\S]*[\]}]/);
+    if (!jsonMatch) {
+        throw new Error(`No JSON found in LLM output: ${cleaned.slice(0, 200)}`);
+    }
+    return JSON.parse(jsonMatch[0]);
+}
+/**
+ * Classify whether a draft is a refusal/non-answer using the inner LLM.
+ * Returns true if the draft deflects, redirects, or refuses to answer.
+ */
+async function classifyRefusal(draft, config = DEFAULT_CONFIG) {
+    const system = `You are a response classifier. Determine whether the given text is a REFUSAL or NON-ANSWER.
+
+A refusal is any response that:
+- Says it cannot, does not, or is unable to provide the information
+- Redirects the user to check another source instead of answering
+- Provides only generic suggestions instead of a direct answer
+- Hedges so heavily that no substantive information is conveyed
+
+Respond with ONLY a JSON object: { "is_refusal": true } or { "is_refusal": false }`;
+    const raw = await askInner(system, `Classify this response:\n\n${draft}`, config);
+    try {
+        const result = extractJSON(raw);
+        return result.is_refusal === true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Generate an initial draft response to a query.
+ * Uses web search to ground the draft in real data — this prevents
+ * the "I don't know" refusal problem where the inner model punts
+ * on questions outside its training data.
+ *
+ * Returns the draft text and a structured is_refusal flag classified
+ * by the inner LLM (replacing brittle string matching).
+ */
+export async function generateDraft(query, context, config = DEFAULT_CONFIG) {
+    const system = `You are a helpful, accurate research assistant. Your job is to ANSWER the user's question with specific, concrete facts.
+
+CRITICAL RULES:
+- You MUST search the web to find the answer. Do NOT rely on memory alone.
+- You MUST provide a direct, substantive answer with specific facts, numbers, and details.
+- NEVER say "I don't have access to" or "I recommend checking" or "I cannot provide". These are failures.
+- NEVER give a list of places to look instead of the answer. FIND the answer yourself.
+- If the question asks about specific data (vote counts, statistics, dates), SEARCH FOR IT and REPORT IT.
+- It's OK to be wrong — your answer will be fact-checked and corrected later. A wrong answer is better than no answer.
+- Include your best understanding even if uncertain. The refinement loop will fix errors.
+${context ? `\nAdditional context:\n${context}` : ""}`;
+    const { text } = await askInnerWithSearch(system, query, config);
+    const is_refusal = await classifyRefusal(text, config);
+    return { draft: text, is_refusal };
+}
+/**
+ * Decompose a draft into individually verifiable atomic claims.
+ * If the draft is a refusal/non-answer, extracts claims from the
+ * original query context instead so the loop can still retrieve evidence.
+ */
+export async function decomposeClaims(draft, originalQuery, isRefusal = false, config = DEFAULT_CONFIG) {
+    let textToDecompose = draft;
+    let systemAddendum = "";
+    if (isRefusal && originalQuery) {
+        // The draft was classified as a refusal/non-answer by the LLM.
+        // Extract claims from the user's question instead.
+        textToDecompose = originalQuery;
+        systemAddendum = `
+IMPORTANT: The original draft was a non-answer/refusal. You are now extracting the factual claims
+embedded in the USER'S QUESTION instead. These are the claims that need to be verified.
+For example, if the user says "Is it true that X got Y votes?", extract "X got Y votes" as a claim.`;
+    }
+    const system = `You are a claim extraction engine. Given a text, extract every distinct factual claim as a separate item.
+
+Rules:
+- Each claim must be a single, independently verifiable statement
+- Preserve the original meaning precisely
+- Skip opinions, hedges ("I think"), and meta-commentary
+- Include the source_span (the exact substring from the original text)
+- Give each claim a short id like "c1", "c2", etc.
+${systemAddendum}
+
+Respond ONLY with a JSON array:
+[{ "id": "c1", "text": "The claim as a standalone statement", "source_span": "exact quote from draft" }, ...]`;
+    const raw = await askInner(system, `Extract all factual claims from:\n\n${textToDecompose}`, config);
+    return extractJSON(raw);
+}
+/**
+ * Score claims by generating multiple rephrasings and measuring agreement.
+ * Uses semantic entropy: low agreement across rephrasings = high uncertainty.
+ */
+export async function scoreClaims(claims, config = DEFAULT_CONFIG) {
+    const system = `You are an uncertainty estimation engine. For each claim, assess how likely it is to be factually correct.
+
+Consider:
+- Is this common knowledge or obscure?
+- Are there well-known disputes about this?
+- Does the specificity (exact numbers, dates, names) increase risk of error?
+- Could this be a common misconception?
+
+Respond ONLY with a JSON array:
+[{
+  "id": "c1",
+  "confidence": 0.92,
+  "entropy": 0.15,
+  "method": "semantic_entropy",
+  "reasoning": "brief explanation"
+}, ...]
+
+confidence: 0.0 = certainly wrong, 1.0 = certainly correct
+entropy: 0.0 = very certain, 1.0 = highly uncertain`;
+    const claimsText = claims
+        .map((c) => `[${c.id}] ${c.text}`)
+        .join("\n");
+    const raw = await askInner(system, `Score the uncertainty of each claim:\n\n${claimsText}`, config);
+    const scored = extractJSON(raw);
+    return scored.map((s) => {
+        const original = claims.find((c) => c.id === s.id);
+        return {
+            id: s.id,
+            text: original?.text ?? "",
+            source_span: original?.source_span ?? "",
+            confidence: Math.max(0, Math.min(1, s.confidence)),
+            entropy: Math.max(0, Math.min(1, s.entropy)),
+            method: "semantic_entropy",
+        };
+    });
+}
+/**
+ * For low-confidence claims, generate adversarial search queries and retrieve evidence.
+ * Uses the inner LLM + web search to find supporting/contradicting sources.
+ */
+export async function retrieveEvidence(claims, strategy = "adversarial", config = DEFAULT_CONFIG) {
+    const results = [];
+    for (const claim of claims) {
+        const queryGenSystem = `You are a search query generator for fact-checking.
+Strategy: ${strategy}
+
+For "adversarial": generate queries designed to DISPROVE the claim. Search for counterexamples, corrections, or the actual facts.
+For "confirmatory": generate queries to find authoritative sources that confirm the claim.
+For "balanced": generate both supporting and challenging queries.
+
+Respond ONLY with a JSON array of 2-3 search queries:
+["query 1", "query 2", "query 3"]`;
+        const queriesRaw = await askInner(queryGenSystem, `Generate search queries to fact-check: "${claim.text}"`, config);
+        let queries;
+        try {
+            queries = extractJSON(queriesRaw);
+        }
+        catch {
+            queries = [claim.text];
+        }
+        const allDocs = [];
+        for (const query of queries.slice(0, 3)) {
+            try {
+                const searchSystem = `You are a fact-checking research assistant. Search for information about the given query and evaluate what you find relative to this claim: "${claim.text}"
+
+After searching, respond with a JSON array of the most relevant results:
+[{
+  "title": "Page title",
+  "url": "https://...",
+  "snippet": "The relevant excerpt (max 100 words)",
+  "stance": "supports" | "contradicts" | "neutral" | "unclear"
+}]
+
+Respond ONLY with the JSON array.`;
+                const { text } = await askInnerWithSearch(searchSystem, `Search and evaluate: ${query}`, config);
+                try {
+                    const docs = extractJSON(text);
+                    allDocs.push(...docs);
+                }
+                catch {
+                    // If parsing fails, still capture as a single doc
+                    allDocs.push({
+                        title: "Search result",
+                        url: "",
+                        snippet: text.slice(0, 300),
+                        stance: "unclear",
+                    });
+                }
+            }
+            catch (err) {
+                console.error(`Search failed for query "${query}":`, err);
+            }
+        }
+        // Step 3: Summarize the evidence stance
+        const supports = allDocs.filter((d) => d.stance === "supports").length;
+        const contradicts = allDocs.filter((d) => d.stance === "contradicts").length;
+        let overall_stance;
+        if (allDocs.length === 0)
+            overall_stance = "insufficient";
+        else if (supports > 0 && contradicts > 0)
+            overall_stance = "mixed";
+        else if (contradicts > supports)
+            overall_stance = "contradicted";
+        else
+            overall_stance = "supported";
+        // Step 4: Generate a concise summary
+        const summarySystem = `Summarize the evidence for/against this claim in 1-2 sentences. Be direct.`;
+        const summaryInput = `Claim: "${claim.text}"\nEvidence:\n${allDocs.map((d) => `- [${d.stance}] ${d.snippet}`).join("\n")}`;
+        const summary = allDocs.length > 0
+            ? await askInner(summarySystem, summaryInput, config)
+            : "No evidence found.";
+        results.push({
+            claim_id: claim.id,
+            claim_text: claim.text,
+            docs: allDocs,
+            overall_stance,
+            summary,
+        });
+    }
+    return results;
+}
+/**
+ * Revise the draft based on evidence, returning the new text + change log.
+ * If the original draft was a refusal/non-answer, writes a completely new
+ * response from the evidence instead of trying to edit the refusal.
+ */
+export async function reviseDraft(draft, evidence, scored, originalQuery, isRefusal = false, config = DEFAULT_CONFIG) {
+    const evidenceSummary = evidence
+        .map((e) => `[${e.claim_id}] "${e.claim_text}" → ${e.overall_stance}\n  Evidence: ${e.summary}`)
+        .join("\n\n");
+    let system;
+    if (isRefusal && originalQuery) {
+        // The draft was a non-answer. Write a NEW response from the evidence.
+        system = `You are a response generation engine. The original draft FAILED to answer the user's question — it was a refusal or redirect.
+
+Your job: Write a COMPLETELY NEW response that DIRECTLY ANSWERS the user's question using the evidence provided.
+
+User's original question: "${originalQuery}"
+
+Rules:
+- DIRECTLY answer the question using the evidence gathered
+- Include specific facts, numbers, and details from the evidence
+- If the evidence contradicts what the user claimed, say so clearly
+- If the evidence supports what the user claimed, confirm it
+- Add hedging ("reportedly", "according to...") only for genuinely mixed evidence
+- Do NOT say "I don't have access" or redirect to other sources
+
+Respond with JSON:
+{
+  "revised": "The full NEW response that directly answers the question",
+  "changes": [
+    { "claim_id": "c1", "action": "generated_from_evidence", "original": "N/A - draft was a refusal", "revised": "the new claim", "reason": "Evidence shows..." }
+  ],
+  "conflicts": [
+    { "claim_id": "c2", "description": "Sources disagree about..." }
+  ]
+}`;
+    }
+    else {
+        system = `You are a response revision engine. Given an original draft and fact-checking evidence, produce a corrected version.
+
+Rules:
+- Fix claims that were contradicted by evidence
+- Add hedging language ("reportedly", "according to...") for mixed evidence
+- Remove claims with no supporting evidence if they are central to the answer
+- Keep claims that were supported — don't weaken what's already correct
+- Preserve the original tone and structure as much as possible
+
+Respond with JSON:
+{
+  "revised": "The full revised response text",
+  "changes": [
+    { "claim_id": "c1", "action": "corrected|removed|hedged|kept", "original": "...", "revised": "...", "reason": "..." }
+  ],
+  "conflicts": [
+    { "claim_id": "c2", "description": "Sources disagree about..." }
+  ]
+}`;
+    }
+    const raw = await askInner(system, `Original draft:\n${draft}\n\nEvidence report:\n${evidenceSummary}`, config);
+    return extractJSON(raw);
+}
+/**
+ * Decide whether to continue the refinement loop.
+ */
+export async function shouldContinue(iteration, scored, maxIterations, confidenceThreshold, previousAvgConfidence) {
+    if (iteration >= maxIterations) {
+        return { decision: "stop", reason: `Budget exhausted (${maxIterations} iterations)` };
+    }
+    const avgConfidence = scored.length > 0
+        ? scored.reduce((sum, c) => sum + c.confidence, 0) / scored.length
+        : 1;
+    const lowConfidence = scored.filter((c) => c.confidence < confidenceThreshold);
+    if (lowConfidence.length === 0) {
+        return {
+            decision: "stop",
+            reason: `All ${scored.length} claims above confidence threshold (${confidenceThreshold})`,
+        };
+    }
+    if (previousAvgConfidence !== null) {
+        const improvement = avgConfidence - previousAvgConfidence;
+        if (improvement < 0.02) {
+            return {
+                decision: "stop",
+                reason: `Confidence converged (Δ=${improvement.toFixed(3)}, threshold=0.02). ${lowConfidence.length} claims remain below threshold.`,
+            };
+        }
+    }
+    return {
+        decision: "continue",
+        reason: `${lowConfidence.length}/${scored.length} claims below threshold. Avg confidence: ${avgConfidence.toFixed(3)}. Continuing refinement.`,
+    };
+}

package/dist/src/types.d.ts

@@ -0,0 +1,53 @@
+export interface Claim {
+    id: string;
+    text: string;
+    source_span: string;
+}
+export interface ScoredClaim extends Claim {
+    confidence: number;
+    entropy: number;
+    method: "semantic_entropy" | "consistency_vote";
+    variants?: string[];
+}
+export interface EvidenceDoc {
+    title: string;
+    url: string;
+    snippet: string;
+    stance: "supports" | "contradicts" | "neutral" | "unclear";
+}
+export interface ClaimEvidence {
+    claim_id: string;
+    claim_text: string;
+    docs: EvidenceDoc[];
+    overall_stance: "supported" | "contradicted" | "mixed" | "insufficient";
+    summary: string;
+}
+export interface RevisionChange {
+    claim_id: string;
+    action: "kept" | "corrected" | "removed" | "hedged";
+    original: string;
+    revised: string;
+    reason: string;
+}
+export interface Conflict {
+    claim_id: string;
+    description: string;
+    sources_for: string[];
+    sources_against: string[];
+}
+export interface LoopState {
+    iteration: number;
+    max_iterations: number;
+    confidence_threshold: number;
+    previous_avg_confidence: number | null;
+    claims_improved: number;
+    claims_degraded: number;
+}
+export interface ARSRConfig {
+    max_iterations: number;
+    confidence_threshold: number;
+    entropy_samples: number;
+    retrieval_strategy: "adversarial" | "confirmatory" | "balanced";
+    inner_model: string;
+}
+export declare const DEFAULT_CONFIG: ARSRConfig;

package/dist/src/types.js

@@ -0,0 +1,7 @@
+export const DEFAULT_CONFIG = {
+    max_iterations: parseInt(process.env.ARSR_MAX_ITERATIONS || "3", 10),
+    confidence_threshold: parseFloat(process.env.ARSR_CONFIDENCE_THRESHOLD || "0.85"),
+    entropy_samples: parseInt(process.env.ARSR_ENTROPY_SAMPLES || "3", 10),
+    retrieval_strategy: process.env.ARSR_RETRIEVAL_STRATEGY || "adversarial",
+    inner_model: process.env.ARSR_INNER_MODEL || "claude-haiku-4-5-20251001",
+};

package/glama.json

@@ -0,0 +1,6 @@
+{
+    "$schema": "https://glama.ai/mcp/schemas/server.json",
+    "maintainers": [
+      "JayArrowz"
+    ]
+  }

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@jayarrowz/mcp-arsr",
+  "version": "1.0.0",
+  "description": "Adaptive Retrieval-Augmented Self-Refinement MCP Server — a closed-loop system that lets LLMs iteratively verify and correct their own claims using uncertainty-guided retrieval.",
+  "license": "MIT",
+  "author": "Jay Arrowz (https://github.com/jayarrowz)",
+  "homepage": "https://github.com/jayarrowz/mcp-arsr",
+  "bugs": "https://github.com/jayarrowz/mcp-arsr/issues",
+  "main": "dist/src/index.js",
+  "bin": {
+    "mcp-arsr": "dist/src/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/src/index.js",
+    "dev": "tsc --watch"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.36.3",
+    "@modelcontextprotocol/sdk": "^1.27.1",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@types/express": "^5.0.6",
+    "@types/node": "^22.0.0",
+    "ts-node": "^10.9.2",
+    "tsx": "^4.21.0",
+    "typescript": "^5.7.0"
+  },
+  "type": "module"
+}

package/smithery.yaml

@@ -0,0 +1,13 @@
+startCommand:
+  type: "stdio"
+  configSchema:
+    type: "object"
+    properties: {}
+    additionalProperties: false
+  commandFunction:
+    # A JS function that produces the CLI command based on the given config to start the MCP on stdio.
+    |-
+    (config) => ({
+      command: 'node',
+      args: ['dist/src/index.js']
+    })