@vespermcp/mcp-server 1.2.22 → 1.2.25

package/build/web/fusion-engine.js

@@ -0,0 +1,483 @@
+import { S3Client, ListObjectsV2Command, GetObjectCommand } from "@aws-sdk/client-s3";
+import { fromEnv, fromTemporaryCredentials } from "@aws-sdk/credential-providers";
+import crypto from "crypto";
+import { estimateQualityScore } from "../metadata/quality.js";
+;
+function identityKey(doc) {
+    const mj = doc.metadata_json || {};
+    switch (doc.source_type) {
+        case "arxiv":
+            return typeof mj.arxiv_id === "string" ? mj.arxiv_id : null;
+        case "github":
+            return typeof mj.full_name === "string" ? mj.full_name : null;
+        case "semantic_scholar":
+            return typeof mj.paper_id === "string" ? mj.paper_id : null;
+        case "hackernews":
+            return typeof mj.item_id !== "undefined" ? String(mj.item_id) : null;
+        case "s3":
+            if (typeof mj.bucket === "string" && typeof mj.key === "string") {
+                return `s3://${mj.bucket}/${mj.key}`;
+            }
+            return typeof doc.source_url === "string" ? doc.source_url : null;
+        default:
+            return null;
+    }
+}
+function safeContentForEmbedding(content) {
+    // Keep embeddings cheap and consistent.
+    const text = String(content || "").replace(/\s+/g, " ").trim();
+    return text.slice(0, 4000);
+}
+function normalizedHash(content) {
+    const base = String(content || "")
+        .toLowerCase()
+        .replace(/\s+/g, " ")
+        .trim();
+    return crypto.createHash("sha256").update(base).digest("hex");
+}
+function tokenize(content) {
+    const words = String(content || "")
+        .toLowerCase()
+        .replace(/[^a-z0-9\s]/g, " ")
+        .split(/\s+/)
+        .filter((w) => w.length >= 3);
+    return new Set(words);
+}
+function titleTokens(doc) {
+    const mj = doc.metadata_json || {};
+    const raw = typeof mj.title === "string" ? mj.title : "";
+    return tokenize(raw);
+}
+function isSuspiciousPair(a, b) {
+    // semantic fallback should be selective; do cheap prefilter first
+    const aLen = a.content.length;
+    const bLen = b.content.length;
+    const maxLen = Math.max(aLen, bLen, 1);
+    const lenRatio = Math.abs(aLen - bLen) / maxLen;
+    // Loosened again to allow abstract-vs-summary style comparisons.
+    if (lenRatio > 0.8)
+        return false;
+    // Fast path: same normalized title-like prefix often indicates same research object.
+    const aPrefix = a.content.slice(0, 140).toLowerCase().replace(/[^a-z0-9\s]/g, " ").trim();
+    const bPrefix = b.content.slice(0, 140).toLowerCase().replace(/[^a-z0-9\s]/g, " ").trim();
+    if (aPrefix && bPrefix && (aPrefix.includes(bPrefix) || bPrefix.includes(aPrefix))) {
+        return true;
+    }
+    // Cross-source papers often have close titles even if abstracts differ.
+    const aTitle = titleTokens(a);
+    const bTitle = titleTokens(b);
+    if (aTitle.size > 0 && bTitle.size > 0) {
+        let tInter = 0;
+        for (const t of aTitle)
+            if (bTitle.has(t))
+                tInter++;
+        const tUnion = aTitle.size + bTitle.size - tInter;
+        const tJaccard = tUnion > 0 ? tInter / tUnion : 0;
+        if (tJaccard >= 0.25)
+            return true;
+    }
+    const aTokens = tokenize(a.content);
+    const bTokens = tokenize(b.content);
+    if (aTokens.size === 0 || bTokens.size === 0)
+        return false;
+    let inter = 0;
+    for (const t of aTokens)
+        if (bTokens.has(t))
+            inter++;
+    const union = aTokens.size + bTokens.size - inter;
+    const jaccard = union > 0 ? inter / union : 0;
+    // Loosened from 0.12 -> 0.08 to let semantic stage inspect more borderline matches.
+    return jaccard >= 0.08;
+}
+function normalizeStars(doc) {
+    const mj = doc.metadata_json || {};
+    const starsLike = mj.stars ?? mj.citation_count ?? mj.points;
+    const n = Number(starsLike || 0);
+    return Number.isFinite(n) ? n : 0;
+}
+function asChain(doc) {
+    const qc = Number(doc.quality_score || 0);
+    return [
+        {
+            source_type: doc.source_type,
+            source_url: doc.source_url,
+            collected_at: doc.collected_at,
+            quality_score: qc,
+        },
+    ];
+}
+function dedupExactKeepBest(docs) {
+    const out = new Map();
+    for (const d of docs) {
+        const key = identityKey(d) || `content:${d.content}`;
+        const prev = out.get(key);
+        if (!prev || (d.quality_score || 0) > (prev.quality_score || 0)) {
+            out.set(key, d);
+        }
+    }
+    return out;
+}
+function sortByQualityDesc(docs) {
+    return [...docs].sort((a, b) => (b.quality_score || 0) - (a.quality_score || 0));
+}
+export class WebFusionEngine {
+    deps;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    async fuse(input) {
+        if (!Array.isArray(input?.sources) || input.sources.length < 2) {
+            throw new Error("vesper.fuse: sources must contain at least 2 entries.");
+        }
+        const mergeStrategy = input.merge_strategy || "union";
+        const deduplication = input.deduplication || "semantic";
+        const sourcesRequested = input.sources;
+        const requestedTypes = Array.from(new Set(sourcesRequested.map(s => s.type)));
+        // For better semantic dedup accuracy, use richer content when enabled.
+        const arxivFullText = deduplication === "semantic";
+        const githubIncludeReadme = deduplication === "semantic";
+        const telemetry = { per_source: [] };
+        const perSourceDocs = {};
+        const perSourceErrors = [];
+        await Promise.all(sourcesRequested.map(async (spec) => {
+            const start = Date.now();
+            try {
+                let docs = [];
+                let cacheHit = false;
+                let latencyMs = 0;
+                if (spec.type === "s3") {
+                    const out = await this.collectFromS3(spec);
+                    docs = out.docs;
+                    cacheHit = out.cacheHit;
+                    latencyMs = out.latencyMs;
+                }
+                else {
+                    const res = await this.deps.webCoreEngine.find({
+                        query: spec.query,
+                        sources: [spec.type],
+                        limit: Math.max(1, Number(spec.max_results || 10)),
+                        arxiv_full_text: arxivFullText,
+                        github_include_readme: githubIncludeReadme,
+                    });
+                    docs = res.results || [];
+                    const perSrcTel = res.telemetry?.per_source?.find((t) => t.source === spec.type);
+                    cacheHit = perSrcTel ? !!perSrcTel.cache_hit : false;
+                    latencyMs = perSrcTel ? Number(perSrcTel.latency_ms) : Date.now() - start;
+                }
+                const filtered = spec.min_stars !== undefined
+                    ? docs.filter((d) => normalizeStars(d) >= Number(spec.min_stars))
+                    : docs;
+                perSourceDocs[spec.type] = (perSourceDocs[spec.type] || []).concat(filtered);
+                telemetry.per_source.push({
+                    source_type: spec.type,
+                    cache_hit: cacheHit,
+                    latency_ms: latencyMs || (Date.now() - start),
+                    result_count: filtered.length,
+                });
+            }
+            catch (e) {
+                const errMsg = e?.message || String(e);
+                perSourceErrors.push(errMsg);
+                if (spec.type === "s3") {
+                    // Graceful S3 error row for downstream agents.
+                    const now = new Date().toISOString();
+                    const errorDoc = {
+                        source_type: "s3",
+                        source_url: spec.bucket ? `s3://${spec.bucket}/${spec.path || ""}` : "s3://",
+                        content: `S3 error: ${errMsg}`,
+                        metadata_json: {
+                            error: true,
+                            message: errMsg,
+                            bucket: spec.bucket,
+                            path: spec.path,
+                            region: spec.region,
+                        },
+                        quality_score: 0,
+                        collected_at: now,
+                        content_type: "text",
+                        source_chain: [
+                            { source_type: "s3", source_url: spec.bucket ? `s3://${spec.bucket}/${spec.path || ""}` : "s3://", collected_at: now, quality_score: 0 },
+                        ],
+                    };
+                    perSourceDocs[spec.type] = (perSourceDocs[spec.type] || []).concat([errorDoc]);
+                }
+                telemetry.per_source.push({
+                    source_type: spec.type,
+                    cache_hit: false,
+                    latency_ms: Date.now() - start,
+                    result_count: spec.type === "s3" ? 1 : 0,
+                    error: errMsg,
+                });
+            }
+        }));
+        const allDocs = Object.values(perSourceDocs).flat();
+        const inputDocuments = allDocs.length;
+        if (inputDocuments === 0) {
+            return {
+                input: { merge_strategy: mergeStrategy, deduplication, sources_requested: sourcesRequested },
+                results: [],
+                stats: { input_documents: 0, output_documents: 0, groups: 0, duplicates_removed: 0 },
+                telemetry,
+            };
+        }
+        // 1) Group/dedup
+        let groups = [];
+        let duplicatesRemoved = 0;
+        const shouldDedup = mergeStrategy === "dedup" || deduplication !== "none";
+        if (!shouldDedup) {
+            groups = allDocs.map((d) => [d]);
+        }
+        else if (deduplication === "exact") {
+            const map = dedupExactKeepBest(allDocs);
+            groups = Array.from(map.values()).map((d) => [d]);
+        }
+        else {
+            // Hash-first grouping (exact normalized-content duplicates).
+            const hashBuckets = new Map();
+            for (const d of allDocs) {
+                const h = normalizedHash(d.content);
+                const bucket = hashBuckets.get(h) || [];
+                bucket.push(d);
+                hashBuckets.set(h, bucket);
+            }
+            const hashRepresentatives = [];
+            const seededGroups = [];
+            for (const bucket of hashBuckets.values()) {
+                const best = bucket.slice().sort((a, b) => (b.quality_score || 0) - (a.quality_score || 0))[0];
+                hashRepresentatives.push(best);
+                seededGroups.push(bucket);
+            }
+            // Semantic fallback on suspicious pairs only.
+            const threshold = 0.93;
+            const docsSorted = sortByQualityDesc(hashRepresentatives);
+            const accepted = [];
+            let candidatesChecked = 0;
+            let suspiciousPairsChecked = 0;
+            let suspiciousPrefilterRejected = 0;
+            for (const d of docsSorted) {
+                const dContent = safeContentForEmbedding(d.content);
+                const dVec = await this.deps.embedder.embed(dContent);
+                let matchedIndex = null;
+                // Try match against existing representatives first.
+                for (let i = 0; i < accepted.length; i++) {
+                    // Cross-source focus: keep intra-source docs separate unless exact hash matched.
+                    if (accepted[i].rep.source_type === d.source_type) {
+                        suspiciousPrefilterRejected++;
+                        continue;
+                    }
+                    if (!isSuspiciousPair(d, accepted[i].rep)) {
+                        suspiciousPrefilterRejected++;
+                        continue;
+                    }
+                    suspiciousPairsChecked++;
+                    candidatesChecked++;
+                    const sim = dotProduct(dVec, accepted[i].repVec);
+                    if (sim >= threshold) {
+                        // If identity keys disagree and similarity is only moderately high, be conservative.
+                        const dk = identityKey(d);
+                        const ak = identityKey(accepted[i].rep);
+                        if (dk && ak && dk !== ak)
+                            continue;
+                        matchedIndex = i;
+                        break;
+                    }
+                }
+                if (matchedIndex === null) {
+                    accepted.push({ rep: d, repVec: dVec, members: [d] });
+                }
+                else {
+                    accepted[matchedIndex].members.push(d);
+                }
+            }
+            groups = accepted.map(a => a.members);
+            duplicatesRemoved = inputDocuments - groups.reduce((sum, g) => sum + 1, 0);
+            telemetry.fusion_dedup_semantic = {
+                threshold,
+                embedding_dim: undefined,
+                candidates_checked: candidatesChecked,
+                hash_groups: hashBuckets.size,
+                suspicious_pairs_checked: suspiciousPairsChecked,
+                suspicious_prefilter_rejected: suspiciousPrefilterRejected,
+            };
+        }
+        if (deduplication !== "semantic" || !shouldDedup) {
+            duplicatesRemoved = inputDocuments - groups.length;
+        }
+        // 2) Merge strategy
+        const results = [];
+        for (const g of groups) {
+            const chainEntries = g.flatMap(asChain);
+            // union / dedup: one per group (best representative), always preserve provenance.
+            const best = g.slice().sort((a, b) => (b.quality_score || 0) - (a.quality_score || 0))[0];
+            results.push({
+                ...best,
+                source_chain: chainEntries,
+                metadata_json: {
+                    ...best.metadata_json,
+                    fused_from: g.map(d => ({
+                        source_type: d.source_type,
+                        source_url: d.source_url,
+                        quality_score: d.quality_score,
+                    })),
+                },
+            });
+        }
+        const outputDocuments = results.length;
+        return {
+            input: { merge_strategy: mergeStrategy, deduplication, sources_requested: sourcesRequested },
+            results,
+            stats: {
+                input_documents: inputDocuments,
+                output_documents: outputDocuments,
+                groups: groups.length,
+                duplicates_removed: duplicatesRemoved,
+            },
+            telemetry,
+        };
+    }
+    async collectFromS3(spec) {
+        const started = Date.now();
+        const bucket = String(spec.bucket || "").trim();
+        const prefix = String(spec.path || "").trim().replace(/^\/+/, "");
+        const region = String(spec.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || "us-east-1");
+        const max = Math.max(1, Math.min(50, Number(spec.max_results || 10)));
+        if (!bucket)
+            throw new Error("s3 source requires bucket");
+        const hasPassThroughCreds = !!(spec.credentials?.accessKeyId && spec.credentials?.secretAccessKey) || !!spec.credentials?.roleArn;
+        const hasEnvCreds = !!(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY);
+        if (!hasPassThroughCreds && !hasEnvCreds) {
+            throw new Error("S3 fusion requires AWS credentials for bucket listing. Provide sources[].credentials (accessKeyId/secretAccessKey or roleArn) or set AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY.");
+        }
+        const cacheKey = `vesper:fuse:s3:list:${bucket}:${prefix}:max=${max}:region=${region}`;
+        const cached = await this.deps.cache?.getJson(cacheKey);
+        if (cached) {
+            return { docs: cached, cacheHit: true, latencyMs: Date.now() - started };
+        }
+        const client = this.makeS3Client(region, spec.credentials);
+        const listed = [];
+        let continuationToken = undefined;
+        while (listed.length < max) {
+            const listResp = await client.send(new ListObjectsV2Command({
+                Bucket: bucket,
+                Prefix: prefix || undefined,
+                MaxKeys: Math.min(1000, max - listed.length),
+                ContinuationToken: continuationToken,
+            }));
+            const contents = listResp.Contents || [];
+            for (const obj of contents) {
+                if (!obj.Key)
+                    continue;
+                listed.push({
+                    key: obj.Key,
+                    size: obj.Size,
+                    etag: obj.ETag,
+                    last_modified: obj.LastModified ? obj.LastModified.toISOString() : undefined,
+                });
+                if (listed.length >= max)
+                    break;
+            }
+            if (!listResp.IsTruncated || !listResp.NextContinuationToken)
+                break;
+            continuationToken = listResp.NextContinuationToken;
+        }
+        const docs = [];
+        for (const obj of listed) {
+            const key = obj.key;
+            const objCacheKey = `vesper:fuse:s3:obj:${bucket}:${key}`;
+            const cachedText = await this.deps.cache?.getJson(objCacheKey);
+            let text = cachedText || "";
+            if (!cachedText) {
+                text = await this.getS3ObjectText(client, bucket, key);
+                if (text) {
+                    await this.deps.cache?.setJson(objCacheKey, text, 21600); // 6h
+                }
+            }
+            if (!text)
+                continue;
+            const truncated = text.slice(0, 50_000);
+            const datePresent = !!obj.last_modified;
+            const quality = estimateQualityScore({
+                abstractLength: truncated.length,
+                authorsPresent: false,
+                datePresent,
+                contentDepth: truncated.length,
+            });
+            const collectedAt = new Date().toISOString();
+            docs.push({
+                source_type: "s3",
+                source_url: `s3://${bucket}/${key}`,
+                content: truncated,
+                metadata_json: {
+                    bucket,
+                    key,
+                    size: obj.size,
+                    etag: obj.etag,
+                    last_modified: obj.last_modified,
+                    prefix,
+                    region,
+                },
+                quality_score: quality,
+                collected_at: collectedAt,
+                content_type: "text",
+                source_chain: [
+                    { source_type: "s3", source_url: `s3://${bucket}/${key}`, collected_at: collectedAt, quality_score: quality },
+                ],
+            });
+        }
+        await this.deps.cache?.setJson(cacheKey, docs, 21600); // 6h
+        return { docs, cacheHit: false, latencyMs: Date.now() - started };
+    }
+    makeS3Client(region, credentials) {
+        // Pass-through only: use explicit credentials if provided, otherwise env/instance.
+        if (credentials?.roleArn) {
+            return new S3Client({
+                region,
+                credentials: fromTemporaryCredentials({
+                    params: { RoleArn: credentials.roleArn, RoleSessionName: "vesper-fuse" },
+                    clientConfig: { region },
+                }),
+            });
+        }
+        if (credentials?.accessKeyId && credentials?.secretAccessKey) {
+            return new S3Client({
+                region,
+                credentials: {
+                    accessKeyId: credentials.accessKeyId,
+                    secretAccessKey: credentials.secretAccessKey,
+                    sessionToken: credentials.sessionToken,
+                },
+            });
+        }
+        return new S3Client({ region, credentials: fromEnv() });
+    }
+    async getS3ObjectText(client, bucket, key) {
+        // Only attempt to read likely-text files.
+        const lower = key.toLowerCase();
+        const isLikelyText = /\.(txt|md|json|jsonl|csv|tsv|yaml|yml|log)$/i.test(lower);
+        if (!isLikelyText)
+            return "";
+        const resp = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+        const body = resp.Body;
+        if (!body)
+            return "";
+        const chunks = [];
+        let total = 0;
+        for await (const chunk of body) {
+            const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            chunks.push(buf);
+            total += buf.length;
+            if (total > 2_000_000)
+                break; // cap read at ~2MB
+        }
+        const buffer = Buffer.concat(chunks);
+        return buffer.toString("utf-8");
+    }
+}
+function dotProduct(a, b) {
+    const n = Math.min(a.length, b.length);
+    let sum = 0;
+    for (let i = 0; i < n; i++)
+        sum += a[i] * b[i];
+    // embedder normalizes vectors, so dot product approximates cosine similarity.
+    return sum;
+}

package/build/web/types.js

@@ -0,0 +1 @@
+export {};