@vespermcp/mcp-server 1.2.22 → 1.2.24

package/build/web/extract-web.js

@@ -0,0 +1,297 @@
+const WHITELIST = [
+    "sec.gov",
+    "www.sec.gov",
+    "wikipedia.org",
+    "en.wikipedia.org",
+    "producthunt.com",
+    "www.producthunt.com",
+    "crunchbase.com",
+    "www.crunchbase.com",
+];
+function stripTags(input) {
+    return input.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
+}
+function decodeHtml(input) {
+    return String(input || "")
+        .replace(/&nbsp;/g, " ")
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, "\"")
+        .replace(/&#39;/g, "'");
+}
+function extractTables(html) {
+    const tables = [];
+    const tableMatches = html.match(/<table[\s\S]*?<\/table>/gi) || [];
+    for (const table of tableMatches) {
+        const rows = [];
+        const rowMatches = table.match(/<tr[\s\S]*?<\/tr>/gi) || [];
+        for (const row of rowMatches) {
+            const cells = row.match(/<(td|th)[\s\S]*?<\/(td|th)>/gi) || [];
+            const line = cells.map(c => decodeHtml(stripTags(c))).filter(Boolean).join(" | ");
+            if (line)
+                rows.push(line);
+        }
+        if (rows.length > 0)
+            tables.push(rows);
+    }
+    return tables;
+}
+function extractLists(html) {
+    const out = [];
+    const listBlocks = html.match(/<(ul|ol)[\s\S]*?<\/(ul|ol)>/gi) || [];
+    for (const block of listBlocks) {
+        const items = block.match(/<li[\s\S]*?<\/li>/gi) || [];
+        for (const item of items) {
+            const text = decodeHtml(stripTags(item));
+            if (text)
+                out.push(text);
+        }
+    }
+    return out;
+}
+function extractInfobox(html) {
+    const box = html.match(/<table[^>]*class\s*=\s*["'][^"']*infobox[^"']*["'][\s\S]*?<\/table>/i)?.[0]
+        || html.match(/<table[^>]*class=[^>\s]*infobox[^>\s]*[\s\S]*?<\/table>/i)?.[0];
+    if (!box)
+        return {};
+    const rows = box.match(/<tr[\s\S]*?<\/tr>/gi) || [];
+    const info = {};
+    for (const row of rows) {
+        const th = row.match(/<th[\s\S]*?<\/th>/i)?.[0];
+        const td = row.match(/<td[\s\S]*?<\/td>/i)?.[0];
+        const k = th ? decodeHtml(stripTags(th)) : "";
+        const v = td ? decodeHtml(stripTags(td)) : "";
+        if (k && v)
+            info[k] = v;
+    }
+    return info;
+}
+function validateSchema(data, schema) {
+    const required = schema?.required_fields || [];
+    const missing = required.filter((f) => data[f] === undefined || data[f] === null || data[f] === "");
+    return { ok: missing.length === 0, missing };
+}
+function toIsoDateLoose(v) {
+    const s = String(v || "").trim();
+    if (!s)
+        return null;
+    const d = new Date(s);
+    if (Number.isNaN(d.getTime()))
+        return null;
+    return d.toISOString().slice(0, 10);
+}
+function parseUsdLoose(v) {
+    const s = String(v || "").trim().replace(/,/g, "");
+    if (!s)
+        return null;
+    const m = s.match(/\$?\s*([\d.]+)\s*([KMB])?/i);
+    if (!m)
+        return null;
+    const n = Number(m[1]);
+    if (!Number.isFinite(n))
+        return null;
+    const suffix = (m[2] || "").toUpperCase();
+    const mul = suffix === "K" ? 1_000 : suffix === "M" ? 1_000_000 : suffix === "B" ? 1_000_000_000 : 1;
+    return Math.round(n * mul);
+}
+function applyDomainTransforms(domain, data) {
+    const out = { ...data };
+    if (domain.endsWith("sec.gov")) {
+        if (out.infobox && typeof out.infobox === "object") {
+            const normalized = { ...out.infobox };
+            for (const [k, v] of Object.entries(normalized)) {
+                const date = toIsoDateLoose(String(v));
+                if (date && /(date|filed|accepted|period|fiscal)/i.test(k)) {
+                    normalized[`${k}_iso`] = date;
+                }
+            }
+            out.infobox = normalized;
+        }
+        if (Array.isArray(out.items)) {
+            out.items = out.items.map((line) => {
+                const date = toIsoDateLoose(line);
+                return date ? `${line} [date_iso=${date}]` : line;
+            });
+        }
+    }
+    if (domain.endsWith("crunchbase.com")) {
+        if (out.infobox && typeof out.infobox === "object") {
+            const normalized = { ...out.infobox };
+            for (const [k, v] of Object.entries(normalized)) {
+                const usd = parseUsdLoose(String(v));
+                if (usd !== null && /(fund|raised|valuation|revenue|price|amount|usd|\$)/i.test(k)) {
+                    normalized[`${k}_usd`] = usd;
+                }
+            }
+            out.infobox = normalized;
+        }
+        if (Array.isArray(out.items)) {
+            out.items = out.items.map((line) => {
+                const usd = parseUsdLoose(line);
+                return usd !== null ? `${line} [usd=${usd}]` : line;
+            });
+        }
+    }
+    return out;
+}
+function strictSchemaForDomain(domain, contentType) {
+    // P1 strict schema contracts (minimal but explicit per domain)
+    if (domain.endsWith("sec.gov")) {
+        return contentType === "table" ? ["tables"] : ["items"];
+    }
+    if (domain.endsWith("crunchbase.com")) {
+        return contentType === "infobox" ? ["infobox"] : ["items"];
+    }
+    if (domain.endsWith("wikipedia.org")) {
+        return contentType === "infobox" ? ["infobox"] : ["tables"];
+    }
+    if (domain.endsWith("producthunt.com")) {
+        return ["items"];
+    }
+    return [];
+}
+function toError(code, message, details) {
+    return { code, message, details };
+}
+export class WebExtractorEngine {
+    cache;
+    constructor(cache) {
+        this.cache = cache;
+    }
+    async extract(input) {
+        let parsed;
+        try {
+            parsed = new URL(String(input.url || "").trim());
+        }
+        catch {
+            return {
+                ok: false,
+                url: String(input.url || ""),
+                domain: "",
+                mode: input.mode || "auto",
+                extracted_at: new Date().toISOString(),
+                content_type: "list",
+                data: {},
+                error: toError("INVALID_URL", "Invalid URL"),
+                cache: { hit: false, fallback_used: false, max_age_ms: 21600_000 },
+            };
+        }
+        const domain = parsed.hostname.toLowerCase();
+        if (!WHITELIST.some(d => domain === d || domain.endsWith(`.${d}`))) {
+            return {
+                ok: false,
+                url: parsed.toString(),
+                domain,
+                mode: input.mode || "auto",
+                extracted_at: new Date().toISOString(),
+                content_type: "list",
+                data: {},
+                error: toError("DOMAIN_NOT_ALLOWED", `Domain not allowed: ${domain}`, { allowed: WHITELIST }),
+                cache: { hit: false, fallback_used: false, max_age_ms: 21600_000 },
+            };
+        }
+        const mode = input.mode || "auto";
+        const strict = input.strict_schema !== false;
+        const cacheKey = `webextract:${domain}:${parsed.pathname}?${parsed.search}:mode=${mode}`;
+        const cached = await this.cache?.getJson(cacheKey);
+        const maxAgeMs = 21600_000;
+        const cacheAgeMs = cached?.extracted_at ? Math.max(0, Date.now() - new Date(cached.extracted_at).getTime()) : undefined;
+        try {
+            const res = await fetch(parsed.toString(), {
+                headers: {
+                    "User-Agent": "vesper/2.0 (phase3-extract-web)",
+                },
+            });
+            if (!res.ok) {
+                throw toError("HTTP_ERROR", `HTTP ${res.status}`, { status: res.status });
+            }
+            const html = await res.text();
+            let content_type = "list";
+            let data = {};
+            const tables = extractTables(html);
+            const lists = extractLists(html);
+            const infobox = extractInfobox(html);
+            if (mode === "table" || (mode === "auto" && tables.length > 0)) {
+                content_type = "table";
+                data = { tables };
+            }
+            else if (mode === "infobox" || (mode === "auto" && Object.keys(infobox).length > 0)) {
+                content_type = "infobox";
+                data = { infobox };
+            }
+            else {
+                content_type = "list";
+                data = { items: lists };
+            }
+            if ((content_type === "table" && (!data.tables || data.tables.length === 0)) ||
+                (content_type === "list" && (!data.items || data.items.length === 0)) ||
+                (content_type === "infobox" && (!data.infobox || Object.keys(data.infobox).length === 0))) {
+                throw toError("EXTRACTION_EMPTY", "No structured content extracted from static HTML; possible dynamic page or site layout change.");
+            }
+            data = applyDomainTransforms(domain, data);
+            const mergedSchemaRequired = [
+                ...(strict ? strictSchemaForDomain(domain, content_type) : []),
+                ...(input.schema?.required_fields || []),
+            ];
+            const schemaCheck = validateSchema(data, { required_fields: mergedSchemaRequired });
+            if (!schemaCheck.ok) {
+                throw toError("SCHEMA_VALIDATION_FAILED", "Schema validation failed", {
+                    missing: schemaCheck.missing,
+                    required_fields: mergedSchemaRequired,
+                    strict_domain_schema: strict,
+                });
+            }
+            const out = {
+                ok: true,
+                url: parsed.toString(),
+                domain,
+                mode,
+                extracted_at: new Date().toISOString(),
+                content_type,
+                data,
+                cache: {
+                    hit: false,
+                    fallback_used: false,
+                    max_age_ms: maxAgeMs,
+                },
+            };
+            await this.cache?.setJson(cacheKey, out, 21600); // 6h fallback cache
+            return out;
+        }
+        catch (e) {
+            if (cached) {
+                return {
+                    ...cached,
+                    ok: true,
+                    from_cache: true,
+                    warning: `Live extraction failed, served cached content. Reason: ${e?.message || String(e)}`,
+                    cache: {
+                        hit: true,
+                        fallback_used: true,
+                        age_ms: cacheAgeMs,
+                        max_age_ms: maxAgeMs,
+                    },
+                    error: toError("LIVE_EXTRACTION_FAILED", e?.message || String(e)),
+                };
+            }
+            return {
+                ok: false,
+                url: parsed.toString(),
+                domain,
+                mode,
+                extracted_at: new Date().toISOString(),
+                content_type: "list",
+                data: {},
+                cache: {
+                    hit: false,
+                    fallback_used: false,
+                    max_age_ms: maxAgeMs,
+                },
+                error: e?.code
+                    ? e
+                    : toError("LIVE_EXTRACTION_FAILED", e?.message || String(e)),
+            };
+        }
+    }
+}