@xapp/arachne-web-service 1.28.0 → 1.30.0

package/lib/utils/checkMcpServer.d.ts

@@ -0,0 +1,43 @@
+/*! Copyright (c) 2026, XAPP AI */
+/**
+ * - `live`: answered `initialize` as an MCP server
+ * - `auth-required`: answered 401/403, so it exists but wants credentials
+ * - `not-mcp`: answered, but not with JSON-RPC
+ * - `unreachable`: 404, 5xx, timeout or network error
+ * - `skipped`: not a public http(s) URL, so never contacted
+ */
+export type McpServerStatus = "live" | "auth-required" | "not-mcp" | "unreachable" | "skipped";
+export interface McpServerCheck {
+    status: McpServerStatus;
+    /** ISO time of the check. */
+    checkedAt: string;
+    httpStatus?: number;
+    protocolVersion?: string;
+    serverInfo?: {
+        name?: string;
+        version?: string;
+    };
+    /** Names and descriptions from `tools/list`. Tools are listed, never called. */
+    tools?: {
+        name: string;
+        description?: string;
+    }[];
+    error?: string;
+}
+export interface CheckMcpServerOptions {
+    /** Per-request timeout. Defaults to 5000ms. */
+    timeoutMs?: number;
+    /** Resolves a hostname to its addresses. Defaults to `dns.lookup`; replaceable for tests. */
+    resolve?: (hostname: string) => Promise<string[]>;
+}
+/**
+ * True for an http(s) URL whose host is not obviously internal: a literal address or
+ * localhost name. `checkMcpServer` also resolves the hostname before connecting.
+ */
+export declare function isPublicHttpUrl(value: string): boolean;
+/**
+ * Checks whether a URL is a working MCP server over Streamable HTTP: sends `initialize`,
+ * then `tools/list`, then ends the session. It never calls a tool, so it cannot submit a
+ * lead or change anything on the server. Never throws.
+ */
+export declare function checkMcpServer(url: string, options?: CheckMcpServerOptions): Promise<McpServerCheck>;

package/lib/utils/checkMcpServer.js

@@ -0,0 +1,297 @@
+"use strict";
+/*! Copyright (c) 2026, XAPP AI */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isPublicHttpUrl = isPublicHttpUrl;
+exports.checkMcpServer = checkMcpServer;
+const promises_1 = require("dns/promises");
+const net_1 = require("net");
+const stentor_logger_1 = require("stentor-logger");
+function resolveAll(hostname) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return (yield (0, promises_1.lookup)(hostname, { all: true })).map((a) => a.address);
+    });
+}
+const DEFAULT_TIMEOUT_MS = 5000;
+const PROTOCOL_VERSION = "2025-06-18";
+const MAX_TOOLS = 100;
+const MAX_DESCRIPTION_LENGTH = 500;
+/** Replies past this are not read further: a JSON-RPC reply with a tool list is a few KB. */
+const MAX_BODY_BYTES = 256 * 1024;
+/** A generic client identity, so the check does not name the crawler to the site. */
+const CLIENT_INFO = { name: "mcp-client", version: "1.0.0" };
+function isPrivateIPv4(ip) {
+    const [a, b] = ip.split(".").map(Number);
+    return (a === 10 ||
+        a === 127 ||
+        a === 0 ||
+        a >= 224 ||
+        (a === 192 && b === 0 && ip.split(".")[2] === "0") ||
+        (a === 198 && (b === 18 || b === 19)) ||
+        (a === 169 && b === 254) ||
+        (a === 172 && b >= 16 && b <= 31) ||
+        (a === 192 && b === 168) ||
+        (a === 100 && b >= 64 && b <= 127));
+}
+/** True for an IP literal outside the private, loopback and link-local ranges. */
+function isPublicAddress(host) {
+    // `new URL` normalizes [::ffff:127.0.0.1] to [::ffff:7f00:1]; decode it back to IPv4.
+    const mapped = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(host);
+    if (mapped) {
+        const hi = parseInt(mapped[1], 16);
+        const lo = parseInt(mapped[2], 16);
+        return !isPrivateIPv4(`${hi >> 8}.${hi & 255}.${lo >> 8}.${lo & 255}`);
+    }
+    const dotted = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(host);
+    if (dotted) {
+        return !isPrivateIPv4(dotted[1]);
+    }
+    const version = (0, net_1.isIP)(host);
+    if (version === 4) {
+        return !isPrivateIPv4(host);
+    }
+    if (version === 6) {
+        return !(host === "::1" || host === "::" || /^f[cd]/.test(host) || /^fe[89a-f]/.test(host) || /^ff/.test(host));
+    }
+    return true;
+}
+/**
+ * True for an http(s) URL whose host is not obviously internal: a literal address or
+ * localhost name. `checkMcpServer` also resolves the hostname before connecting.
+ */
+function isPublicHttpUrl(value) {
+    let url;
+    try {
+        url = new URL(value);
+    }
+    catch (_a) {
+        return false;
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        return false;
+    }
+    const host = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");
+    if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".internal")) {
+        return false;
+    }
+    return isPublicAddress(host);
+}
+/**
+ * True when every address the hostname resolves to is public, so a name pointed at 10.x or
+ * 127.0.0.1 (or a nip.io-style name) is refused. A rebinding resolver could still answer
+ * differently at connect time; this closes the plain case.
+ */
+function resolvesPublic(url, resolve) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const host = new URL(url).hostname.replace(/^\[|\]$/g, "");
+        if ((0, net_1.isIP)(host)) {
+            return true; // already checked as a literal
+        }
+        try {
+            const addresses = yield resolve(host);
+            return addresses.length > 0 && addresses.every((a) => isPublicAddress(a.toLowerCase()));
+        }
+        catch (_a) {
+            return false;
+        }
+    });
+}
+class HttpStatusError extends Error {
+    constructor(status) {
+        super(`HTTP ${status}`);
+        this.status = status;
+    }
+}
+/** The body as text, or undefined once it passes MAX_BODY_BYTES. */
+function readCapped(response) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a, _b;
+        const reader = (_b = (_a = response.body) === null || _a === void 0 ? void 0 : _a.getReader) === null || _b === void 0 ? void 0 : _b.call(_a);
+        if (!reader) {
+            const text = yield response.text();
+            return text.length > MAX_BODY_BYTES ? undefined : text;
+        }
+        const chunks = [];
+        let size = 0;
+        for (;;) {
+            const { done, value } = yield reader.read();
+            if (done) {
+                break;
+            }
+            size += value.byteLength;
+            if (size > MAX_BODY_BYTES) {
+                yield reader.cancel().catch(() => undefined);
+                return undefined;
+            }
+            chunks.push(value);
+        }
+        return Buffer.concat(chunks).toString("utf8");
+    });
+}
+/** The JSON-RPC message with `id` from a JSON or server-sent-events body. */
+function parseRpcBody(text, contentType, id) {
+    const candidates = contentType.includes("text/event-stream")
+        ? text
+            .split(/\r?\n/)
+            .filter((line) => line.startsWith("data:"))
+            .map((line) => line.slice(5).trim())
+        : [text];
+    for (const candidate of candidates) {
+        try {
+            const parsed = JSON.parse(candidate);
+            if (parsed && typeof parsed === "object" && parsed.jsonrpc === "2.0" && parsed.id === id) {
+                return parsed;
+            }
+        }
+        catch (_a) {
+            // not this one
+        }
+    }
+    return undefined;
+}
+function post(url, body, sessionId, timeoutMs) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a, _b;
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        try {
+            const headers = {
+                "Content-Type": "application/json",
+                Accept: "application/json, text/event-stream",
+                "MCP-Protocol-Version": PROTOCOL_VERSION,
+            };
+            if (sessionId) {
+                headers["Mcp-Session-Id"] = sessionId;
+            }
+            const response = yield fetch(url, {
+                method: "POST",
+                headers,
+                body: JSON.stringify(body),
+                signal: controller.signal,
+                // The URL came from an untrusted page and was vetted by isPublicHttpUrl; a redirect
+                // could point anywhere, so a 3xx counts as a failed check rather than being followed.
+                redirect: "manual",
+            });
+            const reply = {
+                status: response.status,
+                sessionId: (_a = response.headers.get("mcp-session-id")) !== null && _a !== void 0 ? _a : undefined,
+            };
+            if (!response.ok) {
+                throw new HttpStatusError(response.status);
+            }
+            if (typeof body.id === "number") {
+                const text = yield readCapped(response);
+                if (text !== undefined) {
+                    reply.message = parseRpcBody(text, (_b = response.headers.get("content-type")) !== null && _b !== void 0 ? _b : "", body.id);
+                }
+            }
+            return reply;
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    });
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
+}
+function toolsFrom(message) {
+    var _a;
+    const tools = (_a = asRecord(message === null || message === void 0 ? void 0 : message.result)) === null || _a === void 0 ? void 0 : _a.tools;
+    if (!Array.isArray(tools)) {
+        return undefined;
+    }
+    const out = [];
+    for (const raw of tools.slice(0, MAX_TOOLS)) {
+        const tool = asRecord(raw);
+        if (tool && typeof tool.name === "string") {
+            const entry = { name: tool.name };
+            if (typeof tool.description === "string") {
+                entry.description = tool.description.slice(0, MAX_DESCRIPTION_LENGTH);
+            }
+            out.push(entry);
+        }
+    }
+    return out;
+}
+/**
+ * Checks whether a URL is a working MCP server over Streamable HTTP: sends `initialize`,
+ * then `tools/list`, then ends the session. It never calls a tool, so it cannot submit a
+ * lead or change anything on the server. Never throws.
+ */
+function checkMcpServer(url_1) {
+    return __awaiter(this, arguments, void 0, function* (url, options = {}) {
+        var _a, _b;
+        const timeoutMs = typeof options.timeoutMs === "number" ? options.timeoutMs : DEFAULT_TIMEOUT_MS;
+        const checkedAt = new Date().toISOString();
+        if (!isPublicHttpUrl(url) || !(yield resolvesPublic(url, (_a = options.resolve) !== null && _a !== void 0 ? _a : resolveAll))) {
+            return { status: "skipped", checkedAt };
+        }
+        let init;
+        try {
+            init = yield post(url, {
+                jsonrpc: "2.0",
+                id: 1,
+                method: "initialize",
+                params: { protocolVersion: PROTOCOL_VERSION, capabilities: {}, clientInfo: CLIENT_INFO },
+            }, undefined, timeoutMs);
+        }
+        catch (e) {
+            if (e instanceof HttpStatusError) {
+                const status = e.status === 401 || e.status === 403 ? "auth-required" : "unreachable";
+                return { status, checkedAt, httpStatus: e.status };
+            }
+            return { status: "unreachable", checkedAt, error: String(e).slice(0, 200) };
+        }
+        const result = asRecord((_b = init.message) === null || _b === void 0 ? void 0 : _b.result);
+        if (!result) {
+            return { status: "not-mcp", checkedAt, httpStatus: init.status };
+        }
+        const check = { status: "live", checkedAt, httpStatus: init.status };
+        if (typeof result.protocolVersion === "string") {
+            check.protocolVersion = result.protocolVersion;
+        }
+        const serverInfo = asRecord(result.serverInfo);
+        if (serverInfo) {
+            check.serverInfo = {
+                name: typeof serverInfo.name === "string" ? serverInfo.name : undefined,
+                version: typeof serverInfo.version === "string" ? serverInfo.version : undefined,
+            };
+        }
+        const sessionId = init.sessionId;
+        try {
+            yield post(url, { jsonrpc: "2.0", method: "notifications/initialized" }, sessionId, timeoutMs);
+            const list = yield post(url, { jsonrpc: "2.0", id: 2, method: "tools/list", params: {} }, sessionId, timeoutMs);
+            const tools = toolsFrom(list.message);
+            if (tools) {
+                check.tools = tools;
+            }
+        }
+        catch (e) {
+            (0, stentor_logger_1.log)().debug(`MCP tools/list failed for ${url}: ${e}`);
+        }
+        if (sessionId) {
+            // Best effort: let the server drop the session rather than wait for it to expire.
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), timeoutMs);
+            yield fetch(url, {
+                method: "DELETE",
+                headers: { "Mcp-Session-Id": sessionId },
+                signal: controller.signal,
+                redirect: "manual",
+            })
+                .catch(() => undefined)
+                .finally(() => clearTimeout(timer));
+        }
+        return check;
+    });
+}
+//# sourceMappingURL=checkMcpServer.js.map

package/lib/utils/checkMcpServer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkMcpServer.js","sourceRoot":"","sources":["../../src/utils/checkMcpServer.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;AA0FlC,0CAeC;AAsJD,wCAyEC;AAtUD,2CAAsC;AACtC,6BAA2B;AAC3B,mDAAqC;AA8BrC,SAAe,UAAU,CAAC,QAAgB;;QACtC,OAAO,CAAC,MAAM,IAAA,iBAAM,EAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACzE,CAAC;CAAA;AAED,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,gBAAgB,GAAG,YAAY,CAAC;AACtC,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,6FAA6F;AAC7F,MAAM,cAAc,GAAG,GAAG,GAAG,IAAI,CAAC;AAClC,qFAAqF;AACrF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAE7D,SAAS,aAAa,CAAC,EAAU;IAC7B,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzC,OAAO,CACH,CAAC,KAAK,EAAE;QACR,CAAC,KAAK,GAAG;QACT,CAAC,KAAK,CAAC;QACP,CAAC,IAAI,GAAG;QACR,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC;QAClD,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QACrC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;QACxB,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QACjC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;QACxB,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CACrC,CAAC;AACN,CAAC;AAED,kFAAkF;AAClF,SAAS,eAAe,CAAC,IAAY;IACjC,sFAAsF;IACtF,MAAM,MAAM,GAAG,0CAA0C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,MAAM,GAAG,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1D,IAAI,MAAM,EAAE,CAAC;QACT,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,OAAO,GAAG,IAAA,UAAI,EAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,CAAC,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpH,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAgB,eAAe,CAAC,KAAa;IACzC,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACD,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAChE,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACpF,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAe,cAAc,CAAC,GAAW,EAAE,OAAgD;;QACvF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3D,IAAI,IAAA,UAAI,EAAC,IAAI,CAAC,EAAE,CAAC;YACb,OAAO,IAAI,CAAC,CAAC,+BAA+B;QAChD,CAAC;QACD,IAAI,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;YACtC,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5F,CAAC;QAAC,WAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;CAAA;AAED,MAAM,eAAgB,SAAQ,KAAK;IAC/B,YAA4B,MAAc;QACtC,KAAK,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC;QADA,WAAM,GAAN,MAAM,CAAQ;IAE1C,CAAC;CACJ;AAQD,oEAAoE;AACpE,SAAe,UAAU,CAAC,QAAkB;;;QACxC,MAAM,MAAM,GAAG,MAAA,MAAA,QAAQ,CAAC,IAAI,0CAAE,SAAS,kDAAI,CAAC;QAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3D,CAAC;QACD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,SAAS,CAAC;YACN,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI,EAAE,CAAC;gBACP,MAAM;YACV,CAAC;YACD,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;YACzB,IAAI,IAAI,GAAG,cAAc,EAAE,CAAC;gBACxB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC7C,OAAO,SAAS,CAAC;YACrB,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;CAAA;AAED,6EAA6E;AAC7E,SAAS,YAAY,CAAC,IAAY,EAAE,WAAmB,EAAE,EAAU;IAC/D,MAAM,UAAU,GAAa,WAAW,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAClE,CAAC,CAAC,IAAI;aACC,KAAK,CAAC,OAAO,CAAC;aACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;aAC1C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACb,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACjC,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACrC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;gBACvF,OAAO,MAAM,CAAC;YAClB,CAAC;QACL,CAAC;QAAC,WAAM,CAAC;YACL,eAAe;QACnB,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC;AACrB,CAAC;AAED,SAAe,IAAI,CACf,GAAW,EACX,IAA6B,EAC7B,SAA6B,EAC7B,SAAiB;;;QAEjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAC9D,IAAI,CAAC;YACD,MAAM,OAAO,GAA2B;gBACpC,cAAc,EAAE,kBAAkB;gBAClC,MAAM,EAAE,qCAAqC;gBAC7C,sBAAsB,EAAE,gBAAgB;aAC3C,CAAC;YACF,IAAI,SAAS,EAAE,CAAC;gBACZ,OAAO,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;YAC1C,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC9B,MAAM,EAAE,MAAM;gBACd,OAAO;gBACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC1B,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,oFAAoF;gBACpF,sFAAsF;gBACtF,QAAQ,EAAE,QAAQ;aACrB,CAAC,CAAC;YACH,MAAM,KAAK,GAAa;gBACpB,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,SAAS,EAAE,MAAA,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,mCAAI,SAAS;aACjE,CAAC;YACF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC/C,CAAC;YACD,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;gBAC9B,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;gBACxC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACrB,KAAK,CAAC,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,MAAA,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,mCAAI,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC5F,CAAC;YACL,CAAC;YACD,OAAO,KAAK,CAAC;QACjB,CAAC;gBAAS,CAAC;YACP,YAAY,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACL,CAAC;CAAA;AAED,SAAS,QAAQ,CAAC,KAAc;IAC5B,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,SAAS,CAAC;AACxH,CAAC;AAED,SAAS,SAAS,CAAC,OAA4C;;IAC3D,MAAM,KAAK,GAAG,MAAA,QAAQ,CAAC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,CAAC,0CAAE,KAAK,CAAC;IAC/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,MAAM,GAAG,GAAyC,EAAE,CAAC;IACrD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACxC,MAAM,KAAK,GAA2C,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1E,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBACvC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;YAC1E,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,SAAsB,cAAc;yDAAC,GAAW,EAAE,UAAiC,EAAE;;QACjF,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC;QACjG,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE3C,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,cAAc,CAAC,GAAG,EAAE,MAAA,OAAO,CAAC,OAAO,mCAAI,UAAU,CAAC,CAAC,EAAE,CAAC;YACvF,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;QAC5C,CAAC;QAED,IAAI,IAAc,CAAC;QACnB,IAAI,CAAC;YACD,IAAI,GAAG,MAAM,IAAI,CACb,GAAG,EACH;gBACI,OAAO,EAAE,KAAK;gBACd,EAAE,EAAE,CAAC;gBACL,MAAM,EAAE,YAAY;gBACpB,MAAM,EAAE,EAAE,eAAe,EAAE,gBAAgB,EAAE,YAAY,EAAE,EAAE,EAAE,UAAU,EAAE,WAAW,EAAE;aAC3F,EACD,SAAS,EACT,SAAS,CACZ,CAAC;QACN,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,eAAe,EAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtF,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;YACvD,CAAC;YACD,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAChF,CAAC;QAED,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAA,IAAI,CAAC,OAAO,0CAAE,MAAM,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QACrE,CAAC;QAED,MAAM,KAAK,GAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QACrF,IAAI,OAAO,MAAM,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;YAC7C,KAAK,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;QACnD,CAAC;QACD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC/C,IAAI,UAAU,EAAE,CAAC;YACb,KAAK,CAAC,UAAU,GAAG;gBACf,IAAI,EAAE,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACvE,OAAO,EAAE,OAAO,UAAU,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;aACnF,CAAC;QACN,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,2BAA2B,EAAE,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YAC/F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YAChH,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,KAAK,EAAE,CAAC;gBACR,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YACxB,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,6BAA6B,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACZ,kFAAkF;YAClF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;YAC9D,MAAM,KAAK,CAAC,GAAG,EAAE;gBACb,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,EAAE,gBAAgB,EAAE,SAAS,EAAE;gBACxC,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ;aACrB,CAAC;iBACG,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;iBACtB,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;CAAA"}

package/lib/utils/getPrompt.js

@@ -88,7 +88,7 @@ function getIndustryCategoryOptions() {
  * @param source What the content being analyzed is called, "website" or "page".
  */
 function getLicensesInstruction(source) {
-    return `"licenses" - An array of state-issued contractor or trade license numbers found on the ${source}, along with the operating authority and state board license numbers other trades publish. Focus on licenses required to operate, for example: state contractor licenses, HVAC licenses, plumbing licenses, electrical licenses; for movers a USDOT number, an MC (motor carrier) number and state authority such as CAL-T or CPUC in California, a Florida IM number or a state DOT mover registration; for salons, spas and med spas state cosmetology, barber, esthetician, nail salon establishment or massage therapy licenses, and the medical director's license when a med spa publishes one. Do not include general business registrations, ISO certifications, or membership numbers. Use an object with keys: number (just the license number itself), type (the type of license such as HVAC, Plumbing, Electrical, General Contractor, Roofing, USDOT, MC, CAL-T, Cosmetology, Barber, Esthetician, Massage Therapy, Medical Director, etc), state (two-letter state code if identifiable from context, leave it out for federal numbers such as USDOT and MC), fullText (the complete license text exactly as shown on the ${source}).`;
+    return `"licenses" - An array of state-issued contractor or trade license numbers found on the ${source}, along with the operating authority and state board license numbers other trades publish. Focus on licenses required to operate, for example: state contractor licenses, HVAC licenses, plumbing licenses, electrical licenses; for movers a USDOT number, an MC (motor carrier) number and state authority such as CAL-T or CPUC in California, a Florida IM number or a state DOT mover registration; for salons, spas and med spas state cosmetology, barber, esthetician, nail salon establishment or massage therapy licenses, and the medical director's license when a med spa publishes one; for financial advisors and planners a FINRA CRD number (the firm's or the individual's, often shown beside a BrokerCheck link), an SEC file number such as 801-xxxxx or 8-xxxxx, an investment adviser representative (IARD) number, FINRA securities registrations by series (Series 7, 63, 65, 66), the CFP certification, and state insurance producer licenses. Do not include general business registrations, ISO certifications, or membership numbers. For financial firms do not include SIPC or FDIC membership, which every member states and which identifies nobody. Use an object with keys: number (just the license number itself), type (the type of license such as HVAC, Plumbing, Electrical, General Contractor, Roofing, USDOT, MC, CAL-T, Cosmetology, Barber, Esthetician, Massage Therapy, Medical Director, CRD, SEC File Number, IARD, Series 7, Series 65, CFP, Insurance Producer, etc), state (two-letter state code if identifiable from context, leave it out for federal numbers such as USDOT and MC), fullText (the complete license text exactly as shown on the ${source}).`;
 }
 /**
  * Calculates the number of tokens in a prompt and the maximum tokens available for the response.

package/lib/utils/getPrompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"getPrompt.js","sourceRoot":"","sources":["../../src/utils/getPrompt.ts"],"names":[],"mappings":";;;AA6EA,gDAIC;AAyCD,8BAOC;AAED,oDAmCC;AASD,4CAuBC;AAED,kDAmBC;AA3ND,kCAAkC;AAClC,sCAAkE;AAElE,0CAA0C;AAC1C,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAE5C,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;GAGG;AACU,QAAA,uBAAuB,GAAG,CAAC,iBAAiB,CAAU,CAAC;AAEpE;;;;;GAKG;AACU,QAAA,8BAA8B,GAGvC;IACA,qBAAqB,EACjB,6FAA6F;QAC7F,iFAAiF;IACrF,mBAAmB,EAAE,wDAAwD;IAC7E,UAAU,EAAE,uCAAuC;IACnD,oBAAoB,EAAE,sDAAsD;IAC5E,wBAAwB,EAAE,0CAA0C;IACpE,2BAA2B,EACvB,2FAA2F;QAC3F,+EAA+E;IACnF,uBAAuB,EAAE,6DAA6D;IACtF,cAAc,EAAE,uBAAuB;IACvC,SAAS,EAAE,8BAA8B;IACzC,gBAAgB,EAAE,wCAAwC;IAC1D,eAAe,EAAE,wBAAwB;IACzC,aAAa,EACT,sNAAsN;IAC1N,iBAAiB,EAAE,iDAAiD;IACpE,KAAK,EAAE,iCAAiC;IACxC,aAAa,EAAE,gCAAgC;IAC/C,WAAW,EAAE,wEAAwE;IACrF,gBAAgB,EACZ,qFAAqF;QACrF,6EAA6E;IACjF,qBAAqB,EAAE,sCAAsC;IAC7D,MAAM,EAAE,eAAe;IACvB,qBAAqB,EAAE,8BAA8B;CACxD,CAAC;AAEF;;;GAGG;AACH,MAAM,mBAAmB,GAAwB,IAAI,GAAG,CACpD,4BAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAE,+BAA6C,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CACjG,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,SAAgB,kBAAkB,CAAC,KAAgC;IAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACtE,OAAO,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAE,UAA+B,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9F,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B;IAC/B,OAAO,MAAM,CAAC,OAAO,CAAC,sCAA8B,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,QAAQ,QAAQ,OAAO,WAAW,EAAE,CAAC;SACtE,IAAI,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,MAA0B;IACtD,OAAO,0FAA0F,MAAM,4kCAA4kC,MAAM,IAAI,CAAC;AAClsC,CAAC;AAiBD;;;;;GAKG;AACH,SAAgB,SAAS,CAAC,MAAc,EAAE,aAAqB,KAAK;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAErC,4CAA4C;IAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,eAAe,CAAC,CAAC;IAErE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED,SAAgB,oBAAoB,CAAC,QAAgB,EAAE,GAAW;IAC9D,MAAM,MAAM,GAAG;0FACuE,GAAG;;;EAG3F,QAAQ;;;;;;;;;;;;;;;;;;;;EAoBR,0BAA0B,EAAE;EAC5B,sBAAsB,CAAC,SAAS,CAAC;CAClC,CAAC;IAEE,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAEjC,uBACI,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAClB,MAAM,EACX;AACN,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,gBAAgB,CAAC,QAAgB,EAAE,GAAW;IAC1D,MAAM,MAAM,GAAG;0GACuF,GAAG;;;EAG3G,QAAQ;;;;;;;;;EASR,sBAAsB,CAAC,MAAM,CAAC;CAC/B,CAAC;IAEE,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAEjC,uBACI,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAClB,MAAM,EACX;AACN,CAAC;AAED,SAAgB,mBAAmB,CAAC,IAAS,EAAE,GAAW;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAExC,MAAM,MAAM,GAAG;EACjB,UAAU;;kDAEsC,GAAG;;;CAGpD,CAAC;IAEE,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAElC,uBACI,MAAM,EAAE,OAAO,IACZ,MAAM,EACX;AACN,CAAC"}
+{"version":3,"file":"getPrompt.js","sourceRoot":"","sources":["../../src/utils/getPrompt.ts"],"names":[],"mappings":";;;AA6EA,gDAIC;AAyCD,8BAOC;AAED,oDAmCC;AASD,4CAuBC;AAED,kDAmBC;AA3ND,kCAAkC;AAClC,sCAAkE;AAElE,0CAA0C;AAC1C,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAE5C,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;GAGG;AACU,QAAA,uBAAuB,GAAG,CAAC,iBAAiB,CAAU,CAAC;AAEpE;;;;;GAKG;AACU,QAAA,8BAA8B,GAGvC;IACA,qBAAqB,EACjB,6FAA6F;QAC7F,iFAAiF;IACrF,mBAAmB,EAAE,wDAAwD;IAC7E,UAAU,EAAE,uCAAuC;IACnD,oBAAoB,EAAE,sDAAsD;IAC5E,wBAAwB,EAAE,0CAA0C;IACpE,2BAA2B,EACvB,2FAA2F;QAC3F,+EAA+E;IACnF,uBAAuB,EAAE,6DAA6D;IACtF,cAAc,EAAE,uBAAuB;IACvC,SAAS,EAAE,8BAA8B;IACzC,gBAAgB,EAAE,wCAAwC;IAC1D,eAAe,EAAE,wBAAwB;IACzC,aAAa,EACT,sNAAsN;IAC1N,iBAAiB,EAAE,iDAAiD;IACpE,KAAK,EAAE,iCAAiC;IACxC,aAAa,EAAE,gCAAgC;IAC/C,WAAW,EAAE,wEAAwE;IACrF,gBAAgB,EACZ,qFAAqF;QACrF,6EAA6E;IACjF,qBAAqB,EAAE,sCAAsC;IAC7D,MAAM,EAAE,eAAe;IACvB,qBAAqB,EAAE,8BAA8B;CACxD,CAAC;AAEF;;;GAGG;AACH,MAAM,mBAAmB,GAAwB,IAAI,GAAG,CACpD,4BAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAE,+BAA6C,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CACjG,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,SAAgB,kBAAkB,CAAC,KAAgC;IAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACtE,OAAO,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAE,UAA+B,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9F,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B;IAC/B,OAAO,MAAM,CAAC,OAAO,CAAC,sCAA8B,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,QAAQ,QAAQ,OAAO,WAAW,EAAE,CAAC;SACtE,IAAI,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,MAA0B;IACtD,OAAO,0FAA0F,MAAM,6mDAA6mD,MAAM,IAAI,CAAC;AACnuD,CAAC;AAiBD;;;;;GAKG;AACH,SAAgB,SAAS,CAAC,MAAc,EAAE,aAAqB,KAAK;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAErC,4CAA4C;IAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,eAAe,CAAC,CAAC;IAErE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED,SAAgB,oBAAoB,CAAC,QAAgB,EAAE,GAAW;IAC9D,MAAM,MAAM,GAAG;0FACuE,GAAG;;;EAG3F,QAAQ;;;;;;;;;;;;;;;;;;;;EAoBR,0BAA0B,EAAE;EAC5B,sBAAsB,CAAC,SAAS,CAAC;CAClC,CAAC;IAEE,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAEjC,uBACI,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAClB,MAAM,EACX;AACN,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,gBAAgB,CAAC,QAAgB,EAAE,GAAW;IAC1D,MAAM,MAAM,GAAG;0GACuF,GAAG;;;EAG3G,QAAQ;;;;;;;;;EASR,sBAAsB,CAAC,MAAM,CAAC;CAC/B,CAAC;IAEE,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAEjC,uBACI,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAClB,MAAM,EACX;AACN,CAAC;AAED,SAAgB,mBAAmB,CAAC,IAAS,EAAE,GAAW;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAExC,MAAM,MAAM,GAAG;EACjB,UAAU;;kDAEsC,GAAG;;;CAGpD,CAAC;IAEE,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAElC,uBACI,MAAM,EAAE,OAAO,IACZ,MAAM,EACX;AACN,CAAC"}

package/lib/utils/pageAgenticSignals.d.ts

@@ -0,0 +1,71 @@
+/*! Copyright (c) 2026, XAPP AI */
+import { AgenticProvider } from "./agenticProvider";
+import type { McpServerCheck } from "./checkMcpServer";
+/**
+ * An MCP discovery tag in the rendered page, e.g. `<link rel="mcp-server" href="...">` or
+ * `<meta name="mcp-server" content="...">`. For a meta tag, `rel` is its name and `href` its content.
+ */
+export interface McpPageTag {
+    tag: "link" | "meta";
+    rel: string;
+    href: string;
+}
+/** A tool the page registered for in-browser agents through WebMCP (`modelContext`). */
+export interface WebMcpTool {
+    /** Which object it was registered on: `document.modelContext` (the spec) or `navigator.modelContext`. */
+    api: "document" | "navigator";
+    name: string;
+    description?: string;
+}
+/** Where a site's MCP server was advertised. */
+export type McpServerSource = "server-card" | "page-tag";
+/** One MCP server a site points agents at. */
+export interface McpServerRef {
+    url: string;
+    foundIn: McpServerSource[];
+    provider?: AgenticProvider;
+    /** Present only when the crawl asked for `checkMcpServers`. */
+    check?: McpServerCheck;
+}
+/** The window property the recorder writes to. Not enumerable, so page scripts iterating window miss it. */
+export declare const WEBMCP_RECORDER_KEY = "__arachneWebMcp";
+/**
+ * Finds MCP discovery tags in rendered HTML: any `<link>` with a rel token starting with
+ * "mcp" (mcp-server, mcp-manifest, mcp-server-card) and any `<meta>` whose name does.
+ * Pass the HTML from `page.content()` so tags a script injected (the XAPP chat widget
+ * does) are included.
+ */
+export declare function extractMcpPageTags(html: string, baseUrl: string): McpPageTag[];
+/**
+ * Installs a stand-in `modelContext` on `document` and `navigator` that records every tool
+ * the page registers, so a crawl sees a site's WebMCP tools the way an in-browser agent
+ * would. A real `modelContext` is left untouched. Registering does nothing else: no tool is
+ * ever executed.
+ *
+ * Runs in the browser through `page.evaluateOnNewDocument`, so it must stay self-contained.
+ * The `scope` parameter exists for tests; in the page it is the window.
+ */
+export declare function webMcpRecorder(scope?: unknown): void;
+/**
+ * Gives every `requestIdleCallback` a 2s deadline unless the page set one. Headless Chrome
+ * can go a whole crawl without an idle period on a busy page, so scripts deferred to idle
+ * time - Next.js `strategy="lazyOnload"`, which is how portao.one loads the XAPP chat
+ * widget - never run, although a person's browser runs them within a second or two.
+ *
+ * Runs in the browser through `page.evaluateOnNewDocument`, so it must stay self-contained.
+ */
+export declare function idleCallbackDeadline(scope?: unknown): void;
+/** Validates, dedupes and bounds what the recorder captured. Undefined when nothing usable. */
+export declare function sanitizeWebMcpTools(raw: unknown): WebMcpTool[] | undefined;
+/**
+ * Every MCP server a site advertises - in its server card or with an `mcp-server` page tag -
+ * merged by URL. Manifest and server-card links are not servers and are skipped.
+ */
+export declare function collectMcpServers(signals: {
+    mcpServerCard?: {
+        sourceUrl: string;
+        endpointUrl?: string;
+        provider?: AgenticProvider;
+    };
+    mcpTags?: McpPageTag[];
+}): McpServerRef[] | undefined;