@automatelab/citation-intelligence 0.5.0 → 0.6.0

package/dist/tools/check-citations.js

@@ -5,13 +5,23 @@ import { claudeSearch } from "../adapters/anthropic.js";
 import { openaiSearch } from "../adapters/openai.js";
 import { geminiSearch } from "../adapters/gemini.js";
 import { braveSearch } from "../adapters/brave.js";
+import { googleAiModeSearch } from "../adapters/google-ai-mode.js";
 import { envKey } from "../lib/config.js";
 import { getCitations, putCitations } from "../lib/cache.js";
 import { ToolFetchError } from "../lib/fetch.js";
 export const checkCitationsInputSchema = {
     query: z.string().min(1).describe("The search query to test (what would a user ask an AI?)"),
     engine: z
-        .enum(["perplexity", "claude", "openai", "gemini", "bing", "brave", "auto"])
+        .enum([
+        "perplexity",
+        "claude",
+        "openai",
+        "gemini",
+        "bing",
+        "brave",
+        "google_ai_mode",
+        "auto",
+    ])
         .default("auto")
         .describe("AI engine to query. 'auto' picks the first available based on configured API keys."),
     max_results: z
@@ -36,6 +46,8 @@ function pickAutoEngine() {
         return "brave";
     if (envKey("BING_API_KEY"))
         return "bing";
+    if (envKey("SERPAPI_KEY"))
+        return "google_ai_mode";
     return null;
 }
 async function runEngine(engine, query, maxResults) {
@@ -52,6 +64,8 @@ async function runEngine(engine, query, maxResults) {
             return bingSearch(query, maxResults);
         case "brave":
             return braveSearch(query, maxResults);
+        case "google_ai_mode":
+            return googleAiModeSearch(query, maxResults);
         case "auto":
             throw new Error("auto engine should be resolved before runEngine");
     }

package/dist/tools/check-citations.js.map

@@ -1 +1 @@
-{"version":3,"file":"check-citations.js","sourceRoot":"","sources":["../../src/tools/check-citations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,yDAAyD,CAAC;IAC5F,MAAM,EAAE,CAAC;SACN,IAAI,CAAC,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC3E,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CAAC,oFAAoF,CAAC;IACjG,WAAW,EAAE,CAAC;SACX,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CAAC,8BAA8B,CAAC;CAC5C,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;AAExD,SAAS,cAAc;IACrB,IAAI,MAAM,CAAC,oBAAoB,CAAC;QAAE,OAAO,YAAY,CAAC;IACtD,IAAI,MAAM,CAAC,mBAAmB,CAAC;QAAE,OAAO,QAAQ,CAAC;IACjD,IAAI,MAAM,CAAC,gBAAgB,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC9C,IAAI,MAAM,CAAC,gBAAgB,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC9C,IAAI,MAAM,CAAC,eAAe,CAAC;QAAE,OAAO,OAAO,CAAC;IAC5C,IAAI,MAAM,CAAC,cAAc,CAAC;QAAE,OAAO,MAAM,CAAC;IAC1C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,MAAc,EACd,KAAa,EACb,UAAkB;IAElB,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,YAAY;YACf,OAAO,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC7C,KAAK,QAAQ;YACX,OAAO,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACzC,KAAK,QAAQ;YACX,OAAO,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACzC,KAAK,QAAQ;YACX,OAAO,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACzC,KAAK,MAAM;YACT,OAAO,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACvC,KAAK,OAAO;YACV,OAAO,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACxC,KAAK,MAAM;YACT,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAkC;IACrE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,SAAS,GAAW,MAAM,CAAC,MAAM,CAAC;IAExC,IAAI,MAAc,CAAC;IACnB,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;QAChC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,cAAc,CAAC;gBACvB,IAAI,EAAE,qBAAqB;gBAC3B,OAAO,EACL,gIAAgI;aACnI,CAAC,CAAC;QACL,CAAC;QACD,MAAM,GAAG,MAAM,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,SAAS,CAAC;IACrB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACxD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM;YACN,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC;YACxD,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE5C,MAAM,YAAY,CAAC;QACjB,IAAI,EAAE,gBAAgB;QACtB,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU;QACV,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC,CAAC;IAEH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM;QACN,UAAU;QACV,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,MAAM,EAAE,KAAK;KACd,CAAC;AACJ,CAAC"}
+{"version":3,"file":"check-citations.js","sourceRoot":"","sources":["../../src/tools/check-citations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,yDAAyD,CAAC;IAC5F,MAAM,EAAE,CAAC;SACN,IAAI,CAAC;QACJ,YAAY;QACZ,QAAQ;QACR,QAAQ;QACR,QAAQ;QACR,MAAM;QACN,OAAO;QACP,gBAAgB;QAChB,MAAM;KACP,CAAC;SACD,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CAAC,oFAAoF,CAAC;IACjG,WAAW,EAAE,CAAC;SACX,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CAAC,8BAA8B,CAAC;CAC5C,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;AAExD,SAAS,cAAc;IACrB,IAAI,MAAM,CAAC,oBAAoB,CAAC;QAAE,OAAO,YAAY,CAAC;IACtD,IAAI,MAAM,CAAC,mBAAmB,CAAC;QAAE,OAAO,QAAQ,CAAC;IACjD,IAAI,MAAM,CAAC,gBAAgB,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC9C,IAAI,MAAM,CAAC,gBAAgB,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC9C,IAAI,MAAM,CAAC,eAAe,CAAC;QAAE,OAAO,OAAO,CAAC;IAC5C,IAAI,MAAM,CAAC,cAAc,CAAC;QAAE,OAAO,MAAM,CAAC;IAC1C,IAAI,MAAM,CAAC,aAAa,CAAC;QAAE,OAAO,gBAAgB,CAAC;IACnD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,MAAc,EACd,KAAa,EACb,UAAkB;IAElB,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,YAAY;YACf,OAAO,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC7C,KAAK,QAAQ;YACX,OAAO,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACzC,KAAK,QAAQ;YACX,OAAO,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACzC,KAAK,QAAQ;YACX,OAAO,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACzC,KAAK,MAAM;YACT,OAAO,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACvC,KAAK,OAAO;YACV,OAAO,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACxC,KAAK,gBAAgB;YACnB,OAAO,kBAAkB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC/C,KAAK,MAAM;YACT,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAkC;IACrE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,SAAS,GAAW,MAAM,CAAC,MAAM,CAAC;IAExC,IAAI,MAAc,CAAC;IACnB,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;QAChC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,cAAc,CAAC;gBACvB,IAAI,EAAE,qBAAqB;gBAC3B,OAAO,EACL,gIAAgI;aACnI,CAAC,CAAC;QACL,CAAC;QACD,MAAM,GAAG,MAAM,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,SAAS,CAAC;IACrB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACxD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM;YACN,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC;YACxD,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE5C,MAAM,YAAY,CAAC;QACjB,IAAI,EAAE,gBAAgB;QACtB,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU;QACV,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC,CAAC;IAEH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM;QACN,UAAU;QACV,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,MAAM,EAAE,KAAK;KACd,CAAC;AACJ,CAAC"}

package/dist/tools/citation-evidence.d.ts

@@ -0,0 +1,54 @@
+import { z } from "zod";
+export declare const citationEvidenceInputSchema: {
+    query: z.ZodString;
+    engine: z.ZodDefault<z.ZodEnum<["perplexity", "claude", "openai", "gemini", "bing", "brave", "auto"]>>;
+    max_results: z.ZodDefault<z.ZodNumber>;
+    context_chars: z.ZodDefault<z.ZodNumber>;
+};
+declare const inputSchema: z.ZodObject<{
+    query: z.ZodString;
+    engine: z.ZodDefault<z.ZodEnum<["perplexity", "claude", "openai", "gemini", "bing", "brave", "auto"]>>;
+    max_results: z.ZodDefault<z.ZodNumber>;
+    context_chars: z.ZodDefault<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    query: string;
+    engine: "perplexity" | "claude" | "openai" | "gemini" | "bing" | "brave" | "auto";
+    max_results: number;
+    context_chars: number;
+}, {
+    query: string;
+    engine?: "perplexity" | "claude" | "openai" | "gemini" | "bing" | "brave" | "auto" | undefined;
+    max_results?: number | undefined;
+    context_chars?: number | undefined;
+}>;
+export declare function citationEvidence(input: z.infer<typeof inputSchema>): Promise<{
+    query: string;
+    engine: import("../types.js").Engine;
+    fetched_at: string;
+    raw_answer_chars: number;
+    has_raw_answer: boolean;
+    citations_total: number;
+    evidence_found: number;
+    evidence: ({
+        url: string;
+        rank: number;
+        title: string | undefined;
+        found: boolean;
+        snippet: string | null;
+        nearby_quote: null;
+        mention_char: null;
+        matched?: undefined;
+    } | {
+        url: string;
+        rank: number;
+        title: string | undefined;
+        found: boolean;
+        snippet: string;
+        nearby_quote: string | null;
+        mention_char: number;
+        matched: string;
+    })[];
+    note: string;
+}>;
+export {};
+//# sourceMappingURL=citation-evidence.d.ts.map

package/dist/tools/citation-evidence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"citation-evidence.d.ts","sourceRoot":"","sources":["../../src/tools/citation-evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,2BAA2B;;;;;CA2BvC,CAAC;AAEF,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;EAAwC,CAAC;AAkF1D,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDxE"}

package/dist/tools/citation-evidence.js

@@ -0,0 +1,148 @@
+import { z } from "zod";
+import { checkCitations } from "./check-citations.js";
+import { log } from "../lib/log.js";
+export const citationEvidenceInputSchema = {
+    query: z
+        .string()
+        .min(1)
+        .describe("Search query whose AI answer to extract citation evidence from."),
+    engine: z
+        .enum(["perplexity", "claude", "openai", "gemini", "bing", "brave", "auto"])
+        .default("auto")
+        .describe("AI engine to query. Engines without raw_answer (Bing, Brave) return no evidence."),
+    max_results: z
+        .number()
+        .int()
+        .min(1)
+        .max(20)
+        .default(10)
+        .describe("Max citations to extract evidence for."),
+    context_chars: z
+        .number()
+        .int()
+        .min(40)
+        .max(800)
+        .default(240)
+        .describe("Half-width of the snippet window around each citation mention (chars). Total snippet is up to 2x this."),
+};
+const inputSchema = z.object(citationEvidenceInputSchema);
+function candidateMentions(url) {
+    const set = new Set();
+    set.add(url);
+    set.add(url.replace(/^https?:\/\//, ""));
+    set.add(url.replace(/^https?:\/\/(www\.)?/, ""));
+    try {
+        const u = new URL(url);
+        set.add(u.hostname);
+        set.add(u.hostname.replace(/^www\./, ""));
+    }
+    catch {
+        // ignore
+    }
+    return [...set].filter(Boolean);
+}
+function findFirstMention(url, text) {
+    if (!text)
+        return null;
+    let best = null;
+    for (const c of candidateMentions(url)) {
+        const idx = text.indexOf(c);
+        if (idx !== -1 && (!best || idx < best.index))
+            best = { index: idx, matched: c };
+    }
+    return best;
+}
+function extractWindow(text, centerStart, centerLen, halfWidth) {
+    const start = Math.max(0, centerStart - halfWidth);
+    const end = Math.min(text.length, centerStart + centerLen + halfWidth);
+    let snippet = text.slice(start, end);
+    if (start > 0)
+        snippet = `…${snippet}`;
+    if (end < text.length)
+        snippet = `${snippet}…`;
+    return { snippet, window_start: start, window_end: end };
+}
+// Try to extract a quoted span if one wraps the URL mention. Falls back to
+// sentence containing the mention. Otherwise empty.
+function nearestQuotedOrSentence(text, mentionIdx) {
+    if (!text || mentionIdx < 0)
+        return undefined;
+    const QUOTE_CHARS = ['"', "“", "”", "「", "」"];
+    // search backward and forward for nearest matching quote pair
+    for (const q of QUOTE_CHARS) {
+        const left = text.lastIndexOf(q, mentionIdx);
+        if (left === -1)
+            continue;
+        const right = text.indexOf(q, mentionIdx);
+        if (right === -1 || right - left > 800)
+            continue;
+        const inner = text.slice(left + 1, right).trim();
+        if (inner.length >= 8)
+            return inner;
+    }
+    // fallback: containing sentence
+    const sentStart = Math.max(text.lastIndexOf(".", mentionIdx), text.lastIndexOf("!", mentionIdx), text.lastIndexOf("?", mentionIdx), text.lastIndexOf("\n", mentionIdx));
+    const sentEnd = (() => {
+        const candidates = [
+            text.indexOf(".", mentionIdx),
+            text.indexOf("!", mentionIdx),
+            text.indexOf("?", mentionIdx),
+            text.indexOf("\n", mentionIdx),
+        ].filter((i) => i !== -1);
+        return candidates.length > 0 ? Math.min(...candidates) : text.length;
+    })();
+    const sent = text.slice(sentStart + 1, sentEnd + 1).trim();
+    if (sent.length >= 12 && sent.length <= 600)
+        return sent;
+    return undefined;
+}
+export async function citationEvidence(input) {
+    const parsed = inputSchema.parse(input);
+    log.debug("citation_evidence start", { query: parsed.query });
+    const res = await checkCitations({
+        query: parsed.query,
+        engine: parsed.engine,
+        max_results: parsed.max_results,
+    });
+    const raw = res.raw_answer ?? "";
+    const has_raw = raw.length > 0;
+    const evidence = res.citations.map((c) => {
+        const mention = has_raw ? findFirstMention(c.url, raw) : null;
+        if (!mention) {
+            return {
+                url: c.url,
+                rank: c.rank,
+                title: c.title,
+                found: false,
+                snippet: c.snippet ?? null,
+                nearby_quote: null,
+                mention_char: null,
+            };
+        }
+        const { snippet } = extractWindow(raw, mention.index, mention.matched.length, parsed.context_chars);
+        return {
+            url: c.url,
+            rank: c.rank,
+            title: c.title,
+            found: true,
+            snippet,
+            nearby_quote: nearestQuotedOrSentence(raw, mention.index) ?? null,
+            mention_char: mention.index,
+            matched: mention.matched,
+        };
+    });
+    return {
+        query: parsed.query,
+        engine: res.engine,
+        fetched_at: new Date().toISOString(),
+        raw_answer_chars: raw.length,
+        has_raw_answer: has_raw,
+        citations_total: res.citations.length,
+        evidence_found: evidence.filter((e) => e.found).length,
+        evidence,
+        note: has_raw
+            ? "snippet is a window around the first mention of the URL/hostname in the engine's raw_answer. nearby_quote tries to extract a quoted span or containing sentence."
+            : "engine returned no raw_answer (Bing/Brave style listings, or empty answer). evidence is limited to whatever per-citation snippet the adapter parsed.",
+    };
+}
+//# sourceMappingURL=citation-evidence.js.map

package/dist/tools/citation-evidence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"citation-evidence.js","sourceRoot":"","sources":["../../src/tools/citation-evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAC;AAEpC,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CAAC,iEAAiE,CAAC;IAC9E,MAAM,EAAE,CAAC;SACN,IAAI,CAAC,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC3E,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CACP,kFAAkF,CACnF;IACH,WAAW,EAAE,CAAC;SACX,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CAAC,wCAAwC,CAAC;IACrD,aAAa,EAAE,CAAC;SACb,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,EAAE,CAAC;SACP,GAAG,CAAC,GAAG,CAAC;SACR,OAAO,CAAC,GAAG,CAAC;SACZ,QAAQ,CACP,wGAAwG,CACzG;CACJ,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,2BAA2B,CAAC,CAAC;AAE1D,SAAS,iBAAiB,CAAC,GAAW;IACpC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACb,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,CAAC;IACzC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,CAAC;IACjD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACpB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,SAAS;IACX,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,gBAAgB,CACvB,GAAW,EACX,IAAY;IAEZ,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,IAAI,IAAI,GAA8C,IAAI,CAAC;IAC3D,KAAK,MAAM,CAAC,IAAI,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC;YAAE,IAAI,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACnF,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,WAAmB,EACnB,SAAiB,EACjB,SAAiB;IAEjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,SAAS,CAAC,CAAC;IACvE,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACrC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IACvC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM;QAAE,OAAO,GAAG,GAAG,OAAO,GAAG,CAAC;IAC/C,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;AAC3D,CAAC;AAED,2EAA2E;AAC3E,oDAAoD;AACpD,SAAS,uBAAuB,CAC9B,IAAY,EACZ,UAAkB;IAElB,IAAI,CAAC,IAAI,IAAI,UAAU,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9C,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAC7C,IAAI,IAAI,KAAK,CAAC,CAAC;YAAE,SAAS;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAC1C,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,GAAG,IAAI,GAAG,GAAG;YAAE,SAAS;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACjD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;IACtC,CAAC;IACD,gCAAgC;IAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,EACjC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,EACjC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,EACjC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,CACnC,CAAC;IACF,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE;QACpB,MAAM,UAAU,GAAG;YACjB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC;SAC/B,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1B,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IACvE,CAAC,CAAC,EAAE,CAAC;IACL,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3D,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACzD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAkC;IACvE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxC,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAE9D,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;IAE/B,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACvC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9D,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;gBACL,GAAG,EAAE,CAAC,CAAC,GAAG;gBACV,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI;gBAC1B,YAAY,EAAE,IAAI;gBAClB,YAAY,EAAE,IAAI;aACnB,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,OAAO,EAAE,GAAG,aAAa,CAC/B,GAAG,EACH,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,OAAO,CAAC,MAAM,EACtB,MAAM,CAAC,aAAa,CACrB,CAAC;QACF,OAAO;YACL,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,KAAK,EAAE,IAAI;YACX,OAAO;YACP,YAAY,EAAE,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI;YACjE,YAAY,EAAE,OAAO,CAAC,KAAK;YAC3B,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,gBAAgB,EAAE,GAAG,CAAC,MAAM;QAC5B,cAAc,EAAE,OAAO;QACvB,eAAe,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;QACrC,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM;QACtD,QAAQ;QACR,IAAI,EAAE,OAAO;YACX,CAAC,CAAC,kKAAkK;YACpK,CAAC,CAAC,sJAAsJ;KAC3J,CAAC;AACJ,CAAC"}

package/dist/tools/crawler-access-audit.d.ts

@@ -0,0 +1,49 @@
+import { z } from "zod";
+export declare const crawlerAccessAuditInputSchema: {
+    url: z.ZodString;
+    bots: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    fetch_with_ua: z.ZodDefault<z.ZodBoolean>;
+};
+declare const inputSchema: z.ZodObject<{
+    url: z.ZodString;
+    bots: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    fetch_with_ua: z.ZodDefault<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    fetch_with_ua: boolean;
+    bots?: string[] | undefined;
+}, {
+    url: string;
+    bots?: string[] | undefined;
+    fetch_with_ua?: boolean | undefined;
+}>;
+type BotResult = {
+    name: string;
+    ua_token: string;
+    operator: string;
+    purpose: string;
+    robots_allowed: boolean | "unknown";
+    robots_rule: string | null;
+    fetch_status: number | null;
+    fetch_ok: boolean | null;
+    fetch_error: string | null;
+    verdict: "allowed" | "blocked" | "robots_only_allowed" | "robots_only_blocked" | "unknown";
+};
+export declare function crawlerAccessAudit(input: z.infer<typeof inputSchema>): Promise<{
+    url: string;
+    robots_url: string;
+    robots_status: number | null;
+    robots_present: boolean;
+    robots_error: string | null;
+    fetched_at: string;
+    bots: BotResult[];
+    summary: {
+        total: number;
+        allowed: number;
+        blocked: number;
+        unknown: number;
+    };
+    note: string;
+}>;
+export {};
+//# sourceMappingURL=crawler-access-audit.d.ts.map

package/dist/tools/crawler-access-audit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler-access-audit.d.ts","sourceRoot":"","sources":["../../src/tools/crawler-access-audit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAiBxB,eAAO,MAAM,6BAA6B;;;;CAmBzC,CAAC;AAEF,QAAA,MAAM,WAAW;;;;;;;;;;;;EAA0C,CAAC;AA8G5D,KAAK,SAAS,GAAG;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,SAAS,GAAG,SAAS,GAAG,qBAAqB,GAAG,qBAAqB,GAAG,SAAS,CAAC;CAC5F,CAAC;AAEF,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC;;;;;;;;;;;;;;;GAsJ1E"}

package/dist/tools/crawler-access-audit.js

@@ -0,0 +1,256 @@
+import { z } from "zod";
+import robotsParserDefault from "robots-parser";
+import { fetchText } from "../lib/fetch.js";
+import { log } from "../lib/log.js";
+const robotsParser = robotsParserDefault;
+export const crawlerAccessAuditInputSchema = {
+    url: z
+        .string()
+        .url()
+        .describe("Page URL to test for AI crawler access."),
+    bots: z
+        .array(z.string())
+        .min(1)
+        .max(20)
+        .optional()
+        .describe("Override the default bot list. Each entry is a User-Agent token (e.g. 'GPTBot', 'ClaudeBot')."),
+    fetch_with_ua: z
+        .boolean()
+        .default(true)
+        .describe("If true, do a live GET as each bot's User-Agent and report status. Disable to only parse robots.txt (no extra requests)."),
+};
+const inputSchema = z.object(crawlerAccessAuditInputSchema);
+// Curated list of the AI crawlers that matter for LLM citation visibility.
+// User-agent tokens are the canonical strings each operator documents.
+const DEFAULT_BOTS = [
+    {
+        name: "GPTBot",
+        ua_token: "GPTBot",
+        ua_full: "Mozilla/5.0 (compatible; GPTBot/1.0; +https://openai.com/gptbot)",
+        operator: "OpenAI",
+        purpose: "ChatGPT training data",
+    },
+    {
+        name: "OAI-SearchBot",
+        ua_token: "OAI-SearchBot",
+        ua_full: "Mozilla/5.0 (compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)",
+        operator: "OpenAI",
+        purpose: "ChatGPT Search index",
+    },
+    {
+        name: "ChatGPT-User",
+        ua_token: "ChatGPT-User",
+        ua_full: "Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)",
+        operator: "OpenAI",
+        purpose: "Real-time fetch on user prompt",
+    },
+    {
+        name: "ClaudeBot",
+        ua_token: "ClaudeBot",
+        ua_full: "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
+        operator: "Anthropic",
+        purpose: "Claude training data",
+    },
+    {
+        name: "Claude-Web",
+        ua_token: "Claude-Web",
+        ua_full: "Mozilla/5.0 (compatible; Claude-Web/1.0; +https://www.anthropic.com)",
+        operator: "Anthropic",
+        purpose: "Real-time fetch when Claude browses",
+    },
+    {
+        name: "PerplexityBot",
+        ua_token: "PerplexityBot",
+        ua_full: "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://docs.perplexity.ai/guides/bots)",
+        operator: "Perplexity",
+        purpose: "Perplexity index",
+    },
+    {
+        name: "Perplexity-User",
+        ua_token: "Perplexity-User",
+        ua_full: "Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://docs.perplexity.ai/guides/bots)",
+        operator: "Perplexity",
+        purpose: "Real-time fetch on user query",
+    },
+    {
+        name: "CCBot",
+        ua_token: "CCBot",
+        ua_full: "CCBot/2.0 (https://commoncrawl.org/faq/)",
+        operator: "Common Crawl",
+        purpose: "Used by many LLM training corpora",
+    },
+    {
+        name: "Google-Extended",
+        ua_token: "Google-Extended",
+        ua_full: "Google-Extended",
+        operator: "Google",
+        purpose: "Gemini training opt-out token (robots-only, no live fetch)",
+    },
+    {
+        name: "Applebot-Extended",
+        ua_token: "Applebot-Extended",
+        ua_full: "Applebot-Extended",
+        operator: "Apple",
+        purpose: "Apple Intelligence training opt-out (robots-only)",
+    },
+    {
+        name: "Bytespider",
+        ua_token: "Bytespider",
+        ua_full: "Mozilla/5.0 (compatible; Bytespider; spider-feedback@bytedance.com)",
+        operator: "ByteDance",
+        purpose: "Doubao/Volcengine LLM training",
+    },
+    {
+        name: "Meta-ExternalAgent",
+        ua_token: "Meta-ExternalAgent",
+        ua_full: "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
+        operator: "Meta",
+        purpose: "Llama training data fetch",
+    },
+];
+// robots-only (cannot be live-tested because the operator does not actually
+// fetch the URL - it's an opt-out token consumed by their pipeline).
+const ROBOTS_ONLY = new Set(["Google-Extended", "Applebot-Extended"]);
+function robotsUrlFor(target) {
+    const u = new URL(target);
+    return `${u.protocol}//${u.host}/robots.txt`;
+}
+export async function crawlerAccessAudit(input) {
+    const parsed = inputSchema.parse(input);
+    log.debug("crawler_access_audit start", { url: parsed.url });
+    const robotsUrl = robotsUrlFor(parsed.url);
+    let robotsBody = null;
+    let robotsStatus = null;
+    let robotsError = null;
+    try {
+        const r = await fetchText(robotsUrl, { timeoutMs: 15_000 });
+        robotsStatus = r.status;
+        robotsBody = r.status < 400 ? r.text : "";
+    }
+    catch (err) {
+        robotsError = err instanceof Error ? err.message : String(err);
+        robotsBody = "";
+    }
+    const robots = robotsParser(robotsUrl, robotsBody ?? "");
+    const botList = parsed.bots
+        ? parsed.bots.map((token) => {
+            const known = DEFAULT_BOTS.find((b) => b.name.toLowerCase() === token.toLowerCase() || b.ua_token === token);
+            return (known ?? {
+                name: token,
+                ua_token: token,
+                ua_full: token,
+                operator: "unknown",
+                purpose: "user-supplied bot",
+            });
+        })
+        : DEFAULT_BOTS;
+    const results = await Promise.all(botList.map(async (bot) => {
+        let allowed = "unknown";
+        try {
+            const a = robots.isAllowed(parsed.url, bot.ua_token);
+            if (typeof a === "boolean")
+                allowed = a;
+        }
+        catch {
+            allowed = "unknown";
+        }
+        const rule = (() => {
+            try {
+                // robots-parser exposes the matched rule via getMatchingLineNumber, but
+                // not all versions expose it. Keep it simple: report the User-Agent
+                // group we matched if any disallow line exists for this bot.
+                if (!robotsBody)
+                    return null;
+                const re = new RegExp(`(^|\\n)User-agent:\\s*${bot.ua_token}([\\s\\S]*?)(?=\\n\\s*User-agent:|$)`, "i");
+                const m = robotsBody.match(re);
+                if (!m)
+                    return null;
+                const block = m[2]
+                    .split("\n")
+                    .map((line) => line.trim())
+                    .filter((line) => /^(allow|disallow|crawl-delay):/i.test(line))
+                    .join(" | ");
+                return block || null;
+            }
+            catch {
+                return null;
+            }
+        })();
+        const isRobotsOnly = ROBOTS_ONLY.has(bot.name);
+        if (isRobotsOnly || !parsed.fetch_with_ua) {
+            const verdict = allowed === true
+                ? isRobotsOnly
+                    ? "robots_only_allowed"
+                    : "allowed"
+                : allowed === false
+                    ? isRobotsOnly
+                        ? "robots_only_blocked"
+                        : "blocked"
+                    : "unknown";
+            return {
+                name: bot.name,
+                ua_token: bot.ua_token,
+                operator: bot.operator,
+                purpose: bot.purpose,
+                robots_allowed: allowed,
+                robots_rule: rule,
+                fetch_status: null,
+                fetch_ok: null,
+                fetch_error: null,
+                verdict,
+            };
+        }
+        // live UA test
+        let status = null;
+        let err = null;
+        try {
+            const r = await fetchText(parsed.url, {
+                method: "GET",
+                headers: { "user-agent": bot.ua_full },
+                timeoutMs: 15_000,
+            });
+            status = r.status;
+        }
+        catch (e) {
+            err = e instanceof Error ? e.message : String(e);
+        }
+        const fetchOk = status !== null && status < 400;
+        const verdict = allowed === false
+            ? "blocked"
+            : fetchOk
+                ? "allowed"
+                : allowed === true
+                    ? "blocked"
+                    : "unknown";
+        return {
+            name: bot.name,
+            ua_token: bot.ua_token,
+            operator: bot.operator,
+            purpose: bot.purpose,
+            robots_allowed: allowed,
+            robots_rule: rule,
+            fetch_status: status,
+            fetch_ok: fetchOk,
+            fetch_error: err,
+            verdict,
+        };
+    }));
+    const summary = {
+        total: results.length,
+        allowed: results.filter((r) => r.verdict === "allowed" || r.verdict === "robots_only_allowed").length,
+        blocked: results.filter((r) => r.verdict === "blocked" || r.verdict === "robots_only_blocked").length,
+        unknown: results.filter((r) => r.verdict === "unknown").length,
+    };
+    return {
+        url: parsed.url,
+        robots_url: robotsUrl,
+        robots_status: robotsStatus,
+        robots_present: !!(robotsBody && robotsBody.length > 0),
+        robots_error: robotsError,
+        fetched_at: new Date().toISOString(),
+        bots: results,
+        summary,
+        note: "verdict combines robots.txt parsing with a live GET using each bot's User-Agent (unless fetch_with_ua=false or the bot is opt-out-token only, like Google-Extended). 'blocked' = robots.txt forbids OR the page 4xx/5xx'd under that UA.",
+    };
+}
+//# sourceMappingURL=crawler-access-audit.js.map

package/dist/tools/crawler-access-audit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler-access-audit.js","sourceRoot":"","sources":["../../src/tools/crawler-access-audit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,mBAAmB,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAC;AASpC,MAAM,YAAY,GAAG,mBAGN,CAAC;AAEhB,MAAM,CAAC,MAAM,6BAA6B,GAAG;IAC3C,GAAG,EAAE,CAAC;SACH,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,CAAC,yCAAyC,CAAC;IACtD,IAAI,EAAE,CAAC;SACJ,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;SACjB,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CACP,+FAA+F,CAChG;IACH,aAAa,EAAE,CAAC;SACb,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CACP,0HAA0H,CAC3H;CACJ,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,6BAA6B,CAAC,CAAC;AAE5D,2EAA2E;AAC3E,uEAAuE;AACvE,MAAM,YAAY,GAMb;IACH;QACE,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,kEAAkE;QAC3E,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,uBAAuB;KACjC;IACD;QACE,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,eAAe;QACzB,OAAO,EACL,4EAA4E;QAC9E,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,sBAAsB;KAChC;IACD;QACE,IAAI,EAAE,cAAc;QACpB,QAAQ,EAAE,cAAc;QACxB,OAAO,EAAE,qEAAqE;QAC9E,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,gCAAgC;KAC1C;IACD;QACE,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,mEAAmE;QAC5E,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,sBAAsB;KAChC;IACD;QACE,IAAI,EAAE,YAAY;QAClB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,sEAAsE;QAC/E,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,qCAAqC;KAC/C;IACD;QACE,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,eAAe;QACzB,OAAO,EACL,sFAAsF;QACxF,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,kBAAkB;KAC5B;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,QAAQ,EAAE,iBAAiB;QAC3B,OAAO,EACL,wFAAwF;QAC1F,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,+BAA+B;KACzC;IACD;QACE,IAAI,EAAE,OAAO;QACb,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0CAA0C;QACnD,QAAQ,EAAE,cAAc;QACxB,OAAO,EAAE,mCAAmC;KAC7C;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,QAAQ,EAAE,iBAAiB;QAC3B,OAAO,EAAE,iBAAiB;QAC1B,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,4DAA4D;KACtE;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,mBAAmB;QAC7B,OAAO,EAAE,mBAAmB;QAC5B,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,mDAAmD;KAC7D;IACD;QACE,IAAI,EAAE,YAAY;QAClB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,qEAAqE;QAC9E,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,gCAAgC;KAC1C;IACD;QACE,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,oBAAoB;QAC9B,OAAO,EACL,2FAA2F;QAC7F,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,2BAA2B;KACrC;CACF,CAAC;AAEF,4EAA4E;AAC5E,qEAAqE;AACrE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAC,CAAC;AAEtE,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1B,OAAO,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC;AAC/C,CAAC;AAeD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAkC;IACzE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxC,GAAG,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAE7D,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5D,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;QACxB,UAAU,GAAG,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/D,UAAU,GAAG,EAAE,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC;IAEzD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI;QACzB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACxB,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAC7B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,QAAQ,KAAK,KAAK,CAC5E,CAAC;YACF,OAAO,CACL,KAAK,IAAI;gBACP,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,KAAK;gBACf,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,SAAS;gBACnB,OAAO,EAAE,mBAAmB;aAC7B,CACF,CAAC;QACJ,CAAC,CAAC;QACJ,CAAC,CAAC,YAAY,CAAC;IAEjB,MAAM,OAAO,GAAgB,MAAM,OAAO,CAAC,GAAG,CAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAsB,EAAE;QAC5C,IAAI,OAAO,GAAwB,SAAS,CAAC;QAC7C,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;YACrD,IAAI,OAAO,CAAC,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAC,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,SAAS,CAAC;QACtB,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE;YACjB,IAAI,CAAC;gBACH,wEAAwE;gBACxE,oEAAoE;gBACpE,6DAA6D;gBAC7D,IAAI,CAAC,UAAU;oBAAE,OAAO,IAAI,CAAC;gBAC7B,MAAM,EAAE,GAAG,IAAI,MAAM,CACnB,yBAAyB,GAAG,CAAC,QAAQ,sCAAsC,EAC3E,GAAG,CACJ,CAAC;gBACF,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC/B,IAAI,CAAC,CAAC;oBAAE,OAAO,IAAI,CAAC;gBACpB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;qBACf,KAAK,CAAC,IAAI,CAAC;qBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;qBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,iCAAiC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBAC9D,IAAI,CAAC,KAAK,CAAC,CAAC;gBACf,OAAO,KAAK,IAAI,IAAI,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,YAAY,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,OAAO,GACX,OAAO,KAAK,IAAI;gBACd,CAAC,CAAC,YAAY;oBACZ,CAAC,CAAC,qBAAqB;oBACvB,CAAC,CAAC,SAAS;gBACb,CAAC,CAAC,OAAO,KAAK,KAAK;oBACjB,CAAC,CAAC,YAAY;wBACZ,CAAC,CAAC,qBAAqB;wBACvB,CAAC,CAAC,SAAS;oBACb,CAAC,CAAC,SAAS,CAAC;YAClB,OAAO;gBACL,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,cAAc,EAAE,OAAO;gBACvB,WAAW,EAAE,IAAI;gBACjB,YAAY,EAAE,IAAI;gBAClB,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,IAAI;gBACjB,OAAO;aACR,CAAC;QACJ,CAAC;QAED,eAAe;QACf,IAAI,MAAM,GAAkB,IAAI,CAAC;QACjC,IAAI,GAAG,GAAkB,IAAI,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE;gBACpC,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,EAAE,YAAY,EAAE,GAAG,CAAC,OAAO,EAAE;gBACtC,SAAS,EAAE,MAAM;aAClB,CAAC,CAAC;YACH,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;QACpB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,KAAK,IAAI,IAAI,MAAM,GAAG,GAAG,CAAC;QAChD,MAAM,OAAO,GACX,OAAO,KAAK,KAAK;YACf,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,OAAO;gBACP,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,OAAO,KAAK,IAAI;oBAChB,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,SAAS,CAAC;QACpB,OAAO;YACL,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,cAAc,EAAE,OAAO;YACvB,WAAW,EAAE,IAAI;YACjB,YAAY,EAAE,MAAM;YACpB,QAAQ,EAAE,OAAO;YACjB,WAAW,EAAE,GAAG;YAChB,OAAO;SACR,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,OAAO,GAAG;QACd,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB,CAAC,CAAC,MAAM;QACrG,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB,CAAC,CAAC,MAAM;QACrG,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,MAAM;KAC/D,CAAC;IAEF,OAAO;QACL,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,UAAU,EAAE,SAAS;QACrB,aAAa,EAAE,YAAY;QAC3B,cAAc,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;QACvD,YAAY,EAAE,WAAW;QACzB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,IAAI,EAAE,OAAO;QACb,OAAO;QACP,IAAI,EACF,0OAA0O;KAC7O,CAAC;AACJ,CAAC"}

package/dist/tools/run-panel.d.ts

@@ -2,19 +2,19 @@ import { z } from "zod";
 export declare const runPanelInputSchema: {
     name: z.ZodString;
     domain: z.ZodOptional<z.ZodString>;
-    engine: z.ZodDefault<z.ZodEnum<["perplexity", "claude", "openai", "gemini", "bing", "brave", "auto"]>>;
+    engine: z.ZodDefault<z.ZodEnum<["perplexity", "claude", "openai", "gemini", "bing", "brave", "google_ai_mode", "auto"]>>;
 };
 declare const inputSchema: z.ZodObject<{
     name: z.ZodString;
     domain: z.ZodOptional<z.ZodString>;
-    engine: z.ZodDefault<z.ZodEnum<["perplexity", "claude", "openai", "gemini", "bing", "brave", "auto"]>>;
+    engine: z.ZodDefault<z.ZodEnum<["perplexity", "claude", "openai", "gemini", "bing", "brave", "google_ai_mode", "auto"]>>;
 }, "strip", z.ZodTypeAny, {
-    engine: "perplexity" | "claude" | "openai" | "gemini" | "bing" | "brave" | "auto";
+    engine: "perplexity" | "claude" | "openai" | "gemini" | "bing" | "brave" | "google_ai_mode" | "auto";
     name: string;
     domain?: string | undefined;
 }, {
     name: string;
-    engine?: "perplexity" | "claude" | "openai" | "gemini" | "bing" | "brave" | "auto" | undefined;
+    engine?: "perplexity" | "claude" | "openai" | "gemini" | "bing" | "brave" | "google_ai_mode" | "auto" | undefined;
     domain?: string | undefined;
 }>;
 export declare function runPanel(input: z.infer<typeof inputSchema>): Promise<{
@@ -26,7 +26,7 @@ export declare function runPanel(input: z.infer<typeof inputSchema>): Promise<{
     snapshot: {
         panel: string;
         domain: string;
-        engine: "perplexity" | "claude" | "openai" | "gemini" | "bing" | "brave" | "auto";
+        engine: "perplexity" | "claude" | "openai" | "gemini" | "bing" | "brave" | "google_ai_mode" | "auto";
         taken_at: string;
         per_query: {
             query: string;

package/dist/tools/run-panel.js

@@ -11,7 +11,7 @@ export const runPanelInputSchema = {
         .optional()
         .describe("Override the panel's default domain for this run."),
     engine: z
-        .enum(["perplexity", "claude", "openai", "gemini", "bing", "brave", "auto"])
+        .enum(["perplexity", "claude", "openai", "gemini", "bing", "brave", "google_ai_mode", "auto"])
         .default("auto")
         .describe("AI engine to query."),
 };