@jmp-technologies/analytics 0.1.38 → 0.1.40

package/README.md

@@ -136,7 +136,11 @@ export async function GET() {
 }
 ```
 
-Place that in `app/robots.txt/route.ts` for Next.js App Router sites. The policy uses robots.txt `Disallow` rules for known AI crawler user agents; crawlers that ignore robots.txt or use undisclosed names may still access public pages.
+Place that in `app/robots.txt/route.ts` for Next.js App Router sites. Use `mergeJmpRobotsTxt()` if the site also needs private-path disallows and a sitemap line. Pass `ensureAiCrawlerDisallows: true` only when this site should always list AI `Disallow` rules even if the JMP dashboard toggle is off. The policy uses robots.txt `Disallow` rules for known AI crawler user agents.
+
+**Enforcement:** robots.txt is advisory. When this JMP setting is on, also add a `vercel.json` WAF deny (`jmpAiCrawlerVercelWafRoute()` or `create-jmp-client --block-ai-crawlers`) and set Vercel **AI Bots** to **deny**. Crawlers that spoof a normal browser or use undisclosed names may still access public pages. Generic Applebot is not WAF-denied so Apple Search can still crawl; `Applebot-Extended` is denied.
+
+Use **≥ 0.1.40** for the shared crawler list, robots merge helper, and WAF route helper.
 
 ---
 
@@ -355,6 +359,6 @@ After publish, on client sites:
 npm install @jmp-technologies/analytics@latest
 ```
 
-Use **≥ 0.1.8** for `getJmpSiteProfile` / `extras`. Use **≥ 0.1.10** for `email_click` tracking and `JmpTrackedMailtoLink`. Use **≥ 0.1.37** so server content fetches and server page views send `Origin` from `siteUrl` / `NEXT_PUBLIC_SITE_URL` (required for capabilities writes after the matching dashboard deploy). Use **≥ 0.1.38** so `contentManifest.routes` with wrapper `legal` or `pages` reports page URLs to the dashboard Pages tab.
+Use **≥ 0.1.8** for `getJmpSiteProfile` / `extras`. Use **≥ 0.1.10** for `email_click` tracking and `JmpTrackedMailtoLink`. Use **≥ 0.1.37** so server content fetches and server page views send `Origin` from `siteUrl` / `NEXT_PUBLIC_SITE_URL` (required for capabilities writes after the matching dashboard deploy). Use **≥ 0.1.38** so `contentManifest.routes` with wrapper `legal` or `pages` reports page URLs to the dashboard Pages tab. Use **≥ 0.1.39** so JMP content `fetch` uses the `jmp-content` cache tag (client webhooks revalidate it). Use **≥ 0.1.40** for AI crawler robots helpers and the Vercel WAF route helper.
 
 Ensure the **dashboard** is deployed with migrations **020** and **021** and `/api/content/site` is live before relying on `getJmpSiteProfile` in production.

package/ai-crawler-agents.json

@@ -0,0 +1,63 @@
+{
+  "robotsUserAgents": [
+    "GPTBot",
+    "ChatGPT-User",
+    "OAI-SearchBot",
+    "OAI-AdsBot",
+    "ClaudeBot",
+    "Claude-User",
+    "Claude-SearchBot",
+    "Claude-Web",
+    "PerplexityBot",
+    "Perplexity-User",
+    "Google-Extended",
+    "GoogleOther",
+    "GoogleOther-Image",
+    "GoogleOther-Video",
+    "Google-CloudVertexBot",
+    "Meta-ExternalAgent",
+    "Meta-ExternalFetcher",
+    "FacebookBot",
+    "Applebot",
+    "Applebot-Extended",
+    "CCBot",
+    "Bytespider",
+    "cohere-ai",
+    "cohere-training-data-crawler",
+    "YouBot",
+    "MistralAI-User",
+    "GrokBot",
+    "Grok-DeepSearch",
+    "xAI-Grok",
+    "xAI-Bot"
+  ],
+  "wafUserAgents": [
+    "GPTBot",
+    "ChatGPT-User",
+    "OAI-SearchBot",
+    "OAI-AdsBot",
+    "ClaudeBot",
+    "Claude-User",
+    "Claude-SearchBot",
+    "Claude-Web",
+    "PerplexityBot",
+    "Perplexity-User",
+    "Google-Extended",
+    "GoogleOther",
+    "Google-CloudVertexBot",
+    "Meta-ExternalAgent",
+    "Meta-ExternalFetcher",
+    "FacebookBot",
+    "Applebot-Extended",
+    "CCBot",
+    "Bytespider",
+    "cohere-ai",
+    "cohere-training-data-crawler",
+    "YouBot",
+    "MistralAI-User",
+    "GrokBot",
+    "Grok-DeepSearch",
+    "xAI-Grok",
+    "xAI-Bot"
+  ]
+}

package/dist/astro/client.cjs

@@ -308,7 +308,7 @@ function extractMailtoTarget(href) {
 }
 
 // package.json
-var version = "0.1.38";
+var version = "0.1.40";
 
 // src/constants.ts
 var JMP_ANALYTICS_SDK_VERSION = version;
@@ -951,7 +951,8 @@ function normalizeJmpContentManifest(input) {
   const externalSources = normalizeExternalSources(input.externalSources);
   const trust = normalizeTrust(input.trust);
   const contact = normalizeContact(input.contact);
-  if (routes.length === 0 && !externalSources && !trust && !contact) {
+  const defaults = normalizeManifestDefaults(input.defaults);
+  if (routes.length === 0 && !externalSources && !trust && !contact && !defaults) {
     return null;
   }
   return {
@@ -959,7 +960,8 @@ function normalizeJmpContentManifest(input) {
     ...routes.length > 0 ? { routes } : {},
     ...externalSources ? { externalSources } : {},
     ...trust ? { trust } : {},
-    ...contact ? { contact } : {}
+    ...contact ? { contact } : {},
+    ...defaults ? { defaults } : {}
   };
 }
 function normalizeRoutes(routes) {
@@ -1023,10 +1025,132 @@ function normalizeTrustLabels(labels) {
   }
   return Object.keys(out).length > 0 ? out : null;
 }
+function readDefaultText(value, max = 500) {
+  if (typeof value !== "string") return void 0;
+  const trimmed = value.trim();
+  if (!trimmed) return void 0;
+  return trimmed.slice(0, max);
+}
+function normalizeManifestDefaults(defaults) {
+  if (!defaults || typeof defaults !== "object") return null;
+  const contactRaw = defaults.contact;
+  const contact = {};
+  if (contactRaw && typeof contactRaw === "object") {
+    const displayName = readDefaultText(contactRaw.displayName, 120);
+    const email = readDefaultText(contactRaw.email, 200);
+    const phone = readDefaultText(contactRaw.phone, 40);
+    const address = readDefaultText(contactRaw.address, 400);
+    const hours = readDefaultText(contactRaw.hours, 400);
+    const serviceArea = readDefaultText(contactRaw.serviceArea, 200);
+    const googleMapsUrl = readDefaultText(contactRaw.googleMapsUrl, 2048);
+    const linkedinUrl = readDefaultText(contactRaw.linkedinUrl, 2048);
+    const instagramUrl = readDefaultText(contactRaw.instagramUrl, 2048);
+    const facebookUrl = readDefaultText(contactRaw.facebookUrl, 2048);
+    const xUrl = readDefaultText(contactRaw.xUrl, 2048);
+    const youtubeUrl = readDefaultText(contactRaw.youtubeUrl, 2048);
+    if (displayName) contact.displayName = displayName;
+    if (email) contact.email = email;
+    if (phone) contact.phone = phone;
+    if (address) contact.address = address;
+    if (hours) contact.hours = hours;
+    if (serviceArea) contact.serviceArea = serviceArea;
+    if (googleMapsUrl) contact.googleMapsUrl = googleMapsUrl;
+    if (linkedinUrl) contact.linkedinUrl = linkedinUrl;
+    if (instagramUrl) contact.instagramUrl = instagramUrl;
+    if (facebookUrl) contact.facebookUrl = facebookUrl;
+    if (xUrl) contact.xUrl = xUrl;
+    if (youtubeUrl) contact.youtubeUrl = youtubeUrl;
+    if (contactRaw.extras && typeof contactRaw.extras === "object") {
+      const extras = {};
+      for (const [rawKey, rawValue] of Object.entries(contactRaw.extras)) {
+        if (Object.keys(extras).length >= 10) break;
+        const key = normalizeKey(rawKey);
+        const value = readDefaultText(rawValue, 200);
+        if (key && isValidExtraKey(key) && value) extras[key] = value;
+      }
+      if (Object.keys(extras).length > 0) contact.extras = extras;
+    }
+  }
+  const faq = [];
+  if (Array.isArray(defaults.faq)) {
+    for (const raw of defaults.faq) {
+      if (!raw || typeof raw !== "object" || faq.length >= 40) continue;
+      const question = readDefaultText(raw.question, 300);
+      const answer = readDefaultText(raw.answer, 8e3);
+      if (!question || !answer) continue;
+      const category = readDefaultText(raw.category, 80);
+      faq.push({ question, answer, ...category ? { category } : {} });
+    }
+  }
+  const trustRaw = defaults.trust;
+  const trust = {};
+  if (trustRaw && typeof trustRaw === "object") {
+    const assign = (key, max) => {
+      const value = readDefaultText(trustRaw[key], max);
+      if (value) trust[key] = value;
+    };
+    assign("googleReviewsUrl", 2048);
+    assign("googlePlaceId", 128);
+    assign("yelpUrl", 2048);
+    assign("trustpilotUrl", 2048);
+    assign("disclaimer", 1e3);
+  }
+  const hasContact = Object.keys(contact).length > 0;
+  const hasTrust = Object.keys(trust).length > 0;
+  if (!hasContact && faq.length === 0 && !hasTrust) return null;
+  return {
+    ...hasContact ? { contact } : {},
+    ...faq.length > 0 ? { faq } : {},
+    ...hasTrust ? { trust } : {}
+  };
+}
 function normalizeContact(contact) {
   if (!contact || typeof contact !== "object") return null;
   const extras = normalizeContactExtras(contact.extras);
-  return extras.length > 0 ? { extras } : null;
+  const leadInbox = normalizeLeadInbox(contact.leadInbox);
+  if (extras.length === 0 && !leadInbox) return null;
+  return {
+    ...extras.length > 0 ? { extras } : {},
+    ...leadInbox ? { leadInbox } : {}
+  };
+}
+function normalizeLeadInbox(raw) {
+  if (!raw || typeof raw !== "object") return null;
+  const list = normalizeInboxKeyList(raw.list, 8);
+  const detail = normalizeInboxKeyList(raw.detail, 24);
+  const labels = normalizeInboxLabels(raw.labels);
+  if (!list && !detail && !labels) return null;
+  return {
+    ...list ? { list } : {},
+    ...detail ? { detail } : {},
+    ...labels ? { labels } : {}
+  };
+}
+function normalizeInboxKeyList(raw, max) {
+  if (!Array.isArray(raw)) return void 0;
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const item of raw) {
+    if (typeof item !== "string" || out.length >= max) continue;
+    const key = item.trim();
+    const token = key.replace(/[^a-zA-Z0-9]+/g, "").toLowerCase();
+    if (!key || !token || seen.has(token)) continue;
+    seen.add(token);
+    out.push(key);
+  }
+  return out.length > 0 ? out : void 0;
+}
+function normalizeInboxLabels(raw) {
+  if (!raw || typeof raw !== "object") return void 0;
+  const out = {};
+  for (const [key, value] of Object.entries(raw)) {
+    if (typeof value !== "string") continue;
+    const label = value.trim().slice(0, 60);
+    const token = key.trim();
+    if (!token || !label) continue;
+    out[token] = label;
+  }
+  return Object.keys(out).length > 0 ? out : void 0;
 }
 function normalizeContactExtras(extras) {
   if (!Array.isArray(extras)) return [];
@@ -1056,7 +1180,7 @@ function inferWrapperSlotsFromManifest(manifest) {
     slots.add(`trust.${block.section}`);
     slots.add("trust");
   }
-  if (manifest?.contact?.extras?.length) {
+  if (manifest?.contact?.extras?.length || manifest?.contact?.leadInbox) {
     slots.add("contact");
   }
   return [...slots];
@@ -1090,7 +1214,65 @@ function getDiscoveredContentWrappers() {
   return sortWrapperSlots([...discovered]);
 }
 function resolveContentManifestForReport(config) {
-  return normalizeJmpContentManifest(config.contentManifest);
+  const site = getJmpSiteDefinition();
+  const fallback = site?.contactFallback;
+  const existing = config.contentManifest;
+  const fromFallback = fallbackToManifestDefaults(fallback);
+  if (!existing && !fromFallback) {
+    return null;
+  }
+  const merged = {
+    ...existing ?? {},
+    defaults: {
+      ...fromFallback ?? {},
+      ...existing?.defaults ?? {},
+      contact: {
+        ...fromFallback?.contact ?? {},
+        ...existing?.defaults?.contact ?? {},
+        extras: {
+          ...fromFallback?.contact?.extras ?? {},
+          ...existing?.defaults?.contact?.extras ?? {}
+        }
+      },
+      trust: {
+        ...fromFallback?.trust ?? {},
+        ...existing?.defaults?.trust ?? {}
+      },
+      faq: existing?.defaults?.faq?.length ? existing.defaults.faq : fromFallback?.faq
+    }
+  };
+  return normalizeJmpContentManifest(merged);
+}
+function fallbackToManifestDefaults(fallback) {
+  if (!fallback) return null;
+  const contact = {};
+  if (fallback.displayName?.trim()) contact.displayName = fallback.displayName.trim();
+  if (fallback.email?.trim()) contact.email = fallback.email.trim();
+  if (typeof fallback.phone === "string" && fallback.phone.trim()) {
+    contact.phone = fallback.phone.trim();
+  }
+  if (fallback.address?.trim()) contact.address = fallback.address.trim();
+  if (fallback.hours?.trim()) contact.hours = fallback.hours.trim();
+  if (fallback.serviceArea?.trim()) contact.serviceArea = fallback.serviceArea.trim();
+  if (fallback.googleMapsUrl?.trim()) {
+    contact.googleMapsUrl = fallback.googleMapsUrl.trim();
+  }
+  if (fallback.linkedin?.trim()) contact.linkedinUrl = fallback.linkedin.trim();
+  if (fallback.instagram?.trim()) contact.instagramUrl = fallback.instagram.trim();
+  if (fallback.facebook?.trim()) contact.facebookUrl = fallback.facebook.trim();
+  if (fallback.x?.trim()) contact.xUrl = fallback.x.trim();
+  if (fallback.youtube?.trim()) contact.youtubeUrl = fallback.youtube.trim();
+  const trust = {};
+  if (fallback.googleMapsUrl?.trim()) {
+    trust.googleReviewsUrl = fallback.googleMapsUrl.trim();
+  }
+  const hasContact = Object.keys(contact).length > 0;
+  const hasTrust = Object.keys(trust).length > 0;
+  if (!hasContact && !hasTrust) return null;
+  return {
+    ...hasContact ? { contact } : {},
+    ...hasTrust ? { trust } : {}
+  };
 }
 function resolveContentWrappersForReport(config) {
   const explicit = config.contentWrappers ?? [];
@@ -1186,6 +1368,75 @@ async function registerJmpContentWrappers(config) {
 }
 setContentWrapperSyncHandler(registerJmpContentWrappers);
 
+// ai-crawler-agents.json
+var ai_crawler_agents_default = {
+  robotsUserAgents: [
+    "GPTBot",
+    "ChatGPT-User",
+    "OAI-SearchBot",
+    "OAI-AdsBot",
+    "ClaudeBot",
+    "Claude-User",
+    "Claude-SearchBot",
+    "Claude-Web",
+    "PerplexityBot",
+    "Perplexity-User",
+    "Google-Extended",
+    "GoogleOther",
+    "GoogleOther-Image",
+    "GoogleOther-Video",
+    "Google-CloudVertexBot",
+    "Meta-ExternalAgent",
+    "Meta-ExternalFetcher",
+    "FacebookBot",
+    "Applebot",
+    "Applebot-Extended",
+    "CCBot",
+    "Bytespider",
+    "cohere-ai",
+    "cohere-training-data-crawler",
+    "YouBot",
+    "MistralAI-User",
+    "GrokBot",
+    "Grok-DeepSearch",
+    "xAI-Grok",
+    "xAI-Bot"
+  ],
+  wafUserAgents: [
+    "GPTBot",
+    "ChatGPT-User",
+    "OAI-SearchBot",
+    "OAI-AdsBot",
+    "ClaudeBot",
+    "Claude-User",
+    "Claude-SearchBot",
+    "Claude-Web",
+    "PerplexityBot",
+    "Perplexity-User",
+    "Google-Extended",
+    "GoogleOther",
+    "Google-CloudVertexBot",
+    "Meta-ExternalAgent",
+    "Meta-ExternalFetcher",
+    "FacebookBot",
+    "Applebot-Extended",
+    "CCBot",
+    "Bytespider",
+    "cohere-ai",
+    "cohere-training-data-crawler",
+    "YouBot",
+    "MistralAI-User",
+    "GrokBot",
+    "Grok-DeepSearch",
+    "xAI-Grok",
+    "xAI-Bot"
+  ]
+};
+
+// src/ai-crawlers.ts
+var AI_CRAWLER_ROBOTS_USER_AGENTS = ai_crawler_agents_default.robotsUserAgents;
+var JMP_AI_CRAWLER_WAF_USER_AGENTS = ai_crawler_agents_default.wafUserAgents;
+
 // src/astro/client.ts
 var import_meta = {};
 var jmpAnalyticsSingleton = null;