@jmp-technologies/analytics 0.1.39 → 0.1.41

package/README.md

@@ -136,7 +136,11 @@ export async function GET() {
 }
 ```
 
-Place that in `app/robots.txt/route.ts` for Next.js App Router sites. The policy uses robots.txt `Disallow` rules for known AI crawler user agents; crawlers that ignore robots.txt or use undisclosed names may still access public pages.
+Place that in `app/robots.txt/route.ts` for Next.js App Router sites. Use `mergeJmpRobotsTxt()` if the site also needs private-path disallows and a sitemap line. Pass `ensureAiCrawlerDisallows: true` only when this site should always list AI `Disallow` rules even if the JMP dashboard toggle is off. The policy uses robots.txt `Disallow` rules for known AI crawler user agents.
+
+**Enforcement:** robots.txt is advisory. When this JMP setting is on, also add a `vercel.json` WAF deny (`jmpAiCrawlerVercelWafRoute()` or `create-jmp-client --block-ai-crawlers`) and set Vercel **AI Bots** to **deny**. Crawlers that spoof a normal browser or use undisclosed names may still access public pages. Generic Applebot is not WAF-denied so Apple Search can still crawl; `Applebot-Extended` is denied.
+
+Use **≥ 0.1.41** for the shared crawler list, `mergeJmpRobotsTxt`, and WAF route helper (0.1.40 source had them; the published types did not).
 
 ---
 
@@ -355,6 +359,6 @@ After publish, on client sites:
 npm install @jmp-technologies/analytics@latest
 ```
 
-Use **≥ 0.1.8** for `getJmpSiteProfile` / `extras`. Use **≥ 0.1.10** for `email_click` tracking and `JmpTrackedMailtoLink`. Use **≥ 0.1.37** so server content fetches and server page views send `Origin` from `siteUrl` / `NEXT_PUBLIC_SITE_URL` (required for capabilities writes after the matching dashboard deploy). Use **≥ 0.1.38** so `contentManifest.routes` with wrapper `legal` or `pages` reports page URLs to the dashboard Pages tab. Use **≥ 0.1.39** so JMP content `fetch` uses the `jmp-content` cache tag (client webhooks revalidate it).
+Use **≥ 0.1.8** for `getJmpSiteProfile` / `extras`. Use **≥ 0.1.10** for `email_click` tracking and `JmpTrackedMailtoLink`. Use **≥ 0.1.37** so server content fetches and server page views send `Origin` from `siteUrl` / `NEXT_PUBLIC_SITE_URL` (required for capabilities writes after the matching dashboard deploy). Use **≥ 0.1.38** so `contentManifest.routes` with wrapper `legal` or `pages` reports page URLs to the dashboard Pages tab. Use **≥ 0.1.39** so JMP content `fetch` uses the `jmp-content` cache tag (client webhooks revalidate it). Use **≥ 0.1.41** so npm types export `mergeJmpRobotsTxt`, `buildAiCrawlerRobotsTxt`, and the Vercel WAF route helper.
 
 Ensure the **dashboard** is deployed with migrations **020** and **021** and `/api/content/site` is live before relying on `getJmpSiteProfile` in production.

package/ai-crawler-agents.json

@@ -0,0 +1,63 @@
+{
+  "robotsUserAgents": [
+    "GPTBot",
+    "ChatGPT-User",
+    "OAI-SearchBot",
+    "OAI-AdsBot",
+    "ClaudeBot",
+    "Claude-User",
+    "Claude-SearchBot",
+    "Claude-Web",
+    "PerplexityBot",
+    "Perplexity-User",
+    "Google-Extended",
+    "GoogleOther",
+    "GoogleOther-Image",
+    "GoogleOther-Video",
+    "Google-CloudVertexBot",
+    "Meta-ExternalAgent",
+    "Meta-ExternalFetcher",
+    "FacebookBot",
+    "Applebot",
+    "Applebot-Extended",
+    "CCBot",
+    "Bytespider",
+    "cohere-ai",
+    "cohere-training-data-crawler",
+    "YouBot",
+    "MistralAI-User",
+    "GrokBot",
+    "Grok-DeepSearch",
+    "xAI-Grok",
+    "xAI-Bot"
+  ],
+  "wafUserAgents": [
+    "GPTBot",
+    "ChatGPT-User",
+    "OAI-SearchBot",
+    "OAI-AdsBot",
+    "ClaudeBot",
+    "Claude-User",
+    "Claude-SearchBot",
+    "Claude-Web",
+    "PerplexityBot",
+    "Perplexity-User",
+    "Google-Extended",
+    "GoogleOther",
+    "Google-CloudVertexBot",
+    "Meta-ExternalAgent",
+    "Meta-ExternalFetcher",
+    "FacebookBot",
+    "Applebot-Extended",
+    "CCBot",
+    "Bytespider",
+    "cohere-ai",
+    "cohere-training-data-crawler",
+    "YouBot",
+    "MistralAI-User",
+    "GrokBot",
+    "Grok-DeepSearch",
+    "xAI-Grok",
+    "xAI-Bot"
+  ]
+}

package/dist/astro/client.cjs

@@ -308,7 +308,7 @@ function extractMailtoTarget(href) {
 }
 
 // package.json
-var version = "0.1.39";
+var version = "0.1.41";
 
 // src/constants.ts
 var JMP_ANALYTICS_SDK_VERSION = version;
@@ -1107,7 +1107,50 @@ function normalizeManifestDefaults(defaults) {
 function normalizeContact(contact) {
   if (!contact || typeof contact !== "object") return null;
   const extras = normalizeContactExtras(contact.extras);
-  return extras.length > 0 ? { extras } : null;
+  const leadInbox = normalizeLeadInbox(contact.leadInbox);
+  if (extras.length === 0 && !leadInbox) return null;
+  return {
+    ...extras.length > 0 ? { extras } : {},
+    ...leadInbox ? { leadInbox } : {}
+  };
+}
+function normalizeLeadInbox(raw) {
+  if (!raw || typeof raw !== "object") return null;
+  const list = normalizeInboxKeyList(raw.list, 8);
+  const detail = normalizeInboxKeyList(raw.detail, 24);
+  const labels = normalizeInboxLabels(raw.labels);
+  if (!list && !detail && !labels) return null;
+  return {
+    ...list ? { list } : {},
+    ...detail ? { detail } : {},
+    ...labels ? { labels } : {}
+  };
+}
+function normalizeInboxKeyList(raw, max) {
+  if (!Array.isArray(raw)) return void 0;
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const item of raw) {
+    if (typeof item !== "string" || out.length >= max) continue;
+    const key = item.trim();
+    const token = key.replace(/[^a-zA-Z0-9]+/g, "").toLowerCase();
+    if (!key || !token || seen.has(token)) continue;
+    seen.add(token);
+    out.push(key);
+  }
+  return out.length > 0 ? out : void 0;
+}
+function normalizeInboxLabels(raw) {
+  if (!raw || typeof raw !== "object") return void 0;
+  const out = {};
+  for (const [key, value] of Object.entries(raw)) {
+    if (typeof value !== "string") continue;
+    const label = value.trim().slice(0, 60);
+    const token = key.trim();
+    if (!token || !label) continue;
+    out[token] = label;
+  }
+  return Object.keys(out).length > 0 ? out : void 0;
 }
 function normalizeContactExtras(extras) {
   if (!Array.isArray(extras)) return [];
@@ -1137,7 +1180,7 @@ function inferWrapperSlotsFromManifest(manifest) {
     slots.add(`trust.${block.section}`);
     slots.add("trust");
   }
-  if (manifest?.contact?.extras?.length) {
+  if (manifest?.contact?.extras?.length || manifest?.contact?.leadInbox) {
     slots.add("contact");
   }
   return [...slots];
@@ -1325,6 +1368,75 @@ async function registerJmpContentWrappers(config) {
 }
 setContentWrapperSyncHandler(registerJmpContentWrappers);
 
+// ai-crawler-agents.json
+var ai_crawler_agents_default = {
+  robotsUserAgents: [
+    "GPTBot",
+    "ChatGPT-User",
+    "OAI-SearchBot",
+    "OAI-AdsBot",
+    "ClaudeBot",
+    "Claude-User",
+    "Claude-SearchBot",
+    "Claude-Web",
+    "PerplexityBot",
+    "Perplexity-User",
+    "Google-Extended",
+    "GoogleOther",
+    "GoogleOther-Image",
+    "GoogleOther-Video",
+    "Google-CloudVertexBot",
+    "Meta-ExternalAgent",
+    "Meta-ExternalFetcher",
+    "FacebookBot",
+    "Applebot",
+    "Applebot-Extended",
+    "CCBot",
+    "Bytespider",
+    "cohere-ai",
+    "cohere-training-data-crawler",
+    "YouBot",
+    "MistralAI-User",
+    "GrokBot",
+    "Grok-DeepSearch",
+    "xAI-Grok",
+    "xAI-Bot"
+  ],
+  wafUserAgents: [
+    "GPTBot",
+    "ChatGPT-User",
+    "OAI-SearchBot",
+    "OAI-AdsBot",
+    "ClaudeBot",
+    "Claude-User",
+    "Claude-SearchBot",
+    "Claude-Web",
+    "PerplexityBot",
+    "Perplexity-User",
+    "Google-Extended",
+    "GoogleOther",
+    "Google-CloudVertexBot",
+    "Meta-ExternalAgent",
+    "Meta-ExternalFetcher",
+    "FacebookBot",
+    "Applebot-Extended",
+    "CCBot",
+    "Bytespider",
+    "cohere-ai",
+    "cohere-training-data-crawler",
+    "YouBot",
+    "MistralAI-User",
+    "GrokBot",
+    "Grok-DeepSearch",
+    "xAI-Grok",
+    "xAI-Bot"
+  ]
+};
+
+// src/ai-crawlers.ts
+var AI_CRAWLER_ROBOTS_USER_AGENTS = ai_crawler_agents_default.robotsUserAgents;
+var JMP_AI_CRAWLER_WAF_USER_AGENTS = ai_crawler_agents_default.wafUserAgents;
+
 // src/astro/client.ts
 var import_meta = {};
 var jmpAnalyticsSingleton = null;