@sonordev/site-kit 6.3.3 → 6.3.4

package/dist/chunk-NRBITVDK.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/robots/next-internals.ts","../src/llms/aiRobots.ts"],"names":["list"],"mappings":";;;AAiBA,IAAM,mBAAA,GAAsB,uBAAA;AAErB,SAAS,oBAAoB,IAAA,EAAuB;AACzD,EAAA,OAAO,mBAAA,CAAoB,IAAA,CAAK,IAAA,CAAK,IAAA,EAAM,CAAA;AAC7C;AAEA,IAAM,MAAA,uBAAa,GAAA,EAAY;AAE/B,SAAS,YAAY,KAAA,EAAuB;AAC1C,EAAA,KAAA,MAAW,KAAK,KAAA,EAAO;AACrB,IAAA,IAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,EAAG;AACnB,IAAA,MAAA,CAAO,IAAI,CAAC,CAAA;AACZ,IAAA,OAAA,CAAQ,IAAA;AAAA,MACN,wCAAwC,CAAC,CAAA,4DAAA;AAAA,KAC3C;AAAA,EACF;AACF;AAOO,SAAS,mBACd,KAAA,EAC+B;AAC/B,EAAA,IAAI,KAAA,KAAU,QAAW,OAAO,MAAA;AAChC,EAAA,MAAMA,QAAO,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,GAAI,KAAA,GAAQ,CAAC,KAAK,CAAA;AAClD,EAAA,MAAM,OAAA,GAAUA,KAAAA,CAAK,MAAA,CAAO,mBAAmB,CAAA;AAC/C,EAAA,IAAI,OAAA,CAAQ,MAAA,KAAW,CAAA,EAAG,OAAO,KAAA;AACjC,EAAA,WAAA,CAAY,OAAO,CAAA;AACnB,EAAA,MAAM,IAAA,GAAOA,MAAK,MAAA,CAAO,CAAC,MAAM,CAAC,mBAAA,CAAoB,CAAC,CAAC,CAAA;AACvD,EAAA,IAAI,IAAA,CAAK,MAAA,KAAW,CAAA,EAAG,OAAO,MAAA;AAC9B,EAAA,OAAO,MAAM,OAAA,CAAQ,KAAK,CAAA,GAAI,IAAA,GAAO,KAAK,CAAC,CAAA;AAC7C;;;ACjBO,IAAM,qBAAA,GAAwB;AAAA,EACnC,eAAA;AAAA,EACA,cAAA;AAAA,EACA,kBAAA;AAAA,EACA,aAAA;AAAA,EACA,eAAA;AAAA,EACA,iBAAA;AAAA,EACA,aAAA;AAAA,EACA,UAAA;AAAA,EACA,eAAA;AAAA,EACA,gBAAA;AAAA,EACA;AACF;AASO,IAAM,oBAAA,GAAuB;AAAA,EAClC,QAAA;AAAA,EACA,WAAA;AAAA,EACA,iBAAA;AAAA,EACA,mBAAA;AAAA,EACA,oBAAA;AAAA,EACA,aAAA;AAAA,EACA,OAAA;AAAA,EACA,YAAA;AAAA,EACA,WAAA;AAAA,EACA,YAAA;AAAA,EACA;AACF;AA2BO,SAAS,mBAAA,CACd,OAAA,GAAiC,EAAC,EACpB;AACd,EAAA,MAAM,EAAE,KAAA,GAAQ,GAAA,EAAK,QAAA,GAAW,SAAQ,GAAI,OAAA;AAC5C,EAAA,MAAM,QAAA,GAAW,kBAAA,CAAmB,OAAA,CAAQ,QAAQ,CAAA;AACpD,EAAA,MAAM,SAAA,GAA0B,qBAAA,CAAsB,GAAA,CAAI,CAAC,SAAA,MAAe;AAAA,IACxE,SAAA;AAAA,IACA,KAAA;AAAA,IACA,GAAI,QAAA,GAAW,EAAE,QAAA,KAAa;AAAC,GACjC,CAAE,CAAA;AACF,EAAA,MAAM,gBAA8B,oBAAA,CAAqB,GAAA;AAAA,IAAI,CAAC,SAAA,KAC5D,QAAA,KAAa,UACT,EAAE,SAAA,EAAW,UAAU,GAAA,EAAI,GAC3B,EAAE,SAAA,EAAW,OAAO,GAAI,QAAA,GAAW,EAAE,QAAA,EAAS,GAAI,EAAC;AAAG,GAC5D;AACA,EAAA,OAAO,CAAC,GAAG,SAAA,EAAW,GAAG,aAAa,CAAA;AACxC;AAWA,IAAM,EAAA,GAAK,CAAC,CAAA,KAAgB,CAAA,GAAI,KAAA,GAAQ,IAAA;AAOjC,SAAS,oBAAA,CACd,OAAA,GAAiC,EAAC,EAC1B;AACR,EAAA,MAAM,EAAE,MAAA,GAAS,IAAA,EAAM,UAAU,IAAA,EAAM,OAAA,GAAU,MAAK,GAAI,OAAA;AAC1D,EAAA,OAAO;AAAA,IACL,0EAAA;AAAA,IACA,mFAAA;AAAA,IACA,CAAA,uBAAA,EAA0B,EAAA,CAAG,MAAM,CAAC,CAAA,WAAA,EAAc,EAAA,CAAG,OAAO,CAAC,CAAA,WAAA,EAAc,EAAA,CAAG,OAAO,CAAC,CAAA;AAAA,GACxF,CAAE,KAAK,IAAI,CAAA;AACb;AAaA,IAAM,IAAA,GAAO,CAAC,CAAA,KACZ,CAAA,KAAM,MAAA,GAAY,EAAC,GAAI,KAAA,CAAM,OAAA,CAAQ,CAAC,CAAA,GAAI,CAAA,GAAI,CAAC,CAAC,CAAA;AAc3C,SAAS,uBAAuB,OAAA,EAAkC;AACvE,EAAA,MAAM,EAAE,KAAA,EAAO,cAAA,EAAgB,OAAA,EAAS,MAAK,GAAI,OAAA;AACjD,EAAA,MAAM,SAAmB,EAAC;AAC1B,EAAA,IAAI,cAAA,EAAgB,MAAA,CAAO,IAAA,CAAK,oBAAA,CAAqB,cAAc,CAAC,CAAA;AACpE,EAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,IAAA,MAAM,KAAA,GAAQ,CAAC,CAAA,YAAA,EAAe,IAAA,CAAK,SAAS,CAAA,CAAE,CAAA;AAC9C,IAAA,KAAA,MAAW,CAAA,IAAK,KAAK,IAAA,CAAK,KAAK,GAAG,KAAA,CAAM,IAAA,CAAK,CAAA,OAAA,EAAU,CAAC,CAAA,CAAE,CAAA;AAC1D,IAAA,KAAA,MAAW,CAAA,IAAK,IAAA,CAAK,kBAAA,CAAmB,IAAA,CAAK,QAAQ,CAAC,CAAA,EAAG,KAAA,CAAM,IAAA,CAAK,CAAA,UAAA,EAAa,CAAC,CAAA,CAAE,CAAA;AACpF,IAAA,MAAA,CAAO,IAAA,CAAK,KAAA,CAAM,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,EAC9B;AACA,EAAA,MAAM,OAAiB,EAAC;AACxB,EAAA,KAAA,MAAW,CAAA,IAAK,KAAK,OAAO,CAAA,OAAQ,IAAA,CAAK,CAAA,SAAA,EAAY,CAAC,CAAA,CAAE,CAAA;AACxD,EAAA,IAAI,IAAA,EAAM,IAAA,CAAK,IAAA,CAAK,CAAA,MAAA,EAAS,IAAI,CAAA,CAAE,CAAA;AACnC,EAAA,IAAI,KAAK,MAAA,EAAQ,MAAA,CAAO,KAAK,IAAA,CAAK,IAAA,CAAK,IAAI,CAAC,CAAA;AAC5C,EAAA,MAAM,IAAA,GAAO,CAAA,EAAG,MAAA,CAAO,IAAA,CAAK,MAAM,CAAC;AAAA,CAAA;AAEnC,EAAA,OAAO,eAAe,GAAA,GAAyB;AAC7C,IAAA,OAAO,IAAI,SAAS,IAAA,EAAM;AAAA,MACxB,MAAA,EAAQ,GAAA;AAAA,MACR,OAAA,EAAS;AAAA,QACP,cAAA,EAAgB,2BAAA;AAAA,QAChB,eAAA,EAAiB;AAAA;AACnB,KACD,CAAA;AAAA,EACH,CAAA;AACF","file":"chunk-NRBITVDK.js","sourcesContent":["/**\n * Keeps `/_next/` out of every robots.txt site-kit emits.\n *\n * Googlebot renders pages with the CSS, JS chunks and optimized images Next\n * serves from `/_next/`, and `/_next/image` is how a site's photos reach\n * Google Images. A `Disallow: /_next/` makes Google judge an unstyled,\n * image-less page, and a 2026-09 fleet sweep found it on ten sites. Sites\n * passed it in by hand, so the builders drop it on the way out rather than\n * trusting callers to leave it off.\n *\n * Single source of truth for that rule — used by `createRobots`\n * (`robots/index.ts`) and by `buildAiCrawlerRules` + `createRobotsTxtHandler`\n * (`llms/aiRobots.ts`). Route any new robots builder through it.\n */\n\n// `/_next`, `/_next/`, `/_next/*`, `/_next/static/`, `/_next/image`, `/_next$`,\n// and the slashless `_next/` typo. `/_nextjs-docs/` is a real path; leave it.\nconst NEXT_INTERNALS_PATH = /^\\/?_next(?:[/*?$]|$)/\n\nexport function isNextInternalsPath(path: string): boolean {\n  return NEXT_INTERNALS_PATH.test(path.trim())\n}\n\nconst warned = new Set<string>()\n\nfunction warnDropped(paths: string[]): void {\n  for (const p of paths) {\n    if (warned.has(p)) continue\n    warned.add(p)\n    console.warn(\n      `[site-kit/robots] Dropped \"Disallow: ${p}\". Googlebot needs /_next/ to render pages and index images.`,\n    )\n  }\n}\n\n/**\n * Removes `/_next` paths from a disallow list, keeping its shape: returns the\n * input untouched when there is nothing to drop, and `undefined` when nothing\n * is left.\n */\nexport function stripNextInternals(\n  paths: string | string[] | undefined,\n): string | string[] | undefined {\n  if (paths === undefined) return undefined\n  const list = Array.isArray(paths) ? paths : [paths]\n  const dropped = list.filter(isNextInternalsPath)\n  if (dropped.length === 0) return paths\n  warnDropped(dropped)\n  const kept = list.filter((p) => !isNextInternalsPath(p))\n  if (kept.length === 0) return undefined\n  return Array.isArray(paths) ? kept : kept[0]\n}\n","/**\n * @sonordev/site-kit/llms — AI crawler robots policy\n *\n * 2026 practice separates AI RETRIEVAL crawlers (they fetch pages to answer\n * live queries and cite the source — visibility with attribution) from AI\n * TRAINING crawlers (they ingest content into future model weights). A site\n * can welcome one and decline the other. This module provides:\n *\n * - Curated user-agent lists for both classes (keep current with vendor docs)\n * - buildAiCrawlerRules(): rules for Next's MetadataRoute robots convention\n * - formatContentSignals(): the Cloudflare Content Signals Policy directive\n *   (search / ai-input / ai-train) — declarative, not enforcement, and NOT\n *   expressible through Next's robots.js convention (it drops unknown\n *   directives), hence:\n * - createRobotsTxtHandler(): a plain-text robots.txt route handler for\n *   sites that want Content-Signal lines in the emitted file.\n *\n * Every builder here drops `/_next` disallow paths (see\n * robots/next-internals.ts): Googlebot needs them to render pages.\n *\n * Canonical import: `@sonordev/site-kit/llms`. `@sonordev/site-kit/robots`\n * re-exports the same functions, because that's where people look for them.\n * Same code either way; the lists live here once.\n */\n\nimport { stripNextInternals } from '../robots/next-internals'\n\n/**\n * Retrieval/search agents — fetch on demand or index for AI search, cite sources.\n *\n * MistralAI-User (Le Chat fetching a page for a user) and YouBot (You.com's\n * search index) were added 2026-09-15: vsf-consulting-nextjs named them in\n * its own robots.txt and had to keep a NOT_YET_IN_KIT list beside this one.\n */\nexport const AI_RETRIEVAL_CRAWLERS = [\n  'OAI-SearchBot',\n  'ChatGPT-User',\n  'Claude-SearchBot',\n  'Claude-User',\n  'PerplexityBot',\n  'Perplexity-User',\n  'GoogleOther',\n  'Applebot',\n  'DuckAssistBot',\n  'MistralAI-User',\n  'YouBot',\n] as const\n\n/**\n * Training crawlers — content may enter future model training corpora.\n *\n * Amazonbot is here rather than in retrieval: Amazon says what it collects\n * may be used to train its AI models, so `training: 'block'` has to cover it.\n * It also feeds Alexa answers, which is the cost of blocking it.\n */\nexport const AI_TRAINING_CRAWLERS = [\n  'GPTBot',\n  'ClaudeBot',\n  'Google-Extended',\n  'Applebot-Extended',\n  'Meta-ExternalAgent',\n  'FacebookBot',\n  'CCBot',\n  'Bytespider',\n  'cohere-ai',\n  'MistralBot',\n  'Amazonbot',\n] as const\n\nexport interface AiCrawlerRulesOptions {\n  /** Paths every AI crawler may fetch (default: everything). */\n  allow?: string | string[]\n  /** Paths withheld from every AI crawler (default: none). `/_next` paths are dropped. */\n  disallow?: string | string[]\n  /**\n   * Policy for TRAINING crawlers. 'allow' (default) welcomes them — the\n   * visibility-maximalist choice for marketing sites that WANT to be in\n   * model knowledge. 'block' opts out of training while retrieval crawlers\n   * stay welcome, so the site is still cited in AI answers.\n   */\n  training?: 'allow' | 'block'\n}\n\ninterface RobotsRule {\n  userAgent: string\n  allow?: string | string[]\n  disallow?: string | string[]\n}\n\n/**\n * Rules for Next's MetadataRoute robots convention (app/robots.(js|ts)).\n * Spread into your existing rules array; order relative to your own rules\n * does not matter (robots matching is per-user-agent, most-specific wins).\n */\nexport function buildAiCrawlerRules(\n  options: AiCrawlerRulesOptions = {},\n): RobotsRule[] {\n  const { allow = '/', training = 'allow' } = options\n  const disallow = stripNextInternals(options.disallow)\n  const retrieval: RobotsRule[] = AI_RETRIEVAL_CRAWLERS.map((userAgent) => ({\n    userAgent,\n    allow,\n    ...(disallow ? { disallow } : {}),\n  }))\n  const trainingRules: RobotsRule[] = AI_TRAINING_CRAWLERS.map((userAgent) =>\n    training === 'block'\n      ? { userAgent, disallow: '/' }\n      : { userAgent, allow, ...(disallow ? { disallow } : {}) },\n  )\n  return [...retrieval, ...trainingRules]\n}\n\nexport interface ContentSignalsOptions {\n  /** May the content be indexed for classic search? Default true. */\n  search?: boolean\n  /** May the content feed a live AI answer (RAG/grounding)? Default true. */\n  aiInput?: boolean\n  /** May the content train future models? Default true. */\n  aiTrain?: boolean\n}\n\nconst yn = (v: boolean) => (v ? 'yes' : 'no')\n\n/**\n * The Cloudflare Content Signals Policy block for robots.txt — a preferences\n * declaration (the IETF AIPREF group is standardizing the same concepts).\n * Emit near the top of the file, before user-agent groups.\n */\nexport function formatContentSignals(\n  options: ContentSignalsOptions = {},\n): string {\n  const { search = true, aiInput = true, aiTrain = true } = options\n  return [\n    '# Content Signals Policy (https://contentsignals.org): usage preferences',\n    '# for this site’s content. Declarative — a preference, not enforcement.',\n    `Content-Signal: search=${yn(search)}, ai-input=${yn(aiInput)}, ai-train=${yn(aiTrain)}`,\n  ].join('\\n')\n}\n\nexport interface RobotsTxtHandlerOptions {\n  /** Robots rules (same shape as MetadataRoute robots rules). */\n  rules: RobotsRule[]\n  /** Content Signals preferences; omit to skip the block entirely. */\n  contentSignals?: ContentSignalsOptions\n  /** Absolute sitemap URL(s). */\n  sitemap?: string | string[]\n  /** Preferred host (emitted as Host:). */\n  host?: string\n}\n\nconst list = (v: string | string[] | undefined): string[] =>\n  v === undefined ? [] : Array.isArray(v) ? v : [v]\n\n/**\n * Plain-text robots.txt route handler (app/robots.txt/route.ts) for sites\n * that want Content-Signal lines — Next's MetadataRoute robots convention\n * cannot emit them. Drop-in:\n *\n *   import { createRobotsTxtHandler, buildAiCrawlerRules } from '@sonordev/site-kit/llms'\n *   export const GET = createRobotsTxtHandler({\n *     rules: [{ userAgent: '*', allow: '/' }, ...buildAiCrawlerRules()],\n *     contentSignals: { search: true, aiInput: true, aiTrain: true },\n *     sitemap: 'https://example.com/sitemap.xml',\n *   })\n */\nexport function createRobotsTxtHandler(options: RobotsTxtHandlerOptions) {\n  const { rules, contentSignals, sitemap, host } = options\n  const blocks: string[] = []\n  if (contentSignals) blocks.push(formatContentSignals(contentSignals))\n  for (const rule of rules) {\n    const lines = [`User-Agent: ${rule.userAgent}`]\n    for (const p of list(rule.allow)) lines.push(`Allow: ${p}`)\n    for (const p of list(stripNextInternals(rule.disallow))) lines.push(`Disallow: ${p}`)\n    blocks.push(lines.join('\\n'))\n  }\n  const tail: string[] = []\n  for (const s of list(sitemap)) tail.push(`Sitemap: ${s}`)\n  if (host) tail.push(`Host: ${host}`)\n  if (tail.length) blocks.push(tail.join('\\n'))\n  const body = `${blocks.join('\\n\\n')}\\n`\n\n  return async function GET(): Promise<Response> {\n    return new Response(body, {\n      status: 200,\n      headers: {\n        'Content-Type': 'text/plain; charset=utf-8',\n        'Cache-Control': 'public, max-age=3600, s-maxage=86400',\n      },\n    })\n  }\n}\n"]}