@se-studio/core-ui 1.36.0 → 1.37.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @se-studio/core-ui
 
+## 1.37.0
+
+### Minor Changes
+
+- 0a1bb04: Add opt-in multi-group robots.txt (preamble, AI_AGENT_USER_AGENTS / ai-agents preset, Content-Signal per group, shared no-drift disallow + allowExtra inheritance) and general llms.txt customisation (customBlocks placements, llmsAccessPolicyBlock preset, size/heading guards). Mistral token corrected to `MistralAI-Index` (vendor docs). Both surfaces remain code-configured only — no CMS fields.
+
 ## 1.36.0
 
 ### Minor Changes

package/README.md

@@ -430,7 +430,7 @@ const previewConfig = mergeCmsRendererConfig(
 #### Utilities
 - **`createRobotsConfig`** - Builds `MetadataRoute.Robots` for `src/app/robots.ts` with shared production disallow defaults; configurable via `pathPrefix`, `apiPrefixes`, `disallowExtra`, `disallowOmit`, and `blockTrackingQueryParams`
 - **`buildRobotsDisallow`** - Returns the production disallow path list (for plain-text robots route handlers)
-- **`formatRobotsTxt` / `createRobotsTxtResponse`** - Plain-text robots.txt including optional Content Signals (`src/app/robots.txt/route.ts`)
+- **`formatRobotsTxt` / `createRobotsTxtResponse`** - Plain-text robots.txt including optional Content Signals, preamble comments, multi-group AI crawler rules (`AI_AGENT_USER_AGENTS` / `userAgents: 'ai-agents'`), and shared no-drift disallow (`src/app/robots.txt/route.ts`). Code-configured only — not CMS.
 - **`resolveMarkdownNegotiationPath` / `prefersMarkdownAccept` / `htmlPathToMarkdownPath`** - Middleware helpers for `Accept: text/markdown` → existing `/{path}.md` rewrites
 - **`apiPathNeedingTrailingSlash`** - Middleware helper: rewrite bare `/api/*` → `/api/*/` when `trailingSlash: true` so MCP POST and other APIs are not 308’d
 - **`appendAgentDiscoveryLinkHeaders` / `buildAgentDiscoveryLinkValues`** - Homepage/document `Link` headers (`describedby` → `/llms.txt`, alternate markdown)

package/dist/index.d.ts

@@ -131,8 +131,8 @@ export { processCollectionLinks } from './utils/processCollectionLinks';
 export { processTextContent } from './utils/processTextContent';
 export { extractTextFromReactNode } from './utils/reactNodeText';
 export { extractRichTextNodeText } from './utils/richTextNodeText';
-export type { ContentSignalPreference, ContentSignals, CreateRobotsConfigOptions, FormatRobotsTxtOptions, } from './utils/robots';
-export { buildRobotsDisallow, createRobotsConfig, createRobotsTxtResponse, DEFAULT_ROBOTS_DISALLOW, formatContentSignalLine, formatRobotsTxt, TRACKING_QUERY_ROBOTS_DISALLOW, } from './utils/robots';
+export type { ContentSignalPreference, ContentSignals, CreateRobotsConfigOptions, FormatRobotsTxtOptions, RobotsTxtGroupOptions, RobotsUserAgentPreset, } from './utils/robots';
+export { AI_AGENT_USER_AGENTS, buildRobotsDisallow, createRobotsConfig, createRobotsTxtResponse, DEFAULT_ROBOTS_DISALLOW, formatContentSignalLine, formatRobotsTxt, resolveRobotsUserAgents, TRACKING_QUERY_ROBOTS_DISALLOW, } from './utils/robots';
 export { scopeCss } from './utils/scopeCss';
 export type { CreateSitemapXmlRouteHandlerOptions, GenerateSitemapOptions, SitemapChangeFrequency, SitemapEntry, SitemapSource, } from './utils/sitemap';
 export { buildSitemap, createSitemapXmlResponse, createSitemapXmlRouteHandler, deduplicateSitemapEntries, generateNextSitemap, SITEMAP_CACHE_TAG, SITEMAP_XML_HEADERS, sitemapToXml, sortSitemapEntries, } from './utils/sitemap';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAIH,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,sCAAsC,CAAC;AAC/F,YAAY,EAAE,yBAAyB,EAAE,MAAM,kCAAkC,CAAC;AAElF,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACvF,YAAY,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAEvE,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,uBAAuB,EACvB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,YAAY,EACZ,aAAa,EACb,sBAAsB,EACtB,8BAA8B,GAC/B,MAAM,6CAA6C,CAAC;AACrD,OAAO,EACL,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,6CAA6C,CAAC;AACrD,YAAY,EACV,gBAAgB,EAChB,qBAAqB,EACrB,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,YAAY,EACV,mCAAmC,EACnC,yBAAyB,EACzB,wBAAwB,EACxB,uBAAuB,EACvB,sBAAsB,EACtB,sBAAsB,EACtB,aAAa,GACd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,4BAA4B,EAC5B,uBAAuB,EACvB,kBAAkB,EAClB,4BAA4B,EAC5B,uBAAuB,EACvB,2BAA2B,GAC5B,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EACV,UAAU,EACV,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,iBAAiB,EACjB,gBAAgB,EAChB,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EACrB,cAAc,GACf,MAAM,qBAAqB,CAAC;AAE7B,YAAY,EACV,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,EACpB,6BAA6B,EAC7B,qBAAqB,EACrB,QAAQ,GACT,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,0BAA0B,EAC1B,uBAAuB,EACvB,gBAAgB,EAChB,eAAe,EACf,cAAc,EACd,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC5F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,YAAY,EACV,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2CAA2C,CAAC;AACpF,OAAO,EACL,kCAAkC,EAClC,uBAAuB,GACxB,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAE9D,YAAY,EACV,aAAa,EACb,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,0BAA0B,GAC3B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,YAAY,EACV,YAAY,EACZ,iBAAiB,EACjB,sBAAsB,EACtB,oBAAoB,EACpB,yBAAyB,GAC1B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,YAAY,EACV,iCAAiC,EACjC,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,4BAA4B,EAAE,MAAM,yBAAyB,CAAC;AACnF,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,YAAY,EACV,4BAA4B,EAC5B,iCAAiC,EACjC,oBAAoB,EACpB,yBAAyB,EACzB,8BAA8B,GAC/B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,8BAA8B,EAAE,MAAM,6CAA6C,CAAC;AAC7F,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EACL,sCAAsC,EACtC,aAAa,EACb,KAAK,kBAAkB,GACxB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,YAAY,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D,YAAY,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EACL,uBAAuB,EACvB,qBAAqB,EACrB,mBAAmB,EACnB,oBAAoB,EACpB,sBAAsB,EACtB,gCAAgC,EAChC,wBAAwB,EACxB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AACtD,OAAO,EACL,+BAA+B,EAC/B,0BAA0B,GAC3B,MAAM,0CAA0C,CAAC;AAClD,YAAY,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,YAAY,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,qCAAqC,CAAC;AAC9E,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAClF,OAAO,EAAE,OAAO,IAAI,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAE3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EACL,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,kBAAkB,EAClB,yBAAyB,EACzB,mBAAmB,EACnB,eAAe,EACf,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,gBAAgB,EAChB,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAIhC,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,cAAc,YAAY,CAAC;AAC3B,YAAY,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EACL,2BAA2B,EAC3B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,8BAA8B,GAC/B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,YAAY,EACV,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,EACjB,gCAAgC,GACjC,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,iBAAiB,EACjB,YAAY,EACZ,wBAAwB,GACzB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,EAC1B,gBAAgB,EAChB,sBAAsB,GACvB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAAE,EAAE,EAAE,MAAM,YAAY,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAChF,YAAY,EAAE,oCAAoC,EAAE,MAAM,sCAAsC,CAAC;AACjG,OAAO,EACL,oCAAoC,EACpC,6BAA6B,EAC7B,+BAA+B,GAChC,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,YAAY,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAC/E,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,GAC3B,MAAM,6BAA6B,CAAC;AAErC,YAAY,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EACL,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,mBAAmB,EACnB,oBAAoB,EACpB,oCAAoC,GACrC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AACnE,YAAY,EACV,uBAAuB,EACvB,cAAc,EACd,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,EACf,8BAA8B,GAC/B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,YAAY,EACV,mCAAmC,EACnC,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EACZ,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,YAAY,EACZ,wBAAwB,EACxB,4BAA4B,EAC5B,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,iCAAiC,EACjC,qBAAqB,GACtB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAEhE,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC9E,OAAO,EACL,+BAA+B,EAC/B,iBAAiB,EACjB,aAAa,EACb,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,oCAAoC,EAAE,MAAM,yBAAyB,CAAC;AAC/E,YAAY,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EACL,oBAAoB,EACpB,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACnB,eAAe,GAChB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,4BAA4B,EAAE,MAAM,sCAAsC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAIH,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,sCAAsC,CAAC;AAC/F,YAAY,EAAE,yBAAyB,EAAE,MAAM,kCAAkC,CAAC;AAElF,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACvF,YAAY,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAEvE,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,uBAAuB,EACvB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,YAAY,EACZ,aAAa,EACb,sBAAsB,EACtB,8BAA8B,GAC/B,MAAM,6CAA6C,CAAC;AACrD,OAAO,EACL,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,6CAA6C,CAAC;AACrD,YAAY,EACV,gBAAgB,EAChB,qBAAqB,EACrB,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,YAAY,EACV,mCAAmC,EACnC,yBAAyB,EACzB,wBAAwB,EACxB,uBAAuB,EACvB,sBAAsB,EACtB,sBAAsB,EACtB,aAAa,GACd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,4BAA4B,EAC5B,uBAAuB,EACvB,kBAAkB,EAClB,4BAA4B,EAC5B,uBAAuB,EACvB,2BAA2B,GAC5B,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EACV,UAAU,EACV,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,iBAAiB,EACjB,gBAAgB,EAChB,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EACrB,cAAc,GACf,MAAM,qBAAqB,CAAC;AAE7B,YAAY,EACV,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,EACpB,6BAA6B,EAC7B,qBAAqB,EACrB,QAAQ,GACT,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,0BAA0B,EAC1B,uBAAuB,EACvB,gBAAgB,EAChB,eAAe,EACf,cAAc,EACd,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC5F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,YAAY,EACV,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2CAA2C,CAAC;AACpF,OAAO,EACL,kCAAkC,EAClC,uBAAuB,GACxB,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAE9D,YAAY,EACV,aAAa,EACb,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,0BAA0B,GAC3B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,YAAY,EACV,YAAY,EACZ,iBAAiB,EACjB,sBAAsB,EACtB,oBAAoB,EACpB,yBAAyB,GAC1B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,YAAY,EACV,iCAAiC,EACjC,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,4BAA4B,EAAE,MAAM,yBAAyB,CAAC;AACnF,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,YAAY,EACV,4BAA4B,EAC5B,iCAAiC,EACjC,oBAAoB,EACpB,yBAAyB,EACzB,8BAA8B,GAC/B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,8BAA8B,EAAE,MAAM,6CAA6C,CAAC;AAC7F,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EACL,sCAAsC,EACtC,aAAa,EACb,KAAK,kBAAkB,GACxB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,YAAY,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D,YAAY,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EACL,uBAAuB,EACvB,qBAAqB,EACrB,mBAAmB,EACnB,oBAAoB,EACpB,sBAAsB,EACtB,gCAAgC,EAChC,wBAAwB,EACxB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AACtD,OAAO,EACL,+BAA+B,EAC/B,0BAA0B,GAC3B,MAAM,0CAA0C,CAAC;AAClD,YAAY,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,YAAY,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,qCAAqC,CAAC;AAC9E,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAClF,OAAO,EAAE,OAAO,IAAI,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAE3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EACL,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,kBAAkB,EAClB,yBAAyB,EACzB,mBAAmB,EACnB,eAAe,EACf,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,gBAAgB,EAChB,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,GAC3B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAIhC,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,cAAc,YAAY,CAAC;AAC3B,YAAY,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EACL,2BAA2B,EAC3B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,8BAA8B,GAC/B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,YAAY,EACV,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,EACjB,gCAAgC,GACjC,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,iBAAiB,EACjB,YAAY,EACZ,wBAAwB,GACzB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,EAC1B,gBAAgB,EAChB,sBAAsB,GACvB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAAE,EAAE,EAAE,MAAM,YAAY,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAChF,YAAY,EAAE,oCAAoC,EAAE,MAAM,sCAAsC,CAAC;AACjG,OAAO,EACL,oCAAoC,EACpC,6BAA6B,EAC7B,+BAA+B,GAChC,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,YAAY,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAC/E,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,GAC3B,MAAM,6BAA6B,CAAC;AAErC,YAAY,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EACL,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,mBAAmB,EACnB,oBAAoB,EACpB,oCAAoC,GACrC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AACnE,YAAY,EACV,uBAAuB,EACvB,cAAc,EACd,yBAAyB,EACzB,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,EACf,uBAAuB,EACvB,8BAA8B,GAC/B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,YAAY,EACV,mCAAmC,EACnC,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EACZ,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,YAAY,EACZ,wBAAwB,EACxB,4BAA4B,EAC5B,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,iCAAiC,EACjC,qBAAqB,GACtB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAEhE,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC9E,OAAO,EACL,+BAA+B,EAC/B,iBAAiB,EACjB,aAAa,EACb,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,oCAAoC,EAAE,MAAM,yBAAyB,CAAC;AAC/E,YAAY,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EACL,oBAAoB,EACpB,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACnB,eAAe,GAChB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,4BAA4B,EAAE,MAAM,sCAAsC,CAAC"}

package/dist/index.js

@@ -112,7 +112,7 @@ export { processCollectionLinks } from './utils/processCollectionLinks';
 export { processTextContent } from './utils/processTextContent';
 export { extractTextFromReactNode } from './utils/reactNodeText';
 export { extractRichTextNodeText } from './utils/richTextNodeText';
-export { buildRobotsDisallow, createRobotsConfig, createRobotsTxtResponse, DEFAULT_ROBOTS_DISALLOW, formatContentSignalLine, formatRobotsTxt, TRACKING_QUERY_ROBOTS_DISALLOW, } from './utils/robots';
+export { AI_AGENT_USER_AGENTS, buildRobotsDisallow, createRobotsConfig, createRobotsTxtResponse, DEFAULT_ROBOTS_DISALLOW, formatContentSignalLine, formatRobotsTxt, resolveRobotsUserAgents, TRACKING_QUERY_ROBOTS_DISALLOW, } from './utils/robots';
 export { scopeCss } from './utils/scopeCss';
 export { buildSitemap, createSitemapXmlResponse, createSitemapXmlRouteHandler, deduplicateSitemapEntries, generateNextSitemap, SITEMAP_CACHE_TAG, SITEMAP_XML_HEADERS, sitemapToXml, sortSitemapEntries, } from './utils/sitemap';
 export { buildStructuredDataContext } from './utils/structuredDataUtils';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAMH,YAAY;AACZ,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAEvF,qBAAqB;AACrB,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,uBAAuB,EACvB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAO9B,OAAO,EACL,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,6CAA6C,CAAC;AAOrD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAUxD,OAAO,EACL,4BAA4B,EAC5B,uBAAuB,EACvB,kBAAkB,EAClB,4BAA4B,EAC5B,uBAAuB,EACvB,2BAA2B,GAC5B,MAAM,mBAAmB,CAAC;AAwB3B,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,0BAA0B,EAC1B,uBAAuB,EACvB,gBAAgB,EAChB,eAAe,EACf,cAAc,EACd,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC5F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAK5E,OAAO,EAAE,uBAAuB,EAAE,MAAM,2CAA2C,CAAC;AACpF,OAAO,EACL,kCAAkC,EAClC,uBAAuB,GACxB,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAS9D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAQ3D,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAMzD,OAAO,EAAE,UAAU,EAAE,4BAA4B,EAAE,MAAM,yBAAyB,CAAC;AACnF,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAQrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,8BAA8B,EAAE,MAAM,6CAA6C,CAAC;AAC7F,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AACpC,uBAAuB;AACvB,iBAAiB;AACjB,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EACL,sCAAsC,EACtC,aAAa,GAEd,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAG7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D,oBAAoB;AACpB,OAAO,EACL,uBAAuB,EACvB,qBAAqB,EACrB,mBAAmB,EACnB,oBAAoB,EACpB,sBAAsB,EACtB,gCAAgC,EAChC,wBAAwB,EACxB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AACtD,OAAO,EACL,+BAA+B,EAC/B,0BAA0B,GAC3B,MAAM,0CAA0C,CAAC;AAElD,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,WAAW;AACX,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,qCAAqC,CAAC;AAC9E,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAClF,OAAO,EAAE,OAAO,IAAI,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,eAAe;AACf,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EACL,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,kBAAkB,EAClB,yBAAyB,EACzB,mBAAmB,EACnB,eAAe,EAIf,gBAAgB,GAGjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAKhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,eAAe;AACf,cAAc,YAAY,CAAC;AAE3B,OAAO,EACL,2BAA2B,EAC3B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,8BAA8B,GAC/B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAOxE,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,EAC1B,gBAAgB,EAChB,sBAAsB,GACvB,MAAM,2BAA2B,CAAC;AACnC,mBAAmB;AACnB,OAAO,EAAE,EAAE,EAAE,MAAM,YAAY,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAEhF,OAAO,EACL,oCAAoC,EACpC,6BAA6B,EAC7B,+BAA+B,GAChC,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AAEzE,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,GAC3B,MAAM,6BAA6B,CAAC;AAGrC,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,OAAO,EACL,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,mBAAmB,EACnB,oBAAoB,EACpB,oCAAoC,GACrC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AAOnE,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,EACf,8BAA8B,GAC/B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAS5C,OAAO,EACL,YAAY,EACZ,wBAAwB,EACxB,4BAA4B,EAC5B,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AAKzB,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,wBAAwB;AACxB,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC9E,OAAO,EACL,+BAA+B,EAC/B,iBAAiB,EACjB,aAAa,EACb,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,oCAAoC,EAAE,MAAM,yBAAyB,CAAC;AAE/E,OAAO,EACL,oBAAoB,EACpB,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACnB,eAAe,GAChB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,4BAA4B,EAAE,MAAM,sCAAsC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAMH,YAAY;AACZ,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAEvF,qBAAqB;AACrB,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,uBAAuB,EACvB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAO9B,OAAO,EACL,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,6CAA6C,CAAC;AAOrD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAUxD,OAAO,EACL,4BAA4B,EAC5B,uBAAuB,EACvB,kBAAkB,EAClB,4BAA4B,EAC5B,uBAAuB,EACvB,2BAA2B,GAC5B,MAAM,mBAAmB,CAAC;AAwB3B,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,0BAA0B,EAC1B,uBAAuB,EACvB,gBAAgB,EAChB,eAAe,EACf,cAAc,EACd,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC5F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAK5E,OAAO,EAAE,uBAAuB,EAAE,MAAM,2CAA2C,CAAC;AACpF,OAAO,EACL,kCAAkC,EAClC,uBAAuB,GACxB,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAS9D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAQ3D,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAMzD,OAAO,EAAE,UAAU,EAAE,4BAA4B,EAAE,MAAM,yBAAyB,CAAC;AACnF,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAQrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,8BAA8B,EAAE,MAAM,6CAA6C,CAAC;AAC7F,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AACpC,uBAAuB;AACvB,iBAAiB;AACjB,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EACL,sCAAsC,EACtC,aAAa,GAEd,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAG7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D,oBAAoB;AACpB,OAAO,EACL,uBAAuB,EACvB,qBAAqB,EACrB,mBAAmB,EACnB,oBAAoB,EACpB,sBAAsB,EACtB,gCAAgC,EAChC,wBAAwB,EACxB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AACtD,OAAO,EACL,+BAA+B,EAC/B,0BAA0B,GAC3B,MAAM,0CAA0C,CAAC;AAElD,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,WAAW;AACX,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,qCAAqC,CAAC;AAC9E,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAClF,OAAO,EAAE,OAAO,IAAI,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,eAAe;AACf,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EACL,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,kBAAkB,EAClB,yBAAyB,EACzB,mBAAmB,EACnB,eAAe,EAIf,gBAAgB,GAGjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAKhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,eAAe;AACf,cAAc,YAAY,CAAC;AAE3B,OAAO,EACL,2BAA2B,EAC3B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,8BAA8B,GAC/B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAOxE,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,EAC1B,gBAAgB,EAChB,sBAAsB,GACvB,MAAM,2BAA2B,CAAC;AACnC,mBAAmB;AACnB,OAAO,EAAE,EAAE,EAAE,MAAM,YAAY,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAEhF,OAAO,EACL,oCAAoC,EACpC,6BAA6B,EAC7B,+BAA+B,GAChC,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AAEzE,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,GAC3B,MAAM,6BAA6B,CAAC;AAGrC,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,OAAO,EACL,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,mBAAmB,EACnB,oBAAoB,EACpB,oCAAoC,GACrC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AASnE,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,EACf,uBAAuB,EACvB,8BAA8B,GAC/B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAS5C,OAAO,EACL,YAAY,EACZ,wBAAwB,EACxB,4BAA4B,EAC5B,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AAKzB,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,wBAAwB;AACxB,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC9E,OAAO,EACL,+BAA+B,EAC/B,iBAAiB,EACjB,aAAa,EACb,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,oCAAoC,EAAE,MAAM,yBAAyB,CAAC;AAE/E,OAAO,EACL,oBAAoB,EACpB,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACnB,eAAe,GAChB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,4BAA4B,EAAE,MAAM,sCAAsC,CAAC"}

package/dist/server/index.d.ts

@@ -10,8 +10,8 @@ export { DEFAULT_RELATED_ARTICLES_FALLBACK_POLICY, getRelatedArticles, getRelate
 export type { BuildListingItemsOptions, ListingFetchHelpers, ListingStructuredDataItem, } from '../utils/listingStructuredData';
 export { buildListingItems, buildListingStructuredDataContext, } from '../utils/listingStructuredData';
 export { CMS_HOMEPAGE_SLUG, controlSlugFromCanonicalPath, resolvePageTestControlSlug, } from '../utils/pageTestControlSlug';
-export type { ContentSignalPreference, ContentSignals, CreateRobotsConfigOptions, FormatRobotsTxtOptions, } from '../utils/robots';
-export { buildRobotsDisallow, createRobotsConfig, createRobotsTxtResponse, DEFAULT_ROBOTS_DISALLOW, formatContentSignalLine, formatRobotsTxt, TRACKING_QUERY_ROBOTS_DISALLOW, } from '../utils/robots';
+export type { ContentSignalPreference, ContentSignals, CreateRobotsConfigOptions, FormatRobotsTxtOptions, RobotsTxtGroupOptions, RobotsUserAgentPreset, } from '../utils/robots';
+export { AI_AGENT_USER_AGENTS, buildRobotsDisallow, createRobotsConfig, createRobotsTxtResponse, DEFAULT_ROBOTS_DISALLOW, formatContentSignalLine, formatRobotsTxt, resolveRobotsUserAgents, TRACKING_QUERY_ROBOTS_DISALLOW, } from '../utils/robots';
 export type { GenerateNotFoundPage, GenerateSlimNotFoundPage, RouteHandlersConfig, } from './route-handlers';
 export { createRouteHandlers } from './route-handlers';
 export type { CmsPagePreparationResult, PrepareArticlePageInput, PrepareArticleTypePageInput, PrepareArticleTypesIndexPageInput, PrepareArticleTypeTagPageInput, PrepareCustomTypePageInput, PreparePageInput, PreparePersonPageInput, PrepareTagPageInput, PrepareTagsIndexPageInput, PrepareTeamIndexPageInput, RouteHandlersPrepareHooks, } from './route-handlers-types';

package/dist/server/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,CAAC;AAErB,YAAY,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAC9F,YAAY,EACV,UAAU,EACV,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,YAAY,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AACtE,OAAO,EACL,2BAA2B,EAC3B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,6BAA6B,EAAE,MAAM,yBAAyB,CAAC;AAC7E,OAAO,EACL,wCAAwC,EACxC,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,oCAAoC,GACrC,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,wBAAwB,EACxB,mBAAmB,EACnB,yBAAyB,GAC1B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,iBAAiB,EACjB,iCAAiC,GAClC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,uBAAuB,EACvB,cAAc,EACd,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,EACf,8BAA8B,GAC/B,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,oBAAoB,EACpB,wBAAwB,EACxB,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,YAAY,EACV,wBAAwB,EACxB,uBAAuB,EACvB,2BAA2B,EAC3B,iCAAiC,EACjC,8BAA8B,EAC9B,0BAA0B,EAC1B,gBAAgB,EAChB,sBAAsB,EACtB,mBAAmB,EACnB,yBAAyB,EACzB,yBAAyB,EACzB,yBAAyB,GAC1B,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,cAAc,EACd,sBAAsB,EACtB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,CAAC;AAErB,YAAY,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAC9F,YAAY,EACV,UAAU,EACV,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,YAAY,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AACtE,OAAO,EACL,2BAA2B,EAC3B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,6BAA6B,EAAE,MAAM,yBAAyB,CAAC;AAC7E,OAAO,EACL,wCAAwC,EACxC,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,oCAAoC,GACrC,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,wBAAwB,EACxB,mBAAmB,EACnB,yBAAyB,GAC1B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,iBAAiB,EACjB,iCAAiC,GAClC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,uBAAuB,EACvB,cAAc,EACd,yBAAyB,EACzB,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,EACf,uBAAuB,EACvB,8BAA8B,GAC/B,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,oBAAoB,EACpB,wBAAwB,EACxB,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,YAAY,EACV,wBAAwB,EACxB,uBAAuB,EACvB,2BAA2B,EAC3B,iCAAiC,EACjC,8BAA8B,EAC9B,0BAA0B,EAC1B,gBAAgB,EAChB,sBAAsB,EACtB,mBAAmB,EACnB,yBAAyB,EACzB,yBAAyB,EACzB,yBAAyB,GAC1B,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,cAAc,EACd,sBAAsB,EACtB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC"}

package/dist/server/index.js

@@ -5,7 +5,7 @@ export { apiPathNeedingTrailingSlash, appendAgentDiscoveryLinkHeaders, buildAgen
 export { DEFAULT_RELATED_ARTICLES_FALLBACK_POLICY, getRelatedArticles, getRelatedPeople, getRelatedTags, resolveRelatedArticlesFallbackPolicy, } from '../utils/componentUtils';
 export { buildListingItems, buildListingStructuredDataContext, } from '../utils/listingStructuredData';
 export { CMS_HOMEPAGE_SLUG, controlSlugFromCanonicalPath, resolvePageTestControlSlug, } from '../utils/pageTestControlSlug';
-export { buildRobotsDisallow, createRobotsConfig, createRobotsTxtResponse, DEFAULT_ROBOTS_DISALLOW, formatContentSignalLine, formatRobotsTxt, TRACKING_QUERY_ROBOTS_DISALLOW, } from '../utils/robots';
+export { AI_AGENT_USER_AGENTS, buildRobotsDisallow, createRobotsConfig, createRobotsTxtResponse, DEFAULT_ROBOTS_DISALLOW, formatContentSignalLine, formatRobotsTxt, resolveRobotsUserAgents, TRACKING_QUERY_ROBOTS_DISALLOW, } from '../utils/robots';
 export { createRouteHandlers } from './route-handlers';
 export { toBreadcrumbOptions } from './route-types';
 export { canRenderArticleTypeTagRoute, canRenderTagIndex } from './tag-index-guards';

package/dist/server/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,CAAC;AAGrB,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAM9F,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAElD,OAAO,EACL,2BAA2B,EAC3B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,wCAAwC,EACxC,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,oCAAoC,GACrC,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EACL,iBAAiB,EACjB,iCAAiC,GAClC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAOtC,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,EACf,8BAA8B,GAC/B,MAAM,iBAAiB,CAAC;AAMzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAuBvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,CAAC;AAGrB,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAM9F,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAElD,OAAO,EACL,2BAA2B,EAC3B,+BAA+B,EAC/B,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,wCAAwC,EACxC,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,oCAAoC,GACrC,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EACL,iBAAiB,EACjB,iCAAiC,GAClC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,iBAAiB,EACjB,4BAA4B,EAC5B,0BAA0B,GAC3B,MAAM,8BAA8B,CAAC;AAStC,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,EACf,uBAAuB,EACvB,8BAA8B,GAC/B,MAAM,iBAAiB,CAAC;AAMzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAuBvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC"}

package/dist/utils/robots.d.ts

@@ -7,6 +7,19 @@ export declare const DEFAULT_ROBOTS_DISALLOW: readonly ['/*?_rsc=*', '/_next/dat
  * help other crawlers and audit tooling but are not a complete Google fix.
  */
 export declare const TRACKING_QUERY_ROBOTS_DISALLOW: readonly ['/*?_gl=*', '/*?nnerid=*', '/*?ref=*'];
+/**
+ * Curated AI / LLM crawler user-agents — single source of truth for all sites.
+ * Living list: bump via changeset when vendors rename tokens; verify against vendor docs.
+ *
+ * Spot-checked 12 Oct 2026:
+ * - Meta trio (`meta-externalagent`, `meta-externalfetcher`, `meta-webindexer`) —
+ *   matches https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
+ * - Mistral: `MistralAI-User` + `MistralAI-Index` (not `MistralAI-User/Index`) —
+ *   matches https://docs.mistral.ai/robots
+ */
+export declare const AI_AGENT_USER_AGENTS: readonly ['ClaudeBot', 'Claude-User', 'Claude-SearchBot', 'GPTBot', 'ChatGPT-User', 'OAI-SearchBot', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-externalfetcher', 'meta-webindexer', 'Amazonbot', 'Amzn-SearchBot', 'Amzn-User', 'MistralAI-User', 'MistralAI-Index', 'DuckAssistBot', 'CCBot', 'Bytespider', 'Diffbot', 'cohere-ai'];
+/** Named UA presets that expand via {@link AI_AGENT_USER_AGENTS}. */
+export type RobotsUserAgentPreset = 'ai-agents';
 /** Yes/no preference for Content Signals (contentsignals.org). */
 export type ContentSignalPreference = 'yes' | 'no';
 /**
@@ -60,14 +73,47 @@ export interface CreateRobotsConfigOptions {
      */
     allowExtra?: readonly string[];
 }
-/** Options for plain-text robots.txt (supports Content-Signal). */
+/**
+ * Additional robots.txt group after `User-agent: *`.
+ * Named groups inherit the shared disallow list (no drift) plus optional group-only extras.
+ */
+export interface RobotsTxtGroupOptions {
+    /** Explicit UAs, or a preset key that expands via {@link AI_AGENT_USER_AGENTS}. */
+    userAgents: readonly string[] | RobotsUserAgentPreset;
+    /** Overrides group Content-Signal; default = top-level `contentSignals`. */
+    contentSignals?: ContentSignals;
+    /**
+     * Disallows for this group only (merged after shared base).
+     * Must never be applied to `*` unless also listed in shared `disallowExtra`.
+     */
+    disallowExtra?: readonly string[];
+    /**
+     * Extra `Allow:` paths for this group (after `Allow: /`).
+     * Appended after top-level `allowExtra` (same no-drift inheritance as disallow).
+     */
+    allowExtra?: readonly string[];
+}
+/** Options for plain-text robots.txt (supports Content-Signal, preamble, multi-group). */
 export interface FormatRobotsTxtOptions extends CreateRobotsConfigOptions {
     /**
-     * When set (production only), emit a Content-Signal line under `User-agent: *`.
+     * When set (production only), emit a Content-Signal line under `User-agent: *`
+     * and as the default for each named group (unless the group overrides).
      * Next.js `MetadataRoute.Robots` cannot express this — use `formatRobotsTxt` /
      * `createRobotsTxtResponse` in `src/app/robots.txt/route.ts` instead of `robots.ts`.
      */
     contentSignals?: ContentSignals;
+    /**
+     * Leading comment lines (e.g. Terms of Use preamble). Strings may omit `# `;
+     * the formatter prefixes `# ` when missing and emits a blank line before the
+     * first `User-agent`. Production only. Site-owned copy — not CMS.
+     */
+    preamble?: string | readonly string[];
+    /**
+     * Additional groups after `*`. Each group emits multiple `User-agent:` lines,
+     * optional Content-Signal, Allow, then the shared disallow list plus
+     * `group.disallowExtra`. Production only; opt-in.
+     */
+    groups?: readonly RobotsTxtGroupOptions[];
 }
 /**
  * Builds the production disallow path list from options.
@@ -80,8 +126,8 @@ export declare function buildRobotsDisallow(options: CreateRobotsConfigOptions):
  * In production: allows `/` and disallows internal/infrastructure paths.
  * In non-production: disallows all crawlers (with optional staging bot allowlist).
  *
- * **Content-Signal:** not supported here (Next metadata type). Use
- * {@link formatRobotsTxt} / {@link createRobotsTxtResponse} in a route handler.
+ * **Content-Signal / preamble / multi-group:** not supported here (Next metadata type).
+ * Use {@link formatRobotsTxt} / {@link createRobotsTxtResponse} in a route handler.
  */
 export declare function createRobotsConfig(options: CreateRobotsConfigOptions): MetadataRoute.Robots;
 /**
@@ -92,11 +138,18 @@ export declare function createRobotsConfig(options: CreateRobotsConfigOptions):
  * // → 'Content-Signal: search=yes, ai-train=no, ai-input=yes'
  */
 export declare function formatContentSignalLine(signals: ContentSignals): string;
+/** Resolves a preset or explicit UA list. Throws if the resolved list is empty. */
+export declare function resolveRobotsUserAgents(userAgents: readonly string[] | RobotsUserAgentPreset): readonly string[];
 /**
- * Builds a full plain-text robots.txt body, including optional Content-Signal.
+ * Builds a full plain-text robots.txt body, including optional Content-Signal,
+ * preamble comments, and additional user-agent groups.
+ *
+ * Prefer this over `createRobotsConfig` when you need Content Signals, comments,
+ * or multi-group AI crawler rules. Serve from `src/app/robots.txt/route.ts`.
  *
- * Prefer this over `createRobotsConfig` when you need Content Signals or other
- * non-standard directives. Serve from `src/app/robots.txt/route.ts`.
+ * **No-drift:** shared disallow is built once via {@link buildRobotsDisallow};
+ * named groups append only `group.disallowExtra`. Top-level `allowExtra` is
+ * inherited by named groups, then `group.allowExtra` is appended.
  */
 export declare function formatRobotsTxt(options: FormatRobotsTxtOptions): string;
 /**
@@ -108,6 +161,8 @@ export declare function formatRobotsTxt(options: FormatRobotsTxtOptions): string
  *     isProduction,
  *     baseUrl,
  *     contentSignals: { search: 'yes', aiTrain: 'no', aiInput: 'yes' },
+ *     preamble: ['Terms of Use apply. See /terms'],
+ *     groups: [{ userAgents: 'ai-agents', disallowExtra: ['/results'] }],
  *   });
  * }
  */

package/dist/utils/robots.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/utils/robots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,MAAM,CAAC;AAE1C,uEAAuE;AACvE,eAAO,MAAM,uBAAuB,YAClC,WAAW,EACX,cAAc,EACd,OAAO,EACP,WAAW,EACX,aAAa,EACb,OAAO,EACP,MAAM,CACE,CAAC;AAEX;;;;GAIG;AACH,eAAO,MAAM,8BAA8B,YAAI,UAAU,EAAE,aAAa,EAAE,UAAU,CAAU,CAAC;AAE/F,kEAAkE;AAClE,MAAM,MAAM,uBAAuB,GAAG,KAAK,GAAG,IAAI,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,wDAAwD;IACxD,MAAM,EAAE,uBAAuB,CAAC;IAChC,yCAAyC;IACzC,OAAO,EAAE,uBAAuB,CAAC;IACjC,6DAA6D;IAC7D,OAAO,EAAE,uBAAuB,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7B,+CAA+C;IAC/C,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,0FAA0F;IAC1F,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC,4CAA4C;IAC5C,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;OAGG;IACH,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChE;;;;;OAKG;IACH,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAChC;AAED,mEAAmE;AACnE,MAAM,WAAW,sBAAuB,SAAQ,yBAAyB;IACvE;;;;OAIG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AA2CD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,GAAG,MAAM,EAAE,CAwBhF;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,aAAa,CAAC,MAAM,CAoC3F;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,cAAc,GAAG,MAAM,CAEvE;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CAoCvE;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,sBAAsB,GAAG,QAAQ,CAIjF"}
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/utils/robots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,MAAM,CAAC;AAE1C,uEAAuE;AACvE,eAAO,MAAM,uBAAuB,YAClC,WAAW,EACX,cAAc,EACd,OAAO,EACP,WAAW,EACX,aAAa,EACb,OAAO,EACP,MAAM,CACE,CAAC;AAEX;;;;GAIG;AACH,eAAO,MAAM,8BAA8B,YAAI,UAAU,EAAE,aAAa,EAAE,UAAU,CAAU,CAAC;AAE/F;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB,YAC/B,WAAW,EACX,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,cAAc,EACd,eAAe,EACf,eAAe,EACf,iBAAiB,EACjB,oBAAoB,EACpB,sBAAsB,EACtB,iBAAiB,EACjB,WAAW,EACX,gBAAgB,EAChB,WAAW,EACX,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,OAAO,EACP,YAAY,EACZ,SAAS,EACT,WAAW,CACH,CAAC;AAEX,qEAAqE;AACrE,MAAM,MAAM,qBAAqB,GAAG,WAAW,CAAC;AAEhD,kEAAkE;AAClE,MAAM,MAAM,uBAAuB,GAAG,KAAK,GAAG,IAAI,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,wDAAwD;IACxD,MAAM,EAAE,uBAAuB,CAAC;IAChC,yCAAyC;IACzC,OAAO,EAAE,uBAAuB,CAAC;IACjC,6DAA6D;IAC7D,OAAO,EAAE,uBAAuB,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7B,+CAA+C;IAC/C,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,0FAA0F;IAC1F,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC,4CAA4C;IAC5C,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;OAGG;IACH,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChE;;;;;OAKG;IACH,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,mFAAmF;IACnF,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,qBAAqB,CAAC;IACtD,4EAA4E;IAC5E,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC;;;OAGG;IACH,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC;;;OAGG;IACH,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAChC;AAED,0FAA0F;AAC1F,MAAM,WAAW,sBAAuB,SAAQ,yBAAyB;IACvE;;;;;OAKG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;IACtC;;;;OAIG;IACH,MAAM,CAAC,EAAE,SAAS,qBAAqB,EAAE,CAAC;CAC3C;AA2CD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,GAAG,MAAM,EAAE,CAwBhF;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,aAAa,CAAC,MAAM,CAoC3F;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,cAAc,GAAG,MAAM,CAEvE;AAED,mFAAmF;AACnF,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,qBAAqB,GACpD,SAAS,MAAM,EAAE,CAMnB;AAuCD;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CA0DvE;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,sBAAsB,GAAG,QAAQ,CAIjF"}

package/dist/utils/robots.js

@@ -14,6 +14,39 @@ export const DEFAULT_ROBOTS_DISALLOW = [
  * help other crawlers and audit tooling but are not a complete Google fix.
  */
 export const TRACKING_QUERY_ROBOTS_DISALLOW = ['/*?_gl=*', '/*?nnerid=*', '/*?ref=*'];
+/**
+ * Curated AI / LLM crawler user-agents — single source of truth for all sites.
+ * Living list: bump via changeset when vendors rename tokens; verify against vendor docs.
+ *
+ * Spot-checked 12 Oct 2026:
+ * - Meta trio (`meta-externalagent`, `meta-externalfetcher`, `meta-webindexer`) —
+ *   matches https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
+ * - Mistral: `MistralAI-User` + `MistralAI-Index` (not `MistralAI-User/Index`) —
+ *   matches https://docs.mistral.ai/robots
+ */
+export const AI_AGENT_USER_AGENTS = [
+    'ClaudeBot',
+    'Claude-User',
+    'Claude-SearchBot',
+    'GPTBot',
+    'ChatGPT-User',
+    'OAI-SearchBot',
+    'PerplexityBot',
+    'Perplexity-User',
+    'meta-externalagent',
+    'meta-externalfetcher',
+    'meta-webindexer',
+    'Amazonbot',
+    'Amzn-SearchBot',
+    'Amzn-User',
+    'MistralAI-User',
+    'MistralAI-Index',
+    'DuckAssistBot',
+    'CCBot',
+    'Bytespider',
+    'Diffbot',
+    'cohere-ai',
+];
 function normalizePathPrefix(prefix) {
     if (!prefix) {
         return undefined;
@@ -76,8 +109,8 @@ export function buildRobotsDisallow(options) {
  * In production: allows `/` and disallows internal/infrastructure paths.
  * In non-production: disallows all crawlers (with optional staging bot allowlist).
  *
- * **Content-Signal:** not supported here (Next metadata type). Use
- * {@link formatRobotsTxt} / {@link createRobotsTxtResponse} in a route handler.
+ * **Content-Signal / preamble / multi-group:** not supported here (Next metadata type).
+ * Use {@link formatRobotsTxt} / {@link createRobotsTxtResponse} in a route handler.
  */
 export function createRobotsConfig(options) {
     const sitemap = options.sitemapUrl ?? `${options.baseUrl}/sitemap.xml`;
@@ -120,11 +153,54 @@ export function createRobotsConfig(options) {
 export function formatContentSignalLine(signals) {
     return `Content-Signal: search=${signals.search}, ai-train=${signals.aiTrain}, ai-input=${signals.aiInput}`;
 }
+/** Resolves a preset or explicit UA list. Throws if the resolved list is empty. */
+export function resolveRobotsUserAgents(userAgents) {
+    const resolved = userAgents === 'ai-agents' ? AI_AGENT_USER_AGENTS : userAgents;
+    if (resolved.length === 0) {
+        throw new Error('robots.txt group userAgents must not be empty');
+    }
+    return resolved;
+}
+/** Merges shared (top-level) allowExtra with group-only extras. */
+function mergeAllowExtra(shared, group) {
+    if (!shared?.length && !group?.length) {
+        return undefined;
+    }
+    return [...(shared ?? []), ...(group ?? [])];
+}
+function formatPreambleLines(preamble) {
+    const raw = typeof preamble === 'string' ? [preamble] : [...preamble];
+    return raw.map((line) => {
+        const trimmed = line.replace(/\s+$/, '');
+        if (trimmed.startsWith('#')) {
+            return trimmed;
+        }
+        return `# ${trimmed}`;
+    });
+}
+function appendAllowLines(lines, allowExtra) {
+    lines.push('Allow: /');
+    if (allowExtra?.length) {
+        for (const path of allowExtra) {
+            lines.push(`Allow: ${path}`);
+        }
+    }
+}
+function appendDisallowLines(lines, paths) {
+    for (const path of paths) {
+        lines.push(`Disallow: ${path}`);
+    }
+}
 /**
- * Builds a full plain-text robots.txt body, including optional Content-Signal.
+ * Builds a full plain-text robots.txt body, including optional Content-Signal,
+ * preamble comments, and additional user-agent groups.
+ *
+ * Prefer this over `createRobotsConfig` when you need Content Signals, comments,
+ * or multi-group AI crawler rules. Serve from `src/app/robots.txt/route.ts`.
  *
- * Prefer this over `createRobotsConfig` when you need Content Signals or other
- * non-standard directives. Serve from `src/app/robots.txt/route.ts`.
+ * **No-drift:** shared disallow is built once via {@link buildRobotsDisallow};
+ * named groups append only `group.disallowExtra`. Top-level `allowExtra` is
+ * inherited by named groups, then `group.allowExtra` is appended.
  */
 export function formatRobotsTxt(options) {
     const sitemap = options.sitemapUrl ?? `${options.baseUrl}/sitemap.xml`;
@@ -138,19 +214,36 @@ export function formatRobotsTxt(options) {
         lines.push('', `Sitemap: ${sitemap}`, '');
         return lines.join('\n');
     }
-    const lines = ['User-agent: *'];
+    const lines = [];
+    if (options.preamble !== undefined) {
+        lines.push(...formatPreambleLines(options.preamble));
+        lines.push('');
+    }
+    const sharedDisallow = buildRobotsDisallow(options);
+    lines.push('User-agent: *');
     if (options.contentSignals) {
         lines.push(formatContentSignalLine(options.contentSignals));
     }
-    lines.push('Allow: /');
-    if (options.allowExtra?.length) {
-        for (const path of options.allowExtra) {
-            lines.push(`Allow: ${path}`);
+    appendAllowLines(lines, options.allowExtra);
+    appendDisallowLines(lines, sharedDisallow);
+    if (options.groups?.length) {
+        for (const group of options.groups) {
+            const userAgents = resolveRobotsUserAgents(group.userAgents);
+            lines.push('');
+            for (const ua of userAgents) {
+                lines.push(`User-agent: ${ua}`);
+            }
+            const groupSignals = group.contentSignals ?? options.contentSignals;
+            if (groupSignals) {
+                lines.push(formatContentSignalLine(groupSignals));
+            }
+            appendAllowLines(lines, mergeAllowExtra(options.allowExtra, group.allowExtra));
+            const groupDisallow = group.disallowExtra?.length
+                ? [...sharedDisallow, ...group.disallowExtra]
+                : sharedDisallow;
+            appendDisallowLines(lines, groupDisallow);
         }
     }
-    for (const path of buildRobotsDisallow(options)) {
-        lines.push(`Disallow: ${path}`);
-    }
     lines.push('', `Sitemap: ${sitemap}`, '');
     return lines.join('\n');
 }
@@ -163,6 +256,8 @@ export function formatRobotsTxt(options) {
  *     isProduction,
  *     baseUrl,
  *     contentSignals: { search: 'yes', aiTrain: 'no', aiInput: 'yes' },
+ *     preamble: ['Terms of Use apply. See /terms'],
+ *     groups: [{ userAgents: 'ai-agents', disallowExtra: ['/results'] }],
  *   });
  * }
  */

package/dist/utils/robots.js.map

@@ -1 +1 @@
-{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/utils/robots.ts"],"names":[],"mappings":"AAEA,uEAAuE;AACvE,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,WAAW;IACX,cAAc;IACd,OAAO;IACP,WAAW;IACX,aAAa;IACb,OAAO;IACP,MAAM;CACE,CAAC;AAEX;;;;GAIG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,UAAU,EAAE,aAAa,EAAE,UAAU,CAAU,CAAC;AAiE/F,SAAS,mBAAmB,CAAC,MAA0B;IACrD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;IACtE,OAAO,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,eAAe,CAAC,IAAY,EAAE,UAA8B;IACnE,IAAI,IAAI,KAAK,MAAM,IAAI,UAAU,EAAE,CAAC;QAClC,OAAO,GAAG,UAAU,MAAM,CAAC;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAe,EAAE,WAA8B;IACzE,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;gBAC5B,WAAW,GAAG,IAAI,CAAC;YACrB,CAAC;YACD,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAkC;IACpE,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3D,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,CAAC;IAErD,IAAI,KAAK,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjF,eAAe,CAAC,IAAI,EAAE,UAAU,CAAC,CAClC,CAAC;IAEF,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO,CAAC,wBAAwB,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,GAAG,8BAA8B,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAkC;IACnE,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,OAAO,CAAC,OAAO,cAAc,CAAC;IAEvE,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAkC;YAC3C;gBACE,SAAS,EAAE,GAAG;gBACd,QAAQ,EAAE,GAAG;aACd;SACF,CAAC;QAEF,IAAI,OAAO,CAAC,iBAAiB,EAAE,MAAM,EAAE,CAAC;YACtC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBAC7C,KAAK,CAAC,IAAI,CAAC;oBACT,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,KAAK,EAAE,IAAI,CAAC,KAAK;iBAClB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GACT,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAE3F,OAAO;QACL,KAAK,EAAE;YACL;gBACE,SAAS,EAAE,GAAG;gBACd,KAAK;gBACL,QAAQ,EAAE,mBAAmB,CAAC,OAAO,CAAC;aACvC;SACF;QACD,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAuB;IAC7D,OAAO,0BAA0B,OAAO,CAAC,MAAM,cAAc,OAAO,CAAC,OAAO,cAAc,OAAO,CAAC,OAAO,EAAE,CAAC;AAC9G,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,OAA+B;IAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,OAAO,CAAC,OAAO,cAAc,CAAC;IAEvE,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAa,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;QAEzD,IAAI,OAAO,CAAC,iBAAiB,EAAE,MAAM,EAAE,CAAC;YACtC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBAC7C,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,eAAe,IAAI,CAAC,SAAS,EAAE,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,KAAK,GAAa,CAAC,eAAe,CAAC,CAAC;IAE1C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEvB,IAAI,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,mBAAmB,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAA+B;IACrE,OAAO,IAAI,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE;QAC5C,OAAO,EAAE,EAAE,cAAc,EAAE,2BAA2B,EAAE;KACzD,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/utils/robots.ts"],"names":[],"mappings":"AAEA,uEAAuE;AACvE,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,WAAW;IACX,cAAc;IACd,OAAO;IACP,WAAW;IACX,aAAa;IACb,OAAO;IACP,MAAM;CACE,CAAC;AAEX;;;;GAIG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,UAAU,EAAE,aAAa,EAAE,UAAU,CAAU,CAAC;AAE/F;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,WAAW;IACX,aAAa;IACb,kBAAkB;IAClB,QAAQ;IACR,cAAc;IACd,eAAe;IACf,eAAe;IACf,iBAAiB;IACjB,oBAAoB;IACpB,sBAAsB;IACtB,iBAAiB;IACjB,WAAW;IACX,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,iBAAiB;IACjB,eAAe;IACf,OAAO;IACP,YAAY;IACZ,SAAS;IACT,WAAW;CACH,CAAC;AAsGX,SAAS,mBAAmB,CAAC,MAA0B;IACrD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;IACtE,OAAO,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,eAAe,CAAC,IAAY,EAAE,UAA8B;IACnE,IAAI,IAAI,KAAK,MAAM,IAAI,UAAU,EAAE,CAAC;QAClC,OAAO,GAAG,UAAU,MAAM,CAAC;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAe,EAAE,WAA8B;IACzE,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;gBAC5B,WAAW,GAAG,IAAI,CAAC;YACrB,CAAC;YACD,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAkC;IACpE,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3D,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,CAAC;IAErD,IAAI,KAAK,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjF,eAAe,CAAC,IAAI,EAAE,UAAU,CAAC,CAClC,CAAC;IAEF,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAE/C,IAAI,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO,CAAC,wBAAwB,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,GAAG,8BAA8B,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAkC;IACnE,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,OAAO,CAAC,OAAO,cAAc,CAAC;IAEvE,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAkC;YAC3C;gBACE,SAAS,EAAE,GAAG;gBACd,QAAQ,EAAE,GAAG;aACd;SACF,CAAC;QAEF,IAAI,OAAO,CAAC,iBAAiB,EAAE,MAAM,EAAE,CAAC;YACtC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBAC7C,KAAK,CAAC,IAAI,CAAC;oBACT,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,KAAK,EAAE,IAAI,CAAC,KAAK;iBAClB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GACT,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAE3F,OAAO;QACL,KAAK,EAAE;YACL;gBACE,SAAS,EAAE,GAAG;gBACd,KAAK;gBACL,QAAQ,EAAE,mBAAmB,CAAC,OAAO,CAAC;aACvC;SACF;QACD,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAuB;IAC7D,OAAO,0BAA0B,OAAO,CAAC,MAAM,cAAc,OAAO,CAAC,OAAO,cAAc,OAAO,CAAC,OAAO,EAAE,CAAC;AAC9G,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,uBAAuB,CACrC,UAAqD;IAErD,MAAM,QAAQ,GAAG,UAAU,KAAK,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,UAAU,CAAC;IAChF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,mEAAmE;AACnE,SAAS,eAAe,CACtB,MAAqC,EACrC,KAAoC;IAEpC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QACtC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAoC;IAC/D,MAAM,GAAG,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;IACtE,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,OAAO,KAAK,OAAO,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAe,EAAE,UAAyC;IAClF,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACvB,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAe,EAAE,KAAwB;IACpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAC,OAA+B;IAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,OAAO,CAAC,OAAO,cAAc,CAAC;IAEvE,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAa,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;QAEzD,IAAI,OAAO,CAAC,iBAAiB,EAAE,MAAM,EAAE,CAAC;YACtC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBAC7C,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,eAAe,IAAI,CAAC,SAAS,EAAE,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAE5B,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC5C,mBAAmB,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IAE3C,IAAI,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,uBAAuB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;YAClC,CAAC;YAED,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC;YACpE,IAAI,YAAY,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,YAAY,CAAC,CAAC,CAAC;YACpD,CAAC;YAED,gBAAgB,CAAC,KAAK,EAAE,eAAe,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;YAE/E,MAAM,aAAa,GAAG,KAAK,CAAC,aAAa,EAAE,MAAM;gBAC/C,CAAC,CAAC,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC,aAAa,CAAC;gBAC7C,CAAC,CAAC,cAAc,CAAC;YACnB,mBAAmB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAA+B;IACrE,OAAO,IAAI,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE;QAC5C,OAAO,EAAE,EAAE,cAAc,EAAE,2BAA2B,EAAE;KACzD,CAAC,CAAC;AACL,CAAC"}

package/docs/llms.md

@@ -253,9 +253,9 @@ Default production disallows: `/*?_rsc=*`, `/_next/data/`, `/api/`, `/preview/`,
 
 **Split-domain sites (Point Me):** pass `pathPrefix` (`SITE_PREFIX`), extra `apiPrefixes` (`/marketing-api/`), `disallowExtra`, `disallowOmit`, and `blockTrackingQueryParams` as needed. Query-string disallows help non-Google crawlers; Google matches path only — pair with strong canonicals.
 
-### Content Signals (recommended for agent readiness)
+### Content Signals + multi-group (recommended for agent readiness)
 
-Next.js `MetadataRoute.Robots` cannot emit `Content-Signal`. Use a route handler:
+Next.js `MetadataRoute.Robots` cannot emit `Content-Signal`, preamble comments, or multi-group AI rules. Use a route handler. **All robots values are code-configured** (site route / constants) — not CMS.
 
 ```ts
 // src/app/robots.txt/route.ts
@@ -267,15 +267,29 @@ export function GET() {
   return createRobotsTxtResponse({
     isProduction,
     baseUrl,
+    // Leading # comments (Terms preamble). Strings may omit `# `.
+    // preamble: ['Terms of Use apply. See /terms'],
     contentSignals: { search: 'yes', aiTrain: 'no', aiInput: 'yes' },
-    // disallowExtra: ['/private/'],
+    // Shared disallow (applied to * and every named group):
+    // disallowExtra: ['/private/', '/flight/'],
     // Public agent APIs under /api/ (Google: specific Allow can win over Disallow: /api/):
     // allowExtra: ['/api/mcp', '/api/search'],
+    // Named groups after *: inherit shared disallow + group-only extras (no drift).
+    // groups: [{ userAgents: 'ai-agents', disallowExtra: ['/results'] }],
   });
 }
 ```
 
-Also: `formatRobotsTxt`, `formatContentSignalLine`, `buildRobotsDisallow`. `allowExtra` emits extra production `Allow:` lines.
+| Option | Purpose |
+|--------|---------|
+| `contentSignals` | Default Content-Signal for `*` and named groups |
+| `preamble` | Leading `#` comment line(s) before first `User-agent` |
+| `groups` | Extra UA groups; each may override Content-Signal and add `disallowExtra` / `allowExtra` |
+| `userAgents: 'ai-agents'` | Expands `AI_AGENT_USER_AGENTS` (Claude, GPT, Perplexity, Meta, Amazon, Mistral, …) |
+
+**No-drift rule:** shared disallow is built once via `buildRobotsDisallow`. `*` uses that list only; named groups use `shared + group.disallowExtra`. Group-only paths (e.g. `/results`) must not appear under `*`. Top-level `allowExtra` is likewise inherited by named groups, then `group.allowExtra` is appended. Empty `userAgents: []` throws.
+
+Also: `formatRobotsTxt`, `formatContentSignalLine`, `buildRobotsDisallow`, `AI_AGENT_USER_AGENTS`, `resolveRobotsUserAgents`. Omit `preamble` + `groups` for byte-compatible legacy output. The UA list is living — bump via changeset when vendors rename tokens.
 
 ## Agent readiness (Band A)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@se-studio/core-ui",
-  "version": "1.36.0",
+  "version": "1.37.0",
   "description": "Shared React UI component library with Tailwind CSS v4 for SE Studio applications",
   "repository": {
     "type": "git",