@aglyn/aglyn 1.0.0-beta.160 → 1.0.0-beta.161

package/src/lib/app-utils/search-indexing.d.ts

@@ -29,7 +29,7 @@ import { HostScreenVisibility } from '../foundation/definitions/platform.types';
  * submission, and submitting a page you have marked noindex is the shape that
  * gets a site flagged for conflicting directives.
  *
- * There are two controls and they compose:
+ * There are three controls and they compose:
  *
  * - **Site-level** `host.seo.discourageSearchEngines` — the staged-launch
  *   switch. Everything about the site goes dark to crawlers at once.
@@ -38,6 +38,11 @@ import { HostScreenVisibility } from '../foundation/definitions/platform.types';
  *   `UNLISTED` is literally `PUBLIC | (1 << 2)`: "public, plus the not-listed
  *   bit". A second per-screen field would be a field that can disagree with
  *   this one, and every render surface would then need a rule for which wins.
+ * - **Per-collection** `collection.excludeEntriesFromSearch` (AGL-3247) — a
+ *   content collection keeps its listing and its category listings indexable
+ *   and withholds its ENTRIES. That is a different axis from the two above,
+ *   not a third copy of either: the subject is the collection, and what it
+ *   answers for is every URL underneath it.
  */
 /** The host fields this module reads; keeps callers free of the full doc. */
 export interface SearchIndexingHost {
@@ -54,6 +59,15 @@ export interface SearchIndexingHost {
 export interface SearchIndexingScreen {
     visibility?: HostScreenVisibility | null;
 }
+/** The content-collection fields this module reads. */
+export interface SearchIndexingCollection {
+    /**
+     * "List the collection, not its entries" (AGL-3247). PERSISTED NAME — flat
+     * on the collection document, where `schemaType` and `categories` already
+     * live, rather than under a `seo` map the collection has never carried.
+     */
+    excludeEntriesFromSearch?: boolean | null;
+}
 /**
  * Site-wide opt-out. Absent means "index me" — the default a site is created
  * with, and the only safe default: a missing field must never be read as
@@ -73,11 +87,32 @@ export declare function isSearchDiscouraged(host: SearchIndexingHost | null | un
  * one whose *name* is about search.
  */
 export declare function isScreenIndexable(screen: SearchIndexingScreen | null | undefined): boolean;
-/** Both controls together — what a render surface actually wants to know. */
+/**
+ * The site and screen controls together — what a SCREEN render wants to know.
+ *
+ * Collection entries do not come through here; their URL belongs to the
+ * collection rather than to a screen, so they ask
+ * {@link areCollectionEntriesIndexable} alongside {@link isSearchDiscouraged}.
+ */
 export declare function isPageIndexable(options: {
     host?: SearchIndexingHost | null;
     screen?: SearchIndexingScreen | null;
 }): boolean;
+/**
+ * Whether a content collection's ENTRIES may be indexed, ignoring the
+ * site-level switch (AGL-3247).
+ *
+ * Absent means "index them" — the default every collection was created with,
+ * and the only safe default, exactly as for {@link isSearchDiscouraged}: a
+ * schema slip must never be readable as "de-index this customer's blog".
+ *
+ * This says nothing about the LISTING at `/{collection}` or the category
+ * listings under it. Those stay indexable on purpose — they are the pages
+ * that make the withheld entries reachable at all, and a collection whose
+ * whole point is that it is too numerous to index is still a collection a
+ * reader should be able to find.
+ */
+export declare function areCollectionEntriesIndexable(collection: SearchIndexingCollection | null | undefined): boolean;
 /**
  * The `robots.txt` body for a host.
  *

package/src/lib/app-utils/search-indexing.js

@@ -39,10 +39,32 @@ import { SITEMAP_INDEX_PATH } from "./sitemap.js";
     if (visibility == null) return true;
     return visibility === HostScreenVisibility.PUBLIC;
 }
-/** Both controls together — what a render surface actually wants to know. */ export function isPageIndexable(options) {
+/**
+ * The site and screen controls together — what a SCREEN render wants to know.
+ *
+ * Collection entries do not come through here; their URL belongs to the
+ * collection rather than to a screen, so they ask
+ * {@link areCollectionEntriesIndexable} alongside {@link isSearchDiscouraged}.
+ */ export function isPageIndexable(options) {
     if (isSearchDiscouraged(options.host)) return false;
     return isScreenIndexable(options.screen);
 }
+/**
+ * Whether a content collection's ENTRIES may be indexed, ignoring the
+ * site-level switch (AGL-3247).
+ *
+ * Absent means "index them" — the default every collection was created with,
+ * and the only safe default, exactly as for {@link isSearchDiscouraged}: a
+ * schema slip must never be readable as "de-index this customer's blog".
+ *
+ * This says nothing about the LISTING at `/{collection}` or the category
+ * listings under it. Those stay indexable on purpose — they are the pages
+ * that make the withheld entries reachable at all, and a collection whose
+ * whole point is that it is too numerous to index is still a collection a
+ * reader should be able to find.
+ */ export function areCollectionEntriesIndexable(collection) {
+    return (collection == null ? void 0 : collection.excludeEntriesFromSearch) !== true;
+}
 /**
  * The `robots.txt` body for a host.
  *

package/src/lib/app-utils/search-indexing.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../../../libs/aglyn/src/lib/app-utils/search-indexing.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { HostScreenVisibility } from '../foundation/definitions/platform.types'\nimport { SITEMAP_INDEX_PATH } from './sitemap'\n\n/**\n * Search-indexing policy (AGL-1263): the single answer to \"may a crawler\n * index this?\", shared by every surface that has to agree about it.\n *\n * Four surfaces answer that question and they must never disagree — the\n * server `generateMetadata`, its client `<Head>` twin, `robots.txt` and\n * `sitemap.xml`. Before this module three of them had their own copy of the\n * rule and the fourth had none: the sitemap listed whatever was in the host's\n * routing map, so an UNLISTED page was simultaneously told \"do not index me\"\n * in its own head and advertised as a canonical URL in the site's sitemap.\n * That is not a harmless duplication — a sitemap entry is an explicit\n * submission, and submitting a page you have marked noindex is the shape that\n * gets a site flagged for conflicting directives.\n *\n * There are two controls and they compose:\n *\n * - **Site-level** `host.seo.discourageSearchEngines` — the staged-launch\n *   switch. Everything about the site goes dark to crawlers at once.\n * - **Per-screen** `screen.visibility` — the existing\n *   {@link HostScreenVisibility} model, NOT a parallel `noindex` flag.\n *   `UNLISTED` is literally `PUBLIC | (1 << 2)`: \"public, plus the not-listed\n *   bit\". A second per-screen field would be a field that can disagree with\n *   this one, and every render surface would then need a rule for which wins.\n */\n\n/** The host fields this module reads; keeps callers free of the full doc. */\nexport interface SearchIndexingHost {\n  seo?: {\n    /**\n     * Site-wide \"discourage search engines\" (AGL-1263). PERSISTED NAME — the\n     * wording matches the switch an author sees, so support conversations and\n     * the document agree.\n     */\n    discourageSearchEngines?: boolean\n  } | null\n}\n\n/** The screen fields this module reads. */\nexport interface SearchIndexingScreen {\n  visibility?: HostScreenVisibility | null\n}\n\n/**\n * Site-wide opt-out. Absent means \"index me\" — the default a site is created\n * with, and the only safe default: a missing field must never be read as\n * \"hide this site\", or a schema slip would quietly de-index every customer.\n */\nexport function isSearchDiscouraged(\n  host: SearchIndexingHost | null | undefined,\n): boolean {\n  return host?.seo?.discourageSearchEngines === true\n}\n\n/**\n * Whether a screen may be indexed on its own merits, ignoring the site-level\n * switch.\n *\n * Only `PUBLIC` — and an absent value, which every screen predating the\n * visibility model carries — is indexable. Everything else is excluded, and\n * that is wider than the old `=== UNLISTED` test on purpose: a\n * password-protected, members-only or private screen has nothing a crawler\n * can reach, so listing it in a sitemap published a URL that answers with a\n * gate. `UNLISTED` was the only one anyone remembered because it is the only\n * one whose *name* is about search.\n */\nexport function isScreenIndexable(\n  screen: SearchIndexingScreen | null | undefined,\n): boolean {\n  const visibility = screen?.visibility\n  if (visibility == null) return true\n  return visibility === HostScreenVisibility.PUBLIC\n}\n\n/** Both controls together — what a render surface actually wants to know. */\nexport function isPageIndexable(options: {\n  host?: SearchIndexingHost | null\n  screen?: SearchIndexingScreen | null\n}): boolean {\n  if (isSearchDiscouraged(options.host)) return false\n  return isScreenIndexable(options.screen)\n}\n\n/**\n * The `robots.txt` body for a host.\n *\n * When search is discouraged the file disallows everything AND the pages\n * carry `noindex`, which reads like a contradiction and is not. `Disallow`\n * only asks a crawler not to FETCH; a URL linked from elsewhere can still be\n * indexed without ever being fetched, and a crawler that obeys the disallow\n * never sees the `noindex` that would have stopped it. Belt and braces is the\n * documented remedy, and it is what every other builder's equivalent switch\n * does.\n *\n * The `Sitemap:` line is dropped in that state rather than kept: handing a\n * crawler an index of the site you just asked it not to crawl is the\n * contradiction the rest of this module exists to remove.\n */\nexport function buildRobotsTxt(options: {\n  host?: SearchIndexingHost | null\n  /** Absolute origin (no trailing slash); omitted when unresolvable. */\n  origin?: string\n}): string {\n  if (isSearchDiscouraged(options.host)) {\n    /*\n      Every named agent group is dropped along with the wildcard. A group that\n      named an agent and allowed it would be a LOUDER permission than the\n      wildcard's refusal — `robots.txt` precedence gives the most specific\n      matching group, so `User-agent: ClaudeBot` beats `User-agent: *` — and a\n      site that asked to be left alone would be inviting exactly the readers it\n      asked to leave.\n    */\n    return 'User-agent: *\\nDisallow: /\\n'\n  }\n  const sitemap = options.origin\n    ? `Sitemap: ${options.origin}${SITEMAP_INDEX_PATH}\\n`\n    : ''\n  /*\n    AI agents are named EXPLICITLY, and the group says exactly what the\n    wildcard above already says.\n\n    That looks redundant and is not. A wildcard `Allow: /` is the absence of a\n    restriction; a named group is a STATEMENT, and several readers only look\n    for the statement: an AI crawler checks for a group bearing its own token\n    before it checks the wildcard, some readiness audits score a site on\n    whether one exists, and an operator reading this file learns that admitting\n    these agents is a decision somebody made rather than a default nobody\n    revisited.\n\n    It is also the file that will be edited when the decision changes. A site\n    owner who wants to exclude one model's crawler edits its line here; before\n    this list existed there was no line to edit, and the only lever was a\n    wildcard `Disallow` that would have taken Google with it.\n\n    ⚠️ `robots.txt` is a REQUEST, not a control. It stops a crawler that reads\n    it and honors it, and nothing else — it is not an access control, and the\n    WAF is where an agent is actually admitted or refused. The two have to\n    agree: a site that names ClaudeBot here and challenges it at the edge has\n    published an invitation it does not honor.\n  */\n  const agents = AI_AGENT_USER_AGENTS.map(\n    (agent) => `User-agent: ${agent}\\nAllow: /\\n`,\n  ).join('\\n')\n  return `User-agent: *\\nAllow: /\\n\\n${agents}\\n${sitemap}`\n}\n\n/**\n * The AI agent user agents a site admits by name.\n *\n * Two kinds, kept in one list because `robots.txt` does not distinguish them\n * and a site owner reading the file should see everything that is admitted:\n *\n *  - **Fetchers** — `ChatGPT-User`, `Claude-User`, `Perplexity-User`,\n *    `Claude-SearchBot`, `ora-agent`. These act for a person who is asking\n *    about the site RIGHT NOW. Refusing one does not protect anything; it\n *    means the answer that person gets is written without the site's own\n *    words in it.\n *  - **Trainers and indexers** — `GPTBot`, `ClaudeBot`, `Google-Extended`,\n *    `PerplexityBot`, `Applebot-Extended`, `CCBot`, `Bytespider`,\n *    `Amazonbot`, `meta-externalagent`, `DeepSeekBot`, `cohere-ai`,\n *    `Diffbot`, `Omgilibot`, `Timpibot`, `YouBot`. These read for a corpus.\n *\n * Admitting the second group is a judgment, and it is the platform DEFAULT\n * rather than a permanent answer: a tenant site is published to be read, and\n * a site that is absent from the corpora is absent from the answers. A site\n * owner who disagrees has `discourageSearchEngines`, which refuses everything\n * here in one switch.\n *\n * Spellings are the vendors' own tokens and are case-sensitive to some\n * readers. Do not \"tidy\" them.\n */\nexport const AI_AGENT_USER_AGENTS: readonly string[] = [\n  'GPTBot',\n  'ChatGPT-User',\n  'OAI-SearchBot',\n  'ClaudeBot',\n  'Claude-User',\n  'Claude-SearchBot',\n  'anthropic-ai',\n  'Google-Extended',\n  'PerplexityBot',\n  'Perplexity-User',\n  'Applebot-Extended',\n  'meta-externalagent',\n  'Amazonbot',\n  'Bytespider',\n  'CCBot',\n  'cohere-ai',\n  'DeepSeekBot',\n  'Diffbot',\n  'MistralAI-User',\n  'Omgilibot',\n  'Timpibot',\n  'YouBot',\n  'ora-agent',\n  /*\n    OUR OWN CLI (AGL-2717), and it belongs in this list rather than beside it.\n\n    `@aglyn/cli` reads a site on a person's behalf — the same job\n    `ChatGPT-User` and `Perplexity-User` do, and the same reason they are here.\n    It names itself `aglyn-cli/<version> (+https://aglyn.com)` so a site\n    operator has something to match on and somewhere to complain to.\n\n    It is in THIS list, and not only in the WAF, because\n    `check:agent-readiness` asserts the two are the same set: a client admitted\n    at the edge but unnamed here is a client nobody reading `robots.txt` knows\n    is admitted. The wildcard above already allows it, so the group changes\n    nothing functionally — it is the statement that matters, exactly as for the\n    twenty-three above.\n  */\n  'aglyn-cli',\n]\n\n/**\n * Screen ids that are PUBLISHED but are HTTP STATUS pages, and must never be\n * handed to a crawler or an agent (AGL-2716).\n *\n * ⚠️ NOT `screen-route.ts`'s `nonPageScreenIds`, which is a BILLING predicate:\n * that one answers \"which live documents does the flat infrastructure cap\n * bound\" and counts email documents and entry templates. Two questions, two\n * answers, and merging them would make a pricing cap decide what a crawler may\n * read. The name here says `status` for that reason.\n *\n * Two kinds, and both were found the hard way:\n *\n * - **Bound error screens.** `aglyn.com` published `/401`, `/404` and `/503`\n *   into its sitemap, which hands a new domain the worst possible first crawl:\n *   the crawler fetches each one, gets the status code it exists to represent,\n *   and logs crawl errors and soft-404s against a site it has never seen.\n * - **A bare status code as the whole path.** An error screen on this platform\n *   is usually just a screen published at the status path with nothing binding\n *   it, so filtering the bound ids alone changed nothing on the site that had\n *   the problem. The pattern is anchored, so a real page at `/404-guide` or\n *   `/products/503` keeps its place.\n *\n * Extracted from `/api/sitemap`, where it was written inline, so `/llms.txt`\n * and every later reader apply the same rule. The one thing this MUST NOT\n * become is a second copy of that filter: the two surfaces disagreeing is how\n * a site advertises to agents the URLs it withheld from crawlers.\n *\n * Template screens are NOT here. They are excluded too, but the source is a\n * Firestore read (`getTemplateScreenIds`) rather than the host document, so\n * they cannot be decided by a pure function.\n */\nexport function statusPageScreenIds(\n  host:\n    | {\n        screens?: Record<string, string | undefined> | null\n        errorScreens?: Record<string, string | undefined> | null\n        notFoundScreenId?: string | null\n      }\n    | null\n    | undefined,\n): Set<string> {\n  const excluded = new Set<string>()\n  for (const bound of [\n    ...Object.values(host?.errorScreens ?? {}),\n    host?.notFoundScreenId,\n  ]) {\n    if (typeof bound === 'string' && bound) excluded.add(bound)\n  }\n  for (const [screenId, path] of Object.entries(host?.screens ?? {})) {\n    if (STATUS_CODE_PATH.test(String(path))) excluded.add(screenId)\n  }\n  return excluded\n}\n\n/** A bare HTTP status code as an entire route path — `/404`, `503`. */\nconst STATUS_CODE_PATH = /^\\/?[1-5][0-9][0-9]$/\n"],"names":["HostScreenVisibility","SITEMAP_INDEX_PATH","isSearchDiscouraged","host","seo","discourageSearchEngines","isScreenIndexable","screen","visibility","PUBLIC","isPageIndexable","options","buildRobotsTxt","sitemap","origin","agents","AI_AGENT_USER_AGENTS","map","agent","join","statusPageScreenIds","excluded","Set","bound","Object","values","errorScreens","notFoundScreenId","add","screenId","path","entries","screens","STATUS_CODE_PATH","test","String"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED,SAASA,oBAAoB,QAAQ,8CAA0C;AAC/E,SAASC,kBAAkB,QAAQ,eAAW;AA4C9C;;;;CAIC,GACD,OAAO,SAASC,oBACdC,IAA2C;QAEpCA;IAAP,OAAOA,CAAAA,yBAAAA,YAAAA,KAAMC,GAAG,qBAATD,UAAWE,uBAAuB,MAAK;AAChD;AAEA;;;;;;;;;;;CAWC,GACD,OAAO,SAASC,kBACdC,MAA+C;IAE/C,MAAMC,aAAaD,0BAAAA,OAAQC,UAAU;IACrC,IAAIA,cAAc,MAAM,OAAO;IAC/B,OAAOA,eAAeR,qBAAqBS,MAAM;AACnD;AAEA,2EAA2E,GAC3E,OAAO,SAASC,gBAAgBC,OAG/B;IACC,IAAIT,oBAAoBS,QAAQR,IAAI,GAAG,OAAO;IAC9C,OAAOG,kBAAkBK,QAAQJ,MAAM;AACzC;AAEA;;;;;;;;;;;;;;CAcC,GACD,OAAO,SAASK,eAAeD,OAI9B;IACC,IAAIT,oBAAoBS,QAAQR,IAAI,GAAG;QACrC;;;;;;;IAOA,GACA,OAAO;IACT;IACA,MAAMU,UAAUF,QAAQG,MAAM,GAC1B,CAAC,SAAS,EAAEH,QAAQG,MAAM,GAAGb,mBAAmB,EAAE,CAAC,GACnD;IACJ;;;;;;;;;;;;;;;;;;;;;;EAsBA,GACA,MAAMc,SAASC,qBAAqBC,GAAG,CACrC,CAACC,QAAU,CAAC,YAAY,EAAEA,MAAM,YAAY,CAAC,EAC7CC,IAAI,CAAC;IACP,OAAO,CAAC,2BAA2B,EAAEJ,OAAO,EAAE,EAAEF,SAAS;AAC3D;AAEA;;;;;;;;;;;;;;;;;;;;;;;;CAwBC,GACD,OAAO,MAAMG,uBAA0C;IACrD;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;;;;;;;;;;;;;;EAcA,GACA;CACD,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8BC,GACD,OAAO,SAASI,oBACdjB,IAOa;;IAEb,MAAMkB,WAAW,IAAIC;IACrB,KAAK,MAAMC,SAAS;WACfC,OAAOC,MAAM,SAACtB,wBAAAA,KAAMuB,YAAY,mBAAI,CAAC;QACxCvB,wBAAAA,KAAMwB,gBAAgB;KACvB,CAAE;QACD,IAAI,OAAOJ,UAAU,YAAYA,OAAOF,SAASO,GAAG,CAACL;IACvD;IACA,KAAK,MAAM,CAACM,UAAUC,KAAK,IAAIN,OAAOO,OAAO,UAAC5B,wBAAAA,KAAM6B,OAAO,oBAAI,CAAC,GAAI;QAClE,IAAIC,iBAAiBC,IAAI,CAACC,OAAOL,QAAQT,SAASO,GAAG,CAACC;IACxD;IACA,OAAOR;AACT;AAEA,qEAAqE,GACrE,MAAMY,mBAAmB"}
+{"version":3,"sources":["../../../../../../libs/aglyn/src/lib/app-utils/search-indexing.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { HostScreenVisibility } from '../foundation/definitions/platform.types'\nimport { SITEMAP_INDEX_PATH } from './sitemap'\n\n/**\n * Search-indexing policy (AGL-1263): the single answer to \"may a crawler\n * index this?\", shared by every surface that has to agree about it.\n *\n * Four surfaces answer that question and they must never disagree — the\n * server `generateMetadata`, its client `<Head>` twin, `robots.txt` and\n * `sitemap.xml`. Before this module three of them had their own copy of the\n * rule and the fourth had none: the sitemap listed whatever was in the host's\n * routing map, so an UNLISTED page was simultaneously told \"do not index me\"\n * in its own head and advertised as a canonical URL in the site's sitemap.\n * That is not a harmless duplication — a sitemap entry is an explicit\n * submission, and submitting a page you have marked noindex is the shape that\n * gets a site flagged for conflicting directives.\n *\n * There are three controls and they compose:\n *\n * - **Site-level** `host.seo.discourageSearchEngines` — the staged-launch\n *   switch. Everything about the site goes dark to crawlers at once.\n * - **Per-screen** `screen.visibility` — the existing\n *   {@link HostScreenVisibility} model, NOT a parallel `noindex` flag.\n *   `UNLISTED` is literally `PUBLIC | (1 << 2)`: \"public, plus the not-listed\n *   bit\". A second per-screen field would be a field that can disagree with\n *   this one, and every render surface would then need a rule for which wins.\n * - **Per-collection** `collection.excludeEntriesFromSearch` (AGL-3247) — a\n *   content collection keeps its listing and its category listings indexable\n *   and withholds its ENTRIES. That is a different axis from the two above,\n *   not a third copy of either: the subject is the collection, and what it\n *   answers for is every URL underneath it.\n */\n\n/** The host fields this module reads; keeps callers free of the full doc. */\nexport interface SearchIndexingHost {\n  seo?: {\n    /**\n     * Site-wide \"discourage search engines\" (AGL-1263). PERSISTED NAME — the\n     * wording matches the switch an author sees, so support conversations and\n     * the document agree.\n     */\n    discourageSearchEngines?: boolean\n  } | null\n}\n\n/** The screen fields this module reads. */\nexport interface SearchIndexingScreen {\n  visibility?: HostScreenVisibility | null\n}\n\n/** The content-collection fields this module reads. */\nexport interface SearchIndexingCollection {\n  /**\n   * \"List the collection, not its entries\" (AGL-3247). PERSISTED NAME — flat\n   * on the collection document, where `schemaType` and `categories` already\n   * live, rather than under a `seo` map the collection has never carried.\n   */\n  excludeEntriesFromSearch?: boolean | null\n}\n\n/**\n * Site-wide opt-out. Absent means \"index me\" — the default a site is created\n * with, and the only safe default: a missing field must never be read as\n * \"hide this site\", or a schema slip would quietly de-index every customer.\n */\nexport function isSearchDiscouraged(\n  host: SearchIndexingHost | null | undefined,\n): boolean {\n  return host?.seo?.discourageSearchEngines === true\n}\n\n/**\n * Whether a screen may be indexed on its own merits, ignoring the site-level\n * switch.\n *\n * Only `PUBLIC` — and an absent value, which every screen predating the\n * visibility model carries — is indexable. Everything else is excluded, and\n * that is wider than the old `=== UNLISTED` test on purpose: a\n * password-protected, members-only or private screen has nothing a crawler\n * can reach, so listing it in a sitemap published a URL that answers with a\n * gate. `UNLISTED` was the only one anyone remembered because it is the only\n * one whose *name* is about search.\n */\nexport function isScreenIndexable(\n  screen: SearchIndexingScreen | null | undefined,\n): boolean {\n  const visibility = screen?.visibility\n  if (visibility == null) return true\n  return visibility === HostScreenVisibility.PUBLIC\n}\n\n/**\n * The site and screen controls together — what a SCREEN render wants to know.\n *\n * Collection entries do not come through here; their URL belongs to the\n * collection rather than to a screen, so they ask\n * {@link areCollectionEntriesIndexable} alongside {@link isSearchDiscouraged}.\n */\nexport function isPageIndexable(options: {\n  host?: SearchIndexingHost | null\n  screen?: SearchIndexingScreen | null\n}): boolean {\n  if (isSearchDiscouraged(options.host)) return false\n  return isScreenIndexable(options.screen)\n}\n\n/**\n * Whether a content collection's ENTRIES may be indexed, ignoring the\n * site-level switch (AGL-3247).\n *\n * Absent means \"index them\" — the default every collection was created with,\n * and the only safe default, exactly as for {@link isSearchDiscouraged}: a\n * schema slip must never be readable as \"de-index this customer's blog\".\n *\n * This says nothing about the LISTING at `/{collection}` or the category\n * listings under it. Those stay indexable on purpose — they are the pages\n * that make the withheld entries reachable at all, and a collection whose\n * whole point is that it is too numerous to index is still a collection a\n * reader should be able to find.\n */\nexport function areCollectionEntriesIndexable(\n  collection: SearchIndexingCollection | null | undefined,\n): boolean {\n  return collection?.excludeEntriesFromSearch !== true\n}\n\n/**\n * The `robots.txt` body for a host.\n *\n * When search is discouraged the file disallows everything AND the pages\n * carry `noindex`, which reads like a contradiction and is not. `Disallow`\n * only asks a crawler not to FETCH; a URL linked from elsewhere can still be\n * indexed without ever being fetched, and a crawler that obeys the disallow\n * never sees the `noindex` that would have stopped it. Belt and braces is the\n * documented remedy, and it is what every other builder's equivalent switch\n * does.\n *\n * The `Sitemap:` line is dropped in that state rather than kept: handing a\n * crawler an index of the site you just asked it not to crawl is the\n * contradiction the rest of this module exists to remove.\n */\nexport function buildRobotsTxt(options: {\n  host?: SearchIndexingHost | null\n  /** Absolute origin (no trailing slash); omitted when unresolvable. */\n  origin?: string\n}): string {\n  if (isSearchDiscouraged(options.host)) {\n    /*\n      Every named agent group is dropped along with the wildcard. A group that\n      named an agent and allowed it would be a LOUDER permission than the\n      wildcard's refusal — `robots.txt` precedence gives the most specific\n      matching group, so `User-agent: ClaudeBot` beats `User-agent: *` — and a\n      site that asked to be left alone would be inviting exactly the readers it\n      asked to leave.\n    */\n    return 'User-agent: *\\nDisallow: /\\n'\n  }\n  const sitemap = options.origin\n    ? `Sitemap: ${options.origin}${SITEMAP_INDEX_PATH}\\n`\n    : ''\n  /*\n    AI agents are named EXPLICITLY, and the group says exactly what the\n    wildcard above already says.\n\n    That looks redundant and is not. A wildcard `Allow: /` is the absence of a\n    restriction; a named group is a STATEMENT, and several readers only look\n    for the statement: an AI crawler checks for a group bearing its own token\n    before it checks the wildcard, some readiness audits score a site on\n    whether one exists, and an operator reading this file learns that admitting\n    these agents is a decision somebody made rather than a default nobody\n    revisited.\n\n    It is also the file that will be edited when the decision changes. A site\n    owner who wants to exclude one model's crawler edits its line here; before\n    this list existed there was no line to edit, and the only lever was a\n    wildcard `Disallow` that would have taken Google with it.\n\n    ⚠️ `robots.txt` is a REQUEST, not a control. It stops a crawler that reads\n    it and honors it, and nothing else — it is not an access control, and the\n    WAF is where an agent is actually admitted or refused. The two have to\n    agree: a site that names ClaudeBot here and challenges it at the edge has\n    published an invitation it does not honor.\n  */\n  const agents = AI_AGENT_USER_AGENTS.map(\n    (agent) => `User-agent: ${agent}\\nAllow: /\\n`,\n  ).join('\\n')\n  return `User-agent: *\\nAllow: /\\n\\n${agents}\\n${sitemap}`\n}\n\n/**\n * The AI agent user agents a site admits by name.\n *\n * Two kinds, kept in one list because `robots.txt` does not distinguish them\n * and a site owner reading the file should see everything that is admitted:\n *\n *  - **Fetchers** — `ChatGPT-User`, `Claude-User`, `Perplexity-User`,\n *    `Claude-SearchBot`, `ora-agent`. These act for a person who is asking\n *    about the site RIGHT NOW. Refusing one does not protect anything; it\n *    means the answer that person gets is written without the site's own\n *    words in it.\n *  - **Trainers and indexers** — `GPTBot`, `ClaudeBot`, `Google-Extended`,\n *    `PerplexityBot`, `Applebot-Extended`, `CCBot`, `Bytespider`,\n *    `Amazonbot`, `meta-externalagent`, `DeepSeekBot`, `cohere-ai`,\n *    `Diffbot`, `Omgilibot`, `Timpibot`, `YouBot`. These read for a corpus.\n *\n * Admitting the second group is a judgment, and it is the platform DEFAULT\n * rather than a permanent answer: a tenant site is published to be read, and\n * a site that is absent from the corpora is absent from the answers. A site\n * owner who disagrees has `discourageSearchEngines`, which refuses everything\n * here in one switch.\n *\n * Spellings are the vendors' own tokens and are case-sensitive to some\n * readers. Do not \"tidy\" them.\n */\nexport const AI_AGENT_USER_AGENTS: readonly string[] = [\n  'GPTBot',\n  'ChatGPT-User',\n  'OAI-SearchBot',\n  'ClaudeBot',\n  'Claude-User',\n  'Claude-SearchBot',\n  'anthropic-ai',\n  'Google-Extended',\n  'PerplexityBot',\n  'Perplexity-User',\n  'Applebot-Extended',\n  'meta-externalagent',\n  'Amazonbot',\n  'Bytespider',\n  'CCBot',\n  'cohere-ai',\n  'DeepSeekBot',\n  'Diffbot',\n  'MistralAI-User',\n  'Omgilibot',\n  'Timpibot',\n  'YouBot',\n  'ora-agent',\n  /*\n    OUR OWN CLI (AGL-2717), and it belongs in this list rather than beside it.\n\n    `@aglyn/cli` reads a site on a person's behalf — the same job\n    `ChatGPT-User` and `Perplexity-User` do, and the same reason they are here.\n    It names itself `aglyn-cli/<version> (+https://aglyn.com)` so a site\n    operator has something to match on and somewhere to complain to.\n\n    It is in THIS list, and not only in the WAF, because\n    `check:agent-readiness` asserts the two are the same set: a client admitted\n    at the edge but unnamed here is a client nobody reading `robots.txt` knows\n    is admitted. The wildcard above already allows it, so the group changes\n    nothing functionally — it is the statement that matters, exactly as for the\n    twenty-three above.\n  */\n  'aglyn-cli',\n]\n\n/**\n * Screen ids that are PUBLISHED but are HTTP STATUS pages, and must never be\n * handed to a crawler or an agent (AGL-2716).\n *\n * ⚠️ NOT `screen-route.ts`'s `nonPageScreenIds`, which is a BILLING predicate:\n * that one answers \"which live documents does the flat infrastructure cap\n * bound\" and counts email documents and entry templates. Two questions, two\n * answers, and merging them would make a pricing cap decide what a crawler may\n * read. The name here says `status` for that reason.\n *\n * Two kinds, and both were found the hard way:\n *\n * - **Bound error screens.** `aglyn.com` published `/401`, `/404` and `/503`\n *   into its sitemap, which hands a new domain the worst possible first crawl:\n *   the crawler fetches each one, gets the status code it exists to represent,\n *   and logs crawl errors and soft-404s against a site it has never seen.\n * - **A bare status code as the whole path.** An error screen on this platform\n *   is usually just a screen published at the status path with nothing binding\n *   it, so filtering the bound ids alone changed nothing on the site that had\n *   the problem. The pattern is anchored, so a real page at `/404-guide` or\n *   `/products/503` keeps its place.\n *\n * Extracted from `/api/sitemap`, where it was written inline, so `/llms.txt`\n * and every later reader apply the same rule. The one thing this MUST NOT\n * become is a second copy of that filter: the two surfaces disagreeing is how\n * a site advertises to agents the URLs it withheld from crawlers.\n *\n * Template screens are NOT here. They are excluded too, but the source is a\n * Firestore read (`getTemplateScreenIds`) rather than the host document, so\n * they cannot be decided by a pure function.\n */\nexport function statusPageScreenIds(\n  host:\n    | {\n        screens?: Record<string, string | undefined> | null\n        errorScreens?: Record<string, string | undefined> | null\n        notFoundScreenId?: string | null\n      }\n    | null\n    | undefined,\n): Set<string> {\n  const excluded = new Set<string>()\n  for (const bound of [\n    ...Object.values(host?.errorScreens ?? {}),\n    host?.notFoundScreenId,\n  ]) {\n    if (typeof bound === 'string' && bound) excluded.add(bound)\n  }\n  for (const [screenId, path] of Object.entries(host?.screens ?? {})) {\n    if (STATUS_CODE_PATH.test(String(path))) excluded.add(screenId)\n  }\n  return excluded\n}\n\n/** A bare HTTP status code as an entire route path — `/404`, `503`. */\nconst STATUS_CODE_PATH = /^\\/?[1-5][0-9][0-9]$/\n"],"names":["HostScreenVisibility","SITEMAP_INDEX_PATH","isSearchDiscouraged","host","seo","discourageSearchEngines","isScreenIndexable","screen","visibility","PUBLIC","isPageIndexable","options","areCollectionEntriesIndexable","collection","excludeEntriesFromSearch","buildRobotsTxt","sitemap","origin","agents","AI_AGENT_USER_AGENTS","map","agent","join","statusPageScreenIds","excluded","Set","bound","Object","values","errorScreens","notFoundScreenId","add","screenId","path","entries","screens","STATUS_CODE_PATH","test","String"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED,SAASA,oBAAoB,QAAQ,8CAA0C;AAC/E,SAASC,kBAAkB,QAAQ,eAAW;AA2D9C;;;;CAIC,GACD,OAAO,SAASC,oBACdC,IAA2C;QAEpCA;IAAP,OAAOA,CAAAA,yBAAAA,YAAAA,KAAMC,GAAG,qBAATD,UAAWE,uBAAuB,MAAK;AAChD;AAEA;;;;;;;;;;;CAWC,GACD,OAAO,SAASC,kBACdC,MAA+C;IAE/C,MAAMC,aAAaD,0BAAAA,OAAQC,UAAU;IACrC,IAAIA,cAAc,MAAM,OAAO;IAC/B,OAAOA,eAAeR,qBAAqBS,MAAM;AACnD;AAEA;;;;;;CAMC,GACD,OAAO,SAASC,gBAAgBC,OAG/B;IACC,IAAIT,oBAAoBS,QAAQR,IAAI,GAAG,OAAO;IAC9C,OAAOG,kBAAkBK,QAAQJ,MAAM;AACzC;AAEA;;;;;;;;;;;;;CAaC,GACD,OAAO,SAASK,8BACdC,UAAuD;IAEvD,OAAOA,CAAAA,8BAAAA,WAAYC,wBAAwB,MAAK;AAClD;AAEA;;;;;;;;;;;;;;CAcC,GACD,OAAO,SAASC,eAAeJ,OAI9B;IACC,IAAIT,oBAAoBS,QAAQR,IAAI,GAAG;QACrC;;;;;;;IAOA,GACA,OAAO;IACT;IACA,MAAMa,UAAUL,QAAQM,MAAM,GAC1B,CAAC,SAAS,EAAEN,QAAQM,MAAM,GAAGhB,mBAAmB,EAAE,CAAC,GACnD;IACJ;;;;;;;;;;;;;;;;;;;;;;EAsBA,GACA,MAAMiB,SAASC,qBAAqBC,GAAG,CACrC,CAACC,QAAU,CAAC,YAAY,EAAEA,MAAM,YAAY,CAAC,EAC7CC,IAAI,CAAC;IACP,OAAO,CAAC,2BAA2B,EAAEJ,OAAO,EAAE,EAAEF,SAAS;AAC3D;AAEA;;;;;;;;;;;;;;;;;;;;;;;;CAwBC,GACD,OAAO,MAAMG,uBAA0C;IACrD;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;;;;;;;;;;;;;;EAcA,GACA;CACD,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8BC,GACD,OAAO,SAASI,oBACdpB,IAOa;;IAEb,MAAMqB,WAAW,IAAIC;IACrB,KAAK,MAAMC,SAAS;WACfC,OAAOC,MAAM,SAACzB,wBAAAA,KAAM0B,YAAY,mBAAI,CAAC;QACxC1B,wBAAAA,KAAM2B,gBAAgB;KACvB,CAAE;QACD,IAAI,OAAOJ,UAAU,YAAYA,OAAOF,SAASO,GAAG,CAACL;IACvD;IACA,KAAK,MAAM,CAACM,UAAUC,KAAK,IAAIN,OAAOO,OAAO,UAAC/B,wBAAAA,KAAMgC,OAAO,oBAAI,CAAC,GAAI;QAClE,IAAIC,iBAAiBC,IAAI,CAACC,OAAOL,QAAQT,SAASO,GAAG,CAACC;IACxD;IACA,OAAOR;AACT;AAEA,qEAAqE,GACrE,MAAMY,mBAAmB"}