@ultimat3/core 27.14.1 → 27.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/core",
-  "version": "27.14.1",
+  "version": "27.16.0",
   "description": "Ultimate's foundation: errors, context, env, config, clock, ids, logging, telemetry, lifecycle",
   "license": "MIT",
   "type": "module",
@@ -37,6 +37,6 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/schema": "27.14.1"
+    "@ultimat3/schema": "27.16.0"
   }
 }

package/src/config-robots.ts

@@ -0,0 +1,59 @@
+// Single responsibility: the refusals `seo.robots` earns — its `disallow` paths and each per-crawler
+// group. Split from `config-site.ts`, which owns the shapes and the merge; this file only judges.
+// `unknown` in throughout: an untyped config reaches here with whatever it wrote.
+
+import { describeValue } from './error-render';
+import { isJsonObject } from './json-object';
+
+/** Every entry of a robots path list that is not a string starting with `/`. */
+export function robotsPathIssues(label: string, paths: unknown): readonly string[] {
+  if (!Array.isArray(paths))
+    return [`${label} must be a list of paths, not ${describeValue(paths)}`];
+  const issues: string[] = [];
+  for (const path of paths as readonly unknown[]) {
+    if (typeof path !== 'string') {
+      issues.push(`${label} entry must be a path string, not ${describeValue(path)}`);
+    } else if (!path.startsWith('/')) {
+      issues.push(`${label} entry "${path}" must start with /`);
+    } else if (/[\r\n]/.test(path)) {
+      // A line break would write a directive of its own into `robots.txt`.
+      issues.push(`${label} entry ${JSON.stringify(path)} must not contain a line break`);
+    }
+  }
+  return issues;
+}
+
+function userAgentIssues(label: string, userAgent: unknown): readonly string[] {
+  const agents: readonly unknown[] = Array.isArray(userAgent) ? userAgent : [userAgent];
+  if (agents.length === 0) return [`${label} must name at least one user agent`];
+  return agents.flatMap((agent) =>
+    typeof agent === 'string' && agent.trim() !== '' && !/[\r\n]/.test(agent)
+      ? []
+      : [
+          `${label} must be a non-empty user-agent string (or a list of them), not ${describeValue(agent)}`,
+        ],
+  );
+}
+
+/** Every refusal `seo.robots.groups` earns, as sentences naming the group by its index. */
+export function robotsGroupIssues(groups: unknown): readonly string[] {
+  if (!Array.isArray(groups)) {
+    return [`seo.robots.groups must be a list of groups, not ${describeValue(groups)}`];
+  }
+  return (groups as readonly unknown[]).flatMap((group, index) => {
+    const at = `seo.robots.groups[${index}]`;
+    if (!isJsonObject(group)) return [`${at} must be an object, not ${describeValue(group)}`];
+    const issues = [...userAgentIssues(`${at}.userAgent`, group['userAgent'])];
+    for (const key of ['allow', 'disallow'] as const) {
+      if (group[key] !== undefined) issues.push(...robotsPathIssues(`${at}.${key}`, group[key]));
+    }
+    const delay = group['crawlDelay'];
+    if (
+      delay !== undefined &&
+      !(typeof delay === 'number' && Number.isFinite(delay) && delay >= 0)
+    ) {
+      issues.push(`${at}.crawlDelay must be a number of seconds ≥ 0, not ${describeValue(delay)}`);
+    }
+    return issues;
+  });
+}

package/src/config-site.ts

@@ -3,6 +3,7 @@
 // Split from `config.ts` for `config-pwa.ts`' reason: that file sits at its 500-line ceiling.
 
 import { type Input, layered } from './config-merge';
+import { robotsGroupIssues, robotsPathIssues } from './config-robots';
 import {
   SITEMAP_PAGE_SIZE_MAX,
   type SitemapSectionConfig,
@@ -20,9 +21,34 @@ export interface SiteConfig {
   readonly origin: string | null;
 }
 
+/**
+ * One `User-agent` group of a production `robots.txt`. A crawler obeys the ONE group naming it most
+ * specifically and ignores `*` beside it, so a group for `GPTBot` replaces `*`'s rules for GPTBot.
+ */
+export interface SeoRobotsGroup {
+  /** `'GPTBot'`, or several sharing one set of rules: `['GPTBot', 'CCBot']`. Never empty. */
+  readonly userAgent: string | readonly string[];
+  /** Paths this group may fetch, each starting with `/`. */
+  readonly allow?: readonly string[] | undefined;
+  /** Paths this group may not fetch, each starting with `/`. `['/']` keeps it out entirely. */
+  readonly disallow?: readonly string[] | undefined;
+  /** Seconds between requests, for the crawlers that read `Crawl-delay`. */
+  readonly crawlDelay?: number | undefined;
+}
+
 export interface SeoRobotsConfig {
-  /** Paths `robots.txt` disallows in production, e.g. `['/panel', '/api']`. Each starts with `/`. */
+  /**
+   * Paths EVERY crawler is kept out of in production, e.g. `['/panel', '/api']` — added to every
+   * group, and left out of the sitemap. Each starts with `/`.
+   */
   readonly disallow: readonly string[];
+  /**
+   * Per-crawler groups. Empty (the default): one `User-agent: *` group allowing `/`. Declared, they
+   * ARE the file's groups — name `*` among them to give everyone else rules of their own.
+   * Optional in the TYPE only, so a `SeoRobotsConfig` literal written before it existed still
+   * compiles; `defineConfig` always fills it.
+   */
+  readonly groups?: readonly SeoRobotsGroup[] | undefined;
 }
 
 /**
@@ -84,7 +110,7 @@ export function mergeSite(layers: readonly SiteSectionsInput[]): SiteSections {
     ),
     seo: {
       robots: layered<SeoRobotsConfig>(
-        { disallow: [] },
+        { disallow: [], groups: [] },
         layers.map((layer) => layer.seo?.robots),
       ),
       sitemap: layered<SeoSitemapConfig>(
@@ -122,15 +148,8 @@ export function siteIssues(config: SiteSections, issues: string[]): void {
     const issue = originIssue(origin);
     if (issue !== undefined) issues.push(issue);
   }
-  // `unknown` entries: an untyped config reaches here with whatever it listed, and `5.startsWith`
-  // was a native `TypeError` thrown by the validator itself.
-  for (const path of config.seo.robots.disallow as readonly unknown[]) {
-    if (typeof path !== 'string') {
-      issues.push(`seo.robots.disallow entry must be a path string, not ${describeValue(path)}`);
-    } else if (!path.startsWith('/')) {
-      issues.push(`seo.robots.disallow entry "${path}" must start with /`);
-    }
-  }
+  issues.push(...robotsPathIssues('seo.robots.disallow', config.seo.robots.disallow));
+  issues.push(...robotsGroupIssues(config.seo.robots.groups ?? []));
   for (const path of config.seo.sitemap.extra as readonly unknown[]) {
     if (typeof path !== 'string') {
       issues.push(`seo.sitemap.extra entry must be a path string, not ${describeValue(path)}`);

package/src/index.ts

@@ -194,6 +194,7 @@ export type {
   SeoConfig,
   SeoConfigInput,
   SeoRobotsConfig,
+  SeoRobotsGroup,
   SeoSitemapConfig,
   SiteConfig,
   SitemapLastmod,