@ultimat3/cli 27.8.1 → 27.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/cli",
-  "version": "27.8.1",
+  "version": "27.9.0",
   "description": "The `x` binary: new, dev, build, verify, generate, db, mcp, doctor, deploy",
   "license": "MIT",
   "type": "module",
@@ -39,33 +39,33 @@
   },
   "dependencies": {
     "@babel/core": "^8.0.1",
-    "@ultimat3/action": "27.8.1",
-    "@ultimat3/admin": "27.8.1",
-    "@ultimat3/ai": "27.8.1",
-    "@ultimat3/auth": "27.8.1",
-    "@ultimat3/cache": "27.8.1",
-    "@ultimat3/core": "27.8.1",
-    "@ultimat3/db": "27.8.1",
-    "@ultimat3/entity": "27.8.1",
-    "@ultimat3/flags": "27.8.1",
-    "@ultimat3/http": "27.8.1",
-    "@ultimat3/i18n": "27.8.1",
-    "@ultimat3/jobs": "27.8.1",
-    "@ultimat3/mail": "27.8.1",
-    "@ultimat3/manifest": "27.8.1",
-    "@ultimat3/mcp": "27.8.1",
-    "@ultimat3/money": "27.8.1",
-    "@ultimat3/notify": "27.8.1",
-    "@ultimat3/policy": "27.8.1",
-    "@ultimat3/pwa": "27.8.1",
-    "@ultimat3/query": "27.8.1",
-    "@ultimat3/realtime": "27.8.1",
-    "@ultimat3/render": "27.8.1",
-    "@ultimat3/schema": "27.8.1",
-    "@ultimat3/seo": "27.8.1",
-    "@ultimat3/storage": "27.8.1",
-    "@ultimat3/testing": "27.8.1",
-    "@ultimat3/time": "27.8.1",
+    "@ultimat3/action": "27.9.0",
+    "@ultimat3/admin": "27.9.0",
+    "@ultimat3/ai": "27.9.0",
+    "@ultimat3/auth": "27.9.0",
+    "@ultimat3/cache": "27.9.0",
+    "@ultimat3/core": "27.9.0",
+    "@ultimat3/db": "27.9.0",
+    "@ultimat3/entity": "27.9.0",
+    "@ultimat3/flags": "27.9.0",
+    "@ultimat3/http": "27.9.0",
+    "@ultimat3/i18n": "27.9.0",
+    "@ultimat3/jobs": "27.9.0",
+    "@ultimat3/mail": "27.9.0",
+    "@ultimat3/manifest": "27.9.0",
+    "@ultimat3/mcp": "27.9.0",
+    "@ultimat3/money": "27.9.0",
+    "@ultimat3/notify": "27.9.0",
+    "@ultimat3/policy": "27.9.0",
+    "@ultimat3/pwa": "27.9.0",
+    "@ultimat3/query": "27.9.0",
+    "@ultimat3/realtime": "27.9.0",
+    "@ultimat3/render": "27.9.0",
+    "@ultimat3/schema": "27.9.0",
+    "@ultimat3/seo": "27.9.0",
+    "@ultimat3/storage": "27.9.0",
+    "@ultimat3/testing": "27.9.0",
+    "@ultimat3/time": "27.9.0",
     "babel-preset-solid": "^1.9.15"
   }
 }

package/src/index.ts

@@ -157,8 +157,9 @@ export {
 export { quoteArg } from './shell-quote';
 export type { SiteSettings } from './site-config';
 export { loadSiteSettings, publicOrigin } from './site-config';
-export type { SiteSeo, SiteSeoOptions } from './site-seo';
+export type { SiteSeo, SiteSeoOptions, SiteSeoSitemap } from './site-seo';
 export { ROBOTS_PATH, SITEMAP_PATH, siteSeo } from './site-seo';
+export { writeSiteSeo } from './site-seo-write';
 export { eachSourceFile, isGenerated, isTest, SOURCE_GLOBS } from './source-files';
 export type { SkippedRoute, StaticReport } from './static-report';
 export { parseStaticReport } from './static-report';

package/src/seo-meta.ts

@@ -5,6 +5,8 @@
 // this is the only tier that can see both: `@ultimat3/seo` is tier 1 and may not import the route
 // registry, which is tier 4.
 
+import { ctxOf, runWithContext } from '@ultimat3/core';
+import { localizedPath } from '@ultimat3/i18n';
 import { metaContextFor, type RouteEntry, routeDataFor, routeEntries } from '@ultimat3/render';
 import type { RouteRecord } from '@ultimat3/seo';
 import { loadApp } from './app-load';
@@ -66,11 +68,19 @@ export async function scanSiteMeta(root: string): Promise<SiteMetaScan> {
 /**
  * The same scan over the registry as it stands, without loading anything.
  *
+ * `locale` runs every `meta` inside that locale's context; omitted, in none (the gate's read).
+ *
  * Split from `scanSiteMeta` so the resolution rules are testable against routes registered by hand:
  * the half worth pinning is which routes are reachable and what happens when one throws, and
  * neither of those is a fact about globbing a directory.
  */
-export async function readSiteMeta(): Promise<SiteMetaScan> {
+export async function readSiteMeta(locale?: string): Promise<SiteMetaScan> {
+  // In a locale: each `meta` runs as that locale's document would — `ctx.locale`, `t` and the URL
+  // are its own — which is what lets a page that is `noindex` in one language say so.
+  const scoped = <T>(run: () => Promise<T>): Promise<T> =>
+    locale === undefined ? run() : runWithContext(ctxOf({ role: 'web', locale }), run);
+  const pathOf = (path: string): string =>
+    locale === undefined ? path : localizedPath(path, locale);
   const records: RouteRecord[] = [];
   const unresolved: UnresolvedRoute[] = [];
   const findings: Finding[] = [];
@@ -82,10 +92,10 @@ export async function readSiteMeta(): Promise<SiteMetaScan> {
       unresolved.push({ path: entry.path, file: entry.file, reason });
       continue;
     }
-    const ctx = { url: `${PROBE_ORIGIN}${entry.path}`, params: {} };
+    const ctx = { url: `${PROBE_ORIGIN}${pathOf(entry.path)}`, params: {} };
     try {
-      const meta = await entry.config.meta(
-        metaContextFor(ctx, await routeDataFor(entry.config, ctx)),
+      const meta = await scoped(async () =>
+        entry.config.meta(metaContextFor(ctx, await routeDataFor(entry.config, ctx))),
       );
       records.push({
         path: entry.path,

package/src/site-config.ts

@@ -11,14 +11,14 @@ export interface SiteSettings {
   readonly origin: string | null;
   /** `seo.robots.disallow` — the paths a production `robots.txt` keeps crawlers out of. */
   readonly disallow: readonly string[];
-  /** `seo.sitemap` — public `app/` pages to list, and where `<lastmod>` comes from. */
+  /** `seo.sitemap` — public `app/` pages to list, where `<lastmod>` comes from, sections, page size. */
   readonly sitemap: SeoSitemapConfig;
 }
 
 export const NO_SITE_SETTINGS: SiteSettings = {
   origin: null,
   disallow: [],
-  sitemap: { extra: [], lastmod: 'none' },
+  sitemap: { extra: [], lastmod: 'none', sections: [], pageSize: 50_000 },
 };
 
 const trimSlash = (origin: string): string => origin.replace(/\/+$/, '');

package/src/site-seo-write.ts

@@ -0,0 +1,16 @@
+// Single responsibility: `siteSeo()`'s answer as files in a static export — `/sitemap.xml`, every
+// part an index names, and `robots.txt` — so a CDN serves the bytes a running web role answers.
+
+// why: Bun exposes no path-join primitive, and every file path is relative to the output root.
+import { join } from 'node:path';
+import { ROBOTS_PATH, type SiteSeo } from './site-seo';
+
+/**
+ * Writes every file of `seo` under `out` and answers their site-relative paths, the sitemap files
+ * first. `Bun.write` creates `sitemaps/` on the way, so an index and its parts land together.
+ */
+export async function writeSiteSeo(out: string, seo: SiteSeo): Promise<readonly string[]> {
+  for (const file of seo.sitemaps) await Bun.write(join(out, file.path), file.xml);
+  await Bun.write(join(out, ROBOTS_PATH), seo.robots);
+  return [...seo.sitemaps.map((file) => file.path), ROBOTS_PATH];
+}

package/src/site-seo.ts

@@ -8,8 +8,8 @@ import { localizedPath, unlocalizedPath } from '@ultimat3/i18n';
 import { appLocaleSet } from '@ultimat3/i18n/app-catalogs';
 import type { RouteEntry } from '@ultimat3/render';
 import { routeEntries } from '@ultimat3/render';
-import { enumeratePrerender, fillPath } from '@ultimat3/render/server';
-import type { RouteRecord, SitemapFile } from '@ultimat3/seo';
+import { enumeratePrerenderEntries, fillPath } from '@ultimat3/render/server';
+import type { RouteRecord, SitemapFile, SitemapPage } from '@ultimat3/seo';
 import { buildRobots, buildSitemap, isDynamic } from '@ultimat3/seo';
 import { readSiteMeta } from './seo-meta';
 import { lastmodOf } from './sitemap-lastmod';
@@ -25,7 +25,8 @@ export interface SiteSeoOptions {
   /**
    * The concrete pages a build EMITTED for a dynamic route. The static export passes its report's,
    * so the sitemap cannot name a page the artifact lacks; absent, `prerender()` is asked — which is
-   * the same list, enumerated by the same function the prerenderer calls.
+   * the same list, enumerated by the same function the prerenderer calls. Given, `prerender()` is
+   * still asked what each page says of ITSELF (`lastmod`, `locales`); the list stays this one.
    */
   readonly pagesFor?: ((routePath: string) => readonly string[]) | undefined;
   /**
@@ -35,14 +36,22 @@ export interface SiteSeoOptions {
   readonly disallow?: readonly string[] | undefined;
   /**
    * `seo.sitemap` from `app.config.ts` (`loadSiteSettings`): the public `app/` pages to list beside
-   * the `site/` ones, and where each `<lastmod>` comes from. Omitted: `site/` pages, no `<lastmod>`.
+   * the `site/` ones, where each `<lastmod>` comes from, and the sections and page size of an
+   * index. Omitted: `site/` pages, no `<lastmod>`, one urlset.
    */
-  readonly sitemap?: SeoSitemapConfig | undefined;
+  readonly sitemap?: SiteSeoSitemap | undefined;
   /** The app root the route files are relative to — read for `lastmod: 'git' | 'mtime'`. */
   readonly root?: string | undefined;
 }
 
-const NO_SITEMAP_SETTINGS: SeoSitemapConfig = { extra: [], lastmod: 'none' };
+/**
+ * `seo.sitemap` as `siteSeo` takes it: `sections` and `pageSize` optional, so a caller that wrote
+ * `{ extra, lastmod }` before either existed still compiles and still gets the one urlset.
+ */
+export type SiteSeoSitemap = Pick<SeoSitemapConfig, 'extra' | 'lastmod'> &
+  Partial<Pick<SeoSitemapConfig, 'sections' | 'pageSize'>>;
+
+const NO_SITEMAP_SETTINGS: SiteSeoSitemap = { extra: [], lastmod: 'none' };
 
 /** A `seo.sitemap.extra` path that names no public page outside `site/`. */
 export class SitemapExtraInvalidError extends UltimateError {
@@ -67,42 +76,87 @@ const isPublicSite = (entry: RouteEntry): boolean =>
   entry.surface === 'site' && entry.config.policy === undefined;
 
 /**
- * A dynamic route's pages, UNPREFIXED and once each. A static export's report lists every locale's
- * copy (`/blog/a` and `/en/blog/a`), and the sitemap localizes each page itself — reading the
- * prefixed copies back as pages of their own listed `/en/en/blog/a`.
+ * A route's pages, UNPREFIXED and once each, with what `prerender()` said of each one. A
+ * static export's report lists every locale's copy (`/blog/a` and `/en/blog/a`), and the sitemap
+ * localizes each page itself — reading the prefixed copies back as pages of their own listed
+ * `/en/en/blog/a`.
  */
-const pagesOf = async (entry: RouteEntry, options: SiteSeoOptions): Promise<readonly string[]> => {
-  if (options.pagesFor !== undefined) {
-    return [...new Set(options.pagesFor(entry.path).map((path) => unlocalizedPath(path)))];
-  }
-  const params = await enumeratePrerender(entry);
-  return params.map((set) => fillPath(entry.pattern.source, set));
+const pagesOf = async (
+  entry: RouteEntry,
+  options: SiteSeoOptions,
+): Promise<readonly SitemapPage[]> => {
+  const said = (await enumeratePrerenderEntries(entry)).map(
+    (page): SitemapPage => ({
+      path: fillPath(entry.pattern.source, page.params),
+      ...(page.lastmod === undefined ? {} : { lastmod: page.lastmod }),
+      ...(page.locales === undefined ? {} : { locales: page.locales }),
+    }),
+  );
+  // Only a DYNAMIC route's list is the report's: a page with no params is listed whether or not
+  // this build emitted it (an `isr` one is rendered by the server), exactly as before.
+  if (options.pagesFor === undefined || !isDynamic(entry.path)) return said;
+  const byPath = new Map(said.map((page) => [page.path, page]));
+  const emitted = new Set(options.pagesFor(entry.path).map((path) => unlocalizedPath(path)));
+  return [...emitted].map((path) => byPath.get(path) ?? { path });
 };
 
+/**
+ * What a page's own `meta` says about being indexed, per locale: the record's `meta` when every
+ * locale agrees, its `locales` when only some may be indexed. `robots: { index: false }` in ONE
+ * locale used to leave that copy listed and every other copy's hreflang pointing at it — the read
+ * was made once, in no locale at all.
+ */
+async function indexability(
+  locales: readonly string[],
+): Promise<(path: string) => Pick<RouteRecord, 'meta' | 'locales'>> {
+  if (locales.length < 2) {
+    const metaByPath = new Map((await readSiteMeta()).records.map((r) => [r.path, r.meta]));
+    return (path) => {
+      const meta = metaByPath.get(path);
+      return meta === undefined ? {} : { meta };
+    };
+  }
+  const scans = new Map<string, Map<string, RouteRecord['meta']>>();
+  // One locale at a time: each scan runs every route's `meta` inside that locale's context.
+  for (const locale of locales) {
+    scans.set(locale, new Map((await readSiteMeta(locale)).records.map((r) => [r.path, r.meta])));
+  }
+  return (path) => {
+    const listed = locales.filter(
+      (locale) => scans.get(locale)?.get(path)?.robots?.index !== false,
+    );
+    if (listed.length === locales.length) return {};
+    // No locale may be indexed: the `noindex` itself is what keeps the route out.
+    if (listed.length === 0) return { meta: { robots: { index: false } } };
+    return { locales: listed };
+  };
+}
+
 /**
  * The public `site/` routes as `@ultimat3/seo` reads them. `meta` is carried where it resolves
  * without a request (`readSiteMeta`), which is what lets a page's own `robots: { index: false }`
  * keep it out of the sitemap — a page that asks crawlers to stay away must not be listed for them.
+ * A route with params or a `load` resolves no `meta` here: its pages say it per entry (`locales`).
  */
 async function publicSiteRoutes(
   options: SiteSeoOptions,
   stamp: Stamp,
+  locales: readonly string[],
 ): Promise<readonly RouteRecord[]> {
-  const metaByPath = new Map((await readSiteMeta()).records.map((r) => [r.path, r.meta]));
+  const indexed = await indexability(locales);
   return routeEntries()
     .filter(isPublicSite)
-    .map((entry) => {
-      const meta = metaByPath.get(entry.path);
-      return {
-        path: entry.path,
-        file: entry.file,
-        surface: 'site' as const,
-        render: entry.config.render,
-        ...(meta === undefined ? {} : { meta }),
-        ...(isDynamic(entry.path) ? { prerender: () => pagesOf(entry, options) } : {}),
-        ...stamp(entry.file),
-      };
-    });
+    .map((entry) => ({
+      path: entry.path,
+      file: entry.file,
+      surface: 'site' as const,
+      render: entry.config.render,
+      ...indexed(entry.path),
+      ...(isDynamic(entry.path) || entry.config.prerender !== undefined
+        ? { prerender: () => pagesOf(entry, options) }
+        : {}),
+      ...stamp(entry.file),
+    }));
 }
 
 type Stamp = (file: string) => { readonly lastmod?: string };
@@ -158,11 +212,15 @@ export async function siteSeo(options: SiteSeoOptions): Promise<SiteSeo> {
     return lastmod === undefined ? {} : { lastmod };
   };
   const routes = [
-    ...(await publicSiteRoutes(options, stamp)),
+    ...(await publicSiteRoutes(options, stamp, locales)),
     ...extraRoutes(settings.extra, stamp),
   ];
   const sitemap = await buildSitemap(routes, {
     baseUrl: options.baseUrl,
+    // A URL `robots.txt` keeps crawlers out of is not handed to them here: one config, two files.
+    ...(options.disallow === undefined ? {} : { disallow: options.disallow }),
+    ...(settings.sections === undefined ? {} : { sections: settings.sections }),
+    ...(settings.pageSize === undefined ? {} : { maxUrls: settings.pageSize }),
     ...(locales.length < 2
       ? {}
       : {
@@ -172,8 +230,8 @@ export async function siteSeo(options: SiteSeoOptions): Promise<SiteSeo> {
             localizedPath(path, locale, defaultLocale),
         }),
   });
-  // Past 50,000 URLs `files` are `/sitemaps/<n>.xml` and the index is `/sitemap.xml`; below it,
-  // `files` is that one file and there is no index.
+  // With sections, or past a protocol cap, `files` are `/sitemaps/*.xml` and the index is
+  // `/sitemap.xml`; otherwise `files` is that one file and there is no index.
   const sitemaps = sitemap.index === undefined ? sitemap.files : [sitemap.index, ...sitemap.files];
   const robots = buildRobots({
     baseUrl: options.baseUrl,

package/src/templates/scaffold-entries.ts

@@ -66,7 +66,13 @@ const prerender =
 // landed on disk — which is what \`x build --target static --json\` reads back.
 
 import { join } from 'node:path';
-import { loadSiteSettings, type PrerenderReport, prerenderSite, siteSeo } from '@ultimat3/cli';
+import {
+  loadSiteSettings,
+  type PrerenderReport,
+  prerenderSite,
+  siteSeo,
+  writeSiteSeo,
+} from '@ultimat3/cli';
 
 const root = join(import.meta.dir, '..', '..');
 const flag = Bun.argv.indexOf('--out');
@@ -84,9 +90,12 @@ const origin = Bun.env.SITE_ORIGIN;
  * page whose \`meta\` says \`robots: { index: false }\` left out — so a crawler reads one sitemap
  * from the CDN and from the container.
  *
- * A DYNAMIC route contributes exactly the URLs this build emitted for it, read back off the report
- * rather than by calling \`prerender()\` a second time: the sitemap then cannot name a page the
- * artifact does not contain.
+ * A DYNAMIC route contributes exactly the URLs this build emitted for it, read back off the report:
+ * the sitemap then cannot name a page the artifact does not contain. \`prerender()\` is asked again
+ * only for what each page says of itself — its \`lastmod\`, the \`locales\` it is listed in.
+ *
+ * With \`seo.sitemap.sections\` the answer is an index and its parts: \`/sitemap.xml\` plus
+ * \`/sitemaps/<name>-<n>.xml\`, every one written here.
  *
  * \`robots.txt\` fails closed — anything that is not \`ULTIMATE_ENV=production\` emits
  * \`Disallow: /\` and advertises no sitemap — so a preview build cannot outrank the real site.
@@ -104,9 +113,7 @@ async function writeSeoFiles(report: PrerenderReport): Promise<readonly string[]
     pagesFor: (route) =>
       report.pages.filter((page) => page.route === route).map((page) => page.path),
   });
-  for (const file of seo.sitemaps) await Bun.write(join(out, file.path), file.xml);
-  await Bun.write(join(out, 'robots.txt'), seo.robots);
-  return [...seo.sitemaps.map((file) => file.path), '/robots.txt'];
+  return await writeSiteSeo(out, seo);
 }
 
 if (import.meta.main) {