@ultimat3/cli 22.9.0 → 22.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/cli",
-  "version": "22.9.0",
+  "version": "22.10.0",
   "description": "The `x` binary: new, dev, build, verify, generate, db, mcp, doctor, deploy",
   "license": "MIT",
   "type": "module",
@@ -38,33 +38,33 @@
   },
   "dependencies": {
     "@babel/core": "^8.0.1",
-    "@ultimat3/action": "22.9.0",
-    "@ultimat3/admin": "22.9.0",
-    "@ultimat3/ai": "22.9.0",
-    "@ultimat3/auth": "22.9.0",
-    "@ultimat3/cache": "22.9.0",
-    "@ultimat3/core": "22.9.0",
-    "@ultimat3/db": "22.9.0",
-    "@ultimat3/entity": "22.9.0",
-    "@ultimat3/flags": "22.9.0",
-    "@ultimat3/http": "22.9.0",
-    "@ultimat3/i18n": "22.9.0",
-    "@ultimat3/jobs": "22.9.0",
-    "@ultimat3/mail": "22.9.0",
-    "@ultimat3/manifest": "22.9.0",
-    "@ultimat3/mcp": "22.9.0",
-    "@ultimat3/money": "22.9.0",
-    "@ultimat3/notify": "22.9.0",
-    "@ultimat3/policy": "22.9.0",
-    "@ultimat3/pwa": "22.9.0",
-    "@ultimat3/query": "22.9.0",
-    "@ultimat3/realtime": "22.9.0",
-    "@ultimat3/render": "22.9.0",
-    "@ultimat3/schema": "22.9.0",
-    "@ultimat3/seo": "22.9.0",
-    "@ultimat3/storage": "22.9.0",
-    "@ultimat3/testing": "22.9.0",
-    "@ultimat3/time": "22.9.0",
+    "@ultimat3/action": "22.10.0",
+    "@ultimat3/admin": "22.10.0",
+    "@ultimat3/ai": "22.10.0",
+    "@ultimat3/auth": "22.10.0",
+    "@ultimat3/cache": "22.10.0",
+    "@ultimat3/core": "22.10.0",
+    "@ultimat3/db": "22.10.0",
+    "@ultimat3/entity": "22.10.0",
+    "@ultimat3/flags": "22.10.0",
+    "@ultimat3/http": "22.10.0",
+    "@ultimat3/i18n": "22.10.0",
+    "@ultimat3/jobs": "22.10.0",
+    "@ultimat3/mail": "22.10.0",
+    "@ultimat3/manifest": "22.10.0",
+    "@ultimat3/mcp": "22.10.0",
+    "@ultimat3/money": "22.10.0",
+    "@ultimat3/notify": "22.10.0",
+    "@ultimat3/policy": "22.10.0",
+    "@ultimat3/pwa": "22.10.0",
+    "@ultimat3/query": "22.10.0",
+    "@ultimat3/realtime": "22.10.0",
+    "@ultimat3/render": "22.10.0",
+    "@ultimat3/schema": "22.10.0",
+    "@ultimat3/seo": "22.10.0",
+    "@ultimat3/storage": "22.10.0",
+    "@ultimat3/testing": "22.10.0",
+    "@ultimat3/time": "22.10.0",
     "babel-preset-solid": "^1.9.15"
   }
 }

package/src/dev-route-table.ts

@@ -127,7 +127,7 @@ export async function devRouteTable(input: DevRouteTableInput): Promise<DevRoute
     // captured at boot would answer 404 for the href the document now carries.
     ...styleRoutes(() => styleBundle()),
     // `robots.txt` and `sitemap.xml`, the same two files the static export writes (`site-seo.ts`).
-    ...seoRoutes({ env: input.env, site }),
+    ...seoRoutes({ env: input.env, site, root: input.root }),
     // `x shot --island`'s harness, in the `/_x` dev namespace so no app route can shadow it. It
     // lives here rather than in a second server because everything it needs is in THIS process:
     // the built chunks, the app's stylesheet registry, and the one embedded Postgres a checkout

package/src/error-codes.ts

@@ -202,6 +202,7 @@ export const CLI_OWNED_ERROR_CODES = [
   'X_VERIFY_SHARD_INVALID',
   'X_VERIFY_MERGE_INCOMPLETE',
   'X_VERIFY_MERGE_INPUT',
+  'X_SITEMAP_EXTRA_INVALID',
 ] as const;
 
 /**
@@ -353,6 +354,7 @@ export const CLI_ERROR_TITLES: Readonly<Record<CliOwnedErrorCode, string>> = {
   X_VERIFY_SHARD_INVALID: 'x verify --shard was given a spec or a step list it cannot split',
   X_VERIFY_MERGE_INCOMPLETE: 'x verify merge was handed parts that do not add up to the whole gate',
   X_VERIFY_MERGE_INPUT: 'a part handed to x verify merge is not an x verify --json document',
+  X_SITEMAP_EXTRA_INVALID: 'a seo.sitemap.extra path names no public page',
 };
 
 // One unconditional call, so a second package claiming one of the CLI's codes throws

package/src/mcp-errors.ts

@@ -251,6 +251,8 @@ const CLI_FIXES: Readonly<Record<CliErrorCode, string>> = {
     'x build --target static --json   # after adding prerender() with one real path to the route',
   X_ROUTE_ASYNC_PAGE:
     'x verify --only budgets --json   # after moving each await in Page into export const load',
+  X_SITEMAP_EXTRA_INVALID:
+    'x routes --json   # then list in seo.sitemap.extra only a path a registered app/ route without a policy answers',
 };
 
 const isCliCode = (code: string): code is CliErrorCode =>

package/src/seo-routes.ts

@@ -10,8 +10,10 @@ import { ROBOTS_PATH, SITEMAP_PATH, siteSeo } from './site-seo';
 
 export interface SeoRoutesOptions {
   readonly env: Readonly<Record<string, string | undefined>>;
-  /** `site.origin` and `seo.robots.disallow` from `app.config.ts` (`loadSiteSettings`). */
+  /** `site.origin`, `seo.robots.disallow` and `seo.sitemap` from `app.config.ts` (`loadSiteSettings`). */
   readonly site?: SiteSettings;
+  /** The app root route files are relative to — `seo.sitemap.lastmod: 'git' | 'mtime'` reads them. */
+  readonly root?: string;
 }
 
 /**
@@ -37,6 +39,8 @@ export function seoRoutes(options: SeoRoutesOptions): readonly Route[] {
       baseUrl: originOf(options, request),
       environment,
       disallow: options.site?.disallow ?? [],
+      ...(options.site === undefined ? {} : { sitemap: options.site.sitemap }),
+      ...(options.root === undefined ? {} : { root: options.root }),
     });
 
   return [

package/src/serve-boot.ts

@@ -198,7 +198,7 @@ async function webSurface(
     // filled, so this process serves exactly the CSS it renders against.
     ...styleRoutes(() => styleBundle()),
     // `robots.txt` and `sitemap.xml`, the same two files the static export writes (`site-seo.ts`).
-    ...seoRoutes({ env: options.env, site }),
+    ...seoRoutes({ env: options.env, site, root: options.root }),
     // The page's one socket: its worker script, served beside the islands for their reason.
     ...sync.routes,
     ...navigation.routes,

package/src/site-config.ts

@@ -5,7 +5,8 @@
 
 // why: Bun exposes no path-join primitive, and the config path is app-root-relative.
 import { join } from 'node:path';
-import type { AppConfig, Environment } from '@ultimat3/core';
+import type { AppConfig, Environment, SeoSitemapConfig, SitemapLastmod } from '@ultimat3/core';
+import { SITEMAP_LASTMOD_SOURCES } from '@ultimat3/core';
 import { APP_CONFIG_EXPORT } from './app-auth';
 import { APP_CONFIG_FILE } from './app-root';
 
@@ -14,9 +15,15 @@ export interface SiteSettings {
   readonly origin: string | null;
   /** `seo.robots.disallow` — the paths a production `robots.txt` keeps crawlers out of. */
   readonly disallow: readonly string[];
+  /** `seo.sitemap` — public `app/` pages to list, and where `<lastmod>` comes from. */
+  readonly sitemap: SeoSitemapConfig;
 }
 
-export const NO_SITE_SETTINGS: SiteSettings = { origin: null, disallow: [] };
+export const NO_SITE_SETTINGS: SiteSettings = {
+  origin: null,
+  disallow: [],
+  sitemap: { extra: [], lastmod: 'none' },
+};
 
 const isRecord = (value: unknown): value is Record<string, unknown> =>
   typeof value === 'object' && value !== null;
@@ -43,6 +50,26 @@ export async function loadSiteSettings(root: string): Promise<SiteSettings> {
   return {
     origin: typeof origin === 'string' && origin !== '' ? trimSlash(origin) : null,
     disallow: [...config.seo.robots.disallow],
+    sitemap: sitemapOf(config.seo),
+  };
+}
+
+/**
+ * Structural for `hasSiteSections`' reason: a config resolved through a core older than
+ * `seo.sitemap` has no such key, and that is the default, not a crash.
+ */
+function sitemapOf(seo: unknown): SeoSitemapConfig {
+  const declared = isRecord(seo) ? seo['sitemap'] : undefined;
+  if (!isRecord(declared)) return NO_SITE_SETTINGS.sitemap;
+  const extra = declared['extra'];
+  const lastmod = declared['lastmod'];
+  return {
+    extra: Array.isArray(extra)
+      ? extra.filter((path): path is string => typeof path === 'string')
+      : [],
+    lastmod: SITEMAP_LASTMOD_SOURCES.includes(lastmod as SitemapLastmod)
+      ? (lastmod as SitemapLastmod)
+      : 'none',
   };
 }
 

package/src/site-seo.ts

@@ -2,7 +2,8 @@
 // (`apps/web/prerender.ts` writes it into the artifact) and the running web role (`seo-routes.ts`
 // serves it), so a crawler reads the same two files from a CDN and from a container.
 
-import type { Environment } from '@ultimat3/core';
+import type { Environment, SeoSitemapConfig } from '@ultimat3/core';
+import { UltimateError } from '@ultimat3/core';
 import { localeConfig, localizedPath, routedLocales, unlocalizedPath } from '@ultimat3/i18n';
 import type { RouteEntry } from '@ultimat3/render';
 import { routeEntries } from '@ultimat3/render';
@@ -10,6 +11,7 @@ import { enumeratePrerender, fillPath } from '@ultimat3/render/server';
 import type { RouteRecord, SitemapFile } from '@ultimat3/seo';
 import { buildRobots, buildSitemap, isDynamic } from '@ultimat3/seo';
 import { readSiteMeta } from './seo-meta';
+import { lastmodOf } from './sitemap-lastmod';
 
 export const ROBOTS_PATH = '/robots.txt';
 export const SITEMAP_PATH = '/sitemap.xml';
@@ -30,6 +32,27 @@ export interface SiteSeoOptions {
    * `User-agent: *` group; a non-production `robots.txt` still disallows everything.
    */
   readonly disallow?: readonly string[] | undefined;
+  /**
+   * `seo.sitemap` from `app.config.ts` (`loadSiteSettings`): the public `app/` pages to list beside
+   * the `site/` ones, and where each `<lastmod>` comes from. Omitted: `site/` pages, no `<lastmod>`.
+   */
+  readonly sitemap?: SeoSitemapConfig | undefined;
+  /** The app root the route files are relative to — read for `lastmod: 'git' | 'mtime'`. */
+  readonly root?: string | undefined;
+}
+
+const NO_SITEMAP_SETTINGS: SeoSitemapConfig = { extra: [], lastmod: 'none' };
+
+/** A `seo.sitemap.extra` path that names no public page outside `site/`. */
+export class SitemapExtraInvalidError extends UltimateError {
+  constructor(path: string, reason: string) {
+    super({
+      code: 'X_SITEMAP_EXTRA_INVALID',
+      cause: `seo.sitemap.extra lists "${path}", which ${reason}`,
+      fix: 'x routes --json   # then list in seo.sitemap.extra only a path a registered app/ route without a policy answers',
+      meta: { path },
+    });
+  }
 }
 
 export interface SiteSeo {
@@ -60,7 +83,10 @@ const pagesOf = async (entry: RouteEntry, options: SiteSeoOptions): Promise<read
  * without a request (`readSiteMeta`), which is what lets a page's own `robots: { index: false }`
  * keep it out of the sitemap — a page that asks crawlers to stay away must not be listed for them.
  */
-async function publicSiteRoutes(options: SiteSeoOptions): Promise<readonly RouteRecord[]> {
+async function publicSiteRoutes(
+  options: SiteSeoOptions,
+  stamp: Stamp,
+): Promise<readonly RouteRecord[]> {
   const metaByPath = new Map((await readSiteMeta()).records.map((r) => [r.path, r.meta]));
   return routeEntries()
     .filter(isPublicSite)
@@ -73,17 +99,70 @@ async function publicSiteRoutes(options: SiteSeoOptions): Promise<readonly Route
         render: entry.config.render,
         ...(meta === undefined ? {} : { meta }),
         ...(isDynamic(entry.path) ? { prerender: () => pagesOf(entry, options) } : {}),
+        ...stamp(entry.file),
       };
     });
 }
 
+type Stamp = (file: string) => { readonly lastmod?: string };
+
+/**
+ * The app's `seo.sitemap.extra` paths as records, each resolved to the route that answers it — the
+ * most specific match, the router's own tie-break. REFUSED, never skipped, when it names no route,
+ * an `api/` route, a gated one or a `site/` page (listed already): a sitemap missing a page its
+ * author listed is the silent drop, and a gated page in it is a URL a crawler cannot open.
+ */
+function extraRoutes(extra: readonly string[], stamp: Stamp): readonly RouteRecord[] {
+  const entries = routeEntries();
+  return [...new Set(extra)].map((path) => {
+    const entry = entries
+      .filter((candidate) => candidate.pattern.regex.test(path))
+      .sort((a, b) => b.pattern.specificity - a.pattern.specificity)[0];
+    if (entry === undefined)
+      throw new SitemapExtraInvalidError(path, 'no registered route answers');
+    if (entry.surface === 'api') {
+      throw new SitemapExtraInvalidError(path, `is the api/ route ${entry.file}, not a page`);
+    }
+    if (entry.surface === 'site') {
+      throw new SitemapExtraInvalidError(
+        path,
+        `is the site/ page ${entry.file} — every public site/ page is listed already`,
+      );
+    }
+    if (entry.config.policy !== undefined) {
+      throw new SitemapExtraInvalidError(
+        path,
+        `is answered by ${entry.file}, which declares a policy — a crawler cannot open it`,
+      );
+    }
+    return {
+      path,
+      file: entry.file,
+      surface: 'app' as const,
+      render: entry.config.render,
+      sitemap: true,
+      ...stamp(entry.file),
+    };
+  });
+}
+
 export async function siteSeo(options: SiteSeoOptions): Promise<SiteSeo> {
   // Every routed locale, with `xhtml:link` alternates: one `<url>` per page per locale, each
   // naming the whole cluster and `x-default`. A single-locale app passes none and gets the plain
   // urlset it always had — an alternates cluster of one is noise.
   const locales = routedLocales();
   const defaultLocale = localeConfig().fallback;
-  const sitemap = await buildSitemap(await publicSiteRoutes(options), {
+  const settings = options.sitemap ?? NO_SITEMAP_SETTINGS;
+  const root = options.root ?? process.cwd();
+  const stamp: Stamp = (file) => {
+    const lastmod = lastmodOf(file, settings.lastmod, root);
+    return lastmod === undefined ? {} : { lastmod };
+  };
+  const routes = [
+    ...(await publicSiteRoutes(options, stamp)),
+    ...extraRoutes(settings.extra, stamp),
+  ];
+  const sitemap = await buildSitemap(routes, {
     baseUrl: options.baseUrl,
     ...(locales.length < 2
       ? {}

package/src/sitemap-lastmod.ts

@@ -0,0 +1,73 @@
+// A sitemap `<lastmod>` per route source file — `seo.sitemap.lastmod` in `app.config.ts`. Read once
+// per file per process: the running web role answers `/sitemap.xml` per request, and a `git log`
+// per route per request is a subprocess per crawler hit for a date that changes on deploy.
+
+// why: Bun has no synchronous stat, and `Bun.file(missing).lastModified` answers a garbage date
+// (measured: year 144683) instead of refusing — the sitemap is built synchronously per route.
+import { statSync } from 'node:fs';
+// why: Bun exposes no path-join primitive, and the route file is app-root-relative.
+import { join } from 'node:path';
+import type { SitemapLastmod } from '@ultimat3/core';
+
+/** The moment this process first built a sitemap — `'build'`'s one timestamp for every URL. */
+let builtAt: string | undefined;
+
+const cache = new Map<string, string | undefined>();
+
+/**
+ * The `<lastmod>` for one route file, W3C Datetime (ISO 8601, UTC), or `undefined` when the source
+ * yields none — a sitemap entry without `<lastmod>` is valid, a wrong one misleads a crawler.
+ *
+ * `'git'` falls back to the file's mtime where there is no work tree (a container image): the
+ * image's copy of the file was written by the build, which is the honest "last changed" there.
+ */
+export function lastmodOf(file: string, mode: SitemapLastmod, root: string): string | undefined {
+  if (mode === 'none') return undefined;
+  if (mode === 'build') {
+    builtAt ??= new Date().toISOString();
+    return builtAt;
+  }
+  const key = `${mode}\u0000${root}\u0000${file}`;
+  if (cache.has(key)) return cache.get(key);
+  const value = (mode === 'git' ? gitDate(file, root) : undefined) ?? mtimeOf(join(root, file));
+  cache.set(key, value);
+  return value;
+}
+
+/** Test seam: forget every date read, so a test can move a file's clock. */
+export function resetLastmodCache(): void {
+  cache.clear();
+  builtAt = undefined;
+}
+
+function gitDate(file: string, root: string): string | undefined {
+  try {
+    const run = Bun.spawnSync(['git', 'log', '-1', '--format=%cI', '--', file], {
+      cwd: root,
+      stdout: 'pipe',
+      stderr: 'ignore',
+    });
+    if (run.exitCode !== 0) return undefined;
+    return isoOf(run.stdout.toString().trim());
+  } catch {
+    // No `git` binary on PATH: the fallback, never a sitemap that cannot be served.
+    return undefined;
+  }
+}
+
+function mtimeOf(path: string): string | undefined {
+  try {
+    const modified = statSync(path, { throwIfNoEntry: false })?.mtimeMs;
+    return modified !== undefined && Number.isFinite(modified) && modified > 0
+      ? new Date(modified).toISOString()
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function isoOf(text: string): string | undefined {
+  if (text === '') return undefined;
+  const time = Date.parse(text);
+  return Number.isFinite(time) ? new Date(time).toISOString() : undefined;
+}

package/src/templates/scaffold-entries.ts

@@ -91,10 +91,15 @@ const origin = Bun.env.SITE_ORIGIN;
  * \`Disallow: /\` and advertises no sitemap — so a preview build cannot outrank the real site.
  */
 async function writeSeoFiles(report: PrerenderReport): Promise<readonly string[]> {
+  // The same disallow list and sitemap settings the running server answers with
+  // (app.config.ts seo.robots.disallow and seo.sitemap).
+  const settings = await loadSiteSettings(root);
   const seo = await siteSeo({
     // The origin the pages were built against, so the sitemap and every canonical agree.
     baseUrl: report.origin,
-    disallow: (await loadSiteSettings(root)).disallow,
+    disallow: settings.disallow,
+    sitemap: settings.sitemap,
+    root,
     pagesFor: (route) =>
       report.pages.filter((page) => page.route === route).map((page) => page.path),
   });