@uniweb/build 1.8.4 → 1.9.0

package/src/site/query-processor.js

@@ -1,17 +1,18 @@
 /**
- * Collection Processor
+ * Query Processor
  *
  * Materializes each named QUERY over the site's records into JSON data.
- * Collections are defined in site.yml and processed at build time.
+ * Queries are declared in `queries.yml` (or `site.yml::queries`) and compiled
+ * at build time.
  *
- * ⛔ A COLLECTION `.md` IS NOT A PAGE-SECTION `.md`. Same extension, unrelated
+ * ⛔ A RECORD `.md` IS NOT A PAGE-SECTION `.md`. Same extension, unrelated
  * meanings, and the sync-side reader states this too
- * (`build/src/uwx/collection-source.js`) because getting it backwards produces
+ * (`build/src/uwx/entity-source.js`) because getting it backwards produces
  * confident nonsense:
  *
- *   - collection record — frontmatter is structured DATA whose shape is the
- *     collection's data schema; the body is the value of ONE declared field
- *     (the Model's content body field). Not "metadata".
+ *   - record — frontmatter is structured DATA whose shape is the record's
+ *     data schema; the body is the value of ONE declared field (the Model's
+ *     content body field). Not "metadata".
  *   - page section — frontmatter is foundation/runtime CONFIG (`type:`, params,
  *     `theme:`); the body is authored content with no schema behind it.
  *
@@ -24,7 +25,7 @@
  *
  * Features:
  * - Discovers markdown (.md), data (.yml/.yaml), JSON (.json), and BibTeX (.bib)
- *   files in collection folders
+ *   record files
  * - Parses frontmatter for record data (markdown), full YAML or JSON (data items),
  *   or BibTeX → CSL-JSON (bibliography items)
  * - Pure-data formats (YAML, JSON, BibTeX) accept either one record per file
@@ -38,7 +39,7 @@
  *   fixed `where` selects compiles
  * - Auto-generates excerpts and extracts first images (markdown items only)
  *
- * @module @uniweb/build/site/collection-processor
+ * @module @uniweb/build/site/query-processor
  *
  * @example
  * // queries.yml
@@ -73,24 +74,7 @@ import { collectNestedRefs } from '@uniweb/schemas/format'
 import { labelIn } from './entry-label.js'
 import { fileRecordValue, makeFilePreview } from './file-records.js'
 import { warnOutsideVocabulary } from './vocabulary-warnings.js'
-
-// Try to import content-reader for markdown parsing
-let markdownToProseMirror
-try {
-  const contentReader = await import('@uniweb/content-reader')
-  markdownToProseMirror = contentReader.markdownToProseMirror
-} catch {
-  // Simplified fallback
-  markdownToProseMirror = (markdown) => ({
-    type: 'doc',
-    content: [
-      {
-        type: 'paragraph',
-        content: [{ type: 'text', text: markdown.trim() }]
-      }
-    ]
-  })
-}
+import { markdownToProseMirror } from '@uniweb/content-reader'
 
 /**
  * Parse collection config from site.yml

package/src/site/record-projections.js

@@ -0,0 +1,77 @@
+/**
+ * A STATICALLY BUILT SITE'S RECORD PAGES IN ITS PROJECTIONS — the agent index, each record page's
+ * markdown, the search index — made by the rule a host makes them with (`@uniweb/projections`:
+ * `recordRoutes`, `generateRecordSearchIndex`, `renderPageMarkdown`), over the files this build
+ * compiled. A parametric page is one template in the site's content, and each of its records is a
+ * page a visitor opens.
+ *
+ * ⛔ Until 2026-10-25 the static build made its projections from the site's pages alone: llms.txt
+ * and the search index held none of a site's books, talks or posts, while a host's held all of them
+ * (measured on a site with 16 record pages: 0 in llms.txt, 0 in the search index).
+ */
+
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { fillRoutePattern, recordHandle } from '@uniweb/core/route-match'
+import { recordRoutes, generateRecordSearchIndex, renderPageMarkdown } from '@uniweb/projections'
+
+/**
+ * A route as a file is named for it: each segment decoded, since a static host decodes a request's
+ * path before it looks for the file (`fillRoutePattern` encodes a record's route for a URL).
+ */
+export function routeOnDisk(route) {
+  return route.split('/').map((segment) => {
+    try {
+      return decodeURIComponent(segment)
+    } catch {
+      return segment
+    }
+  }).join('/')
+}
+
+function readJson(file) {
+  try {
+    return JSON.parse(readFileSync(file, 'utf8'))
+  } catch {
+    return null
+  }
+}
+
+/**
+ * The record pages of a site: each query whose records have a page of their own, with its search
+ * index and each record's page — its route, and its markdown (the parametric page's, then the
+ * record).
+ *
+ * @param {Object} content - one locale's site content
+ * @param {Object} options
+ * @param {string} options.dataRoot - the directory a query's compiled `path` (`/data/talks.json`) is under
+ * @param {string} [options.locale]
+ * @returns {{ indexes: Object[], entries: Object[], pages: Array<{ route: string, markdown: string }> }}
+ */
+export function recordProjections(content, { dataRoot, locale } = {}) {
+  const queries = content?.config?.queries || {}
+  const indexes = []
+  const pages = []
+  for (const { name, pattern, path, field } of recordRoutes(content)) {
+    // No compiled file: a query answered by a records service, or an external one — its records
+    // are not this build's to list.
+    if (!path) continue
+    const records = readJson(join(dataRoot, path))
+    if (!Array.isArray(records)) continue
+    indexes.push(generateRecordSearchIndex(name, { ...(queries[name] || {}), pattern, field }, records, locale))
+
+    const template = (content.pages || []).find((page) => page.route === pattern)
+    if (!template) continue
+    // A record's own file holds it whole; the list holds its brief.
+    const wholeDir = join(dataRoot, path.replace(/\.json$/, ''))
+    for (const brief of records) {
+      const route = fillRoutePattern(pattern, brief, { field: field ?? null })
+      if (!route) continue
+      const handle = recordHandle(brief)
+      const record = (handle && readJson(join(wholeDir, `${handle}.json`))) || brief
+      const markdown = renderPageMarkdown(template, { record })
+      if (markdown) pages.push({ route, markdown })
+    }
+  }
+  return { indexes, entries: indexes.flatMap((index) => index.entries), pages }
+}

package/src/site/seo-files.js

@@ -0,0 +1,190 @@
+/**
+ * `sitemap.xml` and `robots.txt` — the two files a crawler reads about a statically served site,
+ * written by the prerender from the pages it wrote.
+ *
+ * ⭐ THE SITEMAP LISTS THE PAGES A STATIC HOST WILL ANSWER WITH, BY THE URLS THEY STATE. An entry is
+ * a page the prerender wrote — the static ones and each record's own — at its canonical URL with its
+ * alternates, read off `Page#getHeadMeta`, the source its `<head>` was written from, so the sitemap
+ * and the page cannot disagree. ⛔ Until 2026-10-25 the plugin wrote it at bundle time, before
+ * parametric pages were expanded: it listed `/books/:slug` and no book, at URLs that left the base
+ * path out.
+ *
+ * ⭐ `robots.txt` IS READ ONLY AT A HOST'S ROOT. A site served under a base path — a GitHub project
+ * site — cannot have one; it is not written there, and the build says so, rather than shipping a file
+ * no crawler reads that points at a sitemap where none is.
+ */
+
+/** The three defined Content Signals, in the order they are emitted. */
+const CONTENT_SIGNAL_KEYS = ['search', 'ai-input', 'ai-train']
+
+/**
+ * Format the `Content-Signal:` directive from `seo.robots.contentSignals`.
+ *
+ * Content Signals express what a site permits its content to be *used for*,
+ * which is a different axis from `Disallow:` — that governs fetching, this
+ * governs use after fetching. The three defined signals:
+ *
+ * - `search`    — appear in search results
+ * - `ai-input`  — be retrieved at inference time (RAG, grounding)
+ * - `ai-train`  — be used to train a model
+ *
+ * **Emitted only when declared.** There is no default: a preference the site
+ * owner did not state is not ours to assert, in either direction. An absent
+ * signal means "unstated", which is not the same as `no`.
+ *
+ * Unknown keys are ignored rather than passed through — the vocabulary is a
+ * closed set, and forwarding an invented signal would produce a directive no
+ * crawler honors while reading as though it were doing something.
+ *
+ * @param {Object|null} signals - e.g. `{ search: true, 'ai-input': true, 'ai-train': false }`
+ * @returns {string} The directive line, or `''` when nothing is declared
+ */
+export function formatContentSignals(signals) {
+  if (!signals || typeof signals !== 'object') return ''
+
+  const parts = []
+  for (const key of CONTENT_SIGNAL_KEYS) {
+    if (!(key in signals)) continue
+    const value = signals[key]
+    const yes = value === true || value === 'yes'
+    const no = value === false || value === 'no'
+    if (!yes && !no) continue
+    parts.push(`${key}=${yes ? 'yes' : 'no'}`)
+  }
+
+  return parts.length ? `Content-Signal: ${parts.join(', ')}` : ''
+}
+
+/**
+ * `robots.txt` — the site's rules, and where its sitemap is.
+ *
+ * @param {Object} [robots] - `seo.robots`: `allow`, `disallow`, `crawlDelay`, `additionalSitemaps`,
+ *   `contentSignals`
+ * @param {Object} [options]
+ * @param {string|null} [options.sitemap] - the sitemap's absolute URL, when there is one
+ * @returns {string}
+ */
+export function generateRobotsTxt(robots = {}, { sitemap = null } = {}) {
+  const { disallow = [], allow = [], crawlDelay = null, additionalSitemaps = [], contentSignals = null } = robots || {}
+
+  let content = 'User-agent: *\n'
+
+  const signals = formatContentSignals(contentSignals)
+  if (signals) {
+    // Inside the User-agent group, before the rules — the directive applies to
+    // the group it sits in.
+    content += `${signals}\n`
+  }
+  for (const path of allow) content += `Allow: ${path}\n`
+  for (const path of disallow) content += `Disallow: ${path}\n`
+  if (crawlDelay) content += `Crawl-delay: ${crawlDelay}\n`
+
+  const sitemaps = [sitemap, ...additionalSitemaps].filter(Boolean)
+  if (sitemaps.length) content += `\n${sitemaps.map((url) => `Sitemap: ${url}`).join('\n')}\n`
+  return content
+}
+
+/** Whether `seo.robots` says anything a crawler would act on. */
+function hasRules(robots) {
+  if (!robots || typeof robots !== 'object') return false
+  return Boolean(
+    robots.allow?.length ||
+      robots.disallow?.length ||
+      robots.crawlDelay ||
+      robots.additionalSitemaps?.length ||
+      formatContentSignals(robots.contentSignals)
+  )
+}
+
+/**
+ * A rendered page's sitemap entry — its canonical URL, with its alternates — or null when it is not
+ * one to list: kept out of search (`noindex`), naming another URL as its canonical, or on a site
+ * whose address is unknown.
+ *
+ * @param {Object} page - a Page of the Website the prerender rendered it from
+ * @returns {{ loc: string, alternates: Array, lastmod: string|null, changefreq: string|null, priority: (string|number)|null }|null}
+ */
+export function sitemapEntry(page) {
+  const meta = page?.getHeadMeta?.()
+  if (!meta?.canonical || meta.robots?.includes('noindex')) return null
+  const own = page.website?.getPageUrl?.(page)
+  if (!own || meta.canonical !== own) return null
+  return {
+    loc: own,
+    alternates: meta.alternates || [],
+    // A record's page has no file of its own, so no date of its own: none is better than its template's.
+    lastmod: !page.dynamicContext && page.lastModified ? String(page.lastModified).split('T')[0] : null,
+    changefreq: page.seo?.changefreq || null,
+    priority: page.seo?.priority ?? null,
+  }
+}
+
+/** Escape a value for XML text or a double-quoted attribute. */
+function escapeXml(str) {
+  return String(str)
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+}
+
+/**
+ * `sitemap.xml` for entries (`sitemapEntry`), each with its alternates when the site has several
+ * locales.
+ *
+ * @param {Array} entries
+ * @returns {string}
+ */
+export function renderSitemap(entries) {
+  const alternated = entries.some((entry) => entry.alternates?.length)
+  const urls = entries.map((entry) => {
+    const lines = [`    <loc>${escapeXml(entry.loc)}</loc>`]
+    if (entry.lastmod) lines.push(`    <lastmod>${escapeXml(entry.lastmod)}</lastmod>`)
+    if (entry.changefreq) lines.push(`    <changefreq>${escapeXml(entry.changefreq)}</changefreq>`)
+    if (entry.priority !== null && entry.priority !== undefined) lines.push(`    <priority>${escapeXml(entry.priority)}</priority>`)
+    for (const { hreflang, url } of entry.alternates || []) {
+      lines.push(`    <xhtml:link rel="alternate" hreflang="${escapeXml(hreflang)}" href="${escapeXml(url)}" />`)
+    }
+    return `  <url>\n${lines.join('\n')}\n  </url>`
+  })
+  const xmlns = alternated ? ' xmlns:xhtml="http://www.w3.org/1999/xhtml"' : ''
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"${xmlns}>\n${urls.join('\n')}\n</urlset>\n`
+}
+
+/**
+ * The site's crawler files, for what this build serves: the sitemap when the site's address is
+ * known and it has pages to list, `robots.txt` when the site is at its host's root and has rules or a
+ * sitemap to name.
+ *
+ * @param {Object} options
+ * @param {string|null} options.address - the site's address (`config.url`)
+ * @param {string} [options.base] - the base the site is built for
+ * @param {Array} options.entries - `sitemapEntry` results
+ * @param {Object} [options.robots] - `seo.robots`
+ * @returns {{ sitemap: string|null, robots: string|null, notes: string[] }} what to write, and why
+ *   anything was not
+ */
+export function crawlerFiles({ address, base = '/', entries = [], robots = {} }) {
+  const notes = []
+  let sitemap = null
+  if (!address) {
+    notes.push('No sitemap.xml: the site has no address to list its pages at — set `seo.origin` in site.yml, or let its deploy target state one.')
+  } else if (!entries.length) {
+    notes.push('No sitemap.xml: no page was written to list.')
+  } else {
+    sitemap = renderSitemap(entries)
+  }
+  const atRoot = !base || base === '/'
+  const sitemapUrl = sitemap ? new URL('sitemap.xml', address).href : null
+  let robotsTxt = null
+  if (atRoot) {
+    if (sitemapUrl || hasRules(robots)) robotsTxt = generateRobotsTxt(robots, { sitemap: sitemapUrl })
+  } else {
+    notes.push(
+      `No robots.txt: crawlers read it only at a host's root, and this site is served under ${base}.` +
+        (sitemapUrl ? ` Submit ${sitemapUrl} to search engines directly.` : '')
+    )
+  }
+  return { sitemap, robots: robotsTxt, notes }
+}

package/src/site/site-address.js

@@ -0,0 +1,127 @@
+/**
+ * WHERE A STATICALLY BUILT SITE LIVES — the address a build states for it, `config.url`, which the
+ * runtime reads (`@uniweb/core/site-address`) to state every absolute URL: a page's canonical,
+ * `og:url` and hreflang, the sitemap, `robots.txt`, the agent index.
+ *
+ * A static host cannot tell a site where it is served, so the build states it: the origin its deploy
+ * target gives — `ORIGIN_ENV` (the GitHub Pages workflow passes the one GitHub states), or the
+ * production origin a host building the site says it serves it at (Netlify, Vercel:
+ * `detectCiContext`) — else the author's (`site.yml::seo.origin`), joined to the base the site is
+ * built for. A host that serves the site itself states its own, and none of this is consulted.
+ *
+ * ⭐ AN ORIGIN IS A SCHEME AND A HOST; THE PATH IS THE BASE'S. An origin written with a path is
+ * refused: the path would be stated twice, and could disagree with the base the files are built for.
+ * ⛔ Until 2026-10-25 the author wrote `seo.baseUrl`, which the docs called an origin while the
+ * sitemap and the page tags read it as the whole address, so under a base path no value was right.
+ */
+
+import { siteAddress } from '@uniweb/core/site-address'
+import { detectCiContext } from '../hosts/detect-ci-context.js'
+
+/** The origin a deploy target states for the site it serves, e.g. `https://user.github.io`. */
+export const ORIGIN_ENV = 'UNIWEB_ORIGIN'
+
+/** Keys of `seo` that are retired, and what replaced each. */
+const RETIRED_SEO_KEYS = {
+  baseUrl:
+    "`seo.baseUrl` is now `seo.origin`: the origin the site is served from, `https://example.com`. A path is the site's base — `base:` in site.yml, or UNIWEB_BASE from its deploy.",
+  defaultImage: '`seo.defaultImage` is `seo.image`: the social card every page inherits.',
+  locales: "`seo.locales` is retired: a page's hreflang follows the site's `languages:`.",
+}
+
+/** Keys of a site's config that whoever serves the site states, never its author. */
+const STATED_KEYS = {
+  url: "`url:` in site.yml is not read: a site's address is its origin (`seo.origin`) and its base, or what its deploy target states.",
+  trailingSlash: "`trailingSlash:` in site.yml is not read: how a page's URL ends follows how the build lays out its files.",
+}
+
+/**
+ * Stop on what `site.yml` says about where the site lives that nothing reads — a retired `seo` key,
+ * an address or URL form only whoever serves the site states, an origin that is not one. Run
+ * wherever `site.yml` is read for a build or a push, as `refuseRetiredServiceKeys` is.
+ *
+ * @param {Object} siteYml
+ * @param {string} [where]
+ * @throws {Error}
+ */
+export function refuseUnreadableAddress(siteYml, where = 'site.yml') {
+  if (!siteYml || typeof siteYml !== 'object') return
+  for (const [key, message] of Object.entries(STATED_KEYS)) {
+    if (siteYml[key] !== undefined) throw new Error(`[uniweb] ${where}: ${message}`)
+  }
+  const seo = siteYml.seo
+  if (!seo || typeof seo !== 'object') return
+  for (const [key, message] of Object.entries(RETIRED_SEO_KEYS)) {
+    if (seo[key] !== undefined) throw new Error(`[uniweb] ${where}: ${message}`)
+  }
+  readOrigin(seo.origin, `\`seo.origin\` in ${where}`)
+}
+
+/**
+ * A site's effective `seo` block: `site.yml::seo`, with a caller's overrides over it key by key
+ * (`defineSiteConfig({ seo })`). Refused when it holds a retired key, wherever that came from.
+ *
+ * @param {Object} [seo] - `site.yml::seo`
+ * @param {Object} [overrides]
+ * @returns {Object}
+ */
+export function effectiveSeo(seo, overrides = {}) {
+  const merged = { ...(seo || {}), ...(overrides || {}) }
+  for (const [key, message] of Object.entries(RETIRED_SEO_KEYS)) {
+    if (merged[key] !== undefined) throw new Error(message)
+  }
+  return merged
+}
+
+/**
+ * An origin as written, normalized — `https://example.com` — or null when none is written. Throws,
+ * naming `where`, for a value that is not one.
+ *
+ * @param {*} value
+ * @param {string} where - what to call it in the message (`seo.origin`, `UNIWEB_ORIGIN`)
+ * @returns {string|null}
+ */
+export function readOrigin(value, where) {
+  if (value === undefined || value === null || value === '') return null
+  const text = String(value).trim()
+  let url = null
+  try {
+    url = new URL(text)
+  } catch {
+    // reported below
+  }
+  // A string that parses is not yet an origin: `example.com:443` parses, with `example.com:` as its scheme.
+  if (!url || (url.protocol !== 'http:' && url.protocol !== 'https:') || !url.host) {
+    throw new Error(`${where} is "${text}", which is not an origin. Write it with its scheme: https://example.com`)
+  }
+  if (url.pathname !== '/') {
+    throw new Error(
+      `${where} is "${text}", but an origin is a scheme and a host: ${url.origin}. ` +
+        `The path ${url.pathname.replace(/\/?$/, '/')} is the site's base — \`base:\` in site.yml, or UNIWEB_BASE from its deploy.`
+    )
+  }
+  if (url.search || url.hash) {
+    throw new Error(`${where} is "${text}", but an origin is a scheme and a host: ${url.origin}.`)
+  }
+  return url.origin
+}
+
+/**
+ * The address a static build states for its site — the origin from its deploy target or its
+ * author, under the base it is built for — or null when neither states an origin.
+ *
+ * @param {Object} options
+ * @param {Object} [options.config] - the site's collected config, its `seo` effective (`effectiveSeo`)
+ * @param {string} [options.base] - the base the site is built for, `/handbook/`
+ * @param {Object} [options.env]
+ * @returns {string|null}
+ */
+export function resolveSiteAddress({ config = {}, base = '/', env = process.env } = {}) {
+  const ci = detectCiContext(env)
+  const origin =
+    readOrigin(env[ORIGIN_ENV], ORIGIN_ENV) ??
+    readOrigin(ci?.origin, `the production URL ${ci?.host} states`) ??
+    readOrigin(config.seo?.origin, '`seo.origin`')
+  if (!origin) return null
+  return siteAddress(`${origin}${base || '/'}`)
+}

package/src/uwx/site.js

@@ -92,6 +92,7 @@ import {
   refuseRetiredServiceKeys,
   refuseUnreadableServices
 } from './services-request.js'
+import { refuseUnreadableAddress } from '../site/site-address.js'
 import { resolveQueriesConfig } from './queries-config.js'
 import { qualifyRef } from '@uniweb/schemas/refs'
 import { siteSelfScope, refuseOrgOption } from './self-scope.js'
@@ -1418,6 +1419,7 @@ export async function siteProjectToDocument(siteRoot, opts = {}) {
   refuseOrgOption(opts, 'uwx/site')
   const siteYml = await readYamlFile(join(siteRoot, 'site.yml'))
   refuseRetiredServiceKeys(siteYml)
+  refuseUnreadableAddress(siteYml)
   refuseUnreadableServices(siteYml)
   const sourceLocale = opts.sourceLocale || resolveDefaultLocale(siteYml)
   if (!siteYml.name) {