domma-cms 1.8.34 → 1.8.36

package/server/services/renderer.js

@@ -11,6 +11,7 @@ import {applyTransforms} from './hooks.js';
 import {contentsPaint, readContentsOverride, resolveContents} from './pageContents.js';
 import {applyMenuOverride, hasThemeSwitcherItem, resolveLocation, resolveMenuDecorations, resolveOverlaysForPage} from './menus.js';
 import {buildMenuNav} from './menuRender.js';
+import {buildFooterFallback, buildNavFallback} from './nojsLinks.js';
 import {getProjectForPage} from './projects.js';
 import {resolveContextMenusForPage} from './contextMenus.js';
 import {buildCustomThemeStyleTag, buildOverrideStyleTag, getThemeTokenMap, listThemeIds, loadThemeConfig, resolveThemeClasses} from './themeSettings.js';
@@ -75,7 +76,7 @@ function buildSearchAssets() {
     return {
         headTag: `<link rel="stylesheet" href="/public/css/search.css?v=${SEARCH_ASSET_V}">`,
         bodyTag: `<script>window.__CMS_SEARCH__ = ${json};</script>\n`
-            + `<script src="/public/js/search.js?v=${SEARCH_ASSET_V}"></script>`
+            + `<script src="/public/js/search.js?v=${SEARCH_ASSET_V}" defer></script>`
     };
 }
 
@@ -97,7 +98,7 @@ const ANALYTICS_ASSET_V = '20260922-analytics';
 function buildAnalyticsTag() {
     // Unless the site has switched Analytics off (config/tools.json).
     if (!isToolEnabled('analytics')) return '';
-    return `<script src="/public/js/analytics.js?v=${ANALYTICS_ASSET_V}"></script>`;
+    return `<script src="/public/js/analytics.js?v=${ANALYTICS_ASSET_V}" defer></script>`;
 }
 
 /**
@@ -441,6 +442,12 @@ export async function renderPage(page, opts = {}) {
         ...(footerLegal?.class   && {legalClass:   footerLegal.class})
     };
     const footerJson = JSON.stringify(footer).replace(/<\/script>/gi, '<\\/script>');
+    // The navbar's and footer's links as plain HTML too, for robots that run no
+    // JavaScript - built from the same per-viewer data as the globals above;
+    // site.js removes them once it has drawn the real ones (nojsLinks.js).
+    const publicNav = filterHiddenNavItems(navigation);
+    const navFallbackHtml = buildNavFallback(publicNav, site);
+    const footerFallbackHtml = buildFooterFallback(footer, site);
 
     // A preview is not a visit. Without this the editor's preview iframe posts a
     // page view on every re-render - recorded under the srcdoc document's path,
@@ -498,7 +505,9 @@ export async function renderPage(page, opts = {}) {
         contentsStyle: contentsPaint(pageContents).style,
         contentsJson: escapeAttrValue(JSON.stringify(pageContents)),
         contentsCollapsible: pageContents.collapsible === true && (preset.sidebar === true || page.sidebar === true) && !pageContents.hidden,
-        navJson: JSON.stringify(filterHiddenNavItems(navigation)).replace(/<\/script>/gi, '<\\/script>'),
+        navJson: JSON.stringify(publicNav).replace(/<\/script>/gi, '<\\/script>'),
+        navFallbackHtml,
+        footerFallbackHtml,
         footerScript: footerJson ? `window.__CMS_FOOTER__ = ${footerJson};` : '',
         ctxMenusScript: ctxMenusScript,
         ctxMenusModule: ctxMenusModule,
@@ -536,8 +545,8 @@ export async function renderPage(page, opts = {}) {
             searchAssets.bodyTag,
             switcherAssets.bodyTag,
             buildAnalyticsTag(),
-            site.backToTop?.enabled    ? '<script src="/public/js/btt.js"></script>'             : '',
-            site.cookieConsent?.enabled ? '<script src="/public/js/cookie-consent.js"></script>' : '',
+            site.backToTop?.enabled    ? '<script src="/public/js/btt.js" defer></script>'       : '',
+            site.cookieConsent?.enabled ? '<script src="/public/js/cookie-consent.js" defer></script>' : '',
             (page.usedComponents || [])
                 .map(n => `<script type="module" src="/api/components/${n}.js"></script>`)
                 .join('\n')
@@ -1233,6 +1242,12 @@ export async function renderBlogPage(templatePath, data = {}, seoMeta = {}) {
         ...(footerLegal?.class   && {legalClass:   footerLegal.class})
     };
     const footerJson = JSON.stringify(footer).replace(/<\/script>/gi, '<\\/script>');
+    // The navbar's and footer's links as plain HTML too, for robots that run no
+    // JavaScript - built from the same per-viewer data as the globals above;
+    // site.js removes them once it has drawn the real ones (nojsLinks.js).
+    const publicNav = filterHiddenNavItems(navigation);
+    const navFallbackHtml = buildNavFallback(publicNav, site);
+    const footerFallbackHtml = buildFooterFallback(footer, site);
 
     /*
      * Authored context menus. Resolved server-side so a visitor only receives
@@ -1278,7 +1293,9 @@ export async function renderBlogPage(templatePath, data = {}, seoMeta = {}) {
         showFooter: true,
         footerBlocksHtml,
         showSidebar: false,
-        navJson: JSON.stringify(filterHiddenNavItems(navigation)).replace(/<\/script>/gi, '<\\/script>'),
+        navJson: JSON.stringify(publicNav).replace(/<\/script>/gi, '<\\/script>'),
+        navFallbackHtml,
+        footerFallbackHtml,
         footerScript: footerJson ? `window.__CMS_FOOTER__ = ${footerJson};` : '',
         ctxMenusScript: ctxMenusScript,
         ctxMenusModule: ctxMenusModule,

package/server/services/sitemap.js

@@ -35,9 +35,11 @@ function shouldIncludeInSitemap(page) {
         && !page.seo?.noindex;
 }
 
-/** The sitemap entry of a core page. */
+/** The sitemap entry of a core page. `description` is for llms.txt; the XML ignores it. */
 function pageEntry(page) {
-    return {urlPath: page.urlPath, title: page.title || '', lastmod: page.updatedAt || null, source: 'pages'};
+    const description = String(page.seo?.description || page.description || '').trim();
+    return {urlPath: page.urlPath, title: page.title || '', lastmod: page.updatedAt || null,
+        ...(description && {description}), source: 'pages'};
 }
 
 /**
@@ -77,6 +79,7 @@ export async function listSitemapEntries(baseUrl, {log = console} = {}) {
                 ...(CHANGEFREQ.has(item.changefreq) && {changefreq: item.changefreq}),
                 ...(validPriority(item.priority) && {priority: Number(item.priority)}),
                 ...(typeof item.image === 'string' && item.image && {image: item.image}),
+                ...(typeof item.description === 'string' && item.description.trim() && {description: item.description.trim()}),
                 source: source.id
             });
         }
@@ -106,6 +109,24 @@ export async function generate(baseUrl, opts = {}) {
     return wrapInUrlset(entries.map(e => buildUrlEntry(e, normalised)), {withImages});
 }
 
+/**
+ * The public GET endpoints a public page calls while it renders: effects
+ * settings (effects.js, every page), a form's definition (forms.js), a
+ * component's compiled module (the page's <script> tags) and the public
+ * listings an interactive [collection] / [view] pages through. Never a route
+ * that needs a sign-in or writes anything.
+ */
+export const PUBLIC_API_ALLOW = [
+    '/api/effects/settings$',
+    '/api/forms/*/public$',
+    '/api/components/*.js$',
+    // The listing itself, with or without its query string - not /public/<id>.
+    '/api/collections/*/public$',
+    '/api/collections/*/public?',
+    '/api/views/*/public$',
+    '/api/views/*/public?'
+];
+
 /**
  * Build the robots.txt document. Points at /sitemap.xml on the given base URL.
  * The 'robots:lines' transform may change the lines (the SEO plugin's editor).
@@ -123,6 +144,11 @@ export function buildRobotsTxt(baseUrl) {
     let lines = [
         'User-agent: *',
         'Allow: /',
+        // Google renders pages with JavaScript, and a page's scripts read these
+        // public endpoints while it renders - so they are let through ahead of
+        // the /api/ block (the longest matching rule wins). Read-only and
+        // anonymous, each still answering to its own access rules.
+        ...PUBLIC_API_ALLOW.map(p => `Allow: ${p}`),
         'Disallow: /admin/',
         'Disallow: /api/',
         '',
@@ -138,10 +164,82 @@ export function buildRobotsTxt(baseUrl) {
     return lines.join('\n') + '\n';
 }
 
+/**
+ * May /llms.txt be served? Its own switch in the SEO settings (on unless
+ * switched off), and never while robots.txt asks every crawler to stay away.
+ * `settings` is the SEO engine's (config/seo.json over the defaults), or
+ * null when SEO is switched off on the site - then, as with the sitemap,
+ * the core default applies and it is served.
+ *
+ * @param {object|null} settings
+ * @returns {boolean}
+ */
+export function llmsTxtAllowed(settings) {
+    if (!settings) return true;
+    return settings.llms?.enabled !== false && settings.robots?.blockAll !== true;
+}
+
+/**
+ * Build /llms.txt (llmstxt.org): the site's name, its description, then one
+ * section per source - the site's pages first, then each plugin's sitemap
+ * source (blog posts...) - listing `- [Title](absolute url): description`.
+ * The same published, public, indexable set the sitemap lists, so anything
+ * kept out of the sitemap is kept out of here too.
+ *
+ * @param {string} baseUrl - Absolute origin; site.json's baseUrl when set
+ * @param {{log?: {warn: Function}}} [opts]
+ * @returns {Promise<string>}
+ */
+export async function buildLlmsTxt(baseUrl, opts = {}) {
+    const site = getConfig('site') || {};
+    const normalised = normaliseBaseUrl(baseUrl);
+    const entries = await listSitemapEntries(normalised, opts);
+    const labels = new Map(getSitemapSources().map(s => [s.id, s.label || s.id]));
+
+    const title = oneLine(site.title || site.seo?.defaultTitle || 'Website');
+    const description = oneLine(site.seo?.defaultDescription || site.description || '');
+    const out = [`# ${title}`, ''];
+    if (description) out.push(`> ${description}`, '');
+
+    // Sections in first-seen order: core pages lead, then each source.
+    const sections = new Map();
+    for (const e of entries) {
+        const key = e.source || 'pages';
+        if (!sections.has(key)) sections.set(key, []);
+        sections.get(key).push(e);
+    }
+    for (const [key, list] of sections) {
+        out.push(`## ${oneLine(key === 'pages' ? 'Pages' : (labels.get(key) || key))}`, '');
+        for (const e of list) {
+            const url = `${normalised}${e.urlPath}`;
+            const name = linkText(e.title || e.urlPath);
+            const about = oneLine(e.description || '');
+            out.push(`- [${name}](${linkUrl(url)})${about ? `: ${about}` : ''}`);
+        }
+        out.push('');
+    }
+    return out.join('\n').replace(/\n+$/, '') + '\n';
+}
+
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
 
+/** Text on one line, runs of whitespace collapsed. */
+function oneLine(v) {
+    return String(v ?? '').replace(/\s+/g, ' ').trim();
+}
+
+/** An address safe inside a Markdown link's (parentheses): brackets and whitespace percent-encoded. */
+function linkUrl(url) {
+    return String(url).replace(/[()\s]/g, c => '%' + c.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0'));
+}
+
+/** Text safe inside a Markdown link's [brackets]. */
+function linkText(v) {
+    return oneLine(v).replace(/[\\[\]]/g, '\\$&');
+}
+
 function buildUrlEntry(entry, baseUrl) {
     const loc = escapeXml(baseUrl ? `${baseUrl}${entry.urlPath}` : entry.urlPath);
     const when = entry.lastmod ? new Date(entry.lastmod) : null;

package/server/templates/page.html

@@ -62,7 +62,7 @@
     {{/if}}
 
     {{#if showNavbar}}
-    <nav id="site-navbar" class="{{navbarClass}}"></nav>
+    <nav id="site-navbar" class="{{navbarClass}}">{{navFallbackHtml}}</nav>
     {{/if}}
 
     <main class="site-main {{#if showSidebar}}with-sidebar {{contentsClass}}{{/if}}"{{#if contentsStyle}} style="{{contentsStyle}}"{{/if}}>
@@ -91,11 +91,12 @@
     </main>
 
     {{#if showFooter}}
-    <footer id="site-footer" class="page-footer">{{footerBlocksHtml}}</footer>
+    <footer id="site-footer" class="page-footer">{{footerBlocksHtml}}{{footerFallbackHtml}}</footer>
     {{/if}}
 
-    <!-- DOMPurify - must load before DommaJS -->
-    <script src="https://cdn.jsdelivr.net/npm/dompurify@3/dist/purify.min.js"></script>
+    <!-- DOMPurify - must load before DommaJS. Served from this site (public/js/vendor),
+         pinned, rather than a floating CDN version. -->
+    <script src="/public/js/vendor/dompurify/purify.min.js?v=3.4.17"></script>
 
     <!-- DommaJS -->
     <script src="/dist/domma/domma.min.js"></script>
@@ -172,8 +173,17 @@
       {{dconfigScript}}
     </script>
 
+    <!-- Deferred classic scripts. They run once the page is parsed, in document
+         order with the module scripts below - so they stay ABOVE site.js, which
+         reads window.FormLogicEngine and window.DommaCollectionBrowser. -->
+    <!-- Interactive Collection Browser (used by [collection] shortcodes with
+         searchable / filterable / sortable / paginate attributes) -->
+    <script src="/public/js/collection-browser.js?v=20260927-scope-mine" defer></script>
+    <!-- Core Forms logic engine (must run before forms.js and site.js) -->
+    <script src="/public/js/form-logic-engine.js?v=20260927-submit-btn" defer></script>
+
     <!-- Site initialisation -->
-    <script src="/public/js/site.js?v=20261016-footer-blocks" type="module"></script>
+    <script src="/public/js/site.js?v=20261022-nojs-links" type="module"></script>
     <script src="/public/js/collection-context.js?v=20261010-utf8-payloads" type="module"></script>
     {{ctxMenusModule}}
 
@@ -182,16 +192,11 @@
          [menu] shortcodes are wired even on pages with no floating navbar. -->
     <script src="/public/js/menu-float.js?v=20260927-float" type="module"></script>
 
-    <!-- Interactive Collection Browser (used by [collection] shortcodes with
-         searchable / filterable / sortable / paginate attributes) -->
-    <script src="/public/js/collection-browser.js?v=20260927-scope-mine"></script>
-
-    <!-- Core Forms (logic engine must load before renderer) -->
-    <script src="/public/js/form-logic-engine.js?v=20260927-submit-btn"></script>
+    <!-- Core Forms renderer -->
     <script src="/public/js/forms.js?v=20261015-smooth-caret" type="module"></script>
 
     <!-- Core effects runtime -->
-    <script src="/public/js/effects.js?v=20261015-occasions"></script>
+    <script src="/public/js/effects.js?v=20261015-occasions" defer></script>
 
     <!-- Plugin body-end injection -->
     {{bodyEndInject}}