ferst-core 0.7.12 → 0.8.0

package/lib/seo.mjs

@@ -0,0 +1,130 @@
+/**
+ * Search signals every Ferst site gives search engines: one canonical address per page, a
+ * sitemap, robots.txt, and test copies kept out of search results. Pure helpers, shared by
+ * the ferst() integration (build time, Node) and its injected routes.
+ *
+ * The site's public address comes from `ferst-site.json` `siteUrl` (the platform sets it
+ * at setup and when the domain changes), or the `FERST_SITE_URL` env var. Without one, no
+ * address is guessed: canonical links and the sitemap need absolute addresses, so they're
+ * left out rather than pointed at a wrong host.
+ *
+ * Live site vs test copy: the production branch (`main`, or `productionBranch`) is the live
+ * site; any other branch's deploy is a test copy, the same rule the CMS uses to pick its
+ * git-proxy. A test copy is marked noindex everywhere (an `X-Robots-Tag` header on every
+ * file, and a robots meta on every page) while crawling stays allowed: an engine has to
+ * fetch a page to see its noindex.
+ */
+
+/**
+ * An address's origin (`https://host`), or null when it isn't a web address. A bare host
+ * gets https; any path or query is dropped, since it would make every canonical wrong.
+ * @param {unknown} value
+ * @returns {string | null}
+ */
+export function normaliseSiteUrl(value) {
+  if (typeof value !== 'string') return null;
+  const raw = value.trim();
+  if (!raw) return null;
+  const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(raw) ? raw : `https://${raw}`;
+  let url;
+  try {
+    url = new URL(withScheme);
+  } catch {
+    return null;
+  }
+  if (url.protocol !== 'https:' && url.protocol !== 'http:') return null;
+  if (!url.hostname.includes('.') && url.hostname !== 'localhost') return null;
+  return url.origin;
+}
+
+/**
+ * The site's address and whether this build is the live site.
+ * @param {{ siteMeta?: { siteUrl?: string, productionBranch?: string } | null, branch: string, env?: Record<string, string | undefined> }} p
+ * @returns {{ siteUrl: string | null, production: boolean }}
+ */
+export function seoContext({ siteMeta, branch, env = {} }) {
+  const siteUrl = normaliseSiteUrl(env.FERST_SITE_URL || siteMeta?.siteUrl);
+  const prodFromMeta = typeof siteMeta?.productionBranch === 'string' ? siteMeta.productionBranch.trim() : '';
+  const productionBranch = (env.FERST_PROD_BRANCH || prodFromMeta || 'main').trim();
+  return { siteUrl, production: branch === productionBranch };
+}
+
+/**
+ * A page's canonical address: the site's origin plus the path it's served at. Pages are
+ * folders (`/clergy/` serves `clergy/index.html`), so a page path gets a trailing slash; a
+ * file path (`/uploads/x.pdf`) is left as it is. Any query string is dropped.
+ * @param {string} siteUrl
+ * @param {string} pathname
+ */
+export function canonicalUrl(siteUrl, pathname) {
+  let path = String(pathname || '/').split(/[?#]/)[0] || '/';
+  if (!path.startsWith('/')) path = `/${path}`;
+  const last = path.split('/').pop() ?? '';
+  const isFile = /\.[a-z0-9]{1,5}$/i.test(last);
+  if (!isFile && !path.endsWith('/')) path = `${path}/`;
+  return `${siteUrl}${path}`;
+}
+
+/**
+ * robots.txt. The live site allows everything and names its sitemap. A test copy also
+ * allows crawling (its pages carry noindex, which engines only see by fetching them) and
+ * names no sitemap.
+ * @param {{ siteUrl: string | null, production: boolean }} p
+ */
+export function robotsTxt({ siteUrl, production }) {
+  if (!production) {
+    return [
+      '# A test copy: every page and file is marked noindex, so search engines leave it out.',
+      '# Crawling stays allowed, or they could not see that noindex.',
+      'User-agent: *',
+      'Allow: /',
+      '',
+    ].join('\n');
+  }
+  const lines = ['User-agent: *', 'Allow: /'];
+  if (siteUrl) lines.push('', `Sitemap: ${siteUrl}/sitemap.xml`);
+  return `${lines.join('\n')}\n`;
+}
+
+const XML_ESCAPES = { '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&apos;' };
+const xmlEscape = (s) => String(s).replace(/[&<>"']/g, (c) => XML_ESCAPES[c]);
+
+/**
+ * A sitemap (sitemaps.org protocol): each address once, with a last-modified date where known.
+ * @param {Array<{ loc: string, lastmod?: Date | string }>} entries
+ */
+export function sitemapXml(entries) {
+  const seen = new Set();
+  const urls = [];
+  for (const { loc, lastmod } of entries) {
+    if (!loc || seen.has(loc)) continue;
+    seen.add(loc);
+    let date = '';
+    if (lastmod) {
+      const d = lastmod instanceof Date ? lastmod : new Date(lastmod);
+      if (!Number.isNaN(d.getTime())) date = d.toISOString().slice(0, 10);
+    }
+    urls.push(`  <url><loc>${xmlEscape(loc)}</loc>${date ? `<lastmod>${date}</lastmod>` : ''}</url>`);
+  }
+  return [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+    ...urls,
+    '</urlset>',
+    '',
+  ].join('\n');
+}
+
+const NOINDEX_RULE = '/*\n  X-Robots-Tag: noindex\n';
+
+/**
+ * A Cloudflare Pages `_headers` file that marks every path noindex (pages and files such as
+ * PDFs alike), keeping any headers the site already sets. Idempotent.
+ * @param {string} existing
+ */
+export function withNoindexHeader(existing) {
+  const current = String(existing ?? '');
+  if (/X-Robots-Tag:\s*noindex/i.test(current)) return current;
+  if (!current.trim()) return NOINDEX_RULE;
+  return `${current.replace(/\s*$/, '')}\n\n${NOINDEX_RULE}`;
+}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "ferst-core",
-  "version": "0.7.12",
+  "version": "0.8.0",
   "type": "module",
-  "description": "Ferst Core \u2014 the shared, brand-agnostic client-site system: Astro components, layouts, content schemas, the layered config resolver, and a neutral styling architecture that every client overrides.",
+  "description": "Ferst Core — the shared, brand-agnostic client-site system: Astro components, layouts, content schemas, the layered config resolver, and a neutral styling architecture that every client overrides.",
   "license": "BUSL-1.1",
   "author": "MyAI4 Ltd (https://ferst.co.uk)",
   "homepage": "https://ferst.co.uk",

package/routes/404.astro

@@ -0,0 +1,36 @@
+---
+/**
+ * The "page not found" page (`/404` → `404.html`): INJECTED by the ferst() integration.
+ * Cloudflare Pages answers a missing address with this page and a real 404 status. Without
+ * a `404.html` it serves the HOME PAGE with a 200 instead (a "soft 404"): search engines
+ * would see the home page at every mistyped or retired address, and a broken link would
+ * look fine to visitors and to the migration's redirect check. A site with its own
+ * `src/pages/404.astro` keeps it (the integration skips this route then).
+ */
+import Base from '../layouts/Base.astro';
+import BlockRenderer from '../components/BlockRenderer.astro';
+import { pageSchema } from '../content/blocks';
+
+const blog = import.meta.env.FERST_ROUTES_BLOG !== false;
+const page = pageSchema.parse({
+  title: 'Page not found',
+  blocks: [
+    {
+      type: 'hero',
+      title: 'Page not found',
+      subtitle: "The page you were looking for isn't here. It may have moved, or the link may be out of date.",
+    },
+    {
+      type: 'prose',
+      variant: 'lead',
+      text: blog ? 'The home page, or the latest news, is a good place to start.' : 'The home page is a good place to start.',
+    },
+    { type: 'button', label: 'Go to the home page', href: '/' },
+    ...(blog ? [{ type: 'button', label: 'Latest news', href: '/posts/', variant: 'secondary' }] : []),
+  ],
+});
+---
+
+<Base title={page.title} description="This page isn't here." indexable={false}>
+  <BlockRenderer blocks={page.blocks} />
+</Base>

package/routes/robots.txt.ts

@@ -0,0 +1,15 @@
+// /robots.txt: injected by the ferst() integration (lib/seo.mjs explains the rules). The
+// live site names its sitemap; a test copy stays crawlable so engines see its noindex.
+import type { APIRoute } from 'astro';
+import { robotsTxt } from '../lib/seo.mjs';
+
+export const prerender = true;
+
+export const GET: APIRoute = () =>
+  new Response(
+    robotsTxt({
+      siteUrl: import.meta.env.FERST_SITE_URL || null,
+      production: import.meta.env.FERST_SEO_PRODUCTION !== false,
+    }),
+    { headers: { 'Content-Type': 'text/plain; charset=utf-8' } },
+  );

package/routes/sitemap.xml.ts

@@ -0,0 +1,37 @@
+// /sitemap.xml: injected by the ferst() integration once the site has an address
+// (`ferst-site.json` `siteUrl`). It lists every published page, the posts index, each post
+// and each tag page in use, and the calendar when it's set up. Drafts, hidden tags, later
+// listing pages (/posts/page/2…) and the cookie policy stay out.
+import type { APIRoute } from 'astro';
+import { getPageEntries, pageRouteParams } from '../lib/pages';
+import { canonicalUrl, sitemapXml } from '../lib/seo.mjs';
+
+export const prerender = true;
+
+export const GET: APIRoute = async () => {
+  const site = String(import.meta.env.FERST_SITE_URL || '');
+  const entries: Array<{ loc: string; lastmod?: Date }> = [];
+
+  for (const page of await getPageEntries()) {
+    const { slug } = pageRouteParams(page.id);
+    entries.push({ loc: canonicalUrl(site, slug ? `/${slug}` : '/') });
+  }
+
+  if (import.meta.env.FERST_ROUTES_BLOG !== false) {
+    const { getPublishedPosts, getUsedTags } = await import('../lib/posts');
+    const posts = await getPublishedPosts();
+    if (posts.length) entries.push({ loc: canonicalUrl(site, '/posts'), lastmod: posts[0].data.date });
+    for (const post of posts) entries.push({ loc: canonicalUrl(site, `/posts/${post.slug}`), lastmod: post.data.date });
+    for (const tag of await getUsedTags()) {
+      if (!tag.hidden && tag.count > 0) entries.push({ loc: canonicalUrl(site, `/posts/tag/${tag.slug}`) });
+    }
+  }
+
+  if (import.meta.env.FERST_ROUTES_CALENDAR !== false) {
+    const { getCalendarSettings } = await import('../lib/calendar');
+    const calendar = await getCalendarSettings();
+    if (calendar?.embedUrl) entries.push({ loc: canonicalUrl(site, '/calendar') });
+  }
+
+  return new Response(sitemapXml(entries), { headers: { 'Content-Type': 'application/xml; charset=utf-8' } });
+};