freestruct 1.0.0 → 1.2.0

package/engine/notfound.js

@@ -5,7 +5,7 @@
 
 import fs from 'fs';
 import path from 'path';
-import { injectPage } from './seo.js';
+import { injectPage, escAttr } from './seo.js';
 import { pageUrlFor } from './urls.js';
 import { stripRobotsMeta } from './scan.js';
 
@@ -14,8 +14,12 @@ export function generate404(config, outputDir, buildHash, template, snippetDir)
   const file = path.join(outputDir, '404.html');
   // Links follow the pageUrlFor contract (basePath-aware) - raw site.url
   // guesses like '/docs/' point at 404s on sub-path installs (#3 class).
-  const siteName = config.site.name || 'Site';
-  const homeUrl = pageUrlFor(config, '/');
+  // The site name and the home URL land in HTML text/attribute slots, so they
+  // ride escAttr: a quote-bearing or '<'-bearing site name otherwise broke the
+  // <title> and could inject markup into the error page (the Pass 26.21 walk
+  // claimed this was escaped - it was not).
+  const siteName = escAttr(config.site.name || 'Site');
+  const homeUrl = escAttr(pageUrlFor(config, '/'));
   const notFound = `<!DOCTYPE html>
 <html lang="en">
 <head>
@@ -64,10 +68,22 @@ export function pipeline404(page, context) {
   // drop any pre-existing robots meta so ours is the single source of truth
   // (scan.js's stripper exists for exactly this - one regex, one owner)
   html = stripRobotsMeta(html);
-  html = html.replace(/<\/head>/i, '<meta name="robots" content="noindex, nofollow">\n</head>');
-  fs.writeFileSync(page.file, html);
   page.html = html;
+  fs.writeFileSync(page.file, html);
 
   injectPage(page, context);
+
+  // Assert the noindex AFTER injectPage, not before: injectPage's rerun
+  // hygiene (stripInjectedRegions) deletes everything from the
+  // `<!-- freestruct -->` marker to `</head>`, so a noindex inserted before
+  // the marker on run 1 fell INSIDE that span once the marker existed on
+  // run 2 and was stripped - a second `freestruct` run silently dropped the
+  // 404 noindex and the audit then failed the page. Re-assert it here, past
+  // the strip, and it stays put across reruns.
+  let out = fs.readFileSync(page.file, 'utf8');
+  out = stripRobotsMeta(out);
+  out = out.replace(/<\/head>/i, '<meta name="robots" content="noindex, nofollow">\n</head>');
+  fs.writeFileSync(page.file, out);
+  page.html = out;
   return page;
 }

package/engine/ogimage.js

@@ -0,0 +1,142 @@
+// engine/ogimage.js - per-page OG image generation (issue #22)
+// Post-render OG-image generation: read each page's title/description (the
+// engine already extracts both), render a branded 1200x630 image per page
+// from a consumer-authored SVG template, write it into the output tree, and
+// hand the per-page source to the SEO pipeline so og/twitter tags wire to
+// it (missing-only intact: pages with their own og:image, or a per-page
+// ogImage override, keep theirs).
+//
+// Rasterization honesty (the pitch's own 'hard part'): PNG cannot be made
+// in pure Node. The rasterizer is the OPTIONAL npm package
+// @resvg/resvg-js - loaded lazily with createRequire and loud-skipped when
+// it is not installed (the run never breaks and never ships a half
+// feature). Pure-Node output shape: SVG rendering is deterministic (same
+// page meta -> same bytes; the build hash stays content-derived). Fonts:
+// system fonts are unreliable in build sandboxes, so ogImageGeneration.font
+// loads a TTF/OTF buffer straight into the rasterizer.
+//
+// Config shape (configDir-relative paths):
+//   ogImageGeneration:
+//     template: og-template.svg   # 1200x630 SVG with {{ogTitle}} /
+//                                 # {{ogDescription}} / {{ogUrl}}
+//     font: ./brand/brand-bold.ttf  # optional
+//     width: 1200                  # optional; must be 1200x630 ratio-safe
+//     height: 630                  # optional
+
+import fs from 'fs';
+import path from 'path';
+import { createRequire } from 'module';
+import { escXml } from './xmllike.js';
+import { pageUrlFor } from './urls.js';
+
+const _require = createRequire(import.meta.url);
+
+/** Directory-slug for the image filename: '/' and nested paths flatten. */
+function imageSlug(page) {
+  return page.slug.replace(/^\/+|\/+$/g, '').replace(/\//g, '-') || 'index';
+}
+
+/** Lazily resolve the optional rasterizer CLASS; null when not installed. */
+function resolveRasterizer() {
+  try {
+    const mod = _require('@resvg/resvg-js');
+    // CJS shape: { default?: Resvg, Resvg } ; ESM default handled too.
+    const ctor = mod.Resvg || mod.default || mod;
+    return (typeof ctor === 'function') ? ctor : null;
+  } catch (e) {
+    return null;
+  }
+}
+
+/**
+ * Validate + resolve the generation block. Returns null when it is not
+ * configured; every disqualifying shape logs a loud single warning (config
+ * diagnostics already handled key-name mistakes).
+ */
+export function resolveOgImageGeneration(config, configDir) {
+  const gen = config.ogImageGeneration;
+  if (gen === undefined || gen === null || gen === false) return null;
+  if (typeof gen !== 'object' || Array.isArray(gen)) {
+    console.log('[freestruct] config warning: ogImageGeneration must be a { template, font?, width?, height? } object');
+    return null;
+  }
+  if (typeof gen.template !== 'string' || gen.template === '') {
+    console.log('[freestruct] config warning: ogImageGeneration.template missing - no og image generation');
+    return null;
+  }
+  const templatePath = path.resolve(configDir, gen.template);
+  if (!fs.existsSync(templatePath)) {
+    console.log('[freestruct] config warning: ogImageGeneration.template "'
+      + gen.template + '" not found at ' + templatePath + ' - no og image generation');
+    return null;
+  }
+  const width = Number(gen.width ?? 1200);
+  const height = Number(gen.height ?? 630);
+  if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
+    console.log('[freestruct] config warning: ogImageGeneration width/height must be positive numbers - no og image generation');
+    return null;
+  }
+  const fontPath = gen.font ? path.resolve(configDir, gen.font) : null;
+  if (gen.font && (!fontPath || !fs.existsSync(fontPath))) {
+    console.log('[freestruct] config warning: ogImageGeneration.font "'
+      + gen.font + '" not found - the rasterizer will fall back to system fonts (unreliable in build sandboxes)');
+    return null;
+  }
+  const template = fs.readFileSync(templatePath, 'utf8');
+  const fontBytes = fontPath ? fs.readFileSync(fontPath) : null;
+  if (/<text/i.test(template) && !fontBytes) {
+    console.log('[freestruct] ogimage: template carries <text> but no font is configured - add ogImageGeneration.font so build sandboxes without system fonts do not drop titles');
+  }
+  return { template, templatePath, fontPath, fontBytes, width, height };
+}
+
+/**
+ * Render one per-page image and hand back the tag-ready facts.
+ * Returns null when the rasterizer is not installed (loud skip by caller)
+ * or the SVG renders to no bytes.
+ */
+export function renderPageOgImage(generation, page, config, outputDir) {
+  const Resvg = resolveRasterizer();
+  if (!Resvg) {
+    if (!renderPageOgImage._skippedLoudly) {
+      renderPageOgImage._skippedLoudly = true;
+      console.log('[freestruct] ogimage: @resvg/resvg-js is not installed - generation skips loudly'
+        + ' (pure-Node cannot rasterize; install it with: npm install --save-dev @resvg/resvg-js)');
+    }
+    return null;
+  }
+
+  const pageUrl = pageUrlFor(config, page.slug);
+  const displayTitle = page.title || page.rel || 'Preview';
+  let svg = generation.template
+    .split('{{ogTitle}}').join(escXml(displayTitle))
+    .split('{{ogDescription}}').join(escXml(page.description || ''))
+    .split('{{ogUrl}}').join(escXml(pageUrl));
+
+  const fontBuffers = generation.fontBytes ? [generation.fontBytes] : undefined;
+  let png;
+  try {
+    const rendered = new Resvg(svg, {
+      fitTo: { mode: 'width', value: generation.width },
+      font: fontBuffers ? { fontBuffers, loadSystemFonts: false } : { loadSystemFonts: false },
+    });
+    png = rendered.render().asPng();
+  } catch (e) {
+    console.log('[freestruct] ogimage: page ' + page.rel + ' failed to rasterize: ' + e.message);
+    return null;
+  }
+
+  const name = imageSlug(page);
+  const genDir = path.join(outputDir, 'og');
+  if (!fs.existsSync(genDir)) fs.mkdirSync(genDir, { recursive: true });
+  const file = path.join(genDir, name + '.png');
+  fs.writeFileSync(file, png);
+
+  return {
+    file,
+    src: '/og/' + name + '.png',
+    width: generation.width,
+    height: generation.height,
+    alt: displayTitle + ' - page preview image',
+  };
+}

package/engine/scan.js

@@ -13,31 +13,65 @@ export function getHtmlFiles(dir) {
   for (const item of fs.readdirSync(dir, { withFileTypes: true })) {
     const fp = path.join(dir, item.name);
     if (item.isDirectory()) files.push(...getHtmlFiles(fp));
-    else if (item.name.endsWith('.html')) files.push(fp);
+    // Case-insensitive .html/.htm (Pass 26.10): a consumer SSG that emits
+    // .HTM or upper-case extensions is a page like any other - the old
+    // suffix test silently dropped it from every artifact.
+    else if (/\.html?$/i.test(item.name)) files.push(fp);
   }
   return files;
 }
 
-/**
- * Canonical slug for a file relative to the output root.
+/** Canonical slug for a file relative to the output root.
  *   index.html       -> '/'
  *   guide/index.html -> '/guide/'
  *   a&b.html         -> '/a&b'
+ *   legacy.htm       -> '/legacy' (case-insensitive .html/.htm)
  */
 export function slugFor(rel) {
   let p = String(rel).replace(/\\/g, '/');
-  if (p === '/index.html' || p === 'index.html') return '/';
+  if (/^\/?index\.html?$/i.test(p)) return '/';
   p = '/' + p;
   p = p
-    .replace(/\/index\.html$/, '/')
-    .replace(/\.html$/, '');
+    .replace(/\/index\.html?$/i, '/')
+    .replace(/\.html?$/i, '');
   return p;
 }
 
 /** 404.html at root or any directory level is the 404 pipeline's business. */
 export function is404File(rel) {
   const p = String(rel).replace(/\\/g, '/');
-  return p === '404.html' || p.endsWith('/404.html');
+  return /(^|\/)404\.html?$/i.test(p);
+}
+
+/**
+ * Collapse pages that map to the SAME slug (one URL) to a single entry, so a
+ * tree carrying both `a.html` and `a.htm` (or `index.html` and `index.htm` in
+ * one directory) does not list one URL twice in sitemap.xml / feed.xml /
+ * search.json / llms.txt. Deterministic winner: `.html` over `.htm` (the
+ * pretty-URL resolver's own preference), then the lexicographically smaller
+ * rel path. Order is preserved by first appearance.
+ */
+export function dedupeBySlug(pages) {
+  const bySlug = new Map();
+  for (const page of pages) {
+    const current = bySlug.get(page.slug);
+    if (!current) { bySlug.set(page.slug, page); continue; }
+    const better = (a, b) => {
+      const aHtml = /\.html$/i.test(a.rel) ? 0 : 1;
+      const bHtml = /\.html$/i.test(b.rel) ? 0 : 1;
+      if (aHtml !== bHtml) return aHtml < bHtml ? a : b;
+      return a.rel <= b.rel ? a : b;
+    };
+    bySlug.set(page.slug, better(current, page));
+  }
+  const seen = new Set();
+  const out = [];
+  for (const page of pages) {
+    if (seen.has(page.slug)) continue;
+    seen.add(page.slug);
+    out.push(bySlug.get(page.slug));
+  }
+  return out;
 }
 
 /** Robots noindex detection on (possibly injected) HTML. */
@@ -51,43 +85,135 @@ export function stripRobotsMeta(html) {
   return html.replace(/<meta[^>]*name=["']robots["'][^>]*>/i, '');
 }
 
-/** Extract SEO candidates from raw page HTML. (kept from 0.3.x semantics) */
+/**
+ * Decode the HTML entities an SSG routinely writes into <title>/meta content
+ * (&amp; &lt; &gt; &quot; &#39;). Without this, an entity-encoded source fact
+ * flowed verbatim into the generated artifacts: the feed re-escaped the
+ * already-escaped '&amp;' into '&amp;amp;' (double-encoded text a reader
+ * sees as '&amp;'), and search.json/llms.txt carried raw entities into
+ * plain-text/JSON contexts that render them literally.
+ */
+// Common named entities (the ones that realistically appear in titles and
+// descriptions) plus numeric forms. A title carrying '&#8217;' or '&mdash;'
+// was left literal by the old five-entry map, so the feed double-escaped it
+// ('&amp;#8217;', which renders as the literal entity text) and the length
+// graders counted the markup instead of the character.
+const NAMED_ENTITIES = {
+  lt: '<', gt: '>', quot: '"', apos: "'", amp: '&',
+  nbsp: '\u00a0', mdash: '\u2014', ndash: '\u2013', hellip: '\u2026',
+  lsquo: '\u2018', rsquo: '\u2019', ldquo: '\u201c', rdquo: '\u201d',
+  copy: '\u00a9', reg: '\u00ae', trade: '\u2122', deg: '\u00b0',
+  times: '\u00d7', laquo: '\u00ab', raquo: '\u00bb', middot: '\u00b7',
+  bull: '\u2022', euro: '\u20ac', pound: '\u00a3', yen: '\u00a5',
+  cent: '\u00a2', sect: '\u00a7', para: '\u00b6', dagger: '\u2020',
+  plusmn: '\u00b1', frac12: '\u00bd', frac14: '\u00bc', frac34: '\u00be',
+};
+
+export function decodeEntities(value) {
+  return String(value == null ? '' : value)
+    // Numeric forms first: decimal '&#8217;' and hex '&#x2019;'. Doing these
+    // before the named pass keeps single-pass semantics ('&amp;#65;' stays
+    // the literal '&#65;' after the amp pass, never '<').
+    .replace(/&#x([0-9a-f]+);/gi, (m, hex) => {
+      const cp = parseInt(hex, 16);
+      return cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : m;
+    })
+    .replace(/&#(\d+);/g, (m, dec) => {
+      const cp = parseInt(dec, 10);
+      return cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : m;
+    })
+    .replace(/&([a-z][a-z0-9]*);/gi, (m, name) => {
+      const key = name.toLowerCase();
+      return Object.prototype.hasOwnProperty.call(NAMED_ENTITIES, key) ? NAMED_ENTITIES[key] : m;
+    });
+}
+
+// Quoted-attribute reader (same shape audit.js owns): the closing quote must
+// MATCH the opening one, so a value holding the other quote kind (an
+// apostrophe in a description) is read whole instead of truncated at it.
+function attrValue(tag, attr) {
+  const quoted = tag.match(new RegExp(`\\b${attr}=(["'])([\\s\\S]*?)\\1`, 'i'));
+  if (quoted) return quoted[2];
+  const bare = tag.match(new RegExp(`\\b${attr}=([^\\s>]+)`, 'i'));
+  return bare ? bare[1] : null;
+}
+
+/**
+ * Extract SEO candidates from raw page HTML. Attribute-ORDER tolerant and
+ * quote-agnostic (Pass 26.21): the old fixed `name="title" content="..."`
+ * shape missed a reordered or single-quoted meta, so the page silently fell
+ * back to the site-level value. Values are entity-decoded so downstream
+ * artifacts (feed/search/llms) receive plain text, not raw entities.
+ */
 export function extractTitle(html) {
-  const m = html.match(/<meta name="title" content="([^"]+)"/i);
-  if (m) return m[1];
-  const t = html.match(/<title>([^<]+)<\/title>/i);
-  return t ? t[1].split(' | ')[0] : null;
+  const meta = html.match(/<meta\b[^>]*\bname=["']title["'][^>]*>/i);
+  if (meta) {
+    const v = attrValue(meta[0], 'content');
+    if (v) return decodeEntities(v);
+  }
+  const t = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+  return t ? decodeEntities(t[1]).split(' | ')[0] : null;
 }
 
 export function extractDescription(html) {
-  const m = html.match(/<meta name="description" content="([^"]+)"/i);
-  return m ? m[1] : null;
+  const meta = html.match(/<meta\b[^>]*\bname=["']description["'][^>]*>/i);
+  if (!meta) return null;
+  const v = attrValue(meta[0], 'content');
+  return v ? decodeEntities(v) : null;
 }
 
 /**
  * Does the page already import SEO of the given type from SSG/frontmatter?
  * Used by the missing-only injection gate ('description' added in Phase 1: a
  * plain <meta name="description"> counts as covered).
+ *
+ * Every probe is TAG-ANCHORED (Pass 26.12): substring matches fire on the
+ * page's own PROSE too - a docs page that quotes 'application/ld+json' in
+ * a code span silenced its own structured-data injection, exactly the
+ * configuration-reference page. Markup quotes markup: anchors, never
+ * discussion.
  */
 export function hasSeoType(html, type) {
   if (type === 'og') {
-    return html.includes('property="og:title"') || html.includes('property="og:description"');
+    return /<meta[^>]+property=["']og:(title|description)["']/i.test(html);
   }
   if (type === 'twitter') {
-    return html.includes('name="twitter:card"') || html.includes('name="twitter:title"');
+    return /<meta[^>]+name=["']twitter:(card|title)["']/i.test(html);
   }
   if (type === 'jsonld') {
-    return html.includes('application/ld+json');
+    return /<script[^>]+type=["']application\/ld\+json["']/i.test(html);
   }
   if (type === 'canonical') {
-    return html.includes('rel="canonical"');
+    return /<link[^>]+rel=["']canonical["']/i.test(html);
   }
   if (type === 'description') {
-    return /<meta[^>]*name=["']description["']/.test(html);
+    // Case-insensitive like every branch above: an SSG/author emitting an
+    // upper- or mixed-case meta (<META NAME="description" ...>, valid HTML)
+    // missed this gate, so freestruct injected a SECOND description and its
+    // own audit then failed the page for the duplicate it caused.
+    return /<meta[^>]*name=["']description["']/i.test(html);
   }
   return false;
 }
 
+/**
+ * Visible text of an HTML fragment: script/style bodies and comments drop
+ * first, then every tag, whitespace collapses, and entities decode. Shared by
+ * the search index and the reading-time estimator - both count page prose, and
+ * both were inflated by raw JS/CSS when each stripped tags alone.
+ */
+export function htmlToText(html) {
+  return decodeEntities(
+    String(html)
+      .replace(/<script[\s\S]*?<\/script\s*>/gi, ' ')
+      .replace(/<style[\s\S]*?<\/style\s*>/gi, ' ')
+      .replace(/<!--[\s\S]*?-->/g, ' ')
+      .replace(/<[^>]+>/g, ' ')
+      .replace(/\s+/g, ' ')
+      .trim(),
+  );
+}
+
 /**
  * Snapshot every page once, before freestruct writes anything.
  * mtime is captured HERE so sitemap lastmod / RSS pubDate reflect the last
@@ -141,7 +267,16 @@ export function computeBuildHash(config, pages) {
   const stable = (html) => html
     .replace(/<meta[^>]*name="(freestruct-build|last-modified|reading-time|robots)"[^>]*>/gi, '')
     .replace(/<!-- freestruct -->[\s\S]*?(?=<\/head>)/i, '')
-    .replace(/\?v=[0-9a-f]{8}/g, '')
+    // Body-start/footer snippets (marker-wrapped by seo.js) are the tool's own
+    // output; strip them too or a rerun's body text differs from run 1 and the
+    // content-derived hash churns every pass.
+    .replace(/<!-- freestruct:snippet -->[\s\S]*?<!-- \/freestruct:snippet -->/gi, '')
+    // The tableWrap pass (seo.js addTableWrap) wraps each outermost table in a
+    // tool-owned div AFTER the hash is computed, so a rerun's page content
+    // carries the wrapper the first run did not - without this strip the
+    // content-derived hash churned on every pass for any site with a table.
+    .replace(/<div class="fs-table-wrap"[^>]*>\s*(<table[\s\S]*?<\/table>)\s*<\/div>/gi, '$1')
+    .replace(/[?&]v=[0-9a-f]{8}/g, '')
     .replace(/ loading="lazy"/g, '');
   for (const page of pages.filter(p => !p.is404).sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0))) {
     h.update(page.rel);

package/engine/search.js

@@ -8,6 +8,7 @@
 import fs from 'fs';
 import path from 'path';
 import { pageUrlFor } from './urls.js';
+import { extractTitle, extractDescription, htmlToText } from './scan.js';
 
 export function generateSearchIndex(pages, config, outputDir) {
   const index = [];
@@ -19,18 +20,20 @@ export function generateSearchIndex(pages, config, outputDir) {
     // would search; raw file content at this point is the injected page).
     const html = fs.readFileSync(page.file, 'utf8');
 
-    const titleMatch = html.match(/<title>([^<]+)<\/title>/i);
-    const descMatch = html.match(/<meta[^>]*name="description"[^>]*content="([^"]+)"/i);
-    const h1Match = html.match(/<h1[^>]*>([^<]+)<\/h1>/i);
+    // Entity-decoded, tag-free extraction (Pass 26.21 / 26.24): search.json is
+    // JSON consumed by JS, where '&amp;' renders literally as '&amp;' and a
+    // stray '<em>' renders as nothing. htmlToText drops script/style bodies
+    // and comments, strips every tag, and decodes entities, so the index
+    // carries plain page text (the old tag-only strip shipped raw JS/CSS, and
+    // the heading pass returned the h1's inner HTML with its tags).
+    const h1Match = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
 
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*)<\/body>/i);
-    const content = bodyMatch
-      ? bodyMatch[1].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 500)
-      : '';
+    const content = bodyMatch ? htmlToText(bodyMatch[1]).slice(0, 500) : '';
 
-    const title = titleMatch ? titleMatch[1].split(' | ')[0] : path.basename(page.file, '.html');
-    const description = descMatch ? descMatch[1] : '';
-    const heading = h1Match ? h1Match[1] : title;
+    const title = extractTitle(html) || path.basename(page.file).replace(/\.html?$/i, '');
+    const description = extractDescription(html) || '';
+    const heading = h1Match ? htmlToText(h1Match[1]) : title;
     const url = pageUrlFor(config, page.slug);
 
     index.push({ title, heading, description, content, url });