freestruct 1.1.0 → 1.2.0

package/engine/audit.js

@@ -5,7 +5,10 @@
 // and remapped onto freestruct's own contract:
 //   - meta extraction is attribute-order tolerant (per the donor's notes)
 //   - og:image / twitter:image / twitter:site/creator are config-gated in
-//     this engine (missing-only), so their absence is a WARN, not a FAIL
+//     this engine (missing-only), so their absence is a WARN, not a FAIL.
+//     The og:image / twitter:image absences are CONFIG-shaped (identical on
+//     every page), so they collapse to one line each with a page count
+//     (issue #28); twitter:site/creator are graded only when a handle is set.
 //   - the FAIL og/twitter class is the SHARED core (og:title/og:description,
 //     twitter:card/twitter:title); the rest of the injected set rides WARN
 //     because missing-only means SSG-tagged pages legitimately keep a
@@ -21,6 +24,7 @@
 
 import { readFileSync, existsSync, readdirSync, statSync } from 'fs';
 import path from 'path';
+import { decodeEntities } from './scan.js';
 
 // FAIL class = the tags core to the engine's own contract. The REST of the
 // injected set (og:url/type/site_name/locale, twitter:description) rides as
@@ -37,19 +41,34 @@ const LB_RECOMMENDED = ['name', 'url', 'image', 'logo', 'telephone', 'priceRange
 
 /** Attribute-order tolerant meta extraction: hits aligned multi-space output. */
 function grabMeta(head, attrHasKey, name) {
-  const keyRe = new RegExp(`<meta[^>]${attrHasKey}=["']${name}["'][^>]*>`, 'i');
+  // `<meta\b[^>]*` (not `<meta[^>]`) so a leading attribute - `<meta charset=...
+  // name="description">` - is found too. The single-char class made the audit
+  // false-FAIL "missing meta description" on any page whose description meta
+  // carried another attribute first.
+  const keyRe = new RegExp(`<meta\\b[^>]*\\b${attrHasKey}=["']${name}["'][^>]*>`, 'i');
   const tag = head.match(keyRe);
   if (!tag) return null;
-  const content = tag[0].match(/content=["']([^"']*)["']/i);
-  return content ? content[1] : null;
+  const content = attrValue(tag[0], 'content');
+  return content;
+}
+
+// Quoted-attribute reader. The naive `["']([^"']*)["']` stopped at the FIRST
+// quote of either kind, so any value holding the other quote (a description
+// with an apostrophe - "this repo's docs") was truncated at the apostrophe.
+// The backreference forces the closing quote to match the opening one, so the
+// value survives intact. Unquoted values fall through to a bare read.
+function attrValue(tag, attr) {
+  const quoted = tag.match(new RegExp(`\\b${attr}=(["'])([\\s\\S]*?)\\1`, 'i'));
+  if (quoted) return quoted[2];
+  const bare = tag.match(new RegExp(`\\b${attr}=([^\\s>]+)`, 'i'));
+  return bare ? bare[1] : null;
 }
 
 function grabCanonical(head) {
   const link = head.match(/<link[^>]+rel=["']canonical["'][^>]*>/i)
-    || head.match(/<link[^>]+href=["']([^"']+)["'][^>]+rel=["']canonical["'][^>]*>/i);
+    || head.match(/<link[^>]+href=["'][^"']+["'][^>]+rel=["']canonical["'][^>]*>/i);
   if (!link) return null;
-  const href = link[0].match(/href=["']([^"']+)["']/i);
-  return href ? href[1] : null;
+  return attrValue(link[0], 'href');
 }
 
 /** Count every occurrence of a head shape - the duplicate-class probe. */
@@ -68,11 +87,28 @@ function walkHtml(dir) {
 }
 
 function stripTags(s) {
-  return String(s).replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').trim();
+  return String(s).replace(/<[^>]+>/g, '').trim();
+}
+
+// Title/description length is graded against what a SEARCH ENGINE sees: the
+// DECODED text, not the raw markup. An entity-heavy title ('&amp;', '&#39;',
+// '&#65;') inflated the raw length and tripped the > 65 / > 165 warnings on
+// text that renders well inside the band.
+function decodedLength(s) {
+  return decodeEntities(stripTags(s)).length;
 }
 
-/** Head-referenced URL -> on-disk path under the output dir, or null. */
-function urlToDisk(urlValue, { outputDir, config }) {
+/**
+ * Head-referenced site-local URL -> { resolved, inside } | null.
+ *
+ * null means "not a site-local path" (external host, protocol-relative,
+ * relative, empty) - nothing on disk to check. `inside` is false when the
+ * path resolves OUTSIDE the output tree: a traversal shape like
+ * '/../../etc/hostname', which must never be probed as if it were an
+ * artifact. (Before Pass 26.21 urlToDisk resolved and existsSync'd such a
+ * path, so a value pointing at any file on the host read back as "exists".)
+ */
+function sitePathInfo(urlValue, { outputDir, config }) {
   if (!urlValue) return null;
   let pathName = urlValue.split(/[?#]/)[0];
   if (/^https?:\/\//i.test(pathName)) {
@@ -86,13 +122,20 @@ function urlToDisk(urlValue, { outputDir, config }) {
   } else if (/^\/\//.test(pathName)) {
     return null;
   }
-  if (pathName.startsWith('/')) {
-    if (config?.basePath && pathName.startsWith(config.basePath)) {
-      pathName = pathName.slice(config.basePath.length);
-    }
-    return path.join(outputDir, pathName.replace(/^\//, ''));
+  if (!pathName.startsWith('/')) return null;
+  // basePath normalization mirrors urls.js (issue #32 contract): the
+  // configured prefix may lack its leading slash, and a prefix must match a
+  // whole path segment ('/myproject' never strips '/myprojectx/...').
+  const bp = config?.basePath
+    ? ('/' + String(config.basePath).replace(/^\/+/, '')).replace(/\/+$/, '')
+    : '';
+  if (bp && (pathName === bp || pathName.startsWith(bp + '/'))) {
+    pathName = pathName.slice(bp.length);
   }
-  return null;
+  const resolved = path.resolve(outputDir, pathName.replace(/^\//, ''));
+  const root = path.resolve(outputDir);
+  const inside = resolved === root || resolved.startsWith(root + path.sep);
+  return { resolved, inside };
 }
 
 // ---------- the audit ----------
@@ -110,6 +153,19 @@ export function auditOutput({ outputDir, config, json = false, capture = false }
   const fail = (page, msg) => fails.push({ page, msg });
   const warn = (page, msg) => warns.push({ page, msg });
 
+  // Config-gated absences (issue #28): a condition that can only ever be
+  // true/false at the CONFIG level repeats identically on every page
+  // (og:image / twitter:image absent because no og.image is configured).
+  // Collect affected pages here and flush ONE line each with a count at the
+  // end, instead of N page-suffixed rows that bury the real findings.
+  // Page-shaped classes (title/description length, JSON-LD absence, h1
+  // sanity) still repeat - their findings genuinely differ per page.
+  const configGated = new Map(); // key -> Set(page labels)
+  const configGate = (key, label) => {
+    if (!configGated.has(key)) configGated.set(key, new Set());
+    configGated.get(key).add(label);
+  };
+
   // File-at-path hardening (Pass 26.13): a file at the outputDir path passes
   // existsSync, and walkHtml's readdirSync would arrive here uncaught.
   if (!existsSync(outputDir) || !statSync(outputDir).isDirectory()) {
@@ -137,15 +193,24 @@ export function auditOutput({ outputDir, config, json = false, capture = false }
     const titleMatch = head.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
     if (!titleMatch || !titleMatch[1].trim()) fail(label, 'missing <title>');
     else {
-      const t = stripTags(titleMatch[1]).length;
+      const t = decodedLength(titleMatch[1]);
       if (t > 65) warn(label, 'title ' + t + ' chars (> 65, may truncate)');
       if (t < 10) warn(label, 'title ' + t + ' chars (suspiciously short)');
     }
 
+    // A page that declares `robots noindex` is deliberately out of the index
+    // (a redirect stub, a thank-you page): the content-quality classes below
+    // - description length band, h1 shape, JSON-LD absence - grade indexable
+    // content, so they would only emit cosmetic noise on a page whose whole
+    // point is to not rank. The structural classes (title/canonical presence,
+    // duplicate tags, absolute canonical) still apply.
+    const noindex = /noindex/i.test(grabMeta(head, 'name', 'robots') || '');
+
     const desc = grabMeta(head, 'name', 'description');
     if (!desc) fail(label, 'missing meta description');
-    else if (desc.length < 70 || desc.length > 165) {
-      warn(label, 'description ' + desc.length + ' chars (outside 70-165, may truncate)');
+    else if (!noindex) {
+      const d = decodeEntities(desc).length;
+      if (d < 70 || d > 165) warn(label, 'description ' + d + ' chars (outside 70-165, may truncate)');
     }
 
     const canonical = grabCanonical(head);
@@ -161,12 +226,26 @@ export function auditOutput({ outputDir, config, json = false, capture = false }
       ['duplicate meta description', /<meta[^>]+name=["']description["']/gi],
       ['multiple <title> elements', /<title[^>]*>/gi],
       ['duplicate og:title', /<meta[^>]+property=["']og:title["']/gi],
+      ['duplicate og:description', /<meta[^>]+property=["']og:description["']/gi],
+      ['duplicate twitter:title', /<meta[^>]+name=["']twitter:title["']/gi],
+      ['duplicate twitter:card', /<meta[^>]+name=["']twitter:card["']/gi],
     ]) {
       if (countHead(head, dup[1]) > 1) fail(label, dup[0]);
     }
 
+    // og/twitter CORE class. FAIL only when the page carries NO tag of the
+    // set at all: that is the "the engine failed to inject on a bare page"
+    // case the class exists to catch. A page shipping a PARTIAL set of its
+    // own (an SSG theme emitting only og:title, say) makes the engine's
+    // group-level missing-only gate back off the whole group BY DESIGN, so
+    // its missing companions are report-grade - failing them punished the
+    // very contract (SSG-owned partial sets) the injection rule preserves,
+    // and the audit cannot tell an SSG-owned partial head from an engine one.
+    const ogAny = countHead(head, /<meta[^>]+property=["']og:[^"']*["']/gi);
     for (const prop of OG_CORE) {
-      if (!grabMeta(head, 'property', prop)) fail(label, 'missing ' + prop);
+      if (grabMeta(head, 'property', prop)) continue;
+      if (ogAny === 0) fail(label, 'missing ' + prop);
+      else warn(label, prop + ' absent (page ships a partial og set; SSG-owned partial sets keep contract)');
     }
     for (const prop of OG_SET) {
       if (!grabMeta(head, 'property', prop)) {
@@ -174,19 +253,26 @@ export function auditOutput({ outputDir, config, json = false, capture = false }
       }
     }
     if (!grabMeta(head, 'property', 'og:image')) {
-      warn(label, 'og:image absent (no og.image configured? add one for share previews)');
+      configGate('og:image', label);
     }
+    const twAny = countHead(head, /<meta[^>]+name=["']twitter:[^"']*["']/gi);
     for (const name of TW_CORE) {
-      if (!grabMeta(head, 'name', name)) fail(label, 'missing ' + name);
+      if (grabMeta(head, 'name', name)) continue;
+      if (twAny === 0) fail(label, 'missing ' + name);
+      else warn(label, name + ' absent (page ships a partial twitter set; SSG-owned partial sets keep contract)');
     }
     for (const name of TW_SET) {
       // Symmetry with the engine's own late pass (seo.js): without a
       // configured handle the injector ships neither twitter:site nor
       // twitter:creator, so the audit grades those absences only when the
       // handle exists. The 'config-gated' wording in the warning now holds.
-      if ((name === 'twitter:site' || name === 'twitter:creator') && !config.twitter?.username) continue;
+      if ((name === 'twitter:site' || name === 'twitter:creator') && !config?.twitter?.username) continue;
       if (!grabMeta(head, 'name', name)) {
-        warn(label, name + ' absent (config-gated or SSG-owned partial set; add a twitter username / og.image as needed)');
+        // twitter:image is config-gated (no og.image -> the engine ships no
+        // twitter:image either): collapse it like og:image. The rest of the
+        // twitter set is SSG-owned-partial-set shaped and stays per-page.
+        if (name === 'twitter:image') configGate('twitter:image', label);
+        else warn(label, name + ' absent (config-gated or SSG-owned partial set; add a twitter username / og.image as needed)');
       }
     }
 
@@ -197,16 +283,23 @@ export function auditOutput({ outputDir, config, json = false, capture = false }
     if (ogImage && !grabMeta(head, 'property', 'og:image:width')) {
       warn(label, 'og:image:width/height/alt absent (the og.image object form supplies them)');
     }
-    if (ogImage && urlToDisk(ogImage, { outputDir, config }) && !existsSync(urlToDisk(ogImage, { outputDir, config }))) {
-      // Report-grade on purpose (issue #18 sketch: 'worth having'): an og
-      // image can legitimately be emitted on an SSG-side pass or a same-host
-      // CDN, so a hard FAIL would punish valid setups; the JSON digest
-      // carries it for teams who want to harden it in CI.
-      warn(label, 'og:image points at ' + ogImage + ' which does not exist on disk');
+    if (ogImage) {
+      const info = sitePathInfo(ogImage, { outputDir, config });
+      if (info && !info.inside) {
+        // A site-local og:image that escapes the output tree can never be a
+        // served artifact; report it (same report-grade posture as missing).
+        warn(label, 'og:image points at ' + ogImage + ' which escapes the output directory');
+      } else if (info && !existsSync(info.resolved)) {
+        // Report-grade on purpose (issue #18 sketch: 'worth having'): an og
+        // image can legitimately be emitted on an SSG-side pass or a same-host
+        // CDN, so a hard FAIL would punish valid setups; the JSON digest
+        // carries it for teams who want to harden it in CI.
+        warn(label, 'og:image points at ' + ogImage + ' which does not exist on disk');
+      }
     }
 
     const ldBlocks = [...html.matchAll(/<script type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)];
-    if (!ldBlocks.length) warn(label, 'no JSON-LD structured data');
+    if (!ldBlocks.length && !noindex) warn(label, 'no JSON-LD structured data');
     for (const block of ldBlocks) {
       let parsed;
       try { parsed = JSON.parse(block[1]); } catch (e) {
@@ -225,8 +318,10 @@ export function auditOutput({ outputDir, config, json = false, capture = false }
         // schema-carried logo/image files: same exists-on-disk check as og:image
         for (const key of ['logo', 'image']) {
           const v = node[key];
-          const target = urlToDisk(typeof v === 'string' ? v : null, { outputDir, config });
-          if (target && !existsSync(target)) {
+          const info = sitePathInfo(typeof v === 'string' ? v : null, { outputDir, config });
+          if (info && !info.inside) {
+            warn(label, 'JSON-LD ' + key + ' points at ' + v + ' which escapes the output directory');
+          } else if (info && !existsSync(info.resolved)) {
             warn(label, 'JSON-LD ' + key + ' points at ' + v + ' which does not exist on disk');
           }
         }
@@ -235,15 +330,23 @@ export function auditOutput({ outputDir, config, json = false, capture = false }
 
     const h1s = [...html.matchAll(/<h1[^>]*>([\s\S]*?)<\/h1>/gi)];
     const h1Texts = h1s.map(m => stripTags(m[1])).filter(Boolean);
-    if (!h1Texts.length) warn(label, 'no non-empty <h1>');
+    if (!h1Texts.length && !noindex) warn(label, 'no non-empty <h1>');
     else if (h1Texts.length > 1) warn(label, h1Texts.length + ' <h1> elements (prefer one)');
   }
 
   // ---------- site-level artifacts ----------
 
+  // Honor the config's own generate switches: with `generateSitemap: false`
+  // (or robots), the run never writes the artifact, so its absence is the
+  // documented behavior, not a failure. The audit previously FAILed the
+  // missing file the config explicitly asked not to produce, so a
+  // legitimately-configured site could never pass its own audit. Configless
+  // audits (config === null) keep the strict check - absence is then a real
+  // gap. A DISABLED artifact that is present is still checked for shape.
+  const sitemapDisabled = config?.generateSitemap === false;
   const sitemapPath = path.join(outputDir, 'sitemap.xml');
   if (!existsSync(sitemapPath)) {
-    fail('sitemap.xml', 'missing');
+    if (!sitemapDisabled) fail('sitemap.xml', 'missing');
   } else {
     const s = readFileSync(sitemapPath, 'utf8');
     sitemapLocs = (s.match(/<loc>/g) || []).length;
@@ -251,9 +354,10 @@ export function auditOutput({ outputDir, config, json = false, capture = false }
     if (!s.includes('<lastmod>')) warn('sitemap.xml', 'no <lastmod> dates (engine runs always add them; hand-stripped?)');
   }
 
+  const robotsDisabled = config?.generateRobots === false;
   const robotsPath = path.join(outputDir, 'robots.txt');
   if (!existsSync(robotsPath)) {
-    fail('robots.txt', 'missing');
+    if (!robotsDisabled) fail('robots.txt', 'missing');
   } else {
     const r = readFileSync(robotsPath, 'utf8');
     if (!/Sitemap:/i.test(r)) fail('robots.txt', 'no Sitemap: pointer');
@@ -261,6 +365,34 @@ export function auditOutput({ outputDir, config, json = false, capture = false }
 
   // ---------- report ----------
 
+  // Flush config-gated absences as ONE line each (issue #28), with the
+  // affected-page count. Deterministic order (og:image before twitter:image).
+  // The og:image wording stays truthful when the config DOES supply an image
+  // but not for every page (og.scoped, ogImageGeneration): the line then
+  // reads "absent on N page(s)" rather than the false "not configured".
+  const ogScopedCfg = config?.og?.scoped;
+  const ogImageConfigured = Boolean(config?.og?.image)
+    || Boolean(config?.ogImageGeneration)
+    || Boolean(ogScopedCfg && typeof ogScopedCfg === 'object' && !Array.isArray(ogScopedCfg)
+      && Object.keys(ogScopedCfg).length > 0);
+  for (const [key, pages] of [...configGated.entries()].sort()) {
+    const n = pages.size;
+    let msg;
+    if (key === 'og:image') {
+      msg = ogImageConfigured
+        ? 'og:image absent on ' + n + ' page(s) - no og.image covers them (add one, or extend og.scoped)'
+        : 'og:image not configured - ' + n + ' page(s) affected (add one for share previews)';
+    } else {
+      // twitter:image ships alongside og:image (the injector derives it), so
+      // its absence tracks the same config state: "not configured" only when
+      // NO og image source exists at all, else "absent on N page(s)".
+      msg = ogImageConfigured
+        ? key + ' absent on ' + n + ' page(s) - no og.image covers them (add one, or extend og.scoped)'
+        : key + ' not configured - ' + n + ' page(s) affected (add a twitter username / og.image as needed)';
+    }
+    warn('config', msg);
+  }
+
   if (capture) {
     return { fails, warns, pages: pages.length, sitemapLocs };
   }

package/engine/config.js

@@ -20,6 +20,7 @@ export const KNOWN_CONFIG_KEYS = Object.freeze([
   'site', 'twitter', 'og', 'author', 'jsonLd', 'basePath', 'outputDir', 'minify', 'runHooks',
   'generateSitemap', 'generateFeed', 'generate404', 'generateRobots',
   'readingTime', 'lastModified', 'lazyLoad', 'linkCheck', 'searchIndex',
+  'tableWrap',
   'cacheBusting', 'hooks', 'indexNow', 'ogImageGeneration',
   'search', 'breadcrumb', 'verification', 'llms', 'locales', 'article',
 ]);

package/engine/hooks.js

@@ -42,14 +42,20 @@ export function runPurgeHooks(config, buildHash, outputDir) {
 
     command = substitutePlaceholders(command, { siteUrl, buildHash, outputDir });
 
+    if (!config.runHooks) {
+      // Report EVERY configured hook (not just the first): the old `return`
+      // sat inside the loop, so a multi-hook config printed one line and
+      // silently hid the rest, leaving the operator unsure what a later
+      // runHooks: true would actually execute.
+      console.log('Purge hook ' + name + ': running...');
+      console.log('  Command: ' + command.substring(0, 100) + '...');
+      console.log('  Purge hooks disabled (set runHooks: true to enable)');
+      continue;
+    }
+
     console.log('Purge hook ' + name + ': running...');
     try {
-      if (!config.runHooks) {
-        console.log('  Command: ' + command.substring(0, 100) + '...');
-        console.log('  Purge hooks disabled (set runHooks: true to enable)');
-        return;
-      }
-      execSync(command, { stdio: 'inherit', timeout: 30000 });
+      execSync(command, { stdio: 'inherit', timeout: 30000, env: hookEnv({ siteUrl, buildHash, outputDir }) });
       console.log('  Command: ' + command.substring(0, 100) + '...');
       console.log('Purge hook ' + name + ': done');
     } catch (e) {
@@ -85,9 +91,11 @@ export function runPostBuildHooks(config, buildHash, outputDir) {
     const command = substitutePlaceholders(entry, { siteUrl, buildHash, outputDir });
 
     if (!config.runHooks) {
+      // Report EVERY configured hook (mirror of the purge loop fix): the old
+      // `return` inside the loop hid every entry after the first.
       console.log('PostBuild hook: Command: ' + command.substring(0, 100) + '...');
       console.log('PostBuild hooks disabled (set runHooks: true to enable)');
-      return [];
+      continue;
     }
 
     console.log('PostBuild hook: running ' + command.substring(0, 76) + '...');

package/engine/indexnow.js

@@ -11,6 +11,8 @@
 import fs from 'fs';
 import crypto from 'crypto';
 import path from 'path';
+import { decodeEntities } from './scan.js';
+import { pageUrlFor } from './urls.js';
 
 export const INDEXNOW_API = 'https://api.indexnow.org/indexnow';
 const KEYFILE_HIDDEN = ['indexnow-key.txt'];
@@ -69,8 +71,11 @@ export function readSitemapUrls(outputDir) {
   const sitemapPath = path.join(outputDir, 'sitemap.xml');
   if (!fs.existsSync(sitemapPath)) return null;
   const sitemap = fs.readFileSync(sitemapPath, 'utf8');
+  // The sitemap's <loc> values are XML-escaped by the writer (& -> &amp;), so
+  // they must decode back to the real URL before submission - posting the
+  // escaped form asks the engines to crawl a URL that does not exist.
   const urls = (sitemap.match(/<loc>([\s\S]*?)<\/loc>/g) || [])
-    .map(e => e.replace(/<\/?loc>/g, '').trim())
+    .map(e => decodeEntities(e.replace(/<\/?loc>/g, '').trim()))
     .filter(Boolean);
   return urls.length > 0 ? urls : null;
 }
@@ -107,7 +112,11 @@ export async function submitIndexNow({ config, configDir, outputDir, dryRun }) {
     host,
     key: resolved.key,
     urlList: urls,
-    keyLocation: config.site.url.replace(/\/$/, '') + '/' + resolved.key + '.txt',
+    // The key file is emitted into the OUTPUT ROOT, which a sub-path install
+    // serves under basePath (same contract as sitemap.xml/robots.txt). A raw
+    // site.url + '/' + key.txt pointed at the domain root, so the engines
+    // probed a URL that 404s and refused the submission (the #3 bug class).
+    keyLocation: pageUrlFor(config, resolved.key + '.txt'),
   };
 
   console.log('[freestruct] indexnow: submitting ' + urls.length + ' URL(s) to ' + INDEXNOW_API