@koehler8/cms 1.0.0-beta.9 → 1.1.0

package/scripts/image-variants/classifier.js

@@ -0,0 +1,53 @@
+/**
+ * Classify a file under site/assets/img/** as one of:
+ *   'original' — a hand-authored source image we should generate variants from
+ *   'variant'  — a previously-generated variant matching {name}-{width}.{format}
+ *                where width is one of the configured widths
+ *   'ignore'   — anything else (non-image file, README, etc.)
+ *
+ * The classifier is the bridge between the flat-dir convention and the old
+ * _source/ convention: by recognizing the variant naming pattern, we can keep
+ * originals and variants in the same directory without confusion. Files that
+ * happen to look variant-shaped but use widths outside the configured set
+ * (e.g. `team-2024.jpg`) are treated as originals — the documented escape
+ * hatch is to rename source files that genuinely conflict.
+ */
+
+import path from 'node:path';
+
+export const SOURCE_EXTENSIONS = new Set([
+  '.jpg', '.jpeg', '.png', '.webp', '.avif', '.tiff', '.tif',
+]);
+
+export const FORMAT_OUTPUT_EXT = {
+  avif: 'avif',
+  webp: 'webp',
+  jpg: 'jpg',
+  jpeg: 'jpg',
+  png: 'png',
+};
+
+const VARIANT_FORMAT_EXTS = new Set(['avif', 'webp', 'jpg', 'jpeg']);
+
+/**
+ * @param {string} relPath - File path relative to site/assets/img/
+ * @param {Set<number>} configWidths - Set of widths considered "variant widths"
+ * @returns {'original' | 'variant' | 'ignore'}
+ */
+export function classifyImageFile(relPath, configWidths) {
+  const parsed = path.parse(relPath);
+  const ext = parsed.ext.toLowerCase().slice(1);
+
+  if (ext === 'svg') return 'original';
+  if (!ext) return 'ignore';
+  if (!SOURCE_EXTENSIONS.has(`.${ext}`)) return 'ignore';
+
+  const m = parsed.name.match(/^(.+)-(\d+)$/);
+  if (!m) return 'original';
+  if (!VARIANT_FORMAT_EXTS.has(ext)) return 'original';
+
+  const width = Number.parseInt(m[2], 10);
+  if (!configWidths.has(width)) return 'original';
+
+  return 'variant';
+}

package/scripts/image-variants/config.js

@@ -0,0 +1,54 @@
+/**
+ * Image-variant pipeline configuration.
+ *
+ * Resolves the `imageVariants` block from a site's `site.json` into a
+ * normalized config object: { widths: number[], formats: string[],
+ * quality: { avif, webp, jpg } }. Falls back to sensible marketing-site
+ * defaults when fields are missing or invalid.
+ */
+
+import { FORMAT_OUTPUT_EXT } from './classifier.js';
+
+export const DEFAULT_WIDTHS = [320, 640, 960, 1280, 1920, 2560];
+export const DEFAULT_FORMATS = ['avif', 'webp', 'jpg'];
+export const DEFAULT_QUALITY = { avif: 60, webp: 75, jpg: 80 };
+
+function normalizeWidths(input) {
+  if (!Array.isArray(input)) return [...DEFAULT_WIDTHS];
+  const filtered = input
+    .map((v) => Number.parseInt(v, 10))
+    .filter((v) => Number.isFinite(v) && v > 0);
+  if (!filtered.length) return [...DEFAULT_WIDTHS];
+  return [...new Set(filtered)].sort((a, b) => a - b);
+}
+
+function normalizeFormats(input) {
+  if (!Array.isArray(input)) return [...DEFAULT_FORMATS];
+  const filtered = input
+    .map((v) => (typeof v === 'string' ? v.trim().toLowerCase() : ''))
+    .filter((v) => v && FORMAT_OUTPUT_EXT[v]);
+  if (!filtered.length) return [...DEFAULT_FORMATS];
+  return [...new Set(filtered)];
+}
+
+function normalizeQuality(input) {
+  const merged = { ...DEFAULT_QUALITY };
+  if (input && typeof input === 'object') {
+    for (const [k, v] of Object.entries(input)) {
+      const n = Number.parseInt(v, 10);
+      if (Number.isFinite(n) && n >= 1 && n <= 100) merged[k] = n;
+    }
+  }
+  return merged;
+}
+
+export function resolveImageVariantConfig(siteConfig) {
+  const block = (siteConfig?.imageVariants && typeof siteConfig.imageVariants === 'object')
+    ? siteConfig.imageVariants
+    : {};
+  return {
+    widths: normalizeWidths(block.widths),
+    formats: normalizeFormats(block.formats),
+    quality: normalizeQuality(block.quality),
+  };
+}

package/scripts/image-variants/index.js

@@ -0,0 +1,18 @@
+/**
+ * Public API for the image-variant pipeline.
+ *
+ * Used by:
+ *   - The Vite plugin (vite-plugin.js) — buildStart hook + dev watcher.
+ *
+ * The pipeline runs at build/dev time, generates variants to
+ * node_modules/.cache/@koehler8/cms/image-variants/{siteKey}/, and never
+ * touches the site's source tree.
+ */
+
+export { resolveImageVariantConfig, DEFAULT_WIDTHS, DEFAULT_FORMATS, DEFAULT_QUALITY } from './config.js';
+export { classifyImageFile, SOURCE_EXTENSIONS, FORMAT_OUTPUT_EXT } from './classifier.js';
+export { planVariantJobsFromFlatDir, walkRecursive } from './planner.js';
+export { renderOutput } from './renderer.js';
+export { readManifest, writeManifest, configsEqual, MANIFEST_VERSION } from './manifest.js';
+export { computeVariantCacheDir, siteKeyFor } from './cache.js';
+export { reconcileVariantCache } from './reconcile.js';

package/scripts/image-variants/manifest.js

@@ -0,0 +1,76 @@
+/**
+ * Cache manifest: persisted record of what's in the variant cache directory.
+ *
+ * Format:
+ *   {
+ *     version: 1,
+ *     config:  { widths, formats, quality },
+ *     sources: {
+ *       "img/logo.png": { mtimeMs, variants: ["assets/img/logo-320.webp", ...] },
+ *       ...
+ *     }
+ *   }
+ *
+ * Used by reconcileVariantCache to:
+ *   - Skip regeneration when source mtime is unchanged AND config matches.
+ *   - Detect orphaned entries (source removed) and delete their cached files.
+ *   - Force a full regen when config changes (e.g. user added a new width).
+ */
+
+import fs from 'node:fs';
+
+export const MANIFEST_VERSION = 1;
+
+export function readManifest(manifestPath) {
+  if (!fs.existsSync(manifestPath)) return null;
+  try {
+    const parsed = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+    if (parsed && parsed.version === MANIFEST_VERSION) return parsed;
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+export function writeManifest(manifestPath, manifest) {
+  // Write-to-temp-then-rename: an interrupted run must never leave a
+  // truncated manifest (readManifest would recover, but at the cost of a
+  // full regeneration).
+  const tmpPath = `${manifestPath}.tmp-${process.pid}`;
+  fs.writeFileSync(tmpPath, JSON.stringify(manifest, null, 2));
+  fs.renameSync(tmpPath, manifestPath);
+}
+
+/**
+ * Deep equality check for image-variant config blocks. Returns true when
+ * widths, formats, and quality (per-format) all match. Order-insensitive
+ * for arrays.
+ */
+export function configsEqual(a, b) {
+  if (!a || !b) return false;
+  if (!arraysEqualUnordered(a.widths, b.widths)) return false;
+  if (!arraysEqualUnordered(a.formats, b.formats)) return false;
+  return objectsEqual(a.quality, b.quality);
+}
+
+function arraysEqualUnordered(a, b) {
+  if (!Array.isArray(a) || !Array.isArray(b)) return false;
+  if (a.length !== b.length) return false;
+  const sa = [...a].sort();
+  const sb = [...b].sort();
+  for (let i = 0; i < sa.length; i += 1) {
+    if (sa[i] !== sb[i]) return false;
+  }
+  return true;
+}
+
+function objectsEqual(a, b) {
+  if (!a || !b) return a === b;
+  const ak = Object.keys(a);
+  const bk = Object.keys(b);
+  if (ak.length !== bk.length) return false;
+  for (const k of ak) {
+    if (a[k] !== b[k]) return false;
+  }
+  return true;
+}

package/scripts/image-variants/planner.js

@@ -0,0 +1,95 @@
+/**
+ * Walk site/assets/img/** (recursive), classify each file, and build a list
+ * of variant-generation jobs.
+ *
+ * Each job is { sourceKey, sourcePath, outputs: [{ width, format, cacheRelPath }] }
+ * where:
+ *   sourceKey    - posix path relative to site/assets/ (e.g. "img/logo.png")
+ *   sourcePath   - absolute disk path of the original
+ *   cacheRelPath - posix path relative to the cache dir root, including the
+ *                  `assets/img/` parent — so that the cache directory mirrors
+ *                  the site's `assets/` layout. The `assets/` parent is
+ *                  load-bearing: createAssetResolver's normalizeAssetKey
+ *                  strips up to and including `assets/`, so cache entries
+ *                  surface in the URL map under the same `img/...` keys as
+ *                  flat-dir variants would.
+ *
+ * SVGs are originals but produce no jobs (no variant generation).
+ * Files classified as 'variant' (leftover from the old pipeline) or 'ignore'
+ * are skipped.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { classifyImageFile, FORMAT_OUTPUT_EXT } from './classifier.js';
+
+export function* walkRecursive(dir) {
+  if (!fs.existsSync(dir)) return;
+  const stack = [dir];
+  while (stack.length) {
+    const cur = stack.pop();
+    for (const entry of fs.readdirSync(cur, { withFileTypes: true })) {
+      const full = path.join(cur, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(full);
+        continue;
+      }
+      if (entry.isFile()) yield full;
+    }
+  }
+}
+
+/**
+ * @param {object} args
+ * @param {string} args.siteImgDir - absolute path to site/assets/img/
+ * @param {{ widths: number[], formats: string[] }} args.config
+ * @returns {Array<{sourceKey: string, sourcePath: string, outputs: Array<{width: number, format: string, cacheRelPath: string}>}>}
+ */
+export function planVariantJobsFromFlatDir({ siteImgDir, config }) {
+  if (!fs.existsSync(siteImgDir)) return [];
+
+  const configWidths = new Set(config.widths);
+  const jobs = [];
+
+  for (const sourcePath of walkRecursive(siteImgDir)) {
+    const relFromImg = path.relative(siteImgDir, sourcePath);
+
+    if (relFromImg.split(path.sep)[0] === '_source') continue;
+
+    const cls = classifyImageFile(relFromImg, configWidths);
+    if (cls !== 'original') continue;
+
+    const ext = path.extname(relFromImg).toLowerCase();
+    if (ext === '.svg') continue;
+
+    const parsed = path.parse(relFromImg);
+    const subdir = parsed.dir;
+    const baseName = parsed.name;
+
+    const outputs = [];
+    for (const width of config.widths) {
+      for (const format of config.formats) {
+        const outExt = FORMAT_OUTPUT_EXT[format];
+        if (!outExt) continue;
+        const variantName = `${baseName}-${width}.${outExt}`;
+        const cacheRelPath = path.posix.join(
+          'assets',
+          'img',
+          subdir.split(path.sep).join('/'),
+          variantName,
+        );
+        outputs.push({ width, format, cacheRelPath });
+      }
+    }
+
+    const sourceKey = path.posix.join(
+      'img',
+      relFromImg.split(path.sep).join('/'),
+    );
+
+    jobs.push({ sourceKey, sourcePath, outputs });
+  }
+
+  return jobs;
+}

package/scripts/image-variants/reconcile.js

@@ -0,0 +1,194 @@
+/**
+ * Reconcile the variant cache directory against the planned jobs:
+ *   - Generate variants whose source mtime is newer than cached, or whose
+ *     cached file no longer exists, or when the config block has changed.
+ *   - Skip jobs whose cached variants are all up-to-date.
+ *   - Delete cached variants for sources that no longer exist (orphans).
+ *   - Write the new manifest at the end.
+ *
+ * Sharp is imported lazily — only when there is actual rendering work to do.
+ * This keeps dev-mode cold-start fast when the cache is fully populated.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { readManifest, writeManifest, configsEqual, MANIFEST_VERSION } from './manifest.js';
+import { renderOutput } from './renderer.js';
+
+/**
+ * @param {object} args
+ * @param {Array} args.jobs - planVariantJobsFromFlatDir output
+ * @param {string} args.cacheDir - absolute path to the variant cache root
+ * @param {string} args.manifestPath - absolute path to .manifest.json
+ * @param {{widths, formats, quality}} args.currentConfig
+ * @param {(msg: string) => void} [args.log]
+ * @returns {Promise<{generated: number, skipped: number, evicted: number}>}
+ */
+export async function reconcileVariantCache({
+  jobs,
+  cacheDir,
+  manifestPath,
+  currentConfig,
+  log = () => {},
+}) {
+  await fs.promises.mkdir(cacheDir, { recursive: true });
+
+  const saved = readManifest(manifestPath);
+  const configChanged = !saved || !configsEqual(saved.config, currentConfig);
+
+  const currentSourceKeys = new Set(jobs.map((j) => j.sourceKey));
+
+  let evicted = 0;
+  if (saved) {
+    for (const oldKey of Object.keys(saved.sources || {})) {
+      if (currentSourceKeys.has(oldKey)) continue;
+      const oldEntry = saved.sources[oldKey];
+      for (const variantRel of (oldEntry.variants || [])) {
+        const fullPath = path.join(cacheDir, variantRel);
+        if (fs.existsSync(fullPath)) {
+          await fs.promises.rm(fullPath, { force: true });
+          evicted += 1;
+        }
+      }
+    }
+  }
+
+  const plannedRenders = [];
+  for (const job of jobs) {
+    const previous = saved?.sources?.[job.sourceKey];
+    let sourceMtime;
+    try {
+      sourceMtime = fs.statSync(job.sourcePath).mtimeMs;
+    } catch {
+      continue;
+    }
+
+    // Source is "cached" if the manifest knows it, mtime hasn't changed,
+    // config hasn't changed, AND every variant the previous run wrote is
+    // still on disk. We check `previous.variants` (what was actually
+    // generated) rather than `job.outputs` (everything the planner asked
+    // for) so tiny-source sites — where the renderer skipped over-large
+    // widths to avoid Vite content-dedup — aren't perpetually re-rendered
+    // on each build.
+    const previousVariants = previous?.variants || [];
+    const allCached = !configChanged
+      && previous
+      && previous.mtimeMs >= sourceMtime
+      && previousVariants.every((v) => fs.existsSync(path.join(cacheDir, v)));
+
+    plannedRenders.push({ job, sourceMtime, render: !allCached, previousVariants });
+  }
+
+  const renderJobs = plannedRenders.filter((p) => p.render);
+
+  let generated = 0;
+  let skipped = 0;
+  let sharp = null;
+
+  if (renderJobs.length > 0) {
+    const mod = await import('sharp');
+    sharp = mod.default;
+  }
+
+  // Per-source: filter outputs to widths the source can actually fulfill.
+  // Sharp's `withoutEnlargement: true` clamps oversize requests to the
+  // source dimensions, producing content-identical files for every width
+  // greater than the source. Vite content-hashes those files identically
+  // and emits ONE physical asset under one of the names — but assetUrlMap
+  // still has URL keys for every width, so requests for the deduped
+  // higher-width URLs 404.
+  //
+  // Fix: only generate variants for widths < source.width. Plus, always
+  // generate at exactly source.width (for crisp 1× display) by remapping
+  // any single oversize width down to source.width if no smaller variant
+  // would otherwise exist. The pipeline's natural fallback to the bare
+  // `img/{name}.{ext}` URL (which exists because the original is in the
+  // flat dir) covers tiny sources where no variant width fits.
+  // Only computed for jobs that actually render — a cached job's truth is
+  // `previousVariants` (what an earlier run really wrote). Computing this
+  // filter for cached jobs used to poison the manifest: with no render work
+  // sharp stays unloaded, sourceWidth degraded to MAX_SAFE_INTEGER, and the
+  // manifest recorded the full unfiltered planner matrix — 9 phantom files
+  // for every under-width source — forcing a full re-render every other
+  // build.
+  const filteredOutputsByJob = new Map();
+  for (const { job, render } of plannedRenders) {
+    if (!render) continue;
+    let sourceWidth = Number.MAX_SAFE_INTEGER;
+    if (sharp) {
+      try {
+        const meta = await sharp(job.sourcePath).metadata();
+        if (meta && Number.isFinite(meta.width)) sourceWidth = meta.width;
+      } catch {
+        // If metadata read fails, fall through and render everything —
+        // the failure is benign (just the dedup risk we're trying to avoid).
+      }
+    }
+    const kept = job.outputs.filter((o) => o.width <= sourceWidth);
+    filteredOutputsByJob.set(job.sourceKey, kept);
+  }
+
+  // What each rendered job actually produced. The manifest records only
+  // successful writes: a failed render must never be asserted as an
+  // existing variant. (A transiently-failed output is retried when the
+  // source mtime or config next changes; a permanently-broken source stops
+  // forcing a full re-render on every build.)
+  const successfulByJob = new Map();
+  for (const { job, render, previousVariants } of plannedRenders) {
+    if (!render) {
+      skipped += previousVariants.length;
+      continue;
+    }
+    const outputs = filteredOutputsByJob.get(job.sourceKey) || job.outputs;
+    const successes = [];
+    for (const output of outputs) {
+      const outPath = path.join(cacheDir, output.cacheRelPath);
+      try {
+        await renderOutput(sharp, job.sourcePath, { ...output, outPath }, currentConfig);
+        generated += 1;
+        successes.push(output.cacheRelPath);
+      } catch (err) {
+        log(`failed ${output.cacheRelPath}: ${err.message}`);
+      }
+    }
+    successfulByJob.set(job.sourceKey, successes);
+  }
+
+  // Evict any cached variants that the source can no longer fulfill
+  // (e.g. someone replaced a 4000px source with a 200px one — the
+  // 1280/1920/2560 outputs from the previous run are now stale).
+  for (const { job, render } of plannedRenders) {
+    if (!render) continue;
+    const kept = new Set((filteredOutputsByJob.get(job.sourceKey) || job.outputs).map((o) => o.cacheRelPath));
+    for (const o of job.outputs) {
+      if (kept.has(o.cacheRelPath)) continue;
+      const stalePath = path.join(cacheDir, o.cacheRelPath);
+      if (fs.existsSync(stalePath)) {
+        await fs.promises.rm(stalePath, { force: true });
+        evicted += 1;
+      }
+    }
+  }
+
+  const newManifest = {
+    version: MANIFEST_VERSION,
+    config: currentConfig,
+    sources: {},
+  };
+  for (const { job, sourceMtime, render, previousVariants } of plannedRenders) {
+    newManifest.sources[job.sourceKey] = {
+      mtimeMs: sourceMtime,
+      // Cached job → carry forward what actually exists on disk; rendered
+      // job → record only what this run successfully wrote.
+      variants: render ? (successfulByJob.get(job.sourceKey) || []) : previousVariants,
+    };
+  }
+  writeManifest(manifestPath, newManifest);
+
+  if (generated || evicted) {
+    log(`sources: ${jobs.length}, generated: ${generated}, skipped: ${skipped}, evicted: ${evicted}`);
+  }
+
+  return { generated, skipped, evicted };
+}

package/scripts/image-variants/renderer.js

@@ -0,0 +1,51 @@
+/**
+ * Sharp-powered variant renderer. Reads a source image, resizes to the
+ * requested width (clamped to source size — no enlargement), and writes
+ * the encoded output to outPath.
+ *
+ * Pure function: takes paths and config, performs IO, returns nothing.
+ * Sharp is imported lazily by the caller (see reconcile.js) so dev-mode
+ * cold start doesn't pay sharp's native-binding load cost when there's
+ * nothing to render.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+export async function renderOutput(sharp, sourcePath, output, config) {
+  const { width, format, outPath } = output;
+  await fs.promises.mkdir(path.dirname(outPath), { recursive: true });
+
+  let pipeline = sharp(sourcePath, { failOn: 'truncated' }).rotate();
+  pipeline = pipeline.resize({ width, withoutEnlargement: true });
+
+  switch (format) {
+    case 'avif':
+      pipeline = pipeline.avif({ quality: config.quality.avif });
+      break;
+    case 'webp':
+      pipeline = pipeline.webp({ quality: config.quality.webp });
+      break;
+    case 'jpg':
+    case 'jpeg':
+      pipeline = pipeline.jpeg({ quality: config.quality.jpg, mozjpeg: true });
+      break;
+    case 'png':
+      pipeline = pipeline.png();
+      break;
+    default:
+      throw new Error(`unsupported format: ${format}`);
+  }
+
+  // Render to a temp path and rename: a killed run must never leave a
+  // truncated image that reconciliation would then trust as a valid variant
+  // (it validates presence, not bytes).
+  const tmpPath = `${outPath}.tmp-${process.pid}`;
+  try {
+    await pipeline.toFile(tmpPath);
+    await fs.promises.rename(tmpPath, outPath);
+  } catch (err) {
+    await fs.promises.rm(tmpPath, { force: true });
+    throw err;
+  }
+}

package/scripts/lib/extension-manifests.mjs

@@ -0,0 +1,70 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// framework root = two levels up from scripts/lib/
+const FRAMEWORK_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+
+// Build-time extension-manifest validation. Manifests are static JSON, fixed
+// at build time, so full JSON-Schema validation happens here — failing the
+// build loudly — instead of shipping ajv to every visitor's browser (the
+// runtime loader keeps only cheap structural checks). Spec resolution mirrors
+// how the generated entry imports each extension: './'-relative specs resolve
+// from the project root, package names from the project's node_modules.
+// Exported for tests.
+export async function validateExtensionManifests(extensionPackages, projectRoot, frameworkDir = FRAMEWORK_ROOT) {
+  if (!Array.isArray(extensionPackages) || extensionPackages.length === 0) return [];
+
+  const [{ default: Ajv }, ajvFormats] = await Promise.all([
+    import('ajv/dist/2020.js'),
+    import('ajv-formats'),
+  ]);
+  const addFormats = ajvFormats.default || ajvFormats;
+  const ajv = new Ajv({ allErrors: true, strict: false });
+  addFormats(ajv);
+  const schemaPath = path.join(frameworkDir, 'extensions', 'manifest.schema.json');
+  const validate = ajv.compile(JSON.parse(fs.readFileSync(schemaPath, 'utf-8')));
+
+  const problems = [];
+  const manifests = [];
+  for (const spec of extensionPackages) {
+    const baseDir = spec.startsWith('.') || path.isAbsolute(spec)
+      ? path.resolve(projectRoot, spec)
+      : path.join(projectRoot, 'node_modules', ...spec.split('/'));
+    const manifestPath = path.join(baseDir, 'extension.config.json');
+
+    if (!fs.existsSync(manifestPath)) {
+      // A spec may legitimately carry its manifest elsewhere (the entry
+      // module imports it); the runtime structural check still covers it.
+      console.warn(`[@koehler8/cms] extension "${spec}": no extension.config.json at ${manifestPath} — skipping build-time schema validation.`);
+      continue;
+    }
+
+    let manifest;
+    try {
+      manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+    } catch (err) {
+      problems.push(`- ${spec}: extension.config.json is not valid JSON (${err.message})`);
+      continue;
+    }
+    if (!validate(manifest)) {
+      const details = (validate.errors || [])
+        .map((e) => `    ${e.instancePath || '(root)'} ${e.message}`)
+        .join('\n');
+      problems.push(`- ${spec}: manifest failed schema validation\n${details}`);
+      continue;
+    }
+    manifests.push({
+      spec,
+      slug: manifest.slug,
+      componentNames: new Set(
+        (manifest.components || []).map((c) => c && c.name).filter(Boolean),
+      ),
+    });
+  }
+
+  if (problems.length) {
+    throw new Error(`[@koehler8/cms] Extension manifest validation failed:\n${problems.join('\n')}`);
+  }
+  return manifests;
+}