@jxsuite/compiler 0.25.4 → 0.26.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jxsuite/compiler",
-  "version": "0.25.4",
+  "version": "0.26.0",
   "description": "Jx static HTML compiler, island detector, and site builder",
   "license": "MIT",
   "repository": {
@@ -41,9 +41,9 @@
   },
   "dependencies": {
     "@apidevtools/json-schema-ref-parser": "^15.3.5",
-    "@jxsuite/parser": "^0.25.4",
-    "@jxsuite/runtime": "^0.25.4",
-    "@jxsuite/schema": "^0.25.4",
+    "@jxsuite/parser": "^0.26.0",
+    "@jxsuite/runtime": "^0.26.0",
+    "@jxsuite/schema": "^0.26.0",
     "remark-gfm": "^4.0.1",
     "remark-stringify": "^11.0.0",
     "sharp": "^0.34.5",

package/src/site/content-loader.ts

@@ -2,7 +2,7 @@
  * Content-loader.js — Content type loader
  *
  * Loads content types defined in project.json's `contentTypes` key. Supports Markdown (.md), JSON
- * (.json), and CSV (.csv) source files.
+ * (.json), and CSV (.csv) source files or remote URLs.
  *
  * Phase 2 implementation of site-architecture spec §6.
  *
@@ -180,26 +180,53 @@ function loadJSONEntries(filePath: string) {
  * @param {ContentTypeSchema} [schema] - Content type schema (for type coercion)
  * @returns {ContentLoaderEntry[]} Array of ContentEntry shapes
  */
-function loadCSVEntries(filePath: string, schema?: ContentTypeSchema) {
-  const csv = readFileSync(filePath, "utf-8");
-  const rows = parseCSV(csv);
-  return rows.map((row: Record<string, string>, i: number) => {
-    // Apply type coercion based on schema if available
+/**
+ * Coerce raw CSV row strings to typed values per the schema. - number: strips currency
+ * symbols/commas, returns null for empty cells - boolean: "true" → true, everything else → false -
+ * string: trimmed (already done by parseCSV, but defensively applied)
+ */
+function coerceCSVRows(
+  rows: Record<string, string>[],
+  schema?: ContentTypeSchema,
+): ContentLoaderEntry[] {
+  return rows.map((row, i) => {
     const data: Record<string, unknown> = { ...row };
     if (schema?.properties) {
       for (const [key, def] of Object.entries(schema.properties)) {
-        if (key in data) {
-          if (def.type === "number") data[key] = Number(data[key]);
-          else if (def.type === "boolean") data[key] = data[key] === "true";
+        if (!(key in data)) continue;
+        if (def.type === "number") {
+          const raw = String(data[key] ?? "").trim();
+          if (raw === "") {
+            data[key] = null;
+          } else {
+            const cleaned = raw.replace(/[$€£¥,\s]/g, "");
+            const n = Number(cleaned);
+            data[key] = isNaN(n) ? null : n;
+          }
+        } else if (def.type === "boolean") {
+          data[key] = data[key] === "true";
         }
       }
     }
-    // Use `id` column, `sku` column, or row index as the entry ID
     const id = (data.id as string | undefined) ?? (data.sku as string | undefined) ?? String(i);
     return { id, data, body: null };
   });
 }
 
+function loadCSVEntries(filePath: string, schema?: ContentTypeSchema) {
+  const rows = parseCSV(readFileSync(filePath, "utf-8"));
+  return coerceCSVRows(rows, schema);
+}
+
+/** Fetch a remote CSV URL and return ContentLoaderEntry[]. */
+async function loadRemoteCSV(url: string, schema?: ContentTypeSchema) {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch CSV from ${url}: ${response.status} ${response.statusText}`);
+  }
+  return coerceCSVRows(parseCSV(await response.text()), schema);
+}
+
 // ─── Content Config ───────────────────────────────────────────────────────────
 
 /**
@@ -289,6 +316,18 @@ async function loadContentType(name: string, contentTypeDef: ContentTypeDef, pro
       }
     : undefined;
 
+  // Remote URL source (e.g. Google Sheets CSV export)
+  if (source.startsWith("http://") || source.startsWith("https://")) {
+    try {
+      const entries = await loadRemoteCSV(source, schema);
+      if (schema) validateEntries(entries, schema, name);
+      return entries;
+    } catch (e) {
+      console.warn(`Content type "${name}": ${(e as Error).message}`);
+      return [];
+    }
+  }
+
   // Resolve source path and discover files
   const resolvedSource = resolve(projectRoot, source).split("\\").join("/");
   /** @type {string[]} */

package/src/site/image-cache.ts

@@ -2,11 +2,14 @@
  * Image-cache.js — Content-hash based cache for processed image variants.
  *
  * Stores a manifest of previously processed images so that unchanged sources can skip re-encoding
- * on subsequent builds. Cache lives in .cache/images/.
+ * on subsequent builds. Cache lives in the npm global cache directory under jxsuite-images/ so that
+ * CI environments (e.g. Cloudflare Pages) can persist it via their npm cache layer.
  */
 
 import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
-import { resolve } from "node:path";
+import { resolve, basename } from "node:path";
+import { execSync } from "node:child_process";
+import { tmpdir } from "node:os";
 import { contentHash, configHash } from "./image-optimizer.ts";
 
 import type { ImageManifest } from "./image-optimizer.ts";
@@ -23,6 +26,56 @@ export interface CacheManifest {
   entries: Record<string, CacheEntry>; // Cached entries keyed by content+config hash
 }
 
+// ─── Cache directory resolution ──────────────────────────────────────────────
+
+let _npmCacheBase: string | null | undefined = undefined; // undefined = not yet resolved
+
+/** Reset the memoized npm cache base so the next call re-evaluates. Tests only. */
+export function _testResetNpmCacheBase() {
+  _npmCacheBase = undefined;
+}
+
+/**
+ * Directly set the memoized npm cache base. Pass null to force the project-local fallback. Tests
+ * only.
+ */
+export function _testSetNpmCacheBase(value: string | null) {
+  _npmCacheBase = value;
+}
+
+/**
+ * Return the image cache directory for a project.
+ *
+ * Prefers the npm global cache (`npm config get cache`) so CI environments that cache ~/.npm (e.g.
+ * Cloudflare Pages) automatically persist optimized images across builds. Falls back to
+ * .cache/images inside the project root when npm is unavailable.
+ *
+ * Result is memoized — `npm config get cache` is only spawned once per process.
+ *
+ * @param {string} projectRoot
+ * @returns {string}
+ */
+export function getImageCacheDir(projectRoot: string): string {
+  if (_npmCacheBase === undefined) {
+    try {
+      const result = execSync("npm config get cache", {
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "ignore"],
+        cwd: tmpdir(), // avoid ENOWORKSPACES when cwd is inside an npm workspace
+      }).trim();
+      _npmCacheBase = result && result !== "undefined" ? result : null;
+    } catch {
+      _npmCacheBase = null;
+    }
+  }
+  if (_npmCacheBase) {
+    return resolve(_npmCacheBase, "jxsuite-images", basename(projectRoot));
+  }
+  return resolve(projectRoot, ".cache/images");
+}
+
+// ─── Cache key ───────────────────────────────────────────────────────────────
+
 /**
  * Build a cache key from source file content and config.
  *
@@ -34,6 +87,8 @@ export function cacheKey(srcPath: string, config: ImageConfig) {
   return `${contentHash(srcPath)}:${configHash(config)}`;
 }
 
+// ─── Load / save ─────────────────────────────────────────────────────────────
+
 /**
  * Load the cache manifest from disk, or return an empty one.
  *
@@ -41,7 +96,7 @@ export function cacheKey(srcPath: string, config: ImageConfig) {
  * @returns {CacheManifest}
  */
 export function loadCache(projectRoot: string) {
-  const manifestPath = resolve(projectRoot, ".cache/images/manifest.json");
+  const manifestPath = resolve(getImageCacheDir(projectRoot), "manifest.json");
   if (!existsSync(manifestPath)) {
     return { version: 1, entries: {} };
   }
@@ -59,11 +114,13 @@ export function loadCache(projectRoot: string) {
  * @param {CacheManifest} cache
  */
 export function saveCache(projectRoot: string, cache: CacheManifest) {
-  const cacheDir = resolve(projectRoot, ".cache/images");
+  const cacheDir = getImageCacheDir(projectRoot);
   mkdirSync(cacheDir, { recursive: true });
   writeFileSync(resolve(cacheDir, "manifest.json"), JSON.stringify(cache, null, 2), "utf8");
 }
 
+// ─── Entry access ─────────────────────────────────────────────────────────────
+
 /**
  * Check if a cache entry exists and its output files are still present.
  *

package/src/site/image-optimizer.ts

@@ -118,20 +118,21 @@ export function variantFilename(stem: string, width: number, hash8: string, form
 }
 
 /**
- * Process a single source image: resize to each configured width, encode to each format.
+ * Process a single source image: resize to each configured width, encode to each format. Writes
+ * variants to cacheImgDir/_optimized/ (e.g. .cache/images/_optimized/).
  *
  * @param {string} srcPath - Absolute path to source image
- * @param {string} outDir - Absolute path to the build output directory (dist/)
+ * @param {string} cacheImgDir - Absolute path to the image cache directory (.cache/images/)
  * @param {ImageConfig} config
  * @returns {Promise<ImageManifest>}
  */
-export async function processImage(srcPath: string, outDir: string, config: ImageConfig) {
+export async function processImage(srcPath: string, cacheImgDir: string, config: ImageConfig) {
   const sharp = await getSharp();
   const meta = await getImageMetadata(srcPath);
   const hash8 = contentHash(srcPath);
   const stem = basename(srcPath, extname(srcPath));
 
-  const optimizedDir = resolve(outDir, OPTIMIZED_DIR);
+  const optimizedDir = resolve(cacheImgDir, "_optimized");
   mkdirSync(optimizedDir, { recursive: true });
 
   const variants: ImageVariant[] = [];

package/src/site/image-transform.ts

@@ -9,7 +9,7 @@
 import { existsSync } from "node:fs";
 import { resolve, extname, basename } from "node:path";
 import { processImage, buildSrcset, contentHash, configHash } from "./image-optimizer.ts";
-import { getCached, setCached } from "./image-cache.ts";
+import { getCached, setCached, getImageCacheDir } from "./image-cache.ts";
 
 import type { ImageConfig } from "./image-optimizer.ts";
 import type { ImageManifest } from "./image-optimizer.ts";
@@ -24,7 +24,7 @@ const EXTERNAL_PREFIXES = ["http://", "https://", "data:", "//"];
  * @param {string} absoluteSrc
  * @param {string} src
  * @param {ImageConfig} config
- * @param {string} outDir
+ * @param {string} projectRoot
  * @param {CacheManifest} cache
  * @returns {Promise<ImageManifest>}
  */
@@ -32,7 +32,7 @@ async function resolveManifest(
   absoluteSrc: string,
   src: string,
   config: ImageConfig,
-  outDir: string,
+  projectRoot: string,
   cache: CacheManifest,
 ) {
   const key = `${contentHash(absoluteSrc)}:${configHash(config)}`;
@@ -44,7 +44,7 @@ async function resolveManifest(
   }
 
   if (!cached) console.log(`    Optimizing ${basename(absoluteSrc)}...`);
-  const manifest = await processImage(absoluteSrc, outDir, config);
+  const manifest = await processImage(absoluteSrc, getImageCacheDir(projectRoot), config);
   setCached(cache, key, src, manifest);
   return manifest;
 }
@@ -83,12 +83,12 @@ function resolveImagePath(src: string, projectRoot: string) {
  * Transform image nodes in a Jx document tree.
  *
  * Mutates img nodes in place, injecting srcset, sizes, width, height, loading, and decoding.
- * Returns a set of absolute source paths that need processing.
+ * Optimized variants are written to .cache/images/_optimized/ — the caller is responsible for
+ * copying them to the dist directory after all pages are compiled.
  *
  * @param {JxMutableNode | JxDocument} doc - The Jx document tree (mutated in place)
  * @param {ImageConfig} config
  * @param {string} projectRoot
- * @param {string} outDir
  * @param {CacheManifest} cache
  * @returns {Promise<{ imageRefs: Map<string, ImageManifest> }>}
  */
@@ -96,14 +96,13 @@ export async function transformImageNodes(
   doc: JxMutableNode | JxDocument,
   config: ImageConfig,
   projectRoot: string,
-  outDir: string,
   cache: CacheManifest,
 ) {
   const imageRefs: Map<string, ImageManifest> = new Map();
 
   if (!config.optimize) return { imageRefs };
 
-  await walkAndTransform(doc, config, projectRoot, outDir, cache, imageRefs);
+  await walkAndTransform(doc, config, projectRoot, cache, imageRefs);
 
   return { imageRefs };
 }
@@ -112,7 +111,6 @@ export async function transformImageNodes(
  * @param {JxMutableNode | JxDocument} node
  * @param {ImageConfig} config
  * @param {string} projectRoot
- * @param {string} outDir
  * @param {CacheManifest} cache
  * @param {Map<string, ImageManifest>} imageRefs
  */
@@ -120,14 +118,13 @@ async function walkAndTransform(
   node: JxMutableNode | JxDocument,
   config: ImageConfig,
   projectRoot: string,
-  outDir: string,
   cache: CacheManifest,
   imageRefs: Map<string, ImageManifest>,
 ) {
   if (!node || typeof node !== "object") return;
 
   if (node.tagName === "img") {
-    await transformImgNode(node, config, projectRoot, outDir, cache, imageRefs);
+    await transformImgNode(node, config, projectRoot, cache, imageRefs);
   }
 
   if (typeof node.innerHTML === "string" && node.innerHTML.includes("<img")) {
@@ -135,7 +132,6 @@ async function walkAndTransform(
       node.innerHTML,
       config,
       projectRoot,
-      outDir,
       cache,
       imageRefs,
     );
@@ -144,7 +140,7 @@ async function walkAndTransform(
   if (Array.isArray(node.children)) {
     for (const child of node.children) {
       if (typeof child === "string") continue;
-      await walkAndTransform(child, config, projectRoot, outDir, cache, imageRefs);
+      await walkAndTransform(child, config, projectRoot, cache, imageRefs);
     }
   }
 }
@@ -153,7 +149,6 @@ async function walkAndTransform(
  * @param {JxMutableNode | JxDocument} node
  * @param {ImageConfig} config
  * @param {string} projectRoot
- * @param {string} outDir
  * @param {CacheManifest} cache
  * @param {Map<string, ImageManifest>} imageRefs
  */
@@ -161,7 +156,6 @@ async function transformImgNode(
   node: JxMutableNode | JxDocument,
   config: ImageConfig,
   projectRoot: string,
-  outDir: string,
   cache: CacheManifest,
   imageRefs: Map<string, ImageManifest>,
 ) {
@@ -177,7 +171,7 @@ async function transformImgNode(
   let manifest = imageRefs.get(absoluteSrc);
 
   if (!manifest) {
-    manifest = await resolveManifest(absoluteSrc, src, config, outDir, cache);
+    manifest = await resolveManifest(absoluteSrc, src, config, projectRoot, cache);
     imageRefs.set(absoluteSrc, manifest);
   }
 
@@ -213,7 +207,6 @@ const DATA_NO_OPT_RE = /\bdata-no-optimize\b/;
  * @param {string} html
  * @param {ImageConfig} config
  * @param {string} projectRoot
- * @param {string} outDir
  * @param {CacheManifest} cache
  * @param {Map<string, ImageManifest>} imageRefs
  * @returns {Promise<string>}
@@ -222,7 +215,6 @@ async function transformInnerHtmlImages(
   html: string,
   config: ImageConfig,
   projectRoot: string,
-  outDir: string,
   cache: CacheManifest,
   imageRefs: Map<string, ImageManifest>,
 ) {
@@ -247,7 +239,7 @@ async function transformInnerHtmlImages(
 
     let manifest = imageRefs.get(absoluteSrc);
     if (!manifest) {
-      manifest = await resolveManifest(absoluteSrc, src, config, outDir, cache);
+      manifest = await resolveManifest(absoluteSrc, src, config, projectRoot, cache);
       imageRefs.set(absoluteSrc, manifest);
     }
 

package/src/site/site-build.ts

@@ -14,7 +14,6 @@ import {
   copyFileSync,
   mkdirSync,
   existsSync,
-  renameSync,
   rmSync,
   cpSync,
   readdirSync,
@@ -45,7 +44,7 @@ import { loadContentTypes, loadContentConfig, resolveContentTypeRefs } from "./c
 import { resolvePrototypes } from "./prototype-resolver.ts";
 import { compileMarkdown } from "../targets/compile-markdown.ts";
 import { transformImageNodes } from "./image-transform.ts";
-import { loadCache, saveCache } from "./image-cache.ts";
+import { loadCache, saveCache, getImageCacheDir } from "./image-cache.ts";
 import type {
   JxElement,
   JxMutableNode,
@@ -88,20 +87,7 @@ export async function buildSite(
 
   // ── 2. Clean output directory ───────────────────────────────────────────
   if (clean && existsSync(outDir)) {
-    // Preserve optimized images across builds — sharp re-encoding is expensive
-    const optimizedDir = resolve(outDir, "images/_optimized");
-    const hasOptimized = existsSync(optimizedDir);
-    let tmpOptimized = "";
-    if (hasOptimized) {
-      tmpOptimized = resolve(projectRoot, ".cache/_optimized_tmp");
-      mkdirSync(resolve(projectRoot, ".cache"), { recursive: true });
-      renameSync(optimizedDir, tmpOptimized);
-    }
     rmSync(outDir, { recursive: true, force: true });
-    if (hasOptimized) {
-      mkdirSync(resolve(outDir, "images"), { recursive: true });
-      renameSync(tmpOptimized, optimizedDir);
-    }
   }
   mkdirSync(outDir, { recursive: true });
 
@@ -208,7 +194,6 @@ export async function buildSite(
         projectRoot,
         contentTypes,
         imageCache,
-        outDir,
         componentDefs,
       );
 
@@ -269,9 +254,15 @@ export async function buildSite(
     }
   }
 
-  // ── 6b. Save image cache ─────────────────────────────────────────────
+  // ── 6b. Save image cache and copy variants to dist ──────────────────────
   if (imageCache && projectConfig.images.optimize) {
     saveCache(projectRoot, imageCache);
+    const cacheOptimizedDir = resolve(getImageCacheDir(projectRoot), "_optimized");
+    if (existsSync(cacheOptimizedDir)) {
+      const distOptimizedDir = resolve(outDir, "images/_optimized");
+      mkdirSync(distOptimizedDir, { recursive: true });
+      cpSync(cacheOptimizedDir, distOptimizedDir, { recursive: true });
+    }
     const totalImages = Object.keys(imageCache.entries).length;
     if (totalImages > 0) {
       log(`  Optimized ${totalImages} image(s)`);
@@ -377,7 +368,6 @@ export async function buildSite(
  * @param {string} projectRoot
  * @param {Map<string, ContentLoaderEntry[]>} [contentTypes]
  * @param {import("./image-cache.js").CacheManifest | null} [imageCache]
- * @param {string} [outDir]
  * @param {Map<string, JxElement>} [componentDefs]
  * @returns {Promise<{
  *   html: string;
@@ -392,7 +382,6 @@ async function compilePage(
   projectRoot: string,
   contentTypes: Map<string, ContentLoaderEntry[]> = new Map(),
   imageCache: import("./image-cache.js").CacheManifest | null = null,
-  outDir: string = "",
   componentDefs: Map<string, JxElement> = new Map(),
 ) {
   // Load the raw page document
@@ -483,12 +472,11 @@ async function compilePage(
   }
 
   // Transform <img> nodes for responsive image optimization
-  if (imageCache && projectConfig.images?.optimize && outDir) {
+  if (imageCache && projectConfig.images?.optimize) {
     await transformImageNodes(
       layoutDoc,
       projectConfig.images as import("./image-optimizer").ImageConfig,
       projectRoot,
-      outDir,
       imageCache,
     );
   }