@rettangoli/sites 1.0.2 → 1.1.0

package/README.md

@@ -32,8 +32,8 @@ my-site/
 
 - YAML pages rendered through `jempl` + `yahtml`
 - Markdown pages rendered through `markdown-it` + Shiki (default `rtglMarkdown`)
-- Frontmatter (`template`, `tags`, arbitrary page metadata)
-- Global data (`data/*.yaml`) merged with page frontmatter
+- Frontmatter (`template`, `url`, `tags`, arbitrary page metadata)
+- Global data from `data/*.yaml` and optional inline `sites.config.yaml data`
 - Collections built from page tags
 - `$if`, `$for`, `$partial`, template functions
 - Static file copying from `static/` to `_site/`
@@ -75,6 +75,9 @@ imports:
     docs: https://example.com/templates/docs.yaml
   partials:
     docs/nav: https://example.com/partials/docs-nav.yaml
+data:
+  themeCssHref: /public/theme.css
+  themeBodyClass: dark
 ```
 
 In the default starter template, CDN runtime scripts are controlled via `data/site.yaml`:
@@ -93,10 +96,35 @@ Example mappings:
 - `pages/index.md` -> `_site/index.html` and `_site/index.md`
 - `pages/docs/intro.md` -> `_site/docs/intro/index.html` and `_site/docs/intro.md`
 
+For Markdown pages with a custom `url`, the copied `.md` file follows the custom URL path.
+For example, `url: /guides/start/` writes `_site/guides/start/index.html` and `_site/guides/start.md`.
+
+Pages use their file path as the URL by default:
+- `pages/index.*` -> `/`
+- `pages/about.*` -> `/about/`
+- `pages/docs/intro.*` -> `/docs/intro/`
+
+Set `url` in page frontmatter to override that path:
+
+```md
+---
+title: Company
+url: /company/
+---
+```
+
+`url` is normalized to a site-relative clean URL with a leading and trailing slash, so `company` becomes `/company/`.
+External URLs, query strings, fragments, whitespace, and `.` / `..` path segments are rejected.
+Duplicate page URLs are rejected after normalization.
+
 `imports` lets you map aliases to remote YAML files (HTTP/HTTPS only). Use aliases in pages/templates:
 - page frontmatter: `template: base` or `template: docs`
 - template/page content: `$partial: docs/nav`
 
+Use top-level `data` in `sites.config.yaml` for small global values that do not deserve their own `data/*.yaml` file.
+`sites.config.yaml data` and `data/*.yaml` are merged, with `data/*.yaml` winning on conflicts.
+Inline config data requires `rtgl >= 1.1.4` or `@rettangoli/sites >= 1.0.3`.
+
 Imported files are cached on disk under `.rettangoli/sites/imports/{templates|partials}/` (hashed filenames).
 Alias/url/hash mapping is tracked in `.rettangoli/sites/imports/index.yaml`.
 Build is cache-first: if a cached file exists, it is used without a network request.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rettangoli/sites",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "Generate static sites using Markdown and YAML for docs, blogs, and marketing sites.",
   "author": {
     "name": "Luciano Hanyon Wu",

package/src/cli/build.js

@@ -31,6 +31,7 @@ export const buildSite = async (options = {}) => {
     markdown: config.markdown || {},
     keepMarkdownFiles: config.build?.keepMarkdownFiles === true,
     imports: config.imports || {},
+    data: config.data || {},
     functions: functions || {},
     quiet,
     isScreenshotMode

package/src/createSiteBuilder.js

@@ -17,22 +17,25 @@ const MATTER_OPTIONS = {
 
 // Deep merge utility function
 function deepMerge(target, source) {
-  const output = { ...target };
-  
-  if (isObject(target) && isObject(source)) {
-    Object.keys(source).forEach(key => {
-      if (isObject(source[key])) {
-        if (!(key in target)) {
-          Object.assign(output, { [key]: source[key] });
-        } else {
-          output[key] = deepMerge(target[key], source[key]);
-        }
-      } else {
-        Object.assign(output, { [key]: source[key] });
-      }
-    });
+  if (!isObject(source)) {
+    return source;
+  }
+
+  if (!isObject(target)) {
+    return { ...source };
   }
-  
+
+  const output = { ...target };
+
+  Object.keys(source).forEach(key => {
+    if (isObject(source[key]) && isObject(target[key])) {
+      output[key] = deepMerge(target[key], source[key]);
+      return;
+    }
+
+    Object.assign(output, { [key]: source[key] });
+  });
+
   return output;
 }
 
@@ -225,6 +228,123 @@ function isSchemaSidecarFile(fileName) {
   return fileName.endsWith('.schema.yaml') || fileName.endsWith('.schema.yml');
 }
 
+function hasPageExtension(fileName) {
+  return fileName.endsWith('.yaml') || fileName.endsWith('.yml') || fileName.endsWith('.md');
+}
+
+function hasOwn(object, key) {
+  return Object.prototype.hasOwnProperty.call(object, key);
+}
+
+function normalizeRelativeUrlPath(relativePath) {
+  return relativePath.replace(/\\/g, '/');
+}
+
+function derivePageUrlFromRelativePath(relativePath) {
+  const normalizedPath = normalizeRelativeUrlPath(relativePath);
+  const pathWithoutExtension = normalizedPath.replace(/\.(yaml|yml|md)$/, '');
+  const pageName = path.posix.basename(pathWithoutExtension);
+
+  if (pageName === 'index') {
+    const dirName = path.posix.dirname(pathWithoutExtension);
+    return dirName === '.' ? '/' : `/${dirName}/`;
+  }
+
+  return `/${pathWithoutExtension}/`;
+}
+
+function normalizePageUrlOverride(rawUrl, pagePath) {
+  if (typeof rawUrl !== 'string') {
+    throw new Error(`Invalid url in ${pagePath}: expected a string.`);
+  }
+
+  if (rawUrl === '') {
+    throw new Error(`Invalid url in ${pagePath}: expected a non-empty string.`);
+  }
+
+  if (/[\u0000-\u001F\u007F]/u.test(rawUrl)) {
+    throw new Error(`Invalid url in ${pagePath}: must not contain control characters.`);
+  }
+
+  if (/\s/u.test(rawUrl)) {
+    throw new Error(`Invalid url in ${pagePath}: must not contain whitespace.`);
+  }
+
+  if (/^[A-Za-z][A-Za-z0-9+.-]*:/u.test(rawUrl) || rawUrl.startsWith('//')) {
+    throw new Error(`Invalid url in ${pagePath}: expected a site-relative URL path.`);
+  }
+
+  if (rawUrl.includes('\\')) {
+    throw new Error(`Invalid url in ${pagePath}: must use forward slashes.`);
+  }
+
+  if (rawUrl.includes('?') || rawUrl.includes('#')) {
+    throw new Error(`Invalid url in ${pagePath}: must not include query strings or fragments.`);
+  }
+
+  const withLeadingSlash = rawUrl.startsWith('/') ? rawUrl : `/${rawUrl}`;
+  const collapsedUrl = withLeadingSlash.replace(/\/+/g, '/');
+  const pathWithoutSlashes = collapsedUrl.replace(/^\/+|\/+$/g, '');
+
+  if (pathWithoutSlashes === '') {
+    return '/';
+  }
+
+  const segments = pathWithoutSlashes.split('/');
+  for (const segment of segments) {
+    let decodedSegment;
+    try {
+      decodedSegment = decodeURIComponent(segment);
+    } catch (error) {
+      throw new Error(`Invalid url in ${pagePath}: contains invalid URL encoding.`);
+    }
+
+    if (decodedSegment === '.' || decodedSegment === '..') {
+      throw new Error(`Invalid url in ${pagePath}: must not contain "." or ".." segments.`);
+    }
+
+    if (decodedSegment.includes('/') || decodedSegment.includes('\\')) {
+      throw new Error(`Invalid url in ${pagePath}: must not include encoded slashes or backslashes.`);
+    }
+
+    if (/[\u0000-\u001F\u007F]/u.test(decodedSegment)) {
+      throw new Error(`Invalid url in ${pagePath}: must not contain control characters.`);
+    }
+
+    if (/\s/u.test(decodedSegment)) {
+      throw new Error(`Invalid url in ${pagePath}: must not contain whitespace.`);
+    }
+  }
+
+  return `/${segments.join('/')}/`;
+}
+
+function resolvePageUrl(publicFrontmatter, relativePath, pagePath) {
+  if (hasOwn(publicFrontmatter, 'url')) {
+    return normalizePageUrlOverride(publicFrontmatter.url, pagePath);
+  }
+
+  return derivePageUrlFromRelativePath(relativePath);
+}
+
+function htmlOutputRelativePathFromUrl(url) {
+  if (url === '/') {
+    return 'index.html';
+  }
+
+  return `${url.slice(1, -1)}/index.html`;
+}
+
+function markdownOutputRelativePathFromUrl(url) {
+  if (url === '/') {
+    return 'index.md';
+  }
+
+  const segments = url.slice(1, -1).split('/');
+  const fileName = `${segments.pop()}.md`;
+  return [...segments, fileName].join('/');
+}
+
 async function fetchRemoteYaml(url, fetchImpl, aliasLabel) {
   const effectiveFetch = fetchImpl || globalThis.fetch;
   if (typeof effectiveFetch !== 'function') {
@@ -314,6 +434,7 @@ export function createSiteBuilder({
   markdown = {},
   keepMarkdownFiles = false,
   imports = {},
+  data = {},
   fetchImpl,
   functions = {},
   quiet = false,
@@ -409,9 +530,13 @@ export function createSiteBuilder({
       readPartialsRecursively(partialsDir);
     }
 
+    if (!isObject(data)) {
+      throw new Error('Invalid site data: expected an object.');
+    }
+
     // Read all data files and create a JSON object
     const dataDir = path.join(rootDir, 'data');
-    const globalData = {};
+    const fileData = {};
 
     if (fs.existsSync(dataDir)) {
       const files = fs.readdirSync(dataDir);
@@ -421,11 +546,13 @@ export function createSiteBuilder({
           const fileContent = fs.readFileSync(filePath, 'utf8');
           const nameWithoutExt = path.basename(file, path.extname(file));
           // Load YAML content and store under filename key
-          globalData[nameWithoutExt] = yaml.load(fileContent, { schema: yaml.JSON_SCHEMA });
+          fileData[nameWithoutExt] = yaml.load(fileContent, { schema: yaml.JSON_SCHEMA });
         }
       });
     }
 
+    const globalData = deepMerge(data, fileData);
+
     // Read all templates and create a JSON object
     const templatesDir = path.join(rootDir, 'templates');
     const templates = { ...importedTemplates };
@@ -475,12 +602,11 @@ export function createSiteBuilder({
       };
     }
 
-    // Function to scan all pages and build collections
-    function buildCollections() {
-      const collections = {};
+    function collectPageEntries() {
+      const pageEntries = [];
       const pagesDir = path.join(rootDir, 'pages');
 
-      function scanPages(dir, basePath = '') {
+      function scanPages(basePath = '') {
         const fullDir = path.join(pagesDir, basePath);
         if (!fs.existsSync(fullDir)) return;
 
@@ -492,82 +618,122 @@ export function createSiteBuilder({
           const itemKind = resolveDirentKind(fs, itemPath, item);
 
           if (itemKind === 'directory') {
-            // Recursively scan subdirectories
-            scanPages(dir, relativePath);
-          } else if (itemKind === 'file' && (item.name.endsWith('.yaml') || item.name.endsWith('.yml') || item.name.endsWith('.md'))) {
-            // Extract frontmatter and content
+            scanPages(relativePath);
+          } else if (itemKind === 'file' && hasPageExtension(item.name)) {
             const { frontmatter, content } = extractFrontmatterAndContent(itemPath);
-            const { frontmatter: publicFrontmatter } = splitSystemFrontmatter(frontmatter, globalData, itemPath);
+            const { frontmatter: publicFrontmatter, bindings } = splitSystemFrontmatter(frontmatter, globalData, itemPath);
+            const hasCustomUrl = hasOwn(publicFrontmatter, 'url');
+            const url = resolvePageUrl(publicFrontmatter, relativePath, itemPath);
+            const exposedFrontmatter = hasCustomUrl
+              ? { ...publicFrontmatter, url }
+              : publicFrontmatter;
+
+            pageEntries.push({
+              pagePath: itemPath,
+              relativePath: normalizeRelativeUrlPath(relativePath),
+              isMarkdown: item.name.endsWith('.md'),
+              content,
+              frontmatter: exposedFrontmatter,
+              bindings,
+              url,
+              hasCustomUrl
+            });
+          }
+        }
+      }
 
-            // Calculate URL
-            const baseFileName = item.name.replace(/\.(yaml|yml|md)$/, '');
-            let url;
+      scanPages();
+      return pageEntries;
+    }
 
-            // Special case: index files remain at root, others become directories
-            if (baseFileName === 'index') {
-              url = basePath ? '/' + basePath.replace(/\\/g, '/') : '/';
-              if (url !== '/') {
-                url = url + '/';
-              }
-            } else {
-              const pagePath = basePath ? path.join(basePath, baseFileName) : baseFileName;
-              url = '/' + pagePath.replace(/\\/g, '/') + '/';
-            }
+    function assertUniquePageUrls(pageEntries) {
+      const seen = new Map();
+
+      for (const entry of pageEntries) {
+        const existingEntry = seen.get(entry.url);
+        if (existingEntry) {
+          throw new Error(`Duplicate page URL "${entry.url}" in ${entry.pagePath}; already used by ${existingEntry.pagePath}.`);
+        }
+        seen.set(entry.url, entry);
+      }
+    }
+
+    function assertUniqueMarkdownOutputPaths(pageEntries) {
+      if (!keepMarkdownFiles) {
+        return;
+      }
+
+      const seen = new Map();
+      for (const entry of pageEntries) {
+        if (!entry.isMarkdown) {
+          continue;
+        }
+
+        const markdownOutputRelativePath = entry.hasCustomUrl
+          ? markdownOutputRelativePathFromUrl(entry.url)
+          : entry.relativePath;
+        const existingEntry = seen.get(markdownOutputRelativePath);
+        if (existingEntry) {
+          throw new Error(`Duplicate markdown output path "${markdownOutputRelativePath}" in ${entry.pagePath}; already used by ${existingEntry.pagePath}.`);
+        }
+        seen.set(markdownOutputRelativePath, entry);
+      }
+    }
+
+    // Function to scan all pages and build collections
+    function buildCollections(pageEntries) {
+      const collections = {};
+
+      for (const entry of pageEntries) {
+        const publicFrontmatter = entry.frontmatter;
 
-            // Process tags
-            if (publicFrontmatter.tags) {
-              // Normalize tags to array
-              const tags = Array.isArray(publicFrontmatter.tags) ? publicFrontmatter.tags : [publicFrontmatter.tags];
-
-              // Add to collections
-              tags.forEach(tag => {
-                if (typeof tag === 'string' && tag.trim()) {
-                  const trimmedTag = tag.trim();
-                  if (!collections[trimmedTag]) {
-                    collections[trimmedTag] = [];
-                  }
-                  collections[trimmedTag].push({
-                    data: publicFrontmatter,
-                    url: url,
-                    content: content
-                  });
-                }
+        // Process tags
+        if (publicFrontmatter.tags) {
+          // Normalize tags to array
+          const tags = Array.isArray(publicFrontmatter.tags) ? publicFrontmatter.tags : [publicFrontmatter.tags];
+
+          // Add to collections
+          tags.forEach(tag => {
+            if (typeof tag === 'string' && tag.trim()) {
+              const trimmedTag = tag.trim();
+              if (!collections[trimmedTag]) {
+                collections[trimmedTag] = [];
+              }
+              collections[trimmedTag].push({
+                data: publicFrontmatter,
+                url: entry.url,
+                content: entry.content
               });
             }
-          }
+          });
         }
       }
 
-      scanPages('');
       return collections;
     }
 
+    const pageEntries = collectPageEntries();
+    assertUniquePageUrls(pageEntries);
+    assertUniqueMarkdownOutputPaths(pageEntries);
+
     // Build collections in first pass
     if (!quiet) console.log('Building collections...');
-    const collections = buildCollections();
+    const collections = buildCollections(pageEntries);
 
     // Function to process a single page file
-    async function processPage(pagePath, outputRelativePath, isMarkdown = false, markdownOutputRelativePath = null) {
-      if (!quiet) console.log(`Processing ${pagePath}...`);
-
-      const { frontmatter, content: rawContent } = extractFrontmatterAndContent(pagePath);
-      const { frontmatter: publicFrontmatter, bindings: boundData } = splitSystemFrontmatter(frontmatter, globalData, pagePath);
-
-      // Calculate URL for current page
-      let url;
-      const fileName = path.basename(outputRelativePath, '.html');
-      const basePath = path.dirname(outputRelativePath);
+    async function processPage(pageEntry) {
+      const {
+        pagePath,
+        content: rawContent,
+        frontmatter: publicFrontmatter,
+        bindings: boundData,
+        isMarkdown,
+        url,
+        hasCustomUrl,
+        relativePath
+      } = pageEntry;
 
-      // Special case: index files remain at root, others become directories
-      if (fileName === 'index') {
-        url = basePath && basePath !== '.' ? '/' + basePath.replace(/\\/g, '/') : '/';
-        if (url !== '/') {
-          url = url + '/';
-        }
-      } else {
-        const pagePath = basePath && basePath !== '.' ? path.join(basePath, fileName) : fileName;
-        url = '/' + pagePath.replace(/\\/g, '/') + '/';
-      }
+      if (!quiet) console.log(`Processing ${pagePath}...`);
 
       // Deep merge global data with frontmatter and collections for the page context
       const pageData = deepMerge(globalData, publicFrontmatter);
@@ -638,35 +804,9 @@ export function createSiteBuilder({
         htmlString = convertToHtml(resultArray);
       }
 
-      // Create output directory and file path for new index.html structure
-      const pageFileName = path.basename(outputRelativePath, '.html');
-      const dirPath = path.dirname(outputRelativePath);
-
-      let outputPath, outputDir;
-
-      // Special case: index files remain as index.html, others become directory/index.html
-      if (pageFileName === 'index') {
-        if (dirPath && dirPath !== '.') {
-          // Nested index file: pages/blog/index.yaml -> _site/blog/index.html
-          outputPath = path.join(outputRootDir, dirPath, 'index.html');
-          outputDir = path.join(outputRootDir, dirPath);
-        } else {
-          // Root index file: pages/index.yaml -> _site/index.html
-          outputPath = path.join(outputRootDir, 'index.html');
-          outputDir = path.join(outputRootDir);
-        }
-      } else {
-        // Regular file: pages/test.yaml -> _site/test/index.html
-        if (dirPath && dirPath !== '.') {
-          // Nested regular file: pages/blog/post.yaml -> _site/blog/post/index.html
-          outputPath = path.join(outputRootDir, dirPath, pageFileName, 'index.html');
-          outputDir = path.join(outputRootDir, dirPath, pageFileName);
-        } else {
-          // Root level regular file: pages/test.yaml -> _site/test/index.html
-          outputPath = path.join(outputRootDir, pageFileName, 'index.html');
-          outputDir = path.join(outputRootDir, pageFileName);
-        }
-      }
+      const outputRelativePath = htmlOutputRelativePathFromUrl(url);
+      const outputPath = path.join(outputRootDir, ...outputRelativePath.split('/'));
+      const outputDir = path.dirname(outputPath);
 
       if (!fs.existsSync(outputDir)) {
         fs.mkdirSync(outputDir, { recursive: true });
@@ -676,8 +816,11 @@ export function createSiteBuilder({
       fs.writeFileSync(outputPath, htmlString);
       if (!quiet) console.log(`  -> Written to ${outputPath}`);
 
-      if (isMarkdown && keepMarkdownFiles && typeof markdownOutputRelativePath === 'string') {
-        const markdownOutputPath = path.join(outputRootDir, markdownOutputRelativePath);
+      if (isMarkdown && keepMarkdownFiles) {
+        const markdownOutputRelativePath = hasCustomUrl
+          ? markdownOutputRelativePathFromUrl(url)
+          : relativePath;
+        const markdownOutputPath = path.join(outputRootDir, ...markdownOutputRelativePath.split('/'));
         const markdownOutputDir = path.dirname(markdownOutputPath);
         if (!fs.existsSync(markdownOutputDir)) {
           fs.mkdirSync(markdownOutputDir, { recursive: true });
@@ -687,37 +830,9 @@ export function createSiteBuilder({
       }
     }
 
-    // Process all YAML and Markdown files in pages directory recursively
-    async function processAllPages(dir, basePath = '') {
-      const pagesDir = path.join(rootDir, 'pages');
-      const fullDir = path.join(pagesDir, basePath);
-
-      if (!fs.existsSync(fullDir)) return;
-
-      const items = fs.readdirSync(fullDir, { withFileTypes: true });
-
-      for (const item of items) {
-        const itemPath = path.join(fullDir, item.name);
-        const relativePath = basePath ? path.join(basePath, item.name) : item.name;
-        const itemKind = resolveDirentKind(fs, itemPath, item);
-
-        if (itemKind === 'directory') {
-          // Recursively process subdirectories
-          await processAllPages(dir, relativePath);
-        } else if (itemKind === 'file') {
-          if (item.name.endsWith('.yaml') || item.name.endsWith('.yml')) {
-            // Process YAML file
-            const outputFileName = item.name.replace(/\.(yaml|yml)$/, '.html');
-            const outputRelativePath = basePath ? path.join(basePath, outputFileName) : outputFileName;
-            await processPage(itemPath, outputRelativePath, false);
-          } else if (item.name.endsWith('.md')) {
-            // Process Markdown file
-            const outputFileName = item.name.replace('.md', '.html');
-            const outputRelativePath = basePath ? path.join(basePath, outputFileName) : outputFileName;
-            await processPage(itemPath, outputRelativePath, true, relativePath);
-          }
-          // Ignore other file types
-        }
+    async function processAllPages() {
+      for (const pageEntry of pageEntries) {
+        await processPage(pageEntry);
       }
     }
 
@@ -774,7 +889,7 @@ export function createSiteBuilder({
     copyStaticFiles();
 
     // Process all pages (can overwrite static files)
-    await processAllPages('');
+    await processAllPages();
 
     if (!quiet) console.log('Build complete!');
   };

package/src/utils/loadSiteConfig.js

@@ -2,7 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import yaml from 'js-yaml';
 
-const ALLOWED_TOP_LEVEL_KEYS = new Set(['markdown', 'markdownit', 'build', 'imports']);
+const ALLOWED_TOP_LEVEL_KEYS = new Set(['markdown', 'markdownit', 'build', 'imports', 'data']);
 const MARKDOWN_BOOLEAN_KEYS = new Set(['html', 'linkify', 'typographer', 'breaks', 'xhtmlOut']);
 const MARKDOWN_STRING_KEYS = new Set(['langPrefix', 'quotes', 'preset']);
 const MARKDOWN_NUMBER_KEYS = new Set(['maxNesting']);
@@ -121,6 +121,25 @@ function validateBuildConfig(value, configPath) {
   return { ...value };
 }
 
+function validateDataConfig(value, configPath) {
+  if (!isPlainObject(value)) {
+    throw new Error(`Invalid data config in "${configPath}": expected an object.`);
+  }
+
+  const normalized = {};
+
+  for (const [rawKey, rawValue] of Object.entries(value)) {
+    const key = String(rawKey).trim();
+    if (key === '') {
+      throw new Error(`Invalid data config in "${configPath}": keys must be non-empty.`);
+    }
+
+    normalized[key] = rawValue;
+  }
+
+  return normalized;
+}
+
 function validateImportUrl(url, configPath, groupKey, alias) {
   if (typeof url !== 'string' || url.trim() === '') {
     throw new Error(`Invalid imports.${groupKey} value for alias "${alias}" in "${configPath}": expected a non-empty URL string.`);
@@ -200,7 +219,7 @@ function validateConfig(rawConfig, configPath) {
   for (const key of Object.keys(config)) {
     if (!ALLOWED_TOP_LEVEL_KEYS.has(key)) {
       throw new Error(
-        `Unsupported key "${key}" in "${configPath}". Supported keys: markdownit (recommended), markdown (legacy alias), build, imports.`
+        `Unsupported key "${key}" in "${configPath}". Supported keys: markdownit (recommended), markdown (legacy alias), build, imports, data.`
       );
     }
   }
@@ -292,6 +311,10 @@ function validateConfig(rawConfig, configPath) {
     normalizedConfig.imports = validateImportsConfig(config.imports, configPath);
   }
 
+  if (config.data !== undefined) {
+    normalizedConfig.data = validateDataConfig(config.data, configPath);
+  }
+
   return normalizedConfig;
 }