@uniweb/build 0.1.27 → 0.1.28

package/src/site/collection-processor.js

@@ -0,0 +1,382 @@
+/**
+ * Collection Processor
+ *
+ * Processes content collections from markdown files into JSON data.
+ * Collections are defined in site.yml and processed at build time.
+ *
+ * Features:
+ * - Discovers markdown files in content folders
+ * - Parses frontmatter for metadata
+ * - Converts markdown body to ProseMirror JSON
+ * - Supports filtering, sorting, and limiting
+ * - Auto-generates excerpts and extracts first images
+ *
+ * @module @uniweb/build/site/collection-processor
+ *
+ * @example
+ * // site.yml
+ * collections:
+ *   articles:
+ *     path: content/articles
+ *     sort: date desc
+ *
+ * // Usage
+ * const collections = await processCollections(siteDir, config.collections)
+ * await writeCollectionFiles(siteDir, collections)
+ */
+
+import { readFile, readdir, stat, writeFile, mkdir } from 'node:fs/promises'
+import { join, basename, extname } from 'node:path'
+import { existsSync } from 'node:fs'
+import yaml from 'js-yaml'
+import { applyFilter, applySort } from './data-fetcher.js'
+
+// Try to import content-reader for markdown parsing
+let markdownToProseMirror
+try {
+  const contentReader = await import('@uniweb/content-reader')
+  markdownToProseMirror = contentReader.markdownToProseMirror
+} catch {
+  // Simplified fallback
+  markdownToProseMirror = (markdown) => ({
+    type: 'doc',
+    content: [
+      {
+        type: 'paragraph',
+        content: [{ type: 'text', text: markdown.trim() }]
+      }
+    ]
+  })
+}
+
+/**
+ * Parse collection config from site.yml
+ *
+ * @param {string} name - Collection name
+ * @param {string|Object} config - Simple path string or full config object
+ * @returns {Object} Normalized config
+ *
+ * @example
+ * // Simple form
+ * parseCollectionConfig('articles', 'content/articles')
+ *
+ * // Extended form
+ * parseCollectionConfig('articles', {
+ *   path: 'content/articles',
+ *   sort: 'date desc',
+ *   filter: 'published != false',
+ *   limit: 100
+ * })
+ */
+function parseCollectionConfig(name, config) {
+  if (typeof config === 'string') {
+    return {
+      name,
+      path: config,
+      sort: null,
+      filter: null,
+      limit: 0,
+      excerpt: { maxLength: 160 }
+    }
+  }
+
+  return {
+    name,
+    path: config.path,
+    sort: config.sort || null,
+    filter: config.filter || null,
+    limit: config.limit || 0,
+    excerpt: {
+      maxLength: config.excerpt?.maxLength || 160,
+      field: config.excerpt?.field || null
+    }
+  }
+}
+
+/**
+ * Parse YAML frontmatter from markdown content
+ *
+ * @param {string} raw - Raw file content
+ * @returns {{ frontmatter: Object, body: string }}
+ */
+function parseFrontmatter(raw) {
+  if (!raw.trim().startsWith('---')) {
+    return { frontmatter: {}, body: raw }
+  }
+
+  const parts = raw.split('---\n')
+  if (parts.length < 3) {
+    return { frontmatter: {}, body: raw }
+  }
+
+  try {
+    const frontmatter = yaml.load(parts[1]) || {}
+    const body = parts.slice(2).join('---\n')
+    return { frontmatter, body }
+  } catch (err) {
+    console.warn('[collection-processor] YAML parse error:', err.message)
+    return { frontmatter: {}, body: raw }
+  }
+}
+
+/**
+ * Extract plain text from ProseMirror content
+ *
+ * @param {Object} node - ProseMirror node
+ * @returns {string} Plain text
+ */
+function extractPlainText(node) {
+  if (!node) return ''
+
+  if (node.type === 'text') {
+    return node.text || ''
+  }
+
+  if (Array.isArray(node.content)) {
+    return node.content.map(extractPlainText).join('')
+  }
+
+  return ''
+}
+
+/**
+ * Extract excerpt from content
+ *
+ * @param {Object} frontmatter - Parsed frontmatter
+ * @param {Object} content - ProseMirror content
+ * @param {Object} excerptConfig - Excerpt configuration
+ * @returns {string} Excerpt text
+ */
+function extractExcerpt(frontmatter, content, excerptConfig) {
+  const { maxLength = 160, field = null } = excerptConfig || {}
+
+  // Check for explicit excerpt in frontmatter
+  if (frontmatter.excerpt) {
+    return frontmatter.excerpt.slice(0, maxLength)
+  }
+
+  // Check for alternative field (e.g., 'description')
+  if (field && frontmatter[field]) {
+    return frontmatter[field].slice(0, maxLength)
+  }
+
+  // Auto-extract from content
+  const text = extractPlainText(content)
+  if (!text) return ''
+
+  // Clean and truncate
+  const cleaned = text.replace(/\s+/g, ' ').trim()
+  if (cleaned.length <= maxLength) return cleaned
+
+  // Truncate at word boundary
+  const truncated = cleaned.slice(0, maxLength)
+  const lastSpace = truncated.lastIndexOf(' ')
+  return lastSpace > maxLength * 0.7
+    ? truncated.slice(0, lastSpace) + '...'
+    : truncated + '...'
+}
+
+/**
+ * Extract first image from ProseMirror content
+ *
+ * @param {Object} node - ProseMirror node
+ * @returns {string|null} Image URL or null
+ */
+function extractFirstImage(node) {
+  if (!node) return null
+
+  if (node.type === 'image' && node.attrs?.src) {
+    return node.attrs.src
+  }
+
+  if (Array.isArray(node.content)) {
+    for (const child of node.content) {
+      const img = extractFirstImage(child)
+      if (img) return img
+    }
+  }
+
+  return null
+}
+
+// Filter and sort utilities are imported from data-fetcher.js
+
+/**
+ * Process a single content item from a markdown file
+ *
+ * @param {string} dir - Collection directory path
+ * @param {string} filename - Markdown filename
+ * @param {Object} config - Collection configuration
+ * @returns {Promise<Object|null>} Processed item or null if unpublished
+ */
+async function processContentItem(dir, filename, config) {
+  const filepath = join(dir, filename)
+  const raw = await readFile(filepath, 'utf-8')
+  const slug = basename(filename, extname(filename))
+
+  // Parse frontmatter and body
+  const { frontmatter, body } = parseFrontmatter(raw)
+
+  // Skip unpublished items by default
+  if (frontmatter.published === false) {
+    return null
+  }
+
+  // Parse markdown body to ProseMirror
+  const content = markdownToProseMirror(body)
+
+  // Extract excerpt
+  const excerpt = extractExcerpt(frontmatter, content, config.excerpt)
+
+  // Extract first image (frontmatter takes precedence)
+  const image = frontmatter.image || extractFirstImage(content)
+
+  // Get file stats for lastModified
+  const fileStat = await stat(filepath)
+
+  return {
+    slug,
+    ...frontmatter,
+    excerpt,
+    image,
+    // Include both raw markdown body (for simple rendering)
+    // and ProseMirror content (for rich rendering)
+    body: body.trim(),
+    content,
+    lastModified: fileStat.mtime.toISOString()
+  }
+}
+
+/**
+ * Collect and process all items in a collection folder
+ *
+ * @param {string} siteDir - Site root directory
+ * @param {Object} config - Parsed collection config
+ * @returns {Promise<Array>} Array of processed items
+ */
+async function collectItems(siteDir, config) {
+  const collectionDir = join(siteDir, config.path)
+
+  // Check if collection directory exists
+  if (!existsSync(collectionDir)) {
+    console.warn(`[collection-processor] Collection folder not found: ${config.path}`)
+    return []
+  }
+
+  const files = await readdir(collectionDir)
+  const mdFiles = files.filter(f => f.endsWith('.md') && !f.startsWith('_'))
+
+  // Process all markdown files
+  let items = await Promise.all(
+    mdFiles.map(file => processContentItem(collectionDir, file, config))
+  )
+
+  // Filter out nulls (unpublished items)
+  items = items.filter(Boolean)
+
+  // Apply custom filter
+  if (config.filter) {
+    items = applyFilter(items, config.filter)
+  }
+
+  // Apply sort
+  if (config.sort) {
+    items = applySort(items, config.sort)
+  }
+
+  // Apply limit
+  if (config.limit > 0) {
+    items = items.slice(0, config.limit)
+  }
+
+  return items
+}
+
+/**
+ * Process all content collections defined in site.yml
+ *
+ * @param {string} siteDir - Site root directory
+ * @param {Object} collectionsConfig - Collections config from site.yml
+ * @returns {Promise<Object>} Map of collection name to items array
+ *
+ * @example
+ * const collections = await processCollections('/path/to/site', {
+ *   articles: { path: 'content/articles', sort: 'date desc' },
+ *   products: 'content/products'
+ * })
+ * // { articles: [...], products: [...] }
+ */
+export async function processCollections(siteDir, collectionsConfig) {
+  if (!collectionsConfig || typeof collectionsConfig !== 'object') {
+    return {}
+  }
+
+  const results = {}
+
+  for (const [name, config] of Object.entries(collectionsConfig)) {
+    const parsed = parseCollectionConfig(name, config)
+    const items = await collectItems(siteDir, parsed)
+    results[name] = items
+    console.log(`[collection-processor] Processed ${name}: ${items.length} items`)
+  }
+
+  return results
+}
+
+/**
+ * Write collection data to JSON files in public/data/
+ *
+ * @param {string} siteDir - Site root directory
+ * @param {Object} collections - Map of collection name to items array
+ * @returns {Promise<void>}
+ *
+ * @example
+ * await writeCollectionFiles('/path/to/site', {
+ *   articles: [{ slug: 'hello', title: 'Hello World', ... }]
+ * })
+ * // Creates public/data/articles.json
+ */
+export async function writeCollectionFiles(siteDir, collections) {
+  if (!collections || Object.keys(collections).length === 0) {
+    return
+  }
+
+  const dataDir = join(siteDir, 'public', 'data')
+  await mkdir(dataDir, { recursive: true })
+
+  for (const [name, items] of Object.entries(collections)) {
+    const filepath = join(dataDir, `${name}.json`)
+    await writeFile(filepath, JSON.stringify(items, null, 2))
+    console.log(`[collection-processor] Generated ${filepath} (${items.length} items)`)
+  }
+}
+
+/**
+ * Get last modified time for a collection
+ *
+ * @param {string} siteDir - Site root directory
+ * @param {Object} config - Collection config
+ * @returns {Promise<Date|null>} Most recent modification time
+ */
+export async function getCollectionLastModified(siteDir, config) {
+  const parsed = parseCollectionConfig('temp', config)
+  const collectionDir = join(siteDir, parsed.path)
+
+  if (!existsSync(collectionDir)) {
+    return null
+  }
+
+  const files = await readdir(collectionDir)
+  const mdFiles = files.filter(f => f.endsWith('.md') && !f.startsWith('_'))
+
+  let lastModified = null
+
+  for (const file of mdFiles) {
+    const fileStat = await stat(join(collectionDir, file))
+    if (!lastModified || fileStat.mtime > lastModified) {
+      lastModified = fileStat.mtime
+    }
+  }
+
+  return lastModified
+}

package/src/site/content-collector.js

@@ -12,6 +12,7 @@
  * - preset: Preset configuration name
  * - input: Input field mapping
  * - props: Additional component props (merged with other params)
+ * - fetch: Data fetching configuration (path, url, schema, prerender, merge, transform)
  *
  * Note: `component` is supported as an alias for `type` (deprecated)
  *
@@ -26,6 +27,7 @@ import { join, parse } from 'node:path'
 import { existsSync } from 'node:fs'
 import yaml from 'js-yaml'
 import { collectSectionAssets, mergeAssetCollections } from './assets.js'
+import { parseFetchConfig, singularize } from './data-fetcher.js'
 
 // Try to import content-reader, fall back to simplified parser
 let markdownToProseMirror
@@ -45,6 +47,25 @@ try {
   })
 }
 
+/**
+ * Check if a folder name represents a dynamic route (e.g., [slug], [id])
+ * @param {string} folderName - The folder name to check
+ * @returns {boolean}
+ */
+function isDynamicRoute(folderName) {
+  return /^\[(\w+)\]$/.test(folderName)
+}
+
+/**
+ * Extract the parameter name from a dynamic route folder (e.g., [slug] → slug)
+ * @param {string} folderName - The folder name (e.g., "[slug]")
+ * @returns {string|null} The parameter name or null if not a dynamic route
+ */
+function extractRouteParam(folderName) {
+  const match = folderName.match(/^\[(\w+)\]$/)
+  return match ? match[1] : null
+}
+
 /**
  * Parse YAML string using js-yaml
  */
@@ -140,7 +161,7 @@ async function processMarkdownFile(filePath, id, siteRoot) {
     }
   }
 
-  const { type, component, preset, input, props, ...params } = frontMatter
+  const { type, component, preset, input, props, fetch, ...params } = frontMatter
 
   // Convert markdown to ProseMirror
   const proseMirrorContent = markdownToProseMirror(markdown)
@@ -152,6 +173,7 @@ async function processMarkdownFile(filePath, id, siteRoot) {
     input,
     params: { ...params, ...props },
     content: proseMirrorContent,
+    fetch: parseFetchConfig(fetch),
     subsections: []
   }
 
@@ -293,9 +315,10 @@ async function processExplicitSections(sectionsConfig, pagePath, siteRoot, paren
  * @param {Object} options - Route options
  * @param {boolean} options.isIndex - Whether this page is the index for its parent route
  * @param {string} options.parentRoute - The parent route (e.g., '/' or '/docs')
+ * @param {Object} options.parentFetch - Parent page's fetch config (for dynamic routes)
  * @returns {Object} Page data with assets manifest
  */
-async function processPage(pagePath, pageName, siteRoot, { isIndex = false, parentRoute = '/' } = {}) {
+async function processPage(pagePath, pageName, siteRoot, { isIndex = false, parentRoute = '/', parentFetch = null } = {}) {
   const pageConfig = await readYamlFile(join(pagePath, 'page.yml'))
 
   // Note: We no longer skip hidden pages here - they still exist as valid pages,
@@ -354,9 +377,16 @@ async function processPage(pagePath, pageName, siteRoot, { isIndex = false, pare
   // All pages get their actual folder-based route (no special treatment for index)
   // The isIndex flag marks which page should also be accessible at the parent route
   let route
+  const isDynamic = isDynamicRoute(pageName)
+  const paramName = isDynamic ? extractRouteParam(pageName) : null
+
   if (pageName.startsWith('@')) {
     // Special pages (layout areas) keep their @ prefix
     route = parentRoute === '/' ? `/@${pageName.slice(1)}` : `${parentRoute}/@${pageName.slice(1)}`
+  } else if (isDynamic) {
+    // Dynamic routes: /blog/[slug] → /blog/:slug (for route matching)
+    // The actual routes like /blog/my-post are generated at prerender time
+    route = parentRoute === '/' ? `/:${paramName}` : `${parentRoute}/:${paramName}`
   } else {
     // Normal pages get parent + their name
     route = parentRoute === '/' ? `/${pageName}` : `${parentRoute}/${pageName}`
@@ -365,6 +395,13 @@ async function processPage(pagePath, pageName, siteRoot, { isIndex = false, pare
   // Extract configuration
   const { seo = {}, layout = {}, ...restConfig } = pageConfig
 
+  // For dynamic routes, determine the parent's data schema
+  // This tells prerender which data array to iterate over
+  let parentSchema = null
+  if (isDynamic && parentFetch) {
+    parentSchema = parentFetch.schema
+  }
+
   return {
     page: {
       route,
@@ -375,6 +412,11 @@ async function processPage(pagePath, pageName, siteRoot, { isIndex = false, pare
       order: pageConfig.order,
       lastModified: lastModified?.toISOString(),
 
+      // Dynamic route metadata
+      isDynamic,
+      paramName, // e.g., "slug" from [slug]
+      parentSchema, // e.g., "articles" - the data array to iterate over
+
       // Navigation options
       hidden: pageConfig.hidden || false, // Hide from all navigation
       hideInHeader: pageConfig.hideInHeader || false, // Hide from header nav
@@ -394,6 +436,10 @@ async function processPage(pagePath, pageName, siteRoot, { isIndex = false, pare
         changefreq: seo.changefreq || null,
         priority: seo.priority || null
       },
+
+      // Data fetching
+      fetch: parseFetchConfig(pageConfig.fetch),
+
       sections: hierarchicalSections
     },
     assetCollection: pageAssetCollection
@@ -441,9 +487,10 @@ function determineIndexPage(orderConfig, availableFolders) {
  * @param {string} parentRoute - Parent route (e.g., '/' or '/docs')
  * @param {string} siteRoot - Site root directory for asset resolution
  * @param {Object} orderConfig - { pages: [...], index: 'name' } from parent's config
+ * @param {Object} parentFetch - Parent page's fetch config (for dynamic child routes)
  * @returns {Promise<Object>} { pages, assetCollection, header, footer, left, right }
  */
-async function collectPagesRecursive(dirPath, parentRoute, siteRoot, orderConfig = {}) {
+async function collectPagesRecursive(dirPath, parentRoute, siteRoot, orderConfig = {}, parentFetch = null) {
   const entries = await readdir(dirPath)
   const pages = []
   let assetCollection = {
@@ -487,9 +534,11 @@ async function collectPagesRecursive(dirPath, parentRoute, siteRoot, orderConfig
     const isSpecial = entry.startsWith('@')
 
     // Process this directory as a page
+    // Pass parentFetch so dynamic routes can inherit parent's data schema
     const result = await processPage(entryPath, entry, siteRoot, {
       isIndex: isIndex && !isSpecial,
-      parentRoute
+      parentRoute,
+      parentFetch
     })
 
     if (result) {
@@ -517,7 +566,9 @@ async function collectPagesRecursive(dirPath, parentRoute, siteRoot, orderConfig
       if (!isSpecial) {
         // The child route depends on whether this page is the index
         const childParentRoute = isIndex ? parentRoute : page.route
-        const subResult = await collectPagesRecursive(entryPath, childParentRoute, siteRoot, childOrderConfig)
+        // Pass this page's fetch config to children (for dynamic routes that inherit parent data)
+        const childFetch = page.fetch || parentFetch
+        const subResult = await collectPagesRecursive(entryPath, childParentRoute, siteRoot, childOrderConfig, childFetch)
         pages.push(...subResult.pages)
         assetCollection = mergeAssetCollections(assetCollection, subResult.assetCollection)
       }
@@ -571,7 +622,10 @@ export async function collectSiteContent(sitePath) {
   }
 
   return {
-    config: siteConfig,
+    config: {
+      ...siteConfig,
+      fetch: parseFetchConfig(siteConfig.fetch),
+    },
     theme: themeConfig,
     pages,
     header,