@uniweb/build 0.15.5 → 0.15.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniweb/build",
-  "version": "0.15.5",
+  "version": "0.15.6",
   "description": "Build tooling for the Uniweb Component Web Platform",
   "type": "module",
   "exports": {
@@ -59,12 +59,13 @@
     "js-yaml": "^4.1.0",
     "sharp": "^0.33.2",
     "yaml": "^2.5.0",
+    "@uniweb/projections": "0.1.1",
     "@uniweb/theming": "0.1.14",
-    "@uniweb/content-writer": "0.2.7"
+    "@uniweb/content-writer": "0.2.8"
   },
   "optionalDependencies": {
-    "@uniweb/content-reader": "1.1.15",
-    "@uniweb/runtime": "0.8.36",
+    "@uniweb/content-reader": "1.1.16",
+    "@uniweb/runtime": "0.8.37",
     "@uniweb/schemas": "0.2.4"
   },
   "peerDependencies": {
@@ -74,7 +75,7 @@
     "@tailwindcss/vite": "^4.0.0",
     "@vitejs/plugin-react": "^4.0.0 || ^5.0.0",
     "vite-plugin-svgr": "^4.0.0",
-    "@uniweb/core": "0.7.29"
+    "@uniweb/core": "0.7.30"
   },
   "peerDependenciesMeta": {
     "vite": {

package/src/search/collections.js

@@ -1,54 +1,2 @@
-/**
- * Generate search index for a file-based collection.
- *
- * Collection cascade files (`data/{name}.json`) contain all non-deferred fields.
- * If `search.fetchDetail: true` is set, the caller is responsible for merging
- * per-record detail files into each item before calling this function.
- */
-
-/**
- * @param {string} name - Collection name (e.g. "articles")
- * @param {Object} config - Collection config from site.yml (config.collections[name])
- * @param {Object} collectionData - Parsed cascade JSON (`data/{name}.json`)
- * @param {string} locale - Locale code (e.g. "en")
- * @returns {Object} Collection search index
- */
-export function generateCollectionIndex(name, config, collectionData, locale) {
-  const fields = config.search?.fields || ['title']
-  const weight = config.search?.weight ?? 0.7
-  const items = collectionData?.items || []
-
-  const entries = items.map(item => {
-    const content = fields.map(f => item[f] || '').filter(Boolean).join(' ')
-    const slug = item.slug || item.id || String(item.title || '').toLowerCase().replace(/\s+/g, '-')
-    return {
-      id: `collection:${name}:${slug}`,
-      type: 'collection',
-      collection: name,
-      route: `${config.route}/${slug}`,
-      title: item.title || item.name || slug,
-      content,
-      excerpt: content.length > 160
-        ? content.slice(0, 160).trim() + '…'
-        : content,
-      weight,
-      item: pickDisplayFields(item),
-    }
-  })
-
-  return {
-    type: 'collection',
-    collection: name,
-    locale,
-    generated: new Date().toISOString(),
-    entries,
-  }
-}
-
-function pickDisplayFields(item) {
-  const { slug, title, name, date, image, author, excerpt, role } = item
-  return Object.fromEntries(
-    Object.entries({ slug, title, name, date, image, author, excerpt, role })
-      .filter(([, v]) => v != null)
-  )
-}
+/** @deprecated Import from `@uniweb/projections` — kept so existing paths resolve. */
+export { generateCollectionIndex } from '@uniweb/projections/search'

package/src/search/extract.js

@@ -1,335 +1,2 @@
-/**
- * Extract searchable content from site-content.json
- *
- * Walks through all pages and sections, extracting text content
- * for search indexing. Reuses patterns from i18n extraction.
- */
-
-/**
- * Extract all searchable content from site
- * @param {Object} siteContent - Parsed site-content.json
- * @param {Object} options - Extraction options
- * @param {boolean} [options.pages=true] - Include page metadata
- * @param {boolean} [options.sections=true] - Include section content
- * @param {boolean} [options.headings=true] - Include headings
- * @param {boolean} [options.paragraphs=true] - Include paragraphs
- * @param {boolean} [options.links=true] - Include link labels
- * @param {boolean} [options.lists=true] - Include list items
- * @param {Array<string>} [options.excludeRoutes=[]] - Routes to exclude
- * @param {Array<string>} [options.excludeComponents=[]] - Components to exclude
- * @returns {Array<Object>} Array of search entries
- */
-export function extractSearchContent(siteContent, options = {}) {
-  const {
-    pages: includePagesFlag = true,
-    sections: includeSections = true,
-    headings: includeHeadings = true,
-    paragraphs: includeParagraphs = true,
-    links: includeLinks = true,
-    lists: includeLists = true,
-    excludeRoutes = [],
-    excludeComponents = []
-  } = options
-
-  const entries = []
-
-  for (const page of siteContent.pages || []) {
-    const pageRoute = page.route || '/'
-
-    // Skip excluded routes
-    if (excludeRoutes.some(r => pageRoute.startsWith(r))) {
-      continue
-    }
-
-    // Skip pages marked as noindex
-    if (page.seo?.noindex) {
-      continue
-    }
-
-    // Extract page-level entry
-    if (includePagesFlag) {
-      const pageEntry = extractFromPage(page)
-      if (pageEntry) {
-        entries.push(pageEntry)
-      }
-    }
-
-    // Extract section-level entries
-    if (includeSections) {
-      for (const section of page.sections || []) {
-        const sectionEntries = extractFromSection(section, page, {
-          includeHeadings,
-          includeParagraphs,
-          includeLinks,
-          includeLists,
-          excludeComponents
-        })
-        entries.push(...sectionEntries)
-      }
-    }
-  }
-
-  return entries
-}
-
-/**
- * Extract search entry from page metadata
- * @param {Object} page - Page data
- * @returns {Object|null} Search entry or null
- */
-function extractFromPage(page) {
-  const route = page.route || '/'
-  const title = page.title || ''
-  const description = page.description || ''
-  const keywords = page.keywords || page.seo?.keywords || []
-
-  // Skip pages with no meaningful content
-  if (!title && !description) {
-    return null
-  }
-
-  return {
-    id: `page:${route}`,
-    type: 'page',
-    route,
-    title,
-    description,
-    keywords: Array.isArray(keywords) ? keywords : [keywords].filter(Boolean),
-    content: [title, description].filter(Boolean).join(' '),
-    // Boost factor for search ranking (pages are more important)
-    weight: route === '/' ? 1.0 : 0.8
-  }
-}
-
-/**
- * Extract search entries from a section (and subsections)
- * @param {Object} section - Section data
- * @param {Object} page - Parent page
- * @param {Object} options - Extraction options
- * @returns {Array<Object>} Array of search entries
- */
-function extractFromSection(section, page, options) {
-  const entries = []
-  const {
-    includeHeadings,
-    includeParagraphs,
-    includeLinks,
-    includeLists,
-    excludeComponents
-  } = options
-
-  const sectionId = section.id || 'unknown'
-  const component = section.component || section.type || 'unknown'
-
-  // Skip excluded components
-  if (excludeComponents.includes(component)) {
-    return entries
-  }
-
-  // Extract text from ProseMirror content
-  const textParts = []
-  let sectionTitle = ''
-
-  if (section.content?.type === 'doc') {
-    const extracted = extractFromProseMirrorDoc(section.content, {
-      includeHeadings,
-      includeParagraphs,
-      includeLinks,
-      includeLists
-    })
-
-    sectionTitle = extracted.title || ''
-    textParts.push(...extracted.textParts)
-  }
-
-  // Also check params for title (from YAML frontmatter)
-  if (!sectionTitle && section.params?.title) {
-    sectionTitle = section.params.title
-  }
-
-  // Build content string
-  const content = textParts.join(' ').trim()
-
-  // Create entry if there's meaningful content
-  if (sectionTitle || content) {
-    entries.push({
-      id: `section:${page.route}:${sectionId}`,
-      type: 'section',
-      route: page.route,
-      sectionId,
-      anchor: `Section${sectionId}`,
-      component,
-      title: sectionTitle,
-      pageTitle: page.title || '',
-      content,
-      // Generate excerpt (first ~160 chars)
-      excerpt: generateExcerpt(content, 160),
-      // Section weight is lower than page
-      weight: 0.6
-    })
-  }
-
-  // Recursively process subsections
-  for (const subsection of section.subsections || []) {
-    const subEntries = extractFromSection(subsection, page, options)
-    entries.push(...subEntries)
-  }
-
-  return entries
-}
-
-/**
- * Extract text from ProseMirror document
- * @param {Object} doc - ProseMirror document
- * @param {Object} options - Extraction options
- * @returns {Object} Extracted content { title, textParts }
- */
-function extractFromProseMirrorDoc(doc, options) {
-  const { includeHeadings, includeParagraphs, includeLinks, includeLists } = options
-  const textParts = []
-  let title = ''
-  let foundFirstHeading = false
-
-  if (!doc.content) {
-    return { title, textParts }
-  }
-
-  for (const node of doc.content) {
-    if (node.type === 'heading') {
-      const text = extractTextFromNode(node)
-      if (!text) continue
-
-      // First H1 becomes the title
-      if (!foundFirstHeading && node.attrs?.level === 1) {
-        title = text
-        foundFirstHeading = true
-      }
-
-      if (includeHeadings) {
-        textParts.push(text)
-      }
-    } else if (node.type === 'paragraph') {
-      if (includeParagraphs) {
-        const text = extractTextFromNode(node)
-        if (text) {
-          textParts.push(text)
-        }
-      }
-
-      // Extract link labels separately if requested
-      if (includeLinks) {
-        const links = extractLinksFromNode(node)
-        for (const link of links) {
-          if (link.label) {
-            textParts.push(link.label)
-          }
-        }
-      }
-    } else if ((node.type === 'bulletList' || node.type === 'orderedList') && includeLists) {
-      const listTexts = extractFromList(node)
-      textParts.push(...listTexts)
-    }
-  }
-
-  return { title, textParts }
-}
-
-/**
- * Extract all text content from a node
- * @param {Object} node - ProseMirror node
- * @returns {string} Text content
- */
-function extractTextFromNode(node) {
-  if (!node.content) return ''
-
-  const texts = []
-
-  for (const child of node.content) {
-    if (child.type === 'text') {
-      texts.push(child.text || '')
-    } else if (child.content) {
-      // Recurse for nested nodes
-      texts.push(extractTextFromNode(child))
-    }
-  }
-
-  return texts.join('').trim()
-}
-
-/**
- * Extract links from a paragraph node
- * @param {Object} node - Paragraph node
- * @returns {Array<{label: string, href: string}>} Links
- */
-function extractLinksFromNode(node) {
-  const links = []
-
-  if (!node.content) return links
-
-  for (const child of node.content) {
-    if (child.type === 'text' && child.marks) {
-      const linkMark = child.marks.find(m => m.type === 'link')
-      if (linkMark) {
-        links.push({
-          label: child.text || '',
-          href: linkMark.attrs?.href || ''
-        })
-      }
-    }
-  }
-
-  return links
-}
-
-/**
- * Extract text from list items
- * @param {Object} listNode - List node
- * @returns {Array<string>} List item texts
- */
-function extractFromList(listNode) {
-  const texts = []
-
-  if (!listNode.content) return texts
-
-  for (const listItem of listNode.content) {
-    if (listItem.type === 'listItem' && listItem.content) {
-      for (const child of listItem.content) {
-        if (child.type === 'paragraph') {
-          const text = extractTextFromNode(child)
-          if (text) {
-            texts.push(text)
-          }
-        }
-      }
-    }
-  }
-
-  return texts
-}
-
-/**
- * Generate excerpt from content
- * @param {string} content - Full content
- * @param {number} maxLength - Maximum length
- * @returns {string} Excerpt
- */
-function generateExcerpt(content, maxLength = 160) {
-  if (!content) return ''
-
-  // Normalize whitespace
-  const normalized = content.replace(/\s+/g, ' ').trim()
-
-  if (normalized.length <= maxLength) {
-    return normalized
-  }
-
-  // Find a good break point (word boundary)
-  let breakPoint = normalized.lastIndexOf(' ', maxLength)
-  if (breakPoint < maxLength * 0.5) {
-    breakPoint = maxLength
-  }
-
-  return normalized.slice(0, breakPoint).trim() + '…'
-}
-
-export default extractSearchContent
+/** @deprecated Import from `@uniweb/projections` — kept so existing paths resolve. */
+export { extractSearchContent, extractSearchContent as default } from '@uniweb/projections/search'

package/src/search/generate.js

@@ -1,108 +1,8 @@
-/**
- * Generate search index from site content
- *
- * Creates a JSON search index file that can be loaded at runtime
- * for client-side search functionality.
- */
-
-import { resolveDefaultLocale } from '@uniweb/core'
-import { extractSearchContent } from './extract.js'
-
-/**
- * Generate search index for a site
- * @param {Object} siteContent - Parsed site-content.json
- * @param {Object} options - Generation options
- * @param {string} [options.locale] - Locale code for this index
- * @param {Object} [options.extract] - Options passed to extractSearchContent
- * @param {Object} [options.search] - Search configuration from site.yml
- * @returns {Object} Search index object
- */
-export function generateSearchIndex(siteContent, options = {}) {
-  const {
-    locale = siteContent.config?.activeLocale || resolveDefaultLocale(siteContent.config),
-    extract: extractOptions = {},
-    search: searchConfig = {}
-  } = options
-
-  // Merge search config with extract options
-  const mergedExtractOptions = {
-    ...extractOptions,
-    excludeRoutes: searchConfig.exclude?.routes || extractOptions.excludeRoutes || [],
-    excludeComponents: searchConfig.exclude?.components || extractOptions.excludeComponents || []
-  }
-
-  // Check what to include from config
-  if (searchConfig.include) {
-    mergedExtractOptions.pages = searchConfig.include.pages !== false
-    mergedExtractOptions.sections = searchConfig.include.sections !== false
-    mergedExtractOptions.headings = searchConfig.include.headings !== false
-    mergedExtractOptions.paragraphs = searchConfig.include.paragraphs !== false
-    mergedExtractOptions.links = searchConfig.include.links !== false
-    mergedExtractOptions.lists = searchConfig.include.lists !== false
-  }
-
-  // Extract searchable content
-  const entries = extractSearchContent(siteContent, mergedExtractOptions)
-
-  // Build the index
-  const index = {
-    version: '1.0',
-    locale,
-    generated: new Date().toISOString(),
-    count: entries.length,
-    entries
-  }
-
-  return index
-}
-
-/**
- * Check if search is enabled for a site
- * @param {Object} siteContent - Parsed site-content.json
- * @returns {boolean}
- */
-export function isSearchEnabled(siteContent) {
-  // Search is enabled by default unless explicitly disabled
-  return siteContent.config?.search?.enabled !== false
-}
-
-/**
- * Get search configuration from site content
- * @param {Object} siteContent - Parsed site-content.json
- * @returns {Object} Search configuration
- */
-export function getSearchConfig(siteContent) {
-  const config = siteContent.config?.search || {}
-
-  return {
-    enabled: config.enabled !== false,
-    include: {
-      pages: config.include?.pages !== false,
-      sections: config.include?.sections !== false,
-      headings: config.include?.headings !== false,
-      paragraphs: config.include?.paragraphs !== false,
-      links: config.include?.links !== false,
-      lists: config.include?.lists !== false
-    },
-    exclude: {
-      routes: config.exclude?.routes || [],
-      components: config.exclude?.components || []
-    }
-  }
-}
-
-/**
- * Get the search index filename for a locale
- * @param {string} locale - Locale code
- * @param {string} defaultLocale - Default locale code
- * @returns {string} Filename
- */
-export function getSearchIndexFilename(locale, defaultLocale) {
-  // Default locale uses root filename, others use locale prefix path
-  if (locale === defaultLocale) {
-    return 'search-index.json'
-  }
-  return `${locale}/search-index.json`
-}
-
-export default generateSearchIndex
+/** @deprecated Import from `@uniweb/projections` — kept so existing paths resolve. */
+export {
+  generateSearchIndex,
+  generateSearchIndex as default,
+  isSearchEnabled,
+  getSearchConfig,
+  getSearchIndexFilename
+} from '@uniweb/projections/search'

package/src/search/index.js

@@ -1,34 +1,20 @@
 /**
  * Search Index Generation Module
  *
- * Generates search indexes for Uniweb sites at build time.
- * The generated indexes can be loaded at runtime for client-side search.
+ * Re-export of `@uniweb/projections/search`. The implementation moved there so
+ * one copy serves every publisher of a site — the CLI and the app both import
+ * it, and the backend stores the result rather than generating its own. This
+ * shim keeps `@uniweb/build/search` and every existing call site working.
  *
  * @module @uniweb/build/search
- *
- * @example
- * import { generateSearchIndex, isSearchEnabled } from '@uniweb/build/search'
- *
- * // Check if search is enabled
- * if (isSearchEnabled(siteContent)) {
- *   // Generate index for current locale
- *   const index = generateSearchIndex(siteContent, {
- *     locale: 'en'
- *   })
- *
- *   // Write to file
- *   writeFileSync('dist/search-index.json', JSON.stringify(index))
- * }
  */
 
 export {
   extractSearchContent,
-  extractSearchContent as default
-} from './extract.js'
-
-export {
+  extractSearchContent as default,
   generateSearchIndex,
   isSearchEnabled,
   getSearchConfig,
-  getSearchIndexFilename
-} from './generate.js'
+  getSearchIndexFilename,
+  generateCollectionIndex
+} from '@uniweb/projections/search'

package/src/site/build-site-data.js

@@ -19,15 +19,25 @@
 
 import { writeFile, readFile, mkdir, cp } from 'node:fs/promises'
 import { existsSync } from 'node:fs'
-import { join, resolve } from 'node:path'
+import { join, resolve, dirname } from 'node:path'
 
 import { resolveDefaultLocale } from '@uniweb/core'
 import { collectSiteContent } from './content-collector.js'
 import { processCollections, writeCollectionFiles } from './collection-processor.js'
 import { processAssets, rewriteSiteContentPaths } from './asset-processor.js'
 import { processAdvancedAssets } from './advanced-processors.js'
-import { generateSearchIndex } from '../search/generate.js'
-import { generateCollectionIndex } from '../search/collections.js'
+import {
+  generateSearchIndex,
+  generateCollectionIndex,
+  mergeSearchIndexes,
+  getSearchIndexFilename,
+  renderSiteIndex,
+  renderPageMarkdown,
+  resolveAgentsConfig,
+  selectIndexablePages,
+  pageMarkdownFilename,
+  INDEX_FILENAME
+} from '@uniweb/projections'
 
 /**
  * Build the site data outputs needed by `uniweb deploy` (link-mode).
@@ -215,6 +225,7 @@ export async function buildSiteData({
 
     // Collection indexes — one per routed + search-configured collection
     const collections = finalContent.config?.collections || {}
+    const collectionIndexes = []
     for (const [collName, collConfig] of Object.entries(collections)) {
       if (!collConfig.search?.enabled || !collConfig.route) continue
       const cascadeFile = join(resolvedDistDir, 'data', `${collName}.json`)
@@ -226,9 +237,75 @@ export async function buildSiteData({
         continue
       }
       const collIndex = generateCollectionIndex(collName, collConfig, collectionData, defaultLocale)
+      collectionIndexes.push(collIndex)
       await writeFile(join(searchDir, `${collName}.json`), JSON.stringify(collIndex))
     }
+
+    // The single-file form, for the BROWSER lane.
+    //
+    // The split files above serve a server that loads only the parts a query
+    // needs. Kit's client-side `index` provider needs all of it, and asks for
+    // `search-index.json` — so without this, a site published through this lane
+    // 404s on its own search index and degrades to no results. Emitting both
+    // from the same entries is one extra serialization and means the client
+    // works identically on every lane, with no host configuration describing
+    // where the index lives.
+    await writeFile(
+      join(resolvedDistDir, getSearchIndexFilename(defaultLocale, defaultLocale)),
+      JSON.stringify(mergeSearchIndexes(pagesIndex, collectionIndexes))
+    )
   }
 
+  // 6. Agent projections — `llms.txt` and one `.md` per page.
+  //
+  //    Emitted here as well as in the vite plugin because both lanes publish
+  //    a site, and an artifact derived from site content has to exist
+  //    whichever lane produced it. A projection present after one publish and
+  //    absent after another is worse than none: agents are told to rely on it.
+  //
+  //    Ungated by `features:` — projections are free, so they carry no
+  //    billing intent and no entitlement to check downstream.
+  await writeProjections(finalContent, resolvedDistDir)
+
   return { siteContent: finalContent, distDir: resolvedDistDir }
 }
+
+/**
+ * Write the agent projections into `distDir`.
+ *
+ * Mirrors the vite plugin's `emitProjections`, differing only in how bytes
+ * reach disk (`writeFile` vs. Rollup's `emitFile`) — the generators, options
+ * and filenames come from `@uniweb/projections`, so the two lanes cannot
+ * disagree about what they produce.
+ *
+ * @param {Object} siteContent - Final site content for the default locale
+ * @param {string} distDir - Resolved output directory
+ * @returns {Promise<void>}
+ */
+async function writeProjections(siteContent, distDir) {
+  const agents = resolveAgentsConfig(siteContent?.config)
+  if (!agents.index && !agents.markdown) return
+  if (!siteContent?.pages?.length) return
+
+  const defaultLocale = resolveDefaultLocale(siteContent.config)
+  const options = {
+    baseUrl: siteContent.config?.seo?.baseUrl || '',
+    locale: siteContent.config?.activeLocale || defaultLocale,
+    defaultLocale
+  }
+
+  if (agents.index) {
+    const index = renderSiteIndex(siteContent, { ...options, exclude: agents.exclude })
+    await writeFile(join(distDir, INDEX_FILENAME), index)
+  }
+
+  if (!agents.markdown) return
+
+  for (const page of selectIndexablePages(siteContent.pages, { exclude: agents.exclude })) {
+    const markdown = renderPageMarkdown(page)
+    if (!markdown) continue
+    const target = join(distDir, pageMarkdownFilename(page.route))
+    await mkdir(dirname(target), { recursive: true })
+    await writeFile(target, markdown)
+  }
+}