@uniweb/projections 0.1.1

package/src/pages.js

@@ -0,0 +1,235 @@
+/**
+ * @fileoverview Page-graph helpers: which pages a projection may describe,
+ * how they group, and what URL points at one.
+ *
+ * The exclusion rules here are load-bearing, not tidy-up. An index is *more*
+ * revealing than a sitemap because it describes pages rather than listing
+ * them: an unlinked page becomes discoverable **and** summarized. Projections
+ * are on by default, so weakening these turns the default into a leak.
+ */
+
+import { pageMarkdownFilename, normalizeExclude } from './config.js'
+
+/**
+ * Is this page a structural container rather than a page with content?
+ *
+ * Content-less folders (a `page.yml` with no markdown) exist in the hierarchy
+ * as groups. They have no body to project, so they become index headings.
+ *
+ * @param {Object} page
+ * @returns {boolean}
+ */
+export function isContainer(page) {
+  return page?.hasContent === false
+}
+
+/**
+ * Is this a dynamic route *template* (`/blog/:slug`) rather than a page?
+ *
+ * Templates are expanded into concrete pages by the prerender lane; the
+ * template itself is not a page and must never be described as one.
+ *
+ * @param {Object} page
+ * @returns {boolean}
+ */
+export function isDynamicTemplate(page) {
+  return Boolean(page?.isDynamic) || (page?.route || '').includes(':')
+}
+
+/**
+ * Does any route segment start with `_`? Drafts and private files never reach
+ * the collected content, but a mounted tree can still carry one.
+ *
+ * @param {string} route
+ * @returns {boolean}
+ */
+function hasDraftSegment(route) {
+  return (route || '').split('/').some(segment => segment.startsWith('_'))
+}
+
+/**
+ * Is `route` at or beneath `prefix`?
+ * @param {string} route
+ * @param {string} prefix
+ * @returns {boolean}
+ */
+function isAtOrUnder(route, prefix) {
+  if (prefix === '/') return true
+  return route === prefix || route.startsWith(`${prefix}/`)
+}
+
+/**
+ * Route prefixes whose whole branch is excluded.
+ *
+ * Two sources, with different reach — deliberately:
+ *
+ * - `agents.exclude` cascades. `exclude: [/internal]` plainly means the
+ *   branch, not one page.
+ * - `noindex` / `hidden` on a **container** cascades, because a container is
+ *   pure structure: suppressing the heading while still listing its children
+ *   would orphan them under the wrong group. On a page with content it stays
+ *   per-page, matching how the sitemap reads `noindex`.
+ *
+ * @param {Object[]} pages
+ * @param {string[]} exclude
+ * @returns {string[]}
+ */
+function excludedBranches(pages, exclude) {
+  const branches = normalizeExclude(exclude)
+  for (const page of pages) {
+    if (!isContainer(page)) continue
+    if (page.seo?.noindex || page.hidden) branches.push(page.route)
+  }
+  return branches
+}
+
+/**
+ * Should this page be described in the index / projected to markdown?
+ *
+ * @param {Object} page
+ * @param {string[]} branches - Excluded route prefixes
+ * @returns {boolean}
+ */
+function isIndexable(page, branches) {
+  if (!page?.route) return false
+  if (isDynamicTemplate(page)) return false
+  if (page.seo?.noindex) return false
+  if (page.hidden) return false
+  if (hasDraftSegment(page.route)) return false
+  if (branches.some(prefix => isAtOrUnder(page.route, prefix))) return false
+  return true
+}
+
+/**
+ * The pages a projection may describe: real pages, in build order.
+ *
+ * Containers are excluded — they have no body — but they survive as headings
+ * via {@link groupPagesForIndex}.
+ *
+ * @param {Object[]} pages - `siteContent.pages` (flat, already ordered)
+ * @param {Object} [options]
+ * @param {string[]} [options.exclude] - Additional route prefixes
+ * @returns {Object[]}
+ */
+export function selectIndexablePages(pages = [], { exclude = [] } = {}) {
+  const branches = excludedBranches(pages, exclude)
+  return pages.filter(page => isIndexable(page, branches) && !isContainer(page))
+}
+
+/**
+ * Group indexable pages under their nearest content-less container.
+ *
+ * The grouping comes free from the page graph — no new structure is invented.
+ * Pages with no container ancestor land in a leading, unheaded group.
+ *
+ * @param {Object[]} pages - `siteContent.pages` (flat, already ordered)
+ * @param {Object} [options]
+ * @param {string[]} [options.exclude]
+ * @returns {Array<{heading: string|null, route: string|null, pages: Object[]}>}
+ */
+export function groupPagesForIndex(pages = [], { exclude = [] } = {}) {
+  const branches = excludedBranches(pages, exclude)
+  const containers = pages.filter(p => isContainer(p) && isIndexable(p, branches))
+  const indexable = pages.filter(p => isIndexable(p, branches) && !isContainer(p))
+
+  // Nearest container ancestor = the longest container route the page sits under.
+  const groupFor = page => {
+    let best = null
+    for (const container of containers) {
+      if (container.route === page.route) continue
+      if (!isAtOrUnder(page.route, container.route)) continue
+      if (!best || container.route.length > best.route.length) best = container
+    }
+    return best
+  }
+
+  const root = { heading: null, route: null, pages: [] }
+  const groups = new Map()
+
+  for (const page of indexable) {
+    const container = groupFor(page)
+    if (!container) {
+      root.pages.push(page)
+      continue
+    }
+    if (!groups.has(container.route)) {
+      groups.set(container.route, {
+        heading: container.title || container.label || container.route,
+        route: container.route,
+        pages: [],
+      })
+    }
+    groups.get(container.route).pages.push(page)
+  }
+
+  const ordered = [...groups.values()]
+  return root.pages.length ? [root, ...ordered] : ordered
+}
+
+/**
+ * Translate a route into a locale's URL segments.
+ *
+ * Mirrors the sitemap's behavior so an agent index and a sitemap never
+ * disagree about where a localized page lives.
+ *
+ * @param {string} route
+ * @param {string} locale
+ * @param {Object} [routeTranslations] - `config.i18n.routeTranslations`
+ * @returns {string}
+ */
+export function applyRouteTranslation(route, locale, routeTranslations) {
+  const localeMap = routeTranslations?.[locale]
+  if (!localeMap) return route
+  if (localeMap[route]) return localeMap[route]
+  for (const [canonical, translated] of Object.entries(localeMap)) {
+    if (route.startsWith(`${canonical}/`)) {
+      return translated + route.slice(canonical.length)
+    }
+  }
+  return route
+}
+
+/**
+ * URL of a page's markdown projection.
+ *
+ * Absolute when a `baseUrl` is known, root-relative otherwise. A root-relative
+ * link still resolves for an agent that arrived via the index, which is why an
+ * unset `baseUrl` warns rather than suppressing the artifact the way the
+ * sitemap does.
+ *
+ * @param {string} route
+ * @param {Object} [options]
+ * @param {string} [options.baseUrl] - Site origin, e.g. `https://example.com`
+ * @param {string} [options.basePath] - Subdirectory deploy prefix, e.g. `/docs/`
+ * @param {string} [options.locale] - Active locale
+ * @param {string} [options.defaultLocale] - Locale served at the root
+ * @param {Object} [options.routeTranslations]
+ * @returns {string}
+ */
+export function buildPageUrl(route, options = {}) {
+  const { baseUrl = '', basePath = '', locale, defaultLocale, routeTranslations } = options
+
+  const localized =
+    locale && defaultLocale && locale !== defaultLocale
+      ? applyRouteTranslation(route, locale, routeTranslations)
+      : route
+
+  const localePrefix = locale && defaultLocale && locale !== defaultLocale ? `/${locale}` : ''
+  const filename = pageMarkdownFilename(localized)
+  const prefix = normalizeBasePath(basePath)
+
+  const path = `${prefix}${localePrefix}/${filename}`
+  if (!baseUrl) return path
+  return `${baseUrl.replace(/\/+$/, '')}${path}`
+}
+
+/**
+ * Normalize a base path to `''` or `/segment` (no trailing slash).
+ * @param {string} basePath
+ * @returns {string}
+ */
+function normalizeBasePath(basePath) {
+  if (!basePath || basePath === '/') return ''
+  const withSlash = basePath.startsWith('/') ? basePath : `/${basePath}`
+  return withSlash.replace(/\/+$/, '')
+}

package/src/search/collections.js

@@ -0,0 +1,55 @@
+/**
+ * Generate search index for a file-based collection.
+ *
+ * Collection cascade files (`data/{name}.json`) contain all non-deferred fields.
+ * If `search.fetchDetail: true` is set, the caller is responsible for merging
+ * per-record detail files into each item before calling this function.
+ */
+
+/**
+ * @param {string} name - Collection name (e.g. "articles")
+ * @param {Object} config - Collection config from site.yml (config.collections[name])
+ * @param {Object} collectionData - Parsed cascade JSON (`data/{name}.json`)
+ * @param {string} locale - Locale code (e.g. "en")
+ * @returns {Object} Collection search index
+ */
+export function generateCollectionIndex(name, config, collectionData, locale) {
+  const fields = config.search?.fields || ['title']
+  const weight = config.search?.weight ?? 0.7
+  const items = collectionData?.items || []
+
+  const entries = items.map(item => {
+    const content = fields.map(f => item[f] || '').filter(Boolean).join(' ')
+    const slug = item.slug || item.id || String(item.title || '').toLowerCase().replace(/\s+/g, '-')
+    return {
+      id: `collection:${name}:${slug}`,
+      type: 'collection',
+      collection: name,
+      route: `${config.route}/${slug}`,
+      title: item.title || item.name || slug,
+      content,
+      excerpt: content.length > 160
+        ? content.slice(0, 160).trim() + '…'
+        : content,
+      weight,
+      item: pickDisplayFields(item),
+    }
+  })
+
+  // No `generated` timestamp — see the note in `generate.js`. A clock defeats
+  // content-addressing and byte-parity between publishers.
+  return {
+    type: 'collection',
+    collection: name,
+    locale,
+    entries,
+  }
+}
+
+function pickDisplayFields(item) {
+  const { slug, title, name, date, image, author, excerpt, role } = item
+  return Object.fromEntries(
+    Object.entries({ slug, title, name, date, image, author, excerpt, role })
+      .filter(([, v]) => v != null)
+  )
+}

package/src/search/extract.js

@@ -0,0 +1,335 @@
+/**
+ * Extract searchable content from site-content.json
+ *
+ * Walks through all pages and sections, extracting text content
+ * for search indexing. Reuses patterns from i18n extraction.
+ */
+
+/**
+ * Extract all searchable content from site
+ * @param {Object} siteContent - Parsed site-content.json
+ * @param {Object} options - Extraction options
+ * @param {boolean} [options.pages=true] - Include page metadata
+ * @param {boolean} [options.sections=true] - Include section content
+ * @param {boolean} [options.headings=true] - Include headings
+ * @param {boolean} [options.paragraphs=true] - Include paragraphs
+ * @param {boolean} [options.links=true] - Include link labels
+ * @param {boolean} [options.lists=true] - Include list items
+ * @param {Array<string>} [options.excludeRoutes=[]] - Routes to exclude
+ * @param {Array<string>} [options.excludeComponents=[]] - Components to exclude
+ * @returns {Array<Object>} Array of search entries
+ */
+export function extractSearchContent(siteContent, options = {}) {
+  const {
+    pages: includePagesFlag = true,
+    sections: includeSections = true,
+    headings: includeHeadings = true,
+    paragraphs: includeParagraphs = true,
+    links: includeLinks = true,
+    lists: includeLists = true,
+    excludeRoutes = [],
+    excludeComponents = []
+  } = options
+
+  const entries = []
+
+  for (const page of siteContent.pages || []) {
+    const pageRoute = page.route || '/'
+
+    // Skip excluded routes
+    if (excludeRoutes.some(r => pageRoute.startsWith(r))) {
+      continue
+    }
+
+    // Skip pages marked as noindex
+    if (page.seo?.noindex) {
+      continue
+    }
+
+    // Extract page-level entry
+    if (includePagesFlag) {
+      const pageEntry = extractFromPage(page)
+      if (pageEntry) {
+        entries.push(pageEntry)
+      }
+    }
+
+    // Extract section-level entries
+    if (includeSections) {
+      for (const section of page.sections || []) {
+        const sectionEntries = extractFromSection(section, page, {
+          includeHeadings,
+          includeParagraphs,
+          includeLinks,
+          includeLists,
+          excludeComponents
+        })
+        entries.push(...sectionEntries)
+      }
+    }
+  }
+
+  return entries
+}
+
+/**
+ * Extract search entry from page metadata
+ * @param {Object} page - Page data
+ * @returns {Object|null} Search entry or null
+ */
+function extractFromPage(page) {
+  const route = page.route || '/'
+  const title = page.title || ''
+  const description = page.description || ''
+  const keywords = page.keywords || page.seo?.keywords || []
+
+  // Skip pages with no meaningful content
+  if (!title && !description) {
+    return null
+  }
+
+  return {
+    id: `page:${route}`,
+    type: 'page',
+    route,
+    title,
+    description,
+    keywords: Array.isArray(keywords) ? keywords : [keywords].filter(Boolean),
+    content: [title, description].filter(Boolean).join(' '),
+    // Boost factor for search ranking (pages are more important)
+    weight: route === '/' ? 1.0 : 0.8
+  }
+}
+
+/**
+ * Extract search entries from a section (and subsections)
+ * @param {Object} section - Section data
+ * @param {Object} page - Parent page
+ * @param {Object} options - Extraction options
+ * @returns {Array<Object>} Array of search entries
+ */
+function extractFromSection(section, page, options) {
+  const entries = []
+  const {
+    includeHeadings,
+    includeParagraphs,
+    includeLinks,
+    includeLists,
+    excludeComponents
+  } = options
+
+  const sectionId = section.id || 'unknown'
+  const component = section.component || section.type || 'unknown'
+
+  // Skip excluded components
+  if (excludeComponents.includes(component)) {
+    return entries
+  }
+
+  // Extract text from ProseMirror content
+  const textParts = []
+  let sectionTitle = ''
+
+  if (section.content?.type === 'doc') {
+    const extracted = extractFromProseMirrorDoc(section.content, {
+      includeHeadings,
+      includeParagraphs,
+      includeLinks,
+      includeLists
+    })
+
+    sectionTitle = extracted.title || ''
+    textParts.push(...extracted.textParts)
+  }
+
+  // Also check params for title (from YAML frontmatter)
+  if (!sectionTitle && section.params?.title) {
+    sectionTitle = section.params.title
+  }
+
+  // Build content string
+  const content = textParts.join(' ').trim()
+
+  // Create entry if there's meaningful content
+  if (sectionTitle || content) {
+    entries.push({
+      id: `section:${page.route}:${sectionId}`,
+      type: 'section',
+      route: page.route,
+      sectionId,
+      anchor: `Section${sectionId}`,
+      component,
+      title: sectionTitle,
+      pageTitle: page.title || '',
+      content,
+      // Generate excerpt (first ~160 chars)
+      excerpt: generateExcerpt(content, 160),
+      // Section weight is lower than page
+      weight: 0.6
+    })
+  }
+
+  // Recursively process subsections
+  for (const subsection of section.subsections || []) {
+    const subEntries = extractFromSection(subsection, page, options)
+    entries.push(...subEntries)
+  }
+
+  return entries
+}
+
+/**
+ * Extract text from ProseMirror document
+ * @param {Object} doc - ProseMirror document
+ * @param {Object} options - Extraction options
+ * @returns {Object} Extracted content { title, textParts }
+ */
+function extractFromProseMirrorDoc(doc, options) {
+  const { includeHeadings, includeParagraphs, includeLinks, includeLists } = options
+  const textParts = []
+  let title = ''
+  let foundFirstHeading = false
+
+  if (!doc.content) {
+    return { title, textParts }
+  }
+
+  for (const node of doc.content) {
+    if (node.type === 'heading') {
+      const text = extractTextFromNode(node)
+      if (!text) continue
+
+      // First H1 becomes the title
+      if (!foundFirstHeading && node.attrs?.level === 1) {
+        title = text
+        foundFirstHeading = true
+      }
+
+      if (includeHeadings) {
+        textParts.push(text)
+      }
+    } else if (node.type === 'paragraph') {
+      if (includeParagraphs) {
+        const text = extractTextFromNode(node)
+        if (text) {
+          textParts.push(text)
+        }
+      }
+
+      // Extract link labels separately if requested
+      if (includeLinks) {
+        const links = extractLinksFromNode(node)
+        for (const link of links) {
+          if (link.label) {
+            textParts.push(link.label)
+          }
+        }
+      }
+    } else if ((node.type === 'bulletList' || node.type === 'orderedList') && includeLists) {
+      const listTexts = extractFromList(node)
+      textParts.push(...listTexts)
+    }
+  }
+
+  return { title, textParts }
+}
+
+/**
+ * Extract all text content from a node
+ * @param {Object} node - ProseMirror node
+ * @returns {string} Text content
+ */
+function extractTextFromNode(node) {
+  if (!node.content) return ''
+
+  const texts = []
+
+  for (const child of node.content) {
+    if (child.type === 'text') {
+      texts.push(child.text || '')
+    } else if (child.content) {
+      // Recurse for nested nodes
+      texts.push(extractTextFromNode(child))
+    }
+  }
+
+  return texts.join('').trim()
+}
+
+/**
+ * Extract links from a paragraph node
+ * @param {Object} node - Paragraph node
+ * @returns {Array<{label: string, href: string}>} Links
+ */
+function extractLinksFromNode(node) {
+  const links = []
+
+  if (!node.content) return links
+
+  for (const child of node.content) {
+    if (child.type === 'text' && child.marks) {
+      const linkMark = child.marks.find(m => m.type === 'link')
+      if (linkMark) {
+        links.push({
+          label: child.text || '',
+          href: linkMark.attrs?.href || ''
+        })
+      }
+    }
+  }
+
+  return links
+}
+
+/**
+ * Extract text from list items
+ * @param {Object} listNode - List node
+ * @returns {Array<string>} List item texts
+ */
+function extractFromList(listNode) {
+  const texts = []
+
+  if (!listNode.content) return texts
+
+  for (const listItem of listNode.content) {
+    if (listItem.type === 'listItem' && listItem.content) {
+      for (const child of listItem.content) {
+        if (child.type === 'paragraph') {
+          const text = extractTextFromNode(child)
+          if (text) {
+            texts.push(text)
+          }
+        }
+      }
+    }
+  }
+
+  return texts
+}
+
+/**
+ * Generate excerpt from content
+ * @param {string} content - Full content
+ * @param {number} maxLength - Maximum length
+ * @returns {string} Excerpt
+ */
+function generateExcerpt(content, maxLength = 160) {
+  if (!content) return ''
+
+  // Normalize whitespace
+  const normalized = content.replace(/\s+/g, ' ').trim()
+
+  if (normalized.length <= maxLength) {
+    return normalized
+  }
+
+  // Find a good break point (word boundary)
+  let breakPoint = normalized.lastIndexOf(' ', maxLength)
+  if (breakPoint < maxLength * 0.5) {
+    breakPoint = maxLength
+  }
+
+  return normalized.slice(0, breakPoint).trim() + '…'
+}
+
+export default extractSearchContent