@uniweb/projections 0.2.7 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniweb/projections",
-  "version": "0.2.7",
+  "version": "0.3.0",
   "description": "Projections of a Uniweb site's content — agent index, per-page markdown, search index. Pure JS, runs anywhere.",
   "type": "module",
   "exports": {
@@ -31,8 +31,8 @@
     "node": ">=20.19"
   },
   "dependencies": {
-    "@uniweb/core": "^0.8.4",
-    "@uniweb/content-writer": "^0.3.3"
+    "@uniweb/content-writer": "^0.3.3",
+    "@uniweb/core": "^0.8.5"
   },
   "devDependencies": {
     "vitest": "^4.1.7",

package/src/corpus.js

@@ -0,0 +1,319 @@
+/**
+ * @fileoverview The exploration projection: a whole site as greppable markdown.
+ *
+ * `renderSiteIndex` answers *"what pages exist"* and `renderPageMarkdown`
+ * answers *"what does one page say"*. This answers the third question an agent
+ * asks — *"where in this site is X"* — by producing the artifact `grep`,
+ * `search` and `read_page` run against.
+ *
+ * ── WHY MARKDOWN, AND WHY ONE STRING PER PAGE ───────────────────────────────
+ *
+ * Tools of this kind are commonly built over a plain-text extraction, with
+ * markdown produced separately and left unread. That loses exactly what an
+ * agent navigates by: headings stop being headings, and links, code fences and
+ * list structure vanish from what is searched.
+ *
+ * The subtler failure is line numbers. If a matcher numbers lines *within a
+ * section* while a reader concatenates sections before returning them, a match
+ * at "line 12" does not index into what the reader returns — and an agent
+ * relies on those agreeing, the way they agree in a codebase.
+ *
+ * Both are addressed by *shape* rather than by care: a corpus page is **one
+ * markdown string**, and every tool addresses that same string. The numbers
+ * cannot disagree because there is only one thing to number.
+ *
+ * ── WHY SEGMENTS, RATHER THAN FIXED-SIZE CHUNKS ─────────────────────────────
+ *
+ * Keyword scoring needs a document unit, and a sliding window of ~N words is
+ * the usual choice. Sections are a better one and cost nothing extra: they are
+ * the authored boundary, they are what `search/extract.js` already indexes,
+ * and — decisively — they carry `sectionDomId`, the anchor the renderer
+ * actually emits. So a hit can be cited as `/route#section-3`, a link that
+ * resolves in a browser. A word-window has no anchor, which is why tools built
+ * on one return a route and leave the reader to find the passage.
+ *
+ * ⛔ Do NOT recover anchors by parsing headings out of the finished markdown.
+ * A heading is prose the author wrote; `sectionDomId` is an id the renderer
+ * emits. They are not the same thing and only one of them scrolls anywhere.
+ */
+
+import { collectPageBlocks, BLOCK_SEPARATOR } from './markdown.js'
+import {
+  selectIndexablePages,
+  isContainer,
+  isDynamicTemplate,
+  hasDraftSegment,
+  isAtOrUnder,
+  knowledgeRoots,
+  isKnowledgeRoute,
+} from './pages.js'
+import { normalizeExclude } from './config.js'
+
+/**
+ * Split pages into the agent-only set and the rendered set.
+ *
+ * A page is *knowledge* when it carries `knowledge: true`, or when it sits
+ * beneath a route that does. The cascade is by route prefix, so `/kb` claims
+ * `/kb/auth` and does **not** claim `/kbase`.
+ *
+ * ⚠️ This partition tells a **renderer** what to drop. It does not by itself
+ * decide what the agent sees — {@link selectCorpusPages} does, and it is
+ * deliberately not "everything left over". See the note there.
+ *
+ * @param {Object[]} pages - `siteContent.pages`
+ * @returns {{knowledgePages: Object[], renderedPages: Object[]}}
+ */
+export function partitionKnowledgePages(pages = []) {
+  if (!Array.isArray(pages)) return { knowledgePages: [], renderedPages: [] }
+
+  const roots = knowledgeRoots(pages)
+
+  const knowledgePages = []
+  const renderedPages = []
+
+  for (const page of pages) {
+    if (isKnowledgePage(page, roots)) knowledgePages.push(page)
+    else renderedPages.push(page)
+  }
+
+  return { knowledgePages, renderedPages }
+}
+
+/**
+ * @param {Object} page
+ * @param {string[]} roots - Routes carrying `knowledge: true`
+ * @returns {boolean}
+ */
+function isKnowledgePage(page, roots) {
+  // The flag is checked first rather than folded into the route test: a page
+  // carrying it with no route at all is still not something to render.
+  if (page?.knowledge) return true
+  return isKnowledgeRoute(page?.route, roots)
+}
+
+/**
+ * The pages a corpus may contain — **public by default; agent-only on request.**
+ *
+ * ⛔ **THE ARGUMENT ORDER IS THE SAFETY PROPERTY. Passing nothing gets you the
+ * PUBLIC selection.** A caller who forgets the option, or copies a call from a
+ * public-tier consumer, under-discloses. That is the failure we can afford;
+ * the other direction puts agent-only content on a visitor-facing endpoint.
+ * So `knowledge` defaults to `false` and the agent corpus is the one that has
+ * to ask — the two tiers must not merely pass different arguments to one
+ * selector, the public one must be what you get by passing none.
+ *
+ * ⛔ **AND EVEN WITH `knowledge: true`, THIS IS NOT "the site's full content".**
+ * It is tempting to index every page — the agent is more useful the more it
+ * knows — and to treat `knowledge: true` as purely a render-subtraction. That
+ * is defensible for a private tool an author runs over their own content, and
+ * wrong for a corpus that answers questions **for whoever can reach the site**.
+ * An agent that greps `seo.noindex` pages and quotes them back is a disclosure
+ * the author never agreed to, and `pages.js` already states the principle this
+ * follows from: projections are on by default, so weakening the exclusions
+ * turns the default into a leak.
+ *
+ * With `knowledge: true` the corpus is the public projection **plus** what the
+ * author explicitly marked for agents:
+ *
+ * | signal | reach | why |
+ * |---|---|---|
+ * | the `agents:` selection | included | already the public projection's set |
+ * | `knowledge: true` | **added** | an explicit opt-in to agent-visible content |
+ * | `hidden`, `seo.noindex` | **overridden** for knowledge pages | they mean "not for browsers", and a knowledge page is *by definition* not rendered — honouring them would make `knowledge:` do nothing |
+ * | `agents.exclude` | **always wins** | the one signal that says "keep agents out", so it outranks a conflicting `knowledge: true` |
+ * | `_`-prefixed route | **always wins** | a draft is a draft |
+ * | container, dynamic template | always excluded | shape, not policy — neither is a page with a body |
+ *
+ * The two "always wins" rows are the fail-closed half. Where the author has
+ * contradicted themselves, the cost of honouring the exclusion is that the
+ * agent knows less; the cost of ignoring it is private content on a public
+ * endpoint. Those are not symmetric.
+ *
+ * ⚠️ **The `added` row is load-bearing and was not always doing anything.**
+ * Until `excludedBranches` learned about `knowledge:`, the public half already
+ * contained knowledge pages, so this union was a no-op and every test of it
+ * passed for the wrong reason. That is the shape to watch for when reading a
+ * union: it looks correct whether or not the two halves are actually disjoint.
+ * `corpus.test.js` now pins the public half's *absence* of them directly.
+ *
+ * @param {Object[]} pages - `siteContent.pages`
+ * @param {Object} [options]
+ * @param {string[]} [options.exclude] - Additional excluded route prefixes
+ * @param {boolean} [options.knowledge=false] - Admit `knowledge:` pages. Off by
+ *   default; see the argument-order note above before changing that.
+ * @returns {Object[]} Pages in build order, no duplicates
+ */
+export function selectCorpusPages(pages = [], { exclude = [], knowledge = false } = {}) {
+  const publicPages = selectIndexablePages(pages, { exclude })
+  if (!knowledge) return publicPages
+
+  const publicSet = new Set(publicPages)
+  const branches = normalizeExclude(exclude)
+  const { knowledgePages } = partitionKnowledgePages(pages)
+
+  const admissibleKnowledge = knowledgePages.filter(
+    page =>
+      page?.route &&
+      !isContainer(page) &&
+      !isDynamicTemplate(page) &&
+      !hasDraftSegment(page.route) &&
+      !branches.some(prefix => isAtOrUnder(page.route, prefix))
+  )
+
+  const admitted = new Set([...publicSet, ...admissibleKnowledge])
+
+  // Build order, not selection order — the corpus is read by humans debugging it.
+  return pages.filter(page => admitted.has(page))
+}
+
+/**
+ * Build the greppable corpus for one locale of a site.
+ *
+ * **Public by default.** `knowledge: true` is the agent tier and has to be
+ * asked for — see the argument-order note on {@link selectCorpusPages}.
+ *
+ * @param {Object} siteContent - Parsed site-content.json (one locale)
+ * @param {Object} [options]
+ * @param {string[]} [options.exclude] - Additional excluded route prefixes
+ * @param {boolean} [options.includeChildren=true] - Include nested sections
+ * @param {boolean} [options.knowledge=false] - Admit `knowledge:` pages
+ * @returns {Array<CorpusPage>} Pages with content, in build order
+ */
+export function buildCorpus(
+  siteContent,
+  { exclude, includeChildren = true, knowledge = false } = {}
+) {
+  const pages = siteContent?.pages || []
+  const config = siteContent?.config || {}
+  const excluded = exclude ?? config.agents?.exclude ?? []
+
+  // Only meaningful when knowledge pages were admitted; an empty set otherwise
+  // keeps every `CorpusPage.knowledge` false, which is the truth on that tier.
+  const agentOnly = knowledge
+    ? new Set(partitionKnowledgePages(pages).knowledgePages.map(page => page.route))
+    : new Set()
+
+  const corpus = []
+
+  for (const page of selectCorpusPages(pages, { exclude: excluded, knowledge })) {
+    const built = buildCorpusPage(page, {
+      includeChildren,
+      knowledge: agentOnly.has(page.route),
+    })
+    // A page whose sections carry no projectable content is not a page the
+    // agent can read. Listing it would produce read_page hits that return ''.
+    if (built) corpus.push(built)
+  }
+
+  return corpus
+}
+
+/**
+ * @typedef {Object} CorpusSegment
+ * @property {string} anchor - `sectionDomId`; cite as `${route}#${anchor}`
+ * @property {number} startLine - 1-based, into the page's `markdown`
+ * @property {number} endLine - 1-based inclusive
+ * @property {string} title - Leading heading text, or '' when the block has none
+ * @property {string} markdown - This block alone
+ */
+
+/**
+ * @typedef {Object} CorpusPage
+ * @property {string} route
+ * @property {string} title
+ * @property {string} description
+ * @property {boolean} knowledge - Agent-only (never rendered for a visitor)
+ * @property {string} markdown - The whole page; every tool addresses THIS string
+ * @property {number} lineCount
+ * @property {CorpusSegment[]} segments
+ */
+
+/**
+ * One page, as markdown plus a map of where each section landed in it.
+ *
+ * @param {Object} page
+ * @param {Object} options
+ * @param {boolean} options.includeChildren
+ * @param {boolean} options.knowledge
+ * @returns {CorpusPage|null} `null` when the page projects to nothing
+ */
+function buildCorpusPage(page, { includeChildren, knowledge }) {
+  const blocks = collectPageBlocks(page, { includeChildren })
+  if (blocks.length === 0) return null
+
+  // Joining with a separator holding N newlines puts the next block's first
+  // line N lines below the previous block's last: one newline merely ends that
+  // last line, and each further one opens a blank line.
+  //
+  // ⚠️ Counting the separator's *lines* instead of its *newlines* is an
+  // off-by-one that only shows up from the second block onward, which is why
+  // `segment line ranges index into that exact string` slices every segment
+  // back out of the finished markdown rather than checking arithmetic.
+  const separatorNewlines = (BLOCK_SEPARATOR.match(/\n/g) || []).length
+  const segments = []
+  let line = 1
+
+  for (const block of blocks) {
+    const endLine = line + block.markdown.split('\n').length - 1
+    segments.push({
+      anchor: block.anchor,
+      startLine: line,
+      endLine,
+      title: leadingHeading(block.markdown),
+      markdown: block.markdown,
+    })
+    line = endLine + separatorNewlines
+  }
+
+  return {
+    route: page.route,
+    title: page.title || page.label || page.route,
+    description: page.description || '',
+    knowledge,
+    markdown: blocks.map(block => block.markdown).join(BLOCK_SEPARATOR),
+    lineCount: segments[segments.length - 1].endLine,
+    segments,
+  }
+}
+
+/**
+ * The block's own heading, when it opens with one.
+ *
+ * Only the first line is considered, which is why no fenced-code guard is
+ * needed: a fence cannot open and reach a `#` in the same line. A general
+ * heading scan over the whole block WOULD need one — ``` blocks routinely
+ * contain `# comment` — so do not widen this without adding it.
+ *
+ * @param {string} markdown
+ * @returns {string}
+ */
+function leadingHeading(markdown) {
+  const match = /^ {0,3}(#{1,6})\s+(.*?)\s*#*\s*$/.exec(markdown.split('\n', 1)[0] || '')
+  return match ? match[2] : ''
+}
+
+/**
+ * The corpus table of contents — what `list_pages` answers from.
+ *
+ * Carries no body text, so an agent can hold the whole shape of a site in one
+ * tool call and then read only what it needs. `lines` is what makes it useful
+ * for planning: it is the cost of the `read_page` the agent is deciding whether
+ * to spend.
+ *
+ * @param {CorpusPage[]} corpus
+ * @returns {Array<{route: string, title: string, description: string, knowledge: boolean, lines: number, sections: Array<{anchor: string, title: string, startLine: number}>}>}
+ */
+export function buildCorpusManifest(corpus = []) {
+  return corpus.map(page => ({
+    route: page.route,
+    title: page.title,
+    description: page.description,
+    knowledge: page.knowledge,
+    lines: page.lineCount,
+    sections: page.segments.map(segment => ({
+      anchor: segment.anchor,
+      title: segment.title,
+      startLine: segment.startLine,
+    })),
+  }))
+}

package/src/index.js

@@ -27,6 +27,13 @@ export { renderSiteIndex } from './site-index.js'
 export { renderPageMarkdown } from './markdown.js'
 export { resolvePageDescription } from './description.js'
 
+export {
+  buildCorpus,
+  buildCorpusManifest,
+  selectCorpusPages,
+  partitionKnowledgePages,
+} from './corpus.js'
+
 export {
   INDEX_FILENAME,
   DEFAULT_BRANCH_MIN_PAGES,

package/src/markdown.js

@@ -18,6 +18,14 @@
  */
 
 import { proseMirrorToMarkdown } from '@uniweb/content-writer'
+import { sectionDomId } from '@uniweb/core/section-id'
+
+/**
+ * How blocks are joined into a page. Exported because the corpus projection
+ * reconstructs the same string while tracking line offsets into it, and the
+ * two must not be able to disagree about the separator.
+ */
+export const BLOCK_SEPARATOR = '\n\n'
 
 /**
  * Render one page as markdown.
@@ -28,13 +36,40 @@ import { proseMirrorToMarkdown } from '@uniweb/content-writer'
  * @returns {string} Markdown, or '' when the page has no projectable content
  */
 export function renderPageMarkdown(page, { includeChildren = true } = {}) {
+  return collectPageBlocks(page, { includeChildren })
+    .map(block => block.markdown)
+    .join(BLOCK_SEPARATOR)
+    .trim()
+}
+
+/**
+ * The same blocks, still separate and each tagged with the anchor that links
+ * to it.
+ *
+ * Exists so the corpus projection can say *where in the page* a match landed.
+ * `renderPageMarkdown` throws that away by joining, and reconstructing it by
+ * re-parsing the finished markdown is exactly the mistake this package exists
+ * to prevent: headings in the output are prose, while `sectionDomId` is the id
+ * the renderer actually emits. Only one of those resolves in a browser.
+ *
+ * ⚠️ Blocks are non-empty and individually trimmed, which is what lets the
+ * corpus treat `join(BLOCK_SEPARATOR)` as equal to `renderPageMarkdown`'s
+ * output without re-trimming. `tests/corpus.test.js` asserts that equality
+ * rather than assuming it.
+ *
+ * @param {Object} page - A collected page (`siteContent.pages[n]`)
+ * @param {Object} [options]
+ * @param {boolean} [options.includeChildren=true] - Include nested sections
+ * @returns {Array<{anchor: string, markdown: string}>}
+ */
+export function collectPageBlocks(page, { includeChildren = true } = {}) {
   const blocks = []
 
   for (const section of page?.sections || []) {
-    collectSection(section, blocks, includeChildren)
+    collectSection(section, blocks, includeChildren, null)
   }
 
-  return blocks.join('\n\n').trim()
+  return blocks
 }
 
 /**
@@ -44,17 +79,24 @@ export function renderPageMarkdown(page, { includeChildren = true } = {}) {
  * content that happens to render inside a parent. Skipping them would drop
  * body text an author wrote, which is exactly what retrieval is for.
  *
+ * `ancestorAnchor` is threaded down rather than each child resolving its own,
+ * matching `search/extract.js` — a nested section renders *inside* its parent,
+ * so the parent's id is the fragment that actually scrolls to it. Consequence
+ * worth knowing: a parent and its children share an anchor.
+ *
  * @param {Object} section
- * @param {string[]} blocks
+ * @param {Array<{anchor: string, markdown: string}>} blocks
  * @param {boolean} includeChildren
+ * @param {string|null} ancestorAnchor
  */
-function collectSection(section, blocks, includeChildren) {
+function collectSection(section, blocks, includeChildren, ancestorAnchor) {
+  const anchor = ancestorAnchor || sectionDomId(section)
   const markdown = serializeSectionContent(section)
-  if (markdown) blocks.push(markdown)
+  if (markdown) blocks.push({ anchor, markdown })
 
   if (!includeChildren) return
   for (const child of section?.subsections || []) {
-    collectSection(child, blocks, includeChildren)
+    collectSection(child, blocks, includeChildren, anchor)
   }
 }
 

package/src/pages.js

@@ -43,25 +43,60 @@ export function isDynamicTemplate(page) {
  * @param {string} route
  * @returns {boolean}
  */
-function hasDraftSegment(route) {
+export function hasDraftSegment(route) {
   return (route || '').split('/').some(segment => segment.startsWith('_'))
 }
 
 /**
  * Is `route` at or beneath `prefix`?
+ *
+ * ⚠️ Exported for `corpus.js`, which composes the same exclusions with a
+ * different policy — NOT re-exported from `index.js`. It is a package-internal
+ * primitive, and one shared implementation of "is this route inside that
+ * branch" is the point: two copies of prefix matching is how `/kb` starts
+ * matching `/kbase`.
+ *
  * @param {string} route
  * @param {string} prefix
  * @returns {boolean}
  */
-function isAtOrUnder(route, prefix) {
+export function isAtOrUnder(route, prefix) {
   if (prefix === '/') return true
   return route === prefix || route.startsWith(`${prefix}/`)
 }
 
+/**
+ * The routes carrying `knowledge: true` — the roots of the agent-only branches.
+ *
+ * ⚠️ Defined here, in the lower layer, rather than in `corpus.js` where the
+ * concept reads like it belongs. Both files need it and the dependency only
+ * runs one way, so putting it there would mean a second copy of "is this route
+ * inside that branch" — the failure {@link isAtOrUnder} exists to prevent.
+ *
+ * @param {Object[]} pages
+ * @returns {string[]}
+ */
+export function knowledgeRoots(pages = []) {
+  if (!Array.isArray(pages)) return []
+  return pages.filter(page => page?.knowledge && page.route).map(page => page.route)
+}
+
+/**
+ * Is `route` a knowledge route — carrying the flag, or beneath one that does?
+ *
+ * @param {string} route
+ * @param {string[]} roots - {@link knowledgeRoots}
+ * @returns {boolean}
+ */
+export function isKnowledgeRoute(route, roots) {
+  if (!route) return false
+  return roots.some(root => isAtOrUnder(route, root))
+}
+
 /**
  * Route prefixes whose whole branch is excluded.
  *
- * Two sources, with different reach — deliberately:
+ * Three sources, with different reach — deliberately:
  *
  * - `agents.exclude` cascades. `exclude: [/internal]` plainly means the
  *   branch, not one page.
@@ -69,6 +104,20 @@ function isAtOrUnder(route, prefix) {
  *   pure structure: suppressing the heading while still listing its children
  *   would orphan them under the wrong group. On a page with content it stays
  *   per-page, matching how the sitemap reads `noindex`.
+ * - **`knowledge: true` cascades**, by the same route-prefix rule
+ *   {@link partitionKnowledgePages} renders by. A knowledge page is not
+ *   rendered and cannot be reached by a visitor, so nothing describing the
+ *   *public* site may name it.
+ *
+ * ⛔ That last one is a disclosure boundary, not a tidy-up, and the mistake it
+ * corrects was invisible for a reason worth stating. These projections are the
+ * **free** tier; the agent corpus that knowledge pages exist for is the
+ * **paid** one. Leaving `knowledge` out of this list did not merely list a
+ * page — it published the body an author wrote for a capability they may never
+ * have bought, at `llms.txt`, at `/{route}.md`, and in the search index. It
+ * also made {@link selectCorpusPages} degenerate: that selector is *public ∪
+ * knowledge*, and while knowledge rode in the public half the union added
+ * nothing and read as if it worked.
  *
  * @param {Object[]} pages
  * @param {string[]} exclude
@@ -80,7 +129,7 @@ function excludedBranches(pages, exclude) {
     if (!isContainer(page)) continue
     if (page.seo?.noindex || page.hidden) branches.push(page.route)
   }
-  return branches
+  return branches.concat(knowledgeRoots(pages))
 }
 
 /**

package/src/search/extract.js

@@ -1,14 +1,15 @@
 /**
  * Extract searchable content from site-content.json
  *
- * Walks through all pages and sections, extracting text content
- * for search indexing. Reuses patterns from i18n extraction.
+ * Walks the pages a *visitor* can reach — {@link selectIndexablePages} decides
+ * which those are — and extracts their text for search indexing.
  */
 
 // Leaf subpath, not the bare `@uniweb/core` entry: this package's environment
 // contract forbids the package root (it pulls semantic-parser + theming).
 // Enforced by tests/environment.test.js.
 import { sectionDomId } from '@uniweb/core/section-id'
+import { selectIndexablePages } from '../pages.js'
 
 /**
  * Extract all searchable content from site
@@ -20,7 +21,8 @@ import { sectionDomId } from '@uniweb/core/section-id'
  * @param {boolean} [options.paragraphs=true] - Include paragraphs
  * @param {boolean} [options.links=true] - Include link labels
  * @param {boolean} [options.lists=true] - Include list items
- * @param {Array<string>} [options.excludeRoutes=[]] - Routes to exclude
+ * @param {Array<string>} [options.excludeRoutes=[]] - Route prefixes to
+ *   exclude; the whole branch goes, matching `agents.exclude`
  * @param {Array<string>} [options.excludeComponents=[]] - Components to exclude
  * @returns {Array<Object>} Array of search entries
  */
@@ -38,19 +40,20 @@ export function extractSearchContent(siteContent, options = {}) {
 
   const entries = []
 
-  for (const page of siteContent.pages || []) {
-    const pageRoute = page.route || '/'
-
-    // Skip excluded routes
-    if (excludeRoutes.some(r => pageRoute.startsWith(r))) {
-      continue
-    }
-
-    // Skip pages marked as noindex
-    if (page.seo?.noindex) {
-      continue
-    }
-
+  // Which pages may be described is `pages.js`'s decision, not this file's.
+  //
+  // ⛔ This used to be two inline `continue`s — `search.exclude.routes` and
+  // `seo.noindex` — and the gap between that pair and the shared selector was a
+  // disclosure. A search index is a *visitor-facing* artifact, so it must
+  // describe only pages a visitor can reach; `knowledge:` pages are not
+  // rendered at all, and drafts, `hidden` pages and dynamic route *templates*
+  // are each unreachable or unlinkable in their own way. Reimplementing that
+  // list here is how it drifts from the one `llms.txt` and the per-page
+  // markdown use, which is exactly what happened.
+  //
+  // Note this also fixes the prefix test: `excludeRoutes` was matched with
+  // `startsWith`, so excluding `/kb` silently excluded `/kbase` too.
+  for (const page of selectIndexablePages(siteContent.pages || [], { exclude: excludeRoutes })) {
     // Extract page-level entry
     if (includePagesFlag) {
       const pageEntry = extractFromPage(page)