@uniweb/projections 0.2.8 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniweb/projections",
-  "version": "0.2.8",
+  "version": "0.3.0",
   "description": "Projections of a Uniweb site's content — agent index, per-page markdown, search index. Pure JS, runs anywhere.",
   "type": "module",
   "exports": {
@@ -31,8 +31,8 @@
     "node": ">=20.19"
   },
   "dependencies": {
-    "@uniweb/core": "^0.8.5",
-    "@uniweb/content-writer": "^0.3.3"
+    "@uniweb/content-writer": "^0.3.3",
+    "@uniweb/core": "^0.8.5"
   },
   "devDependencies": {
     "vitest": "^4.1.7",

package/src/corpus.js

@@ -44,6 +44,8 @@ import {
   isDynamicTemplate,
   hasDraftSegment,
   isAtOrUnder,
+  knowledgeRoots,
+  isKnowledgeRoute,
 } from './pages.js'
 import { normalizeExclude } from './config.js'
 
@@ -64,13 +66,13 @@ import { normalizeExclude } from './config.js'
 export function partitionKnowledgePages(pages = []) {
   if (!Array.isArray(pages)) return { knowledgePages: [], renderedPages: [] }
 
-  const knowledgeRoots = pages.filter(page => page?.knowledge).map(page => page.route)
+  const roots = knowledgeRoots(pages)
 
   const knowledgePages = []
   const renderedPages = []
 
   for (const page of pages) {
-    if (isKnowledgePage(page, knowledgeRoots)) knowledgePages.push(page)
+    if (isKnowledgePage(page, roots)) knowledgePages.push(page)
     else renderedPages.push(page)
   }
 
@@ -79,20 +81,28 @@ export function partitionKnowledgePages(pages = []) {
 
 /**
  * @param {Object} page
- * @param {string[]} knowledgeRoots - Routes carrying `knowledge: true`
+ * @param {string[]} roots - Routes carrying `knowledge: true`
  * @returns {boolean}
  */
-function isKnowledgePage(page, knowledgeRoots) {
+function isKnowledgePage(page, roots) {
+  // The flag is checked first rather than folded into the route test: a page
+  // carrying it with no route at all is still not something to render.
   if (page?.knowledge) return true
-  const route = page?.route
-  if (!route) return false
-  return knowledgeRoots.some(root => route !== root && isAtOrUnder(route, root))
+  return isKnowledgeRoute(page?.route, roots)
 }
 
 /**
- * The pages the agent corpus may contain.
+ * The pages a corpus may contain — **public by default; agent-only on request.**
  *
- * ⛔ **THIS IS NOT "the site's full content", AND THE DIFFERENCE IS A LEAK.**
+ * ⛔ **THE ARGUMENT ORDER IS THE SAFETY PROPERTY. Passing nothing gets you the
+ * PUBLIC selection.** A caller who forgets the option, or copies a call from a
+ * public-tier consumer, under-discloses. That is the failure we can afford;
+ * the other direction puts agent-only content on a visitor-facing endpoint.
+ * So `knowledge` defaults to `false` and the agent corpus is the one that has
+ * to ask — the two tiers must not merely pass different arguments to one
+ * selector, the public one must be what you get by passing none.
+ *
+ * ⛔ **AND EVEN WITH `knowledge: true`, THIS IS NOT "the site's full content".**
  * It is tempting to index every page — the agent is more useful the more it
  * knows — and to treat `knowledge: true` as purely a render-subtraction. That
  * is defensible for a private tool an author runs over their own content, and
@@ -102,8 +112,8 @@ function isKnowledgePage(page, knowledgeRoots) {
  * follows from: projections are on by default, so weakening the exclusions
  * turns the default into a leak.
  *
- * So the corpus is the public projection **plus** what the author explicitly
- * marked for agents:
+ * With `knowledge: true` the corpus is the public projection **plus** what the
+ * author explicitly marked for agents:
  *
  * | signal | reach | why |
  * |---|---|---|
@@ -119,13 +129,25 @@ function isKnowledgePage(page, knowledgeRoots) {
  * agent knows less; the cost of ignoring it is private content on a public
  * endpoint. Those are not symmetric.
  *
+ * ⚠️ **The `added` row is load-bearing and was not always doing anything.**
+ * Until `excludedBranches` learned about `knowledge:`, the public half already
+ * contained knowledge pages, so this union was a no-op and every test of it
+ * passed for the wrong reason. That is the shape to watch for when reading a
+ * union: it looks correct whether or not the two halves are actually disjoint.
+ * `corpus.test.js` now pins the public half's *absence* of them directly.
+ *
  * @param {Object[]} pages - `siteContent.pages`
  * @param {Object} [options]
  * @param {string[]} [options.exclude] - Additional excluded route prefixes
+ * @param {boolean} [options.knowledge=false] - Admit `knowledge:` pages. Off by
+ *   default; see the argument-order note above before changing that.
  * @returns {Object[]} Pages in build order, no duplicates
  */
-export function selectCorpusPages(pages = [], { exclude = [] } = {}) {
-  const publicSet = new Set(selectIndexablePages(pages, { exclude }))
+export function selectCorpusPages(pages = [], { exclude = [], knowledge = false } = {}) {
+  const publicPages = selectIndexablePages(pages, { exclude })
+  if (!knowledge) return publicPages
+
+  const publicSet = new Set(publicPages)
   const branches = normalizeExclude(exclude)
   const { knowledgePages } = partitionKnowledgePages(pages)
 
@@ -147,27 +169,36 @@ export function selectCorpusPages(pages = [], { exclude = [] } = {}) {
 /**
  * Build the greppable corpus for one locale of a site.
  *
+ * **Public by default.** `knowledge: true` is the agent tier and has to be
+ * asked for — see the argument-order note on {@link selectCorpusPages}.
+ *
  * @param {Object} siteContent - Parsed site-content.json (one locale)
  * @param {Object} [options]
  * @param {string[]} [options.exclude] - Additional excluded route prefixes
  * @param {boolean} [options.includeChildren=true] - Include nested sections
+ * @param {boolean} [options.knowledge=false] - Admit `knowledge:` pages
  * @returns {Array<CorpusPage>} Pages with content, in build order
  */
-export function buildCorpus(siteContent, { exclude, includeChildren = true } = {}) {
+export function buildCorpus(
+  siteContent,
+  { exclude, includeChildren = true, knowledge = false } = {}
+) {
   const pages = siteContent?.pages || []
   const config = siteContent?.config || {}
   const excluded = exclude ?? config.agents?.exclude ?? []
 
-  const knowledgeRoutes = new Set(
-    partitionKnowledgePages(pages).knowledgePages.map(page => page.route)
-  )
+  // Only meaningful when knowledge pages were admitted; an empty set otherwise
+  // keeps every `CorpusPage.knowledge` false, which is the truth on that tier.
+  const agentOnly = knowledge
+    ? new Set(partitionKnowledgePages(pages).knowledgePages.map(page => page.route))
+    : new Set()
 
   const corpus = []
 
-  for (const page of selectCorpusPages(pages, { exclude: excluded })) {
+  for (const page of selectCorpusPages(pages, { exclude: excluded, knowledge })) {
     const built = buildCorpusPage(page, {
       includeChildren,
-      knowledge: knowledgeRoutes.has(page.route),
+      knowledge: agentOnly.has(page.route),
     })
     // A page whose sections carry no projectable content is not a page the
     // agent can read. Listing it would produce read_page hits that return ''.

package/src/pages.js

@@ -65,10 +65,38 @@ export function isAtOrUnder(route, prefix) {
   return route === prefix || route.startsWith(`${prefix}/`)
 }
 
+/**
+ * The routes carrying `knowledge: true` — the roots of the agent-only branches.
+ *
+ * ⚠️ Defined here, in the lower layer, rather than in `corpus.js` where the
+ * concept reads like it belongs. Both files need it and the dependency only
+ * runs one way, so putting it there would mean a second copy of "is this route
+ * inside that branch" — the failure {@link isAtOrUnder} exists to prevent.
+ *
+ * @param {Object[]} pages
+ * @returns {string[]}
+ */
+export function knowledgeRoots(pages = []) {
+  if (!Array.isArray(pages)) return []
+  return pages.filter(page => page?.knowledge && page.route).map(page => page.route)
+}
+
+/**
+ * Is `route` a knowledge route — carrying the flag, or beneath one that does?
+ *
+ * @param {string} route
+ * @param {string[]} roots - {@link knowledgeRoots}
+ * @returns {boolean}
+ */
+export function isKnowledgeRoute(route, roots) {
+  if (!route) return false
+  return roots.some(root => isAtOrUnder(route, root))
+}
+
 /**
  * Route prefixes whose whole branch is excluded.
  *
- * Two sources, with different reach — deliberately:
+ * Three sources, with different reach — deliberately:
  *
  * - `agents.exclude` cascades. `exclude: [/internal]` plainly means the
  *   branch, not one page.
@@ -76,6 +104,20 @@ export function isAtOrUnder(route, prefix) {
  *   pure structure: suppressing the heading while still listing its children
  *   would orphan them under the wrong group. On a page with content it stays
  *   per-page, matching how the sitemap reads `noindex`.
+ * - **`knowledge: true` cascades**, by the same route-prefix rule
+ *   {@link partitionKnowledgePages} renders by. A knowledge page is not
+ *   rendered and cannot be reached by a visitor, so nothing describing the
+ *   *public* site may name it.
+ *
+ * ⛔ That last one is a disclosure boundary, not a tidy-up, and the mistake it
+ * corrects was invisible for a reason worth stating. These projections are the
+ * **free** tier; the agent corpus that knowledge pages exist for is the
+ * **paid** one. Leaving `knowledge` out of this list did not merely list a
+ * page — it published the body an author wrote for a capability they may never
+ * have bought, at `llms.txt`, at `/{route}.md`, and in the search index. It
+ * also made {@link selectCorpusPages} degenerate: that selector is *public ∪
+ * knowledge*, and while knowledge rode in the public half the union added
+ * nothing and read as if it worked.
  *
  * @param {Object[]} pages
  * @param {string[]} exclude
@@ -87,7 +129,7 @@ function excludedBranches(pages, exclude) {
     if (!isContainer(page)) continue
     if (page.seo?.noindex || page.hidden) branches.push(page.route)
   }
-  return branches
+  return branches.concat(knowledgeRoots(pages))
 }
 
 /**

package/src/search/extract.js

@@ -1,14 +1,15 @@
 /**
  * Extract searchable content from site-content.json
  *
- * Walks through all pages and sections, extracting text content
- * for search indexing. Reuses patterns from i18n extraction.
+ * Walks the pages a *visitor* can reach — {@link selectIndexablePages} decides
+ * which those are — and extracts their text for search indexing.
  */
 
 // Leaf subpath, not the bare `@uniweb/core` entry: this package's environment
 // contract forbids the package root (it pulls semantic-parser + theming).
 // Enforced by tests/environment.test.js.
 import { sectionDomId } from '@uniweb/core/section-id'
+import { selectIndexablePages } from '../pages.js'
 
 /**
  * Extract all searchable content from site
@@ -20,7 +21,8 @@ import { sectionDomId } from '@uniweb/core/section-id'
  * @param {boolean} [options.paragraphs=true] - Include paragraphs
  * @param {boolean} [options.links=true] - Include link labels
  * @param {boolean} [options.lists=true] - Include list items
- * @param {Array<string>} [options.excludeRoutes=[]] - Routes to exclude
+ * @param {Array<string>} [options.excludeRoutes=[]] - Route prefixes to
+ *   exclude; the whole branch goes, matching `agents.exclude`
  * @param {Array<string>} [options.excludeComponents=[]] - Components to exclude
  * @returns {Array<Object>} Array of search entries
  */
@@ -38,19 +40,20 @@ export function extractSearchContent(siteContent, options = {}) {
 
   const entries = []
 
-  for (const page of siteContent.pages || []) {
-    const pageRoute = page.route || '/'
-
-    // Skip excluded routes
-    if (excludeRoutes.some(r => pageRoute.startsWith(r))) {
-      continue
-    }
-
-    // Skip pages marked as noindex
-    if (page.seo?.noindex) {
-      continue
-    }
-
+  // Which pages may be described is `pages.js`'s decision, not this file's.
+  //
+  // ⛔ This used to be two inline `continue`s — `search.exclude.routes` and
+  // `seo.noindex` — and the gap between that pair and the shared selector was a
+  // disclosure. A search index is a *visitor-facing* artifact, so it must
+  // describe only pages a visitor can reach; `knowledge:` pages are not
+  // rendered at all, and drafts, `hidden` pages and dynamic route *templates*
+  // are each unreachable or unlinkable in their own way. Reimplementing that
+  // list here is how it drifts from the one `llms.txt` and the per-page
+  // markdown use, which is exactly what happened.
+  //
+  // Note this also fixes the prefix test: `excludeRoutes` was matched with
+  // `startsWith`, so excluding `/kb` silently excluded `/kbase` too.
+  for (const page of selectIndexablePages(siteContent.pages || [], { exclude: excludeRoutes })) {
     // Extract page-level entry
     if (includePagesFlag) {
       const pageEntry = extractFromPage(page)