@agentskit/doc-bridge 1.7.45 → 1.10.0

package/src/index-builder/build-index.ts

@@ -2,8 +2,14 @@ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
 import { dirname, join } from 'node:path'
 
 import type { DocBridgeConfigV1 } from '../config/schema.js'
+import { applyDocumentationDeclarations } from '../discovery/documentation.js'
+import { discoverRepository } from '../discovery/repository.js'
+import { readBoundedText, type TextReadBudget } from '../lib/bounded-text.js'
 import { toPosix } from '../lib/paths.js'
-import type { DocBridgeIndexV1 } from '../schemas/doc-bridge-index.js'
+import type { DocBridgeIndexV1, KnowledgeEntry } from '../schemas/doc-bridge-index.js'
+import { RETRIEVAL_MAX_ENTRIES } from '../schemas/retrieval-index.js'
+import type { DiscoverySnapshotV1 } from '../schemas/knowledge.js'
+import type { RetrievalIndexV1 } from '../schemas/retrieval-index.js'
 import { buildLookup, collectPackages } from './build-handoffs.js'
 import { renderCapabilitiesJson } from './capabilities.js'
 import { sha256NormalizedV1 } from './content-hash.js'
@@ -11,12 +17,33 @@ import { renderLlmsTxt } from './llms-txt.js'
 import { scanHumanDocs } from './human-adapters/index.js'
 import { discoverNxProjects } from './plugins/nx.js'
 import { discoverPnpmPackages } from './plugins/pnpm-monorepo.js'
+import { repositoryInputs } from './project-corpus.js'
 import { scanAgentCorpus } from './scan-corpus.js'
+import { SEARCH_LEXICON_VERSION } from '../query/text.js'
+import { projectRetrievalIndex, toKnowledgeEntry } from '../retrieval/project.js'
+import { resolveSearchParams, resolveSearchWeights } from '../retrieval/weights.js'
+import { projectEnrichmentOverlay, readEnrichmentOverlay } from '../enrich/overlay.js'
+import type { EnrichmentOverlayV1 } from '../schemas/enrichment.js'
 
 export type BuildIndexOptions = {
   readonly root?: string
   readonly config: DocBridgeConfigV1
   readonly write?: boolean
+  /**
+   * A snapshot to project instead of scanning. The build scans when none is given; a caller that
+   * already holds the snapshot — a workflow stage, a test — passes it so the index and the
+   * artifacts it sits next to describe the same observation.
+   */
+  readonly snapshot?: DiscoverySnapshotV1
+  /**
+   * Which enrichment overlay the projection reads.
+   *
+   * Omitted, the builder reads the one on disk while the Registry is enabled — the normal path.
+   * `'ignore'` builds the deterministic baseline even then, and an overlay object projects that
+   * one instead. Both exist so the retrieval delta can measure the same snapshot twice, with the
+   * overlay and without it, rather than comparing two different repositories.
+   */
+  readonly overlay?: EnrichmentOverlayV1 | 'ignore'
 }
 
 export type BuildIndexResult = {
@@ -37,6 +64,58 @@ const projectName = (root: string, config: DocBridgeConfigV1): string => {
   return toPosix(root.split('/').pop() ?? 'project')
 }
 
+/**
+ * Observe, declare, project.
+ *
+ * The snapshot is the observation; documentation declarations (`covers`, frontmatter relations)
+ * are applied on top of it so a document's declared coverage ranks and routes like an observed
+ * edge; the projection is computed from the result. Document bodies are read once, bounded, and
+ * handed to both steps by path.
+ */
+const projectFromSnapshot = (
+  root: string,
+  config: DocBridgeConfigV1,
+  given: DiscoverySnapshotV1 | undefined,
+  lookup: ReturnType<typeof buildLookup>['lookup'],
+  curated: readonly KnowledgeEntry[],
+  requested: BuildIndexOptions['overlay'],
+): { readonly projection: RetrievalIndexV1 } => {
+  const observed = given ?? discoverRepository({ root, config })
+  const budget: TextReadBudget = { used: 0 }
+  const contents = new Map<string, string>()
+  for (const entity of observed.entities) {
+    if (entity.kind !== 'document' || !entity.path) continue
+    try {
+      contents.set(entity.path, readBoundedText(join(root, entity.path), budget))
+    } catch {
+      // Unreadable now: the entity still projects from what the snapshot recorded about it.
+    }
+  }
+  const declared = applyDocumentationDeclarations(
+    observed,
+    [...contents.entries()].map(([path, content]) => ({ path, content })),
+    { agentRoot: config.corpus.agent.root },
+  ).snapshot
+  /*
+   * The accepted enrichment overlay is consulted only while the Registry is enabled: switching it
+   * off restores the deterministic baseline exactly. The read never writes, and an entry whose
+   * target moved since it was accepted is expired here rather than ranked.
+   */
+  const accepted = requested === 'ignore'
+    ? undefined
+    : (requested ?? (config.intelligence?.registry?.enabled ? readEnrichmentOverlay(root) : undefined))
+  const overlay = accepted ? projectEnrichmentOverlay(accepted, declared) : undefined
+  const projection = projectRetrievalIndex({
+    snapshot: declared,
+    config,
+    routes: lookup,
+    curated: curated.map((entry) => ({ id: entry.id, path: entry.path, title: entry.title, ...(entry.description ? { description: entry.description } : {}) })),
+    ...(overlay ? { overlay } : {}),
+    readDocument: (path) => contents.get(path),
+  })
+  return { projection }
+}
+
 const existingGeneratedAt = (indexPath: string, contentHash: string): string | undefined => {
   try {
     const index = JSON.parse(readFileSync(indexPath, 'utf8')) as {
@@ -59,7 +138,12 @@ export const buildDocBridgeIndex = (opts: BuildIndexOptions): BuildIndexResult =
   const indexPath = join(root, outFile)
 
   const corpus = scanAgentCorpus(root, config)
-  const knowledge = corpus.map(({ absPath: _a, relPath: _r, frontmatter: _f, ...entry }) => entry)
+  const curated = corpus.map(({ absPath: _a, relPath: _r, frontmatter: _f, ...entry }) => entry)
+  const retrieval = {
+    lexiconVersion: SEARCH_LEXICON_VERSION,
+    weights: resolveSearchWeights(config.retrieval?.weights),
+    params: resolveSearchParams(config.retrieval?.params),
+  }
 
   const shouldDiscover =
     config.routing?.plugin === 'pnpm-monorepo' ||
@@ -78,11 +162,39 @@ export const buildDocBridgeIndex = (opts: BuildIndexOptions): BuildIndexResult =
 
   const { lookup, handoffs } = buildLookup(config, packages, corpus, outFile, humanDocs, root)
 
+  /*
+   * Retrieval reads this index, so whatever is missing here is invisible to an agent however well
+   * it is ranked. The projection puts every entity the snapshot observed in it — documents,
+   * modules, areas, packages — next to the routes the configuration declares. It is a function of
+   * the snapshot: the index has no scanner of its own, so a record retrieval can find is an entity
+   * discovery observed, with the same id and the same content hash.
+   */
+  const projected = config.retrieval?.corpus?.enabled === false ? undefined : projectFromSnapshot(root, config, opts.snapshot, lookup, curated, opts.overlay)
+  const projection = projected?.projection
+  const inputs = projected ? repositoryInputs(root, config) : undefined
+  const curatedPaths = new Set(curated.map((entry) => entry.path))
+  /*
+   * `knowledge[]` keeps every reader that predates the projection working: the curated sidecars
+   * first, in reading order, then every projected document and module. Body text lives once, in
+   * the projection, so a projected index carries no bodies here.
+   */
+  const knowledge: KnowledgeEntry[] = projection
+    ? [
+        ...curated.map(({ body: _body, ...entry }) => entry),
+        ...projection.entries
+          .filter((entry) => (entry.kind === 'document' || entry.kind === 'module') && !curatedPaths.has(entry.path))
+          .map(toKnowledgeEntry),
+      ]
+    : curated
+
   const hashPayload = {
     schemaVersion: 1,
     knowledge,
     handoffs,
     lookup,
+    retrieval,
+    ...(inputs ? { inputs } : {}),
+    ...(projection ? { projection: projection.contentHash } : {}),
   }
 
   const contentHash = sha256NormalizedV1(hashPayload)
@@ -95,6 +207,23 @@ export const buildDocBridgeIndex = (opts: BuildIndexOptions): BuildIndexResult =
     knowledge,
     handoffs,
     lookup,
+    ...(inputs ? { inputs } : {}),
+    retrieval,
+    ...(projection ? { projection } : {}),
+  }
+
+  /*
+   * An artifact its own parser refuses is not an artifact. The bound on `knowledge[]` used to be
+   * lower than the bound on the projection it mirrors, so a large repository got an index that
+   * `index` reported writing and every reader — `doctor`, `search`, the MCP server — then rejected,
+   * with a schema dump naming an array instead of a corpus. The bounds are one constant now; this
+   * says so at the point of production, where the numbers and the remedy are both known.
+   */
+  if (knowledge.length > RETRIEVAL_MAX_ENTRIES) {
+    throw new Error(
+      `The index would carry ${knowledge.length} knowledge entries, above the ${RETRIEVAL_MAX_ENTRIES} an index may hold. ` +
+        'Narrow the corpus with `corpus.*.include` or `corpus.*.exclude`, or split the repository into more than one index.',
+    )
   }
 
   if (write) {
@@ -111,7 +240,7 @@ export const buildDocBridgeIndex = (opts: BuildIndexOptions): BuildIndexResult =
     if (write) {
       writeFileSync(
         llmsTxtPath,
-        renderLlmsTxt(config, knowledge, index.project?.name ?? 'project'),
+        renderLlmsTxt(config, knowledge, index.project?.name ?? 'project', { root }),
         'utf8',
       )
     }

package/src/index-builder/llms-txt.ts

@@ -1,5 +1,7 @@
 import type { DocBridgeConfigV1 } from '../config/schema.js'
+import { renderNamedTemplate } from '../render/template-source.js'
 import type { KnowledgeEntry } from '../schemas/doc-bridge-index.js'
+import { isProjectedEntry } from './project-corpus.js'
 
 const routePath = (path: string, pathPrefix: string | undefined): string => {
   const normalizedPath = path.replaceAll('\\', '/').replace(/\.(?:md|mdx)$/, '')
@@ -20,20 +22,58 @@ export const knowledgeUrl = (
   return new URL(relativePath, base).toString()
 }
 
-export const renderLlmsTxt = (
+export type LlmsTxtEntry = {
+  readonly title: string
+  readonly url: string
+  readonly description?: string
+}
+
+export type LlmsTxtVariables = {
+  readonly project: string
+  readonly preamble: string
+  readonly entries: readonly LlmsTxtEntry[]
+}
+
+/** What the `llms.txt` template sees. Exported so a project overriding the template knows the variables. */
+export const llmsTxtVariables = (
   config: DocBridgeConfigV1,
   knowledge: readonly KnowledgeEntry[],
   projectName: string,
-): string => {
+): LlmsTxtVariables => {
   const preamble =
     config.index?.llmsTxt?.preamble ??
     `# ${projectName}\n\n> Agent-readable documentation index generated by ak-docs (@agentskit/doc-bridge).\n`
 
-  const lines = knowledge.slice(0, 500).map((entry) => {
-    const desc = entry.description ? `: ${entry.description}` : ''
-    const url = knowledgeUrl(entry.path, config.index?.llmsTxt)
-    return `- [${entry.title}](${url})${desc}`
-  })
+  /*
+   * llms.txt is a reading order, not an inventory. The index also carries every document and
+   * module the repository projects into it for retrieval; listing those here would bury the
+   * curated entry points an agent is meant to start from. The filter lives in the renderer so
+   * every caller — the builder and the conformance profile that re-renders to check freshness —
+   * agrees on what the file contains.
+   */
+  const entries = knowledge
+    .filter((entry) => !isProjectedEntry(entry))
+    .slice(0, 500)
+    .map((entry) => ({
+      title: entry.title,
+      url: knowledgeUrl(entry.path, config.index?.llmsTxt),
+      ...(entry.description ? { description: entry.description } : {}),
+    }))
 
-  return `${preamble.trim()}\n\n## Knowledge\n\n${lines.join('\n')}\n`
+  return { project: projectName, preamble: preamble.trim(), entries }
 }
+
+/**
+ * Render `llms.txt` from the bundled template, or the project's under `render.templates`.
+ *
+ * `root` resolves a project template path; the builder and the conformance profile both pass
+ * theirs, so the file the builder writes is the file the profile expects. The bundled template
+ * reproduces the previous concatenation byte for byte, which the llms and federation tests hold
+ * it to.
+ */
+export const renderLlmsTxt = (
+  config: DocBridgeConfigV1,
+  knowledge: readonly KnowledgeEntry[],
+  projectName: string,
+  options: { readonly root?: string } = {},
+): string => renderNamedTemplate('llms.txt', llmsTxtVariables(config, knowledge, projectName), config, options.root)

package/src/index-builder/project-corpus.ts

@@ -0,0 +1,111 @@
+import { basename, extname, relative, resolve, sep } from 'node:path'
+import { readFileSync } from 'node:fs'
+
+import type { DocBridgeConfigV1 } from '../config/schema.js'
+import { DOCUMENT_EXTENSIONS, SOURCE_EXTENSIONS, safeWalkOptions } from '../discovery/inputs.js'
+import { toPosix } from '../lib/paths.js'
+import { safeWalkFiles } from '../safety/repository.js'
+import type { KnowledgeEntry } from '../schemas/doc-bridge-index.js'
+import { sha256NormalizedV1 } from './content-hash.js'
+
+/**
+ * The freshness fingerprint of the index.
+ *
+ * This file used to be retrieval's own scanner: it walked the repository, parsed every module and
+ * document a second time, and projected them into index records that shared nothing with the
+ * snapshot but a path. That projection is now a function of the snapshot (`src/retrieval/project.ts`),
+ * and what remains here is the one thing a query needs on every call: a cheap way to tell whether
+ * the index on disk still describes the files on disk. One walk, one hash per file, no parsing.
+ */
+
+/**
+ * Bumped when what the index derives from its inputs changes shape, so an index built by older
+ * code is stale by version even when every file matches.
+ */
+export const CORPUS_PROJECTION_VERSION = 2 as const
+
+/**
+ * Entry types the projection produces into `knowledge[]`. Curated corpus entries are always
+ * `agent-doc`, so this is what distinguishes a record retrieval discovered from one a human wrote
+ * a sidecar for — which surfaces such as llms.txt need, since a reading order is curated, not
+ * enumerated.
+ */
+export const PROJECTED_ENTRY_TYPES = ['document', 'module'] as const
+
+export const isProjectedEntry = (entry: Pick<KnowledgeEntry, 'type'>): boolean =>
+  (PROJECTED_ENTRY_TYPES as readonly string[]).includes(entry.type)
+
+/**
+ * Configuration files the index is derived from. Narrow on purpose: any `.json` would make an
+ * unrelated data file mark the index stale, and a generated artifact could then invalidate the
+ * artifact generated from it.
+ */
+const CONFIG_INPUT_PATTERN =
+  /(?:^|\/)(?:package\.json|pnpm-workspace\.ya?ml|tsconfig(?:\.[\w.-]+)?\.json|jsconfig\.json|meta\.json|doc-bridge\.config\.(?:json|ya?ml|js|ts|mjs|cjs))$/
+
+const INPUT_EXTENSIONS = [...new Set([...SOURCE_EXTENSIONS, ...DOCUMENT_EXTENSIONS, '.json', '.yaml', '.yml'])]
+
+/**
+ * Configuration sections the index is derived from.
+ *
+ * The same files under a different configuration project a different index, so the configuration
+ * belongs in the fingerprint — but only the part of it that can change the artifact. Hashing the
+ * whole configuration would report a stale index when an unrelated section changed (a gate
+ * preset, a report option), which is a false alarm that teaches people to ignore the check.
+ */
+const INDEX_CONFIGURATION_KEYS = ['project', 'corpus', 'index', 'routing', 'safety', 'retrieval', 'analysis'] as const
+
+export const indexConfigurationHash = (config: DocBridgeConfigV1 | undefined): string =>
+  sha256NormalizedV1(
+    Object.fromEntries(
+      INDEX_CONFIGURATION_KEYS.filter((key) => config?.[key] !== undefined).map((key) => [key, config?.[key]]),
+    ),
+  )
+
+export type RepositoryInputsV1 = {
+  /** Hash of every input path and its content. Equal hashes mean an equal projection. */
+  readonly hash: string
+  readonly fileCount: number
+  readonly projectionVersion: number
+  /** True when a safety limit stopped the walk, so the file set is not the whole repository. */
+  readonly incomplete?: boolean
+}
+
+const isInput = (path: string, name: string): boolean => {
+  const extension = extname(name)
+  if ((DOCUMENT_EXTENSIONS as readonly string[]).includes(extension)) return true
+  if ((SOURCE_EXTENSIONS as readonly string[]).includes(extension)) return true
+  return CONFIG_INPUT_PATTERN.test(path)
+}
+
+/**
+ * Walk the repository once and hash every input the index derives from: sources, documents and
+ * the configuration files that decide how they resolve.
+ */
+export const repositoryInputs = (root: string, config: DocBridgeConfigV1 | undefined): RepositoryInputsV1 => {
+  const projectRoot = resolve(root)
+  const walk = safeWalkFiles(projectRoot, { extensions: INPUT_EXTENSIONS, ...safeWalkOptions(config) })
+  const fingerprints: [string, string][] = []
+
+  for (const absPath of walk.files) {
+    const path = toPosix(relative(projectRoot, absPath).split(sep).join('/'))
+    if (!isInput(path, basename(absPath))) continue
+    try {
+      fingerprints.push([path, sha256NormalizedV1(readFileSync(absPath, 'utf8'))])
+    } catch {
+      // An unreadable input cannot be projected, and must not silently change the hash either.
+      fingerprints.push([path, 'unreadable'])
+    }
+  }
+
+  return {
+    hash: sha256NormalizedV1({
+      projectionVersion: CORPUS_PROJECTION_VERSION,
+      configurationHash: indexConfigurationHash(config),
+      files: fingerprints,
+    }),
+    fileCount: fingerprints.length,
+    projectionVersion: CORPUS_PROJECTION_VERSION,
+    ...(walk.incomplete ? { incomplete: true } : {}),
+  }
+}