@agentskit/doc-bridge 1.7.45 → 1.9.0

package/src/bench/baseline.ts

@@ -0,0 +1,198 @@
+import { z } from 'zod'
+
+import { contentHashForArtifactV1 } from '../index-builder/content-hash.js'
+import { RETRIEVAL_BENCH_SCHEMA_VERSION, type RetrievalBenchResultV1, type RetrievalMetrics } from './retrieval.js'
+
+const hash = z.string().regex(/^[a-f0-9]{64}$/)
+
+const BaselineMetricsSchema = z
+  .object({
+    caseCount: z.number().int().nonnegative(),
+    hitAt1: z.number().min(0).max(1),
+    hitAt3: z.number().min(0).max(1),
+    meanReciprocalRank: z.number().min(0).max(1),
+    meanContextBytes: z.number().int().nonnegative(),
+    meanApproxTokens: z.number().int().nonnegative(),
+    zeroResultRate: z.number().min(0).max(1),
+    tokenMethod: z.literal('approximate'),
+  })
+  .strict()
+
+export const RetrievalBaselineV1Schema = z
+  .object({
+    type: z.literal('retrieval-benchmark-baseline'),
+    schemaVersion: z.literal(RETRIEVAL_BENCH_SCHEMA_VERSION),
+    contentHash: hash,
+    contentHashAlgo: z.literal('sha256-normalized-v1'),
+    suite: z
+      .object({
+        name: z.string().min(1).max(128),
+        caseCount: z.number().int().positive(),
+        contentHash: hash,
+      })
+      .strict(),
+    metrics: BaselineMetricsSchema,
+    /**
+     * A baseline exists only as the result of an explicit human action. A normal
+     * `ak-docs bench retrieval` run compares against it and never writes it, so the number
+     * a gate defends cannot drift by accident.
+     */
+    approval: z
+      .object({
+        approvedAt: z.string().datetime(),
+        approvedBy: z.string().min(1).max(256),
+        reason: z.string().min(1).max(1_024).optional(),
+        /** Result the approved figures were measured from, for provenance. */
+        resultHash: hash,
+        /** Index the approved figures were measured on. */
+        indexHash: hash,
+      })
+      .strict(),
+  })
+  .strict()
+
+export type RetrievalBaselineV1 = z.infer<typeof RetrievalBaselineV1Schema>
+
+export const parseRetrievalBaseline = (raw: unknown): RetrievalBaselineV1 => {
+  const baseline = RetrievalBaselineV1Schema.parse(raw)
+  if (contentHashForArtifactV1(baseline) !== baseline.contentHash) {
+    throw new Error('Invalid retrieval benchmark baseline content hash.')
+  }
+  return baseline
+}
+
+export type CreateRetrievalBaselineOptions = {
+  readonly result: RetrievalBenchResultV1
+  readonly approvedBy: string
+  readonly reason?: string
+  readonly approvedAt?: string
+}
+
+/** Record a measured result as the figure future runs are gated against. */
+export const createRetrievalBaseline = (options: CreateRetrievalBaselineOptions): RetrievalBaselineV1 => {
+  const approvedBy = options.approvedBy.trim()
+  if (!approvedBy) throw new Error('A retrieval baseline requires an approver. Pass --by <name>.')
+  const base = {
+    type: 'retrieval-benchmark-baseline' as const,
+    schemaVersion: RETRIEVAL_BENCH_SCHEMA_VERSION,
+    contentHash: '0'.repeat(64),
+    contentHashAlgo: 'sha256-normalized-v1' as const,
+    suite: options.result.suite,
+    metrics: options.result.metrics,
+    approval: {
+      approvedAt: options.approvedAt ?? new Date().toISOString(),
+      approvedBy,
+      ...(options.reason ? { reason: options.reason } : {}),
+      resultHash: options.result.contentHash,
+      indexHash: options.result.index.contentHash,
+    },
+  }
+  return RetrievalBaselineV1Schema.parse({ ...base, contentHash: contentHashForArtifactV1(base) })
+}
+
+export type RetrievalComparisonStatus = 'pass' | 'regressed' | 'suite-changed'
+
+export type RetrievalMetricDelta = {
+  readonly metric: keyof RetrievalMetrics
+  readonly baseline: number
+  readonly current: number
+  readonly delta: number
+}
+
+export type RetrievalComparison = {
+  readonly status: RetrievalComparisonStatus
+  readonly blocking: boolean
+  readonly tolerance: number
+  readonly suiteChanged: boolean
+  /** Reasons the gate fails. Empty when it passes. */
+  readonly regressions: readonly string[]
+  /** Non-blocking movements worth reading before merging. */
+  readonly warnings: readonly string[]
+  readonly improvements: readonly string[]
+  readonly deltas: readonly RetrievalMetricDelta[]
+}
+
+/** Metrics where a lower value is the improvement. */
+const LOWER_IS_BETTER = new Set<keyof RetrievalMetrics>(['meanContextBytes', 'meanApproxTokens', 'zeroResultRate'])
+/** The one metric whose regression blocks. Everything else is reported, not enforced. */
+const BLOCKING_METRIC: keyof RetrievalMetrics = 'hitAt3'
+const COMPARED: readonly (keyof RetrievalMetrics)[] = [
+  'hitAt1',
+  'hitAt3',
+  'meanReciprocalRank',
+  'meanContextBytes',
+  'meanApproxTokens',
+  'zeroResultRate',
+]
+
+const format = (metric: keyof RetrievalMetrics, value: number): string =>
+  metric === 'meanContextBytes' || metric === 'meanApproxTokens' ? String(value) : value.toFixed(3)
+
+export type CompareRetrievalBaselineOptions = {
+  /** Slack allowed on the blocking metric before a drop counts as a regression. Default 0. */
+  readonly tolerance?: number
+}
+
+/**
+ * Compare a fresh result with an approved baseline.
+ *
+ * A changed suite fails the gate rather than being silently compared: the figures were
+ * measured over different questions, so neither a pass nor a regression would mean
+ * anything. Resolving it is an explicit baseline update.
+ */
+export const compareRetrievalBaseline = (
+  result: RetrievalBenchResultV1,
+  baseline: RetrievalBaselineV1,
+  options: CompareRetrievalBaselineOptions = {},
+): RetrievalComparison => {
+  const tolerance = options.tolerance ?? 0
+  if (tolerance < 0) throw new Error('Retrieval baseline tolerance must not be negative.')
+  const suiteChanged = baseline.suite.contentHash !== result.suite.contentHash
+
+  const deltas = COMPARED.map((metric) => ({
+    metric,
+    baseline: baseline.metrics[metric] as number,
+    current: result.metrics[metric] as number,
+    delta: Number(((result.metrics[metric] as number) - (baseline.metrics[metric] as number)).toFixed(6)),
+  }))
+
+  const regressions: string[] = []
+  const warnings: string[] = []
+  const improvements: string[] = []
+
+  if (suiteChanged) {
+    regressions.push(
+      `Suite "${result.suite.name}" changed since the baseline was approved (${baseline.suite.caseCount} case(s) at ${baseline.suite.contentHash.slice(0, 12)}, now ${result.suite.caseCount} at ${result.suite.contentHash.slice(0, 12)}). Re-approve the baseline with --update-baseline --by <name>.`,
+    )
+  }
+
+  for (const entry of deltas) {
+    const improved = LOWER_IS_BETTER.has(entry.metric) ? entry.delta < 0 : entry.delta > 0
+    const worsened = LOWER_IS_BETTER.has(entry.metric) ? entry.delta > 0 : entry.delta < 0
+    const message = `${entry.metric} ${format(entry.metric, entry.baseline)} → ${format(entry.metric, entry.current)}`
+    if (improved) improvements.push(message)
+    if (!worsened) continue
+    if (entry.metric === BLOCKING_METRIC && Math.abs(entry.delta) > tolerance) regressions.push(`${message} (tolerance ${tolerance})`)
+    else warnings.push(message)
+  }
+
+  const status: RetrievalComparisonStatus = suiteChanged ? 'suite-changed' : regressions.length ? 'regressed' : 'pass'
+  return {
+    status,
+    blocking: regressions.length > 0,
+    tolerance,
+    suiteChanged,
+    regressions,
+    warnings,
+    improvements,
+    deltas,
+  }
+}
+
+export const formatRetrievalComparisonText = (comparison: RetrievalComparison): readonly string[] => [
+  `Baseline comparison: ${comparison.status}`,
+  ...comparison.regressions.map((entry) => `  regression: ${entry}`),
+  ...comparison.warnings.map((entry) => `  warning: ${entry}`),
+  ...comparison.improvements.map((entry) => `  improved: ${entry}`),
+  ...(comparison.regressions.length || comparison.warnings.length || comparison.improvements.length ? [] : ['  no change']),
+]

package/src/bench/overlay-delta.ts

@@ -0,0 +1,139 @@
+import type { DocBridgeConfigV1 } from '../config/schema.js'
+import { buildDocBridgeIndex } from '../index-builder/build-index.js'
+import type { EnrichmentOverlayV1 } from '../schemas/enrichment.js'
+import type { DiscoverySnapshotV1 } from '../schemas/knowledge.js'
+import { runRetrievalBench, type RetrievalMetrics, type RetrievalSuite } from './retrieval.js'
+
+/**
+ * Does the overlay earn its cost?
+ *
+ * An enrichment stage that nobody measures is a stage nobody can defend. The question is narrow
+ * and answerable: run the golden suite twice over the same snapshot — once with the accepted
+ * overlay projected, once without it — and report the difference. Everything else about the
+ * agent is opinion; this is the number.
+ *
+ * The rule is asymmetric on purpose. An overlay may leave retrieval unchanged, and it may improve
+ * it, but it must not lower hit@3: aliases and summaries an agent proposed are supposed to help
+ * an agent find things. A drop is reported as a regression of the overlay, never as a new
+ * baseline — it is a finding about the agent, not about the benchmark.
+ */
+
+/** The metrics compared, in the order a reader wants them. */
+export const OVERLAY_DELTA_METRICS = ['hitAt1', 'hitAt3', 'meanReciprocalRank', 'meanContextBytes', 'meanApproxTokens', 'zeroResultRate'] as const
+
+/** Metrics where a lower value is the improvement. */
+const LOWER_IS_BETTER = new Set<keyof RetrievalMetrics>(['meanContextBytes', 'meanApproxTokens', 'zeroResultRate'])
+
+/** The one metric an overlay is not allowed to lower. */
+export const OVERLAY_BLOCKING_METRIC: keyof RetrievalMetrics = 'hitAt3'
+
+export type OverlayMetricDelta = {
+  readonly metric: (typeof OVERLAY_DELTA_METRICS)[number]
+  readonly withoutOverlay: number
+  readonly withOverlay: number
+  readonly delta: number
+  readonly improved: boolean
+  readonly worsened: boolean
+}
+
+export type OverlayRetrievalDelta = {
+  readonly status: 'improved' | 'unchanged' | 'regressed'
+  /** True when the overlay lowered hit@3. The caller fails the run on this. */
+  readonly regression: boolean
+  readonly overlayHash: string
+  readonly suite: { readonly name: string; readonly caseCount: number; readonly contentHash: string }
+  readonly withoutOverlay: RetrievalMetrics
+  readonly withOverlay: RetrievalMetrics
+  readonly deltas: readonly OverlayMetricDelta[]
+  /** Case ids the overlay lost, and gained, at hit@3 — what a regression is actually made of. */
+  readonly lostCases: readonly string[]
+  readonly gainedCases: readonly string[]
+  readonly messages: readonly string[]
+}
+
+export type MeasureOverlayRetrievalDeltaOptions = {
+  readonly root: string
+  readonly config: DocBridgeConfigV1
+  readonly snapshot: DiscoverySnapshotV1
+  readonly overlay: EnrichmentOverlayV1
+  readonly suite: RetrievalSuite
+  readonly limit?: number
+}
+
+const round = (value: number): number => Math.round(value * 1_000_000) / 1_000_000
+
+const format = (metric: keyof RetrievalMetrics, value: number): string =>
+  metric === 'meanContextBytes' || metric === 'meanApproxTokens' ? String(value) : value.toFixed(3)
+
+/**
+ * Measure the suite with and without the overlay.
+ *
+ * Both indexes are built from the one snapshot the caller passes, so the only difference between
+ * the two runs is the overlay — not a re-scan, not a different revision, not a different
+ * configuration.
+ */
+export const measureOverlayRetrievalDelta = (options: MeasureOverlayRetrievalDeltaOptions): OverlayRetrievalDelta => {
+  const { root, config, snapshot, overlay, suite } = options
+  const limit = options.limit === undefined ? {} : { limit: options.limit }
+  const baseline = runRetrievalBench({
+    index: buildDocBridgeIndex({ root, config, write: false, snapshot, overlay: 'ignore' }).index,
+    suite,
+    ...limit,
+  })
+  const enriched = runRetrievalBench({
+    index: buildDocBridgeIndex({ root, config, write: false, snapshot, overlay }).index,
+    suite,
+    ...limit,
+  })
+
+  const deltas = OVERLAY_DELTA_METRICS.map((metric): OverlayMetricDelta => {
+    const before = baseline.metrics[metric]
+    const after = enriched.metrics[metric]
+    const delta = round(after - before)
+    return {
+      metric,
+      withoutOverlay: before,
+      withOverlay: after,
+      delta,
+      improved: LOWER_IS_BETTER.has(metric) ? delta < 0 : delta > 0,
+      worsened: LOWER_IS_BETTER.has(metric) ? delta > 0 : delta < 0,
+    }
+  })
+
+  const hitBefore = new Map(baseline.cases.map((entry) => [entry.id, entry.hitAt3]))
+  const lostCases = enriched.cases.filter((entry) => hitBefore.get(entry.id) === true && !entry.hitAt3).map((entry) => entry.id).sort()
+  const gainedCases = enriched.cases.filter((entry) => hitBefore.get(entry.id) === false && entry.hitAt3).map((entry) => entry.id).sort()
+
+  const blocking = deltas.find((entry) => entry.metric === OVERLAY_BLOCKING_METRIC)
+  const regression = blocking?.worsened === true
+  const status = regression ? 'regressed' : deltas.some((entry) => entry.improved) ? 'improved' : 'unchanged'
+
+  return {
+    status,
+    regression,
+    overlayHash: overlay.contentHash,
+    suite: enriched.suite,
+    withoutOverlay: baseline.metrics,
+    withOverlay: enriched.metrics,
+    deltas,
+    lostCases,
+    gainedCases,
+    messages: [
+      ...(regression
+        ? [`The overlay lowered ${OVERLAY_BLOCKING_METRIC} from ${format(OVERLAY_BLOCKING_METRIC, blocking?.withoutOverlay ?? 0)} to ${format(OVERLAY_BLOCKING_METRIC, blocking?.withOverlay ?? 0)}. This is a finding about the agent, not a new baseline.`]
+        : []),
+      ...(lostCases.length ? [`Cases the overlay lost: ${lostCases.join(', ')}.`] : []),
+      ...(gainedCases.length ? [`Cases the overlay gained: ${gainedCases.join(', ')}.`] : []),
+    ],
+  }
+}
+
+export const formatOverlayRetrievalDeltaText = (delta: OverlayRetrievalDelta): readonly string[] => [
+  `Overlay retrieval delta: ${delta.status} (overlay ${delta.overlayHash.slice(0, 12)}, suite ${delta.suite.name}, ${delta.suite.caseCount} case(s))`,
+  ...delta.deltas.map((entry) => {
+    const sign = entry.delta > 0 ? '+' : ''
+    const mark = entry.improved ? 'improved' : entry.worsened ? 'worse' : 'no change'
+    return `  ${entry.metric}: ${format(entry.metric, entry.withoutOverlay)} → ${format(entry.metric, entry.withOverlay)} (${sign}${format(entry.metric, entry.delta)}, ${mark})`
+  }),
+  ...delta.messages.map((message) => `  ${message}`),
+]

package/src/bench/retrieval.ts

@@ -0,0 +1,319 @@
+import { z } from 'zod'
+
+import { contentHashForArtifactV1, sha256NormalizedV1 } from '../index-builder/content-hash.js'
+import { searchIndex, type SearchMatch } from '../query/search.js'
+import type { DocBridgeIndexV1 } from '../schemas/doc-bridge-index.js'
+
+export const RETRIEVAL_BENCH_SCHEMA_VERSION = 1 as const
+
+/**
+ * Open Eval Format version this suite conforms to (`@agentskit/core/eval-format`).
+ *
+ * The format is mirrored here with Zod rather than imported so `ak-docs bench retrieval`
+ * stays in the deterministic layer: `@agentskit/core` is an optional peer, and the
+ * benchmark must run with no peer installed, no network and no API key. A test
+ * cross-validates every committed suite against the real `validateEvalSuite`, so the
+ * two definitions cannot drift apart silently.
+ */
+export const EVAL_FORMAT_VERSION = '2026-04' as const
+
+const DEFAULT_LIMIT = 20
+/** Payload depth an agent is assumed to read; also the k in hit@k. */
+const CONTEXT_DEPTH = 3
+/** Matches the existing agent-search estimate in src/query/query.ts. */
+const BYTES_PER_TOKEN = 4
+
+const EvalCaseExpectationSchema = z
+  .object({
+    contains: z.string().min(1).max(2_048).optional(),
+    regex: z.object({ body: z.string().min(1).max(2_048), flags: z.string().max(8).optional() }).strict().optional(),
+    equalsNormalized: z.string().min(1).max(2_048).optional(),
+    semanticSimilarity: z.number().min(0).max(1).optional(),
+  })
+  .strict()
+  .refine(
+    (value) => Object.values(value).some((entry) => entry !== undefined),
+    'An expectation must declare at least one rule.',
+  )
+
+/**
+ * A target matches a ranked result whose `id` **or** `path` equals it, so a case stays
+ * valid while entity identity evolves: `docs/mcp.md` keeps working when the index starts
+ * carrying `document:docs/mcp.md`, and a module target keeps working once modules are
+ * projected into the index.
+ */
+const RetrievalCaseMetadataSchema = z
+  .object({
+    expectedTargets: z.array(z.string().min(1).max(512)).min(1).max(32),
+    lang: z.enum(['en', 'pt']).optional(),
+    kind: z.enum(['symbol', 'path', 'question', 'ownership']).optional(),
+    agent: z.boolean().optional(),
+  })
+  // The Open Eval Format allows arbitrary case metadata; keep unknown keys rather than
+  // rejecting a suite a generic runner wrote.
+  .passthrough()
+
+export const RetrievalSuiteCaseSchema = z
+  .object({
+    id: z.string().min(1).max(128),
+    input: z.string().min(1).max(2_048),
+    expected: z.union([EvalCaseExpectationSchema, z.string().min(1).max(2_048)]).optional(),
+    metadata: RetrievalCaseMetadataSchema,
+  })
+  .strict()
+
+export const RetrievalSuiteSchema = z
+  .object({
+    evalFormatVersion: z.literal(EVAL_FORMAT_VERSION),
+    name: z.string().min(1).max(128),
+    description: z.string().max(1_024).optional(),
+    tags: z.array(z.string().min(1).max(64)).max(32).optional(),
+    cases: z.array(RetrievalSuiteCaseSchema).min(1).max(10_000),
+  })
+  .strict()
+  .superRefine((value, context) => {
+    const seen = new Set<string>()
+    for (const [index, entry] of value.cases.entries()) {
+      if (seen.has(entry.id)) {
+        context.addIssue({ code: z.ZodIssueCode.custom, path: ['cases', index, 'id'], message: `Duplicate case id: ${entry.id}` })
+      }
+      seen.add(entry.id)
+    }
+  })
+
+export type RetrievalSuite = z.infer<typeof RetrievalSuiteSchema>
+export type RetrievalSuiteCase = z.infer<typeof RetrievalSuiteCaseSchema>
+
+const MetricsSchema = z
+  .object({
+    caseCount: z.number().int().nonnegative(),
+    hitAt1: z.number().min(0).max(1),
+    hitAt3: z.number().min(0).max(1),
+    meanReciprocalRank: z.number().min(0).max(1),
+    meanContextBytes: z.number().int().nonnegative(),
+    meanApproxTokens: z.number().int().nonnegative(),
+    zeroResultRate: z.number().min(0).max(1),
+    tokenMethod: z.literal('approximate'),
+  })
+  .strict()
+
+export type RetrievalMetrics = z.infer<typeof MetricsSchema>
+
+const CaseOutcomeSchema = z
+  .object({
+    id: z.string().min(1).max(128),
+    input: z.string().min(1).max(2_048),
+    lang: z.enum(['en', 'pt']).optional(),
+    kind: z.enum(['symbol', 'path', 'question', 'ownership']).optional(),
+    expectedTargets: z.array(z.string().min(1).max(512)).min(1),
+    rankedTargets: z.array(z.string().min(1).max(512)).max(64),
+    /** 1-based rank of the first expected target, or null when it is absent from the ranking. */
+    rank: z.number().int().positive().nullable(),
+    hitAt1: z.boolean(),
+    hitAt3: z.boolean(),
+    reciprocalRank: z.number().min(0).max(1),
+    resultCount: z.number().int().nonnegative(),
+    contextBytes: z.number().int().nonnegative(),
+    approxTokens: z.number().int().nonnegative(),
+    /** Result of the portable Open Eval Format expectation, when the case declares one. */
+    expectationMatched: z.boolean().nullable(),
+  })
+  .strict()
+
+export type RetrievalCaseOutcome = z.infer<typeof CaseOutcomeSchema>
+
+export const RetrievalBenchResultV1Schema = z
+  .object({
+    type: z.literal('retrieval-benchmark-result'),
+    schemaVersion: z.literal(RETRIEVAL_BENCH_SCHEMA_VERSION),
+    contentHash: z.string().regex(/^[a-f0-9]{64}$/),
+    contentHashAlgo: z.literal('sha256-normalized-v1'),
+    suite: z
+      .object({
+        name: z.string().min(1).max(128),
+        caseCount: z.number().int().positive(),
+        contentHash: z.string().regex(/^[a-f0-9]{64}$/),
+      })
+      .strict(),
+    index: z
+      .object({
+        contentHash: z.string().regex(/^[a-f0-9]{64}$/),
+        entryCount: z.number().int().nonnegative(),
+      })
+      .strict(),
+    limit: z.number().int().positive().max(1_000),
+    metrics: MetricsSchema,
+    byLang: z.record(z.string().min(1).max(16), MetricsSchema),
+    byKind: z.record(z.string().min(1).max(32), MetricsSchema),
+    cases: z.array(CaseOutcomeSchema).max(10_000),
+  })
+  .strict()
+
+export type RetrievalBenchResultV1 = z.infer<typeof RetrievalBenchResultV1Schema>
+
+export const parseRetrievalSuite = (raw: unknown): RetrievalSuite => RetrievalSuiteSchema.parse(raw)
+
+export const parseRetrievalBenchResult = (raw: unknown): RetrievalBenchResultV1 => {
+  const result = RetrievalBenchResultV1Schema.parse(raw)
+  if (contentHashForArtifactV1(result) !== result.contentHash) {
+    throw new Error('Invalid retrieval benchmark result content hash.')
+  }
+  return result
+}
+
+/** Normalize a target or result value so `./docs/a.md`, `docs/a.md` and a Windows path compare equal. */
+const normalizeTarget = (value: string): string => value.replaceAll('\\', '/').replace(/^\.\//, '').trim()
+
+/**
+ * The identities a ranked result can be addressed by. Both are compared so a suite written
+ * against repository paths keeps working once the index carries snapshot entity ids.
+ */
+const matchTargets = (match: SearchMatch): string[] =>
+  [...new Set([normalizeTarget(match.id), normalizeTarget(match.path)])].filter(Boolean)
+
+/**
+ * The bytes an agent would receive for the top results. Mirrors the `matches` entries of
+ * the agent-search payload in src/query/query.ts, so the figure tracks a real payload
+ * rather than an internal representation.
+ */
+const contextPayload = (matches: readonly SearchMatch[]): unknown =>
+  matches.map((match) => ({
+    type: match.type,
+    id: match.id,
+    path: match.path,
+    ...(match.summary ? { summary: match.summary } : {}),
+  }))
+
+/**
+ * Portable Open Eval Format expectation, evaluated against the newline-joined ranking the
+ * deterministic agent function returns. Kept behaviour-compatible with
+ * `matchesExpectation` from `@agentskit/core/eval-format`.
+ */
+export const matchesRetrievalExpectation = (output: string, expected: RetrievalSuiteCase['expected']): boolean => {
+  if (expected === undefined) return true
+  if (typeof expected === 'string') return output.includes(expected)
+  if (expected.contains !== undefined && !output.includes(expected.contains)) return false
+  if (expected.equalsNormalized !== undefined && output.trim().toLowerCase() !== expected.equalsNormalized.trim().toLowerCase()) return false
+  if (expected.regex !== undefined && !new RegExp(expected.regex.body, expected.regex.flags).test(output)) return false
+  // semanticSimilarity needs an embedder; the deterministic benchmark never evaluates it.
+  return true
+}
+
+/** The deterministic agent function: a query in, the ranked identities out, one per line. */
+export const rankedOutput = (matches: readonly SearchMatch[]): string =>
+  matches.map((match) => matchTargets(match).join(' ')).join('\n')
+
+const mean = (values: readonly number[]): number => (values.length ? values.reduce((total, value) => total + value, 0) / values.length : 0)
+const rate = (count: number, total: number): number => (total ? count / total : 0)
+/** Six decimals keep the artifact hash stable across platforms without losing useful precision. */
+const round = (value: number): number => Math.round(value * 1_000_000) / 1_000_000
+
+const aggregate = (outcomes: readonly RetrievalCaseOutcome[]): RetrievalMetrics => ({
+  caseCount: outcomes.length,
+  hitAt1: round(rate(outcomes.filter((outcome) => outcome.hitAt1).length, outcomes.length)),
+  hitAt3: round(rate(outcomes.filter((outcome) => outcome.hitAt3).length, outcomes.length)),
+  meanReciprocalRank: round(mean(outcomes.map((outcome) => outcome.reciprocalRank))),
+  meanContextBytes: Math.round(mean(outcomes.map((outcome) => outcome.contextBytes))),
+  meanApproxTokens: Math.round(mean(outcomes.map((outcome) => outcome.approxTokens))),
+  zeroResultRate: round(rate(outcomes.filter((outcome) => outcome.resultCount === 0).length, outcomes.length)),
+  tokenMethod: 'approximate',
+})
+
+const groupBy = (
+  outcomes: readonly RetrievalCaseOutcome[],
+  key: (outcome: RetrievalCaseOutcome) => string | undefined,
+): Record<string, RetrievalMetrics> => {
+  const groups = new Map<string, RetrievalCaseOutcome[]>()
+  for (const outcome of outcomes) {
+    const value = key(outcome)
+    if (value === undefined) continue
+    const group = groups.get(value) ?? []
+    group.push(outcome)
+    groups.set(value, group)
+  }
+  return Object.fromEntries([...groups.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([value, group]) => [value, aggregate(group)]))
+}
+
+export type RunRetrievalBenchOptions = {
+  readonly index: DocBridgeIndexV1
+  readonly suite: RetrievalSuite
+  readonly limit?: number
+}
+
+/**
+ * Measure the deterministic retrieval path against a golden suite.
+ *
+ * The result carries no timestamp and no latency, so two runs over the same index and
+ * suite are byte-identical and their content hashes match. Wall time belongs in the
+ * caller's output, never in a comparable artifact.
+ */
+export const runRetrievalBench = (options: RunRetrievalBenchOptions): RetrievalBenchResultV1 => {
+  const limit = options.limit ?? DEFAULT_LIMIT
+  const outcomes: RetrievalCaseOutcome[] = options.suite.cases.map((entry) => {
+    const matches = searchIndex(options.index, entry.input, limit)
+    const expectedTargets = entry.metadata.expectedTargets.map(normalizeTarget)
+    const expected = new Set(expectedTargets)
+    const rankedTargets = matches.map((match) => matchTargets(match).join(' '))
+    const position = matches.findIndex((match) => matchTargets(match).some((target) => expected.has(target)))
+    const rank = position >= 0 ? position + 1 : null
+    const contextBytes = Buffer.byteLength(JSON.stringify(contextPayload(matches.slice(0, CONTEXT_DEPTH))), 'utf8')
+    return CaseOutcomeSchema.parse({
+      id: entry.id,
+      input: entry.input,
+      ...(entry.metadata.lang ? { lang: entry.metadata.lang } : {}),
+      ...(entry.metadata.kind ? { kind: entry.metadata.kind } : {}),
+      expectedTargets,
+      rankedTargets: rankedTargets.slice(0, CONTEXT_DEPTH),
+      rank,
+      hitAt1: rank === 1,
+      hitAt3: rank !== null && rank <= CONTEXT_DEPTH,
+      reciprocalRank: round(rank === null ? 0 : 1 / rank),
+      resultCount: matches.length,
+      contextBytes,
+      approxTokens: Math.ceil(contextBytes / BYTES_PER_TOKEN),
+      // Evaluated over the same top-three window hit@3 uses, so the portable verdict a
+      // generic Open Eval Format runner computes agrees with the metric this repository gates.
+      expectationMatched:
+        entry.expected === undefined ? null : matchesRetrievalExpectation(rankedOutput(matches.slice(0, CONTEXT_DEPTH)), entry.expected),
+    })
+  })
+
+  const base = {
+    type: 'retrieval-benchmark-result' as const,
+    schemaVersion: RETRIEVAL_BENCH_SCHEMA_VERSION,
+    contentHash: '0'.repeat(64),
+    contentHashAlgo: 'sha256-normalized-v1' as const,
+    suite: {
+      name: options.suite.name,
+      caseCount: options.suite.cases.length,
+      contentHash: sha256NormalizedV1(options.suite),
+    },
+    index: {
+      contentHash: options.index.contentHash,
+      entryCount: options.index.knowledge.length,
+    },
+    limit,
+    metrics: aggregate(outcomes),
+    byLang: groupBy(outcomes, (outcome) => outcome.lang),
+    byKind: groupBy(outcomes, (outcome) => outcome.kind),
+    cases: outcomes,
+  }
+  return RetrievalBenchResultV1Schema.parse({ ...base, contentHash: contentHashForArtifactV1(base) })
+}
+
+const percent = (value: number): string => `${(value * 100).toFixed(1)}%`
+
+export const formatRetrievalBenchText = (result: RetrievalBenchResultV1): readonly string[] => {
+  const misses = result.cases.filter((outcome) => !outcome.hitAt3)
+  return [
+    `Retrieval benchmark: ${result.suite.name}`,
+    `Cases: ${result.metrics.caseCount} | Index entries: ${result.index.entryCount}`,
+    `hit@1: ${percent(result.metrics.hitAt1)} | hit@3: ${percent(result.metrics.hitAt3)} | MRR: ${result.metrics.meanReciprocalRank.toFixed(3)}`,
+    `Zero results: ${percent(result.metrics.zeroResultRate)} | Mean context: ${result.metrics.meanContextBytes} bytes (~${result.metrics.meanApproxTokens} tokens, approximate)`,
+    ...Object.entries(result.byLang).map(([lang, metrics]) => `  ${lang}: hit@3 ${percent(metrics.hitAt3)} over ${metrics.caseCount} case(s)`),
+    ...Object.entries(result.byKind).map(([kind, metrics]) => `  ${kind}: hit@3 ${percent(metrics.hitAt3)} over ${metrics.caseCount} case(s)`),
+    ...(misses.length
+      ? [`Missed (${misses.length}):`, ...misses.slice(0, 10).map((outcome) => `  ${outcome.id}: "${outcome.input}" → ${outcome.rankedTargets[0] ?? '(no result)'}`)]
+      : ['Missed: none']),
+  ]
+}