@agentskit/doc-bridge 1.7.44 → 1.9.0

package/src/study/execution.ts

@@ -0,0 +1,350 @@
+import { existsSync, readFileSync, statSync } from 'node:fs'
+import { resolve } from 'node:path'
+import { z } from 'zod'
+
+import { contentHashForArtifactV1, sha256NormalizedV1 } from '../index-builder/content-hash.js'
+import {
+  createControlledStudyLedger,
+  createControlledStudyObservation,
+  parseControlledStudyLedger,
+  persistControlledStudyLedger,
+  runControlledCommand,
+  upsertControlledStudyObservation,
+  type ControlledStudyObservationLedgerV1,
+  type ControlledStudyObservationV1,
+  type ControlledStudyRunPlanV1,
+  type TaskExecutionV1,
+} from './runner.js'
+import { providerForStudyExecution, parseStudyProviderCliConfig, validateStudyProviderCommand, type StudyProviderCliConfigV1 } from './provider-cli.js'
+import { evaluateStudyTask, parseStudyTaskSuite, selectTaskExecutions, type StudyTaskSuiteV1, type StudyTaskV1 } from './task-suite.js'
+
+const PROVIDER_RESPONSE_CONTRACT = 'Return one JSON object matching the output schema. Required keys: taskOutcome, evidenceQuality, safetyOutcome, evidenceIds, clarificationRequests, reworkCount, and measurements. Each measurement is {name:string,value:number>=0}. Run every available acceptance check and report observed acceptanceChecksPassed, acceptanceChecksTotal, and acceptanceChecksExecuted; include firstEvidenceLatencyMs only when observed. Use canonical names when observed: tokensToFirstEvidence (tokens consumed before correct grounded evidence was in hand), registryAgentInputTokens, registryAgentOutputTokens, registryAgentCostUsd, registryAgentRuns (the enrichment agent of the assisted arm, reported apart from your own cost), searchHitRate, acceptanceChecksPassed, acceptanceChecksTotal, acceptanceChecksExecuted, entrypointEvidenceCount, ownershipEvidenceCount, architectureRelationCount, documentationClaimEvidenceCount, sourceComparisonEvidenceCount, verificationEvidenceCount, errorRate, documentationFindingCount, documentationExampleRate, documentationFreshnessRate, documentationCorrectnessRate, documentationCompletenessRate, documentationClarityRate, documentationMaintainabilityRate, timeToFirstEvidenceMs, analysisCostUsd, and agentCostUsd. Omit unknown values; never invent. Output no markdown, prose, logs, token counts, or extra keys; stdout must contain only the JSON object.'
+
+export const STUDY_REPOSITORY_CONFIG_SCHEMA_VERSION = 1 as const
+export const STUDY_REPOSITORY_CONFIG_CONTENT_HASH_ALGO = 'sha256-normalized-v1' as const
+
+const hash = z.string().regex(/^[a-f0-9]{64}$/)
+const identifier = z.string().regex(/^[a-z][a-z0-9-]{0,127}$/)
+const reference = z.string().regex(/^[a-z0-9][a-z0-9._:-]{0,255}$/)
+
+const RepositoryConfigPayloadSchema = z.object({
+  type: z.literal('controlled-study-repository-config'),
+  schemaVersion: z.literal(STUDY_REPOSITORY_CONFIG_SCHEMA_VERSION),
+  configVersion: reference,
+  repositories: z.array(z.object({
+    id: identifier,
+    root: z.string().min(1).max(4_096),
+  }).strict()).min(1).max(64),
+}).strict()
+
+export const StudyRepositoryConfigV1Schema = RepositoryConfigPayloadSchema.extend({
+  contentHash: hash,
+  contentHashAlgo: z.literal(STUDY_REPOSITORY_CONFIG_CONTENT_HASH_ALGO),
+}).strict()
+
+export type StudyRepositoryConfigV1 = z.infer<typeof StudyRepositoryConfigV1Schema>
+
+export type ControlledStudyRunOptions = {
+  readonly plan: ControlledStudyRunPlanV1
+  readonly suite: StudyTaskSuiteV1
+  readonly providers: StudyProviderCliConfigV1
+  readonly repositories: StudyRepositoryConfigV1
+  readonly ledgerPath: string
+  readonly round?: string
+  readonly dryRun?: boolean
+}
+
+/**
+ * Whether the assisted arm can run at all, and why not when it cannot.
+ *
+ * The `registry-assisted` scenario has been reserved since the first suite and has never
+ * executed. It must be possible to run a study without it — a missing Registry is a fact about
+ * the environment, not a reason to lose the other two arms — so the arm reports itself
+ * unavailable and its executions are recorded as unavailable observations.
+ */
+export type AssistedArmStatus = {
+  readonly status: 'ready' | 'unavailable'
+  readonly reason?: string
+  /**
+   * Fields the arm ran without: `promptVersion`, `agentBudget`. It still runs — losing the third
+   * arm over a missing declaration would be worse than running it — but a run that cannot name
+   * the prompt it used, or cost the enrichment agent apart from the model, says so here.
+   */
+  readonly undeclared?: readonly string[]
+  /** Executions recorded as unavailable because the arm could not run. */
+  readonly recorded: number
+}
+
+export type ControlledStudyRunSummary = {
+  readonly status: 'dry-run' | 'completed'
+  readonly runId: string
+  readonly planned: number
+  readonly executed: number
+  readonly skipped: number
+  readonly ledgerPath?: string
+  readonly ledgerHash?: string
+  readonly providerConfigHash: string
+  readonly repositoryConfigHash: string
+  readonly assistedArm: AssistedArmStatus
+}
+
+export const createStudyRepositoryConfig = (input: unknown): StudyRepositoryConfigV1 => {
+  const payload = RepositoryConfigPayloadSchema.parse(input)
+  const hashable = { ...payload, contentHashAlgo: STUDY_REPOSITORY_CONFIG_CONTENT_HASH_ALGO }
+  return StudyRepositoryConfigV1Schema.parse({ ...hashable, contentHash: sha256NormalizedV1(hashable) })
+}
+
+export const parseStudyRepositoryConfig = (input: unknown): StudyRepositoryConfigV1 => {
+  const config = StudyRepositoryConfigV1Schema.parse(input)
+  if (contentHashForArtifactV1(config) !== config.contentHash) throw new Error('Invalid study repository config content hash.')
+  if (new Set(config.repositories.map((repository) => repository.id)).size !== config.repositories.length) throw new Error('Study repository config contains duplicate repository ids.')
+  return config
+}
+
+const emptyLedger = (): ControlledStudyObservationLedgerV1 => createControlledStudyLedger({
+  type: 'controlled-study-observation-ledger',
+  schemaVersion: 1,
+  ledgerVersion: 'v1',
+  observations: [],
+})
+
+const loadLedger = (path: string): ControlledStudyObservationLedgerV1 => {
+  if (!existsSync(path)) return emptyLedger()
+  return parseControlledStudyLedger(JSON.parse(readFileSync(path, 'utf8')) as unknown)
+}
+
+const executionKey = (execution: { readonly taskId: string; readonly repositoryId: string; readonly scenarioId: string; readonly modelId: string; readonly replicate: number; readonly variantId: string }): string => sha256NormalizedV1({
+  taskId: execution.taskId,
+  repositoryId: execution.repositoryId,
+  scenarioId: execution.scenarioId,
+  modelId: execution.modelId,
+  replicate: execution.replicate,
+  variantId: execution.variantId,
+})
+
+export const ASSISTED_SCENARIO = 'registry-assisted' as const
+
+/**
+ * What the assisted arm needs before it is allowed to run: a provider for every model, and a
+ * scenario that declares the agent, its version, its prompt version and its own budget.
+ */
+export const assistedArmReadiness = (
+  plan: ControlledStudyRunPlanV1,
+  providers: StudyProviderCliConfigV1,
+  suite: StudyTaskSuiteV1,
+): Omit<AssistedArmStatus, 'recorded'> => {
+  const scenario = plan.scenarios.find((entry) => entry.id === ASSISTED_SCENARIO)
+  if (!scenario) return { status: 'unavailable', reason: 'The run plan declares no registry-assisted scenario.' }
+  if (!scenario.agentId || !scenario.agentVersion) {
+    return { status: 'unavailable', reason: 'The registry-assisted scenario names no agent identity and version, so nothing it produced could be attributed.' }
+  }
+  const withoutProvider = suite.modelIds.filter((modelId) => !providers.providers.some((provider) => provider.modelId === modelId && provider.scenarioIds.includes(ASSISTED_SCENARIO)))
+  if (withoutProvider.length) {
+    return { status: 'unavailable', reason: `No provider CLI is configured for the registry-assisted scenario and model(s) ${withoutProvider.join(', ')}.` }
+  }
+  const undeclared = [...(scenario.promptVersion ? [] : ['promptVersion']), ...(scenario.agentBudget ? [] : ['agentBudget'])]
+  return {
+    status: 'ready',
+    ...(undeclared.length
+      ? {
+          undeclared,
+          reason: `The registry-assisted scenario declares no ${undeclared.join(' or ')}; the arm runs, but it cannot be ${undeclared.includes('promptVersion') ? 'reproduced' : 'costed'} from this ledger alone.`,
+        }
+      : {}),
+  }
+}
+
+/**
+ * The observation an unavailable arm leaves behind.
+ *
+ * Recorded rather than skipped: a scenario absent from the ledger is indistinguishable from one
+ * that was never planned, and the study's whole purpose is to compare the arms it planned.
+ */
+const unavailableObservation = (
+  plan: ControlledStudyRunPlanV1,
+  execution: TaskExecutionV1,
+  reason: string,
+  round: string | undefined,
+): ControlledStudyObservationV1 => createControlledStudyObservation({
+  type: 'controlled-study-observation',
+  schemaVersion: 1,
+  observationVersion: 'v1',
+  observedAt: new Date().toISOString(),
+  runId: plan.runId,
+  planHash: plan.contentHash,
+  task: execution,
+  model: plan.models.find((model) => model.id === execution.modelId),
+  scenario: plan.scenarios.find((scenario) => scenario.id === execution.scenarioId),
+  execution: { status: 'unavailable', exitCode: null, signal: null, durationMs: 0, responseBytes: 0, stderrBytes: 0, errorCode: 'registry-unavailable' },
+  contextBytes: 0,
+  evidenceIds: [],
+  ...(round === undefined ? {} : { round }),
+  adjudication: { status: 'automated', actor: 'deterministic-rubric-v1', method: 'deterministic-rubric-v1', outcome: 'blocked', reason },
+})
+
+export const adjudicateControlledStudyObservation = (task: StudyTaskV1, observation: ControlledStudyObservationV1): ControlledStudyObservationV1 => {
+  const passed = observation.measurements?.acceptanceChecksPassed
+  const total = observation.measurements?.acceptanceChecksTotal
+  const executed = observation.measurements?.acceptanceChecksExecuted
+  const acceptanceTotal = task.acceptanceChecks.length
+  const blocked = observation.execution.status !== 'completed'
+    || passed === undefined
+    || total !== acceptanceTotal
+    || executed !== acceptanceTotal
+    || passed > executed
+  const evidenceIds = new Set(observation.evidenceIds)
+  const requiredEvidencePresent = task.evidenceRequirements.filter((requirement) => evidenceIds.has(requirement.id)).length
+  const evaluation = evaluateStudyTask(task, {
+    acceptanceChecksPassed: passed ?? 0,
+    evidenceItemsPresent: requiredEvidencePresent,
+    blocked,
+  })
+  const { contentHash: _contentHash, contentHashAlgo: _contentHashAlgo, ...payload } = observation
+  return createControlledStudyObservation({
+    ...payload,
+    adjudication: {
+      status: 'automated',
+      actor: 'deterministic-rubric-v1',
+      method: 'deterministic-rubric-v1',
+      outcome: evaluation.status,
+      reason: 'Independent bounded evaluation of execution status, acceptance metrics, and exact required evidence coverage.',
+    },
+  })
+}
+
+const assertRunInputs = (options: ControlledStudyRunOptions): Map<string, { readonly id: string; readonly root: string }> => {
+  if (options.plan.taskSuiteHash !== options.suite.contentHash) throw new Error('Run plan taskSuiteHash does not match the supplied task suite.')
+  if (options.plan.models.map((model) => model.id).sort().join(',') !== [...options.suite.modelIds].sort().join(',')) throw new Error('Run plan model ids do not match the task suite.')
+  if (options.plan.scenarios.map((scenario) => scenario.id).sort().join(',') !== [...options.suite.scenarioIds].sort().join(',')) throw new Error('Run plan scenario ids do not match the task suite.')
+  if (options.plan.taskIds.slice().sort().join(',') !== options.suite.tasks.map((task) => task.id).sort().join(',')) throw new Error('Run plan task ids do not match the task suite.')
+  const repositories = new Map(options.repositories.repositories.map((repository) => [repository.id, repository]))
+  if (repositories.size !== options.suite.population.length || options.suite.population.some((repositoryId) => !repositories.has(repositoryId))) {
+    throw new Error('Study repository config must contain exactly one root for every task-suite population id.')
+  }
+  for (const repository of options.repositories.repositories) {
+    const root = resolve(repository.root)
+    if (!existsSync(root) || !statSync(root).isDirectory()) throw new Error(`Study repository ${repository.id} is not available at the configured root.`)
+  }
+  const executions = selectTaskExecutions(options.suite, options.plan.sampling.sampleSize, options.plan.sampling)
+  const assisted = assistedArmReadiness(options.plan, options.providers, options.suite)
+  for (const execution of executions) {
+    const repository = repositories.get(execution.repositoryId)
+    if (!repository) throw new Error(`No repository root is configured for ${execution.repositoryId}.`)
+    // An unavailable assisted arm is recorded, not validated: the other two arms still run.
+    if (execution.scenarioId === ASSISTED_SCENARIO && assisted.status === 'unavailable') continue
+    const provider = providerForStudyExecution(options.providers, execution.modelId, execution.scenarioId as 'repository-only' | 'deterministic-doc-bridge' | 'registry-assisted')
+    validateStudyProviderCommand(provider, repository.root)
+    const task = options.suite.tasks.find((item) => item.id === execution.taskId)
+    if (!task) throw new Error(`Task ${execution.taskId} is not present in the task suite.`)
+    const input = JSON.stringify({
+      protocol: 'doc-bridge.study-provider.v1',
+      response: PROVIDER_RESPONSE_CONTRACT,
+      task: { ...execution, difficulty: task.difficulty },
+      objective: task.objective,
+      initialContext: task.initialContext,
+      expectedOutcome: task.expectedOutcome,
+      evidenceRequirements: task.evidenceRequirements,
+      acceptanceChecks: task.acceptanceChecks,
+      allowedTools: task.allowedTools,
+      forbiddenActions: task.forbiddenActions,
+    })
+    if (Buffer.byteLength(input, 'utf8') > provider.maxInputBytes) throw new Error(`Provider CLI input limit ${provider.maxInputBytes} bytes exceeded for ${execution.taskId}.`)
+  }
+  return repositories
+}
+
+export const runControlledStudy = async (options: ControlledStudyRunOptions): Promise<ControlledStudyRunSummary> => {
+  const plan = options.plan
+  const suite = parseStudyTaskSuite(options.suite)
+  const providers = parseStudyProviderCliConfig(options.providers)
+  const repositories = parseStudyRepositoryConfig(options.repositories)
+  const executions = selectTaskExecutions(suite, plan.sampling.sampleSize, plan.sampling)
+  const repositoryMap = assertRunInputs({ ...options, plan, suite, providers, repositories })
+  const assisted = assistedArmReadiness(plan, providers, suite)
+  if (options.dryRun) return {
+    status: 'dry-run',
+    runId: plan.runId,
+    planned: executions.length,
+    executed: 0,
+    skipped: 0,
+    providerConfigHash: providers.contentHash,
+    repositoryConfigHash: repositories.contentHash,
+    assistedArm: { ...assisted, recorded: 0 },
+  }
+
+  let ledger = loadLedger(resolve(options.ledgerPath))
+  if (ledger.observations.some((observation) => observation.runId === plan.runId && observation.planHash !== plan.contentHash)) throw new Error(`Ledger already contains run ${plan.runId} with a different plan hash.`)
+  let executed = 0
+  let skipped = 0
+  let unavailableRecorded = 0
+  for (const execution of executions) {
+    const existing = ledger.observations.find((observation) => observation.runId === plan.runId && executionKey(observation.task) === executionKey(execution))
+    if (existing) { skipped += 1; continue }
+    if (execution.scenarioId === ASSISTED_SCENARIO && assisted.status === 'unavailable') {
+      const typedUnavailable = { ...execution, difficulty: suite.tasks.find((item) => item.id === execution.taskId)?.difficulty, scenarioId: ASSISTED_SCENARIO } as TaskExecutionV1
+      ledger = upsertControlledStudyObservation(ledger, unavailableObservation(plan, typedUnavailable, assisted.reason ?? 'The registry-assisted arm is unavailable.', options.round))
+      persistControlledStudyLedger(options.ledgerPath, ledger)
+      unavailableRecorded += 1
+      continue
+    }
+    const repository = repositoryMap.get(execution.repositoryId)
+    if (!repository) throw new Error(`No repository root is configured for ${execution.repositoryId}.`)
+    const provider = providerForStudyExecution(providers, execution.modelId, execution.scenarioId as 'repository-only' | 'deterministic-doc-bridge' | 'registry-assisted')
+    const task = suite.tasks.find((item) => item.id === execution.taskId)
+    if (!task) throw new Error(`Task ${execution.taskId} is not present in the task suite.`)
+    const input = JSON.stringify({
+      protocol: 'doc-bridge.study-provider.v1',
+      response: PROVIDER_RESPONSE_CONTRACT,
+      task: { ...execution, difficulty: task.difficulty },
+      objective: task.objective,
+      initialContext: task.initialContext,
+      expectedOutcome: task.expectedOutcome,
+      evidenceRequirements: task.evidenceRequirements,
+      acceptanceChecks: task.acceptanceChecks,
+      allowedTools: task.allowedTools,
+      forbiddenActions: task.forbiddenActions,
+    })
+    const typedExecution = { ...execution, difficulty: task.difficulty, scenarioId: execution.scenarioId as 'repository-only' | 'deterministic-doc-bridge' | 'registry-assisted' }
+    const observation = await runControlledCommand({
+      plan,
+      execution: typedExecution,
+      command: provider.command,
+      args: provider.args,
+      cwd: repository.root,
+      input,
+      envAllowlist: provider.envAllowlist,
+      maxOutputBytes: Math.min(provider.maxOutputBytes, plan.budget.maxOutputBytes),
+      contextBytes: Buffer.byteLength(input, 'utf8'),
+      ...(provider.pricing === undefined ? {} : { providerPricing: provider.pricing }),
+      ...(options.round === undefined ? {} : { round: options.round }),
+    })
+    ledger = upsertControlledStudyObservation(ledger, adjudicateControlledStudyObservation(task, observation))
+    persistControlledStudyLedger(options.ledgerPath, ledger)
+    executed += 1
+  }
+  return {
+    status: 'completed',
+    runId: plan.runId,
+    planned: executions.length,
+    executed,
+    skipped,
+    ledgerPath: resolve(options.ledgerPath),
+    ledgerHash: ledger.contentHash,
+    providerConfigHash: providers.contentHash,
+    repositoryConfigHash: repositories.contentHash,
+    assistedArm: { ...assisted, recorded: unavailableRecorded },
+  }
+}
+
+export const formatControlledStudyRunText = (summary: ControlledStudyRunSummary): readonly string[] => [
+  `Study run: ${summary.runId}`,
+  `Status: ${summary.status}`,
+  `Planned: ${summary.planned}`,
+  `Executed: ${summary.executed}`,
+  `Skipped: ${summary.skipped}`,
+  ...(summary.ledgerPath === undefined ? [] : [`Ledger: ${summary.ledgerPath}`, `Ledger hash: ${summary.ledgerHash}`]),
+  `Provider config hash: ${summary.providerConfigHash}`,
+  `Repository config hash: ${summary.repositoryConfigHash}`,
+  `Assisted arm: ${summary.assistedArm.status}${summary.assistedArm.recorded ? ` (${summary.assistedArm.recorded} execution(s) recorded as unavailable)` : ''}${summary.assistedArm.undeclared?.length ? ` (undeclared: ${summary.assistedArm.undeclared.join(', ')})` : ''}`,
+  ...(summary.assistedArm.reason ? [`  ${summary.assistedArm.reason}`] : []),
+]

package/src/study/expectations.ts

@@ -0,0 +1,219 @@
+import { z } from 'zod'
+
+import { contentHashForArtifactV1, sha256NormalizedV1 } from '../index-builder/content-hash.js'
+import { EVAL_FORMAT_VERSION, runRetrievalBench, type RetrievalBenchResultV1, type RetrievalSuite } from '../bench/retrieval.js'
+import type { DocBridgeIndexV1 } from '../schemas/doc-bridge-index.js'
+import { hasRetrievalExpectations, taskRetrievalQueries, type StudyTaskSuiteV1, type StudyTaskV1 } from './task-suite.js'
+
+/**
+ * The mechanical half of a study task, and why it lives in two files.
+ *
+ * The last controlled round recorded zero semantic successes in both arms, which means it could
+ * not tell a working retrieval layer from a broken one. The cause was structural: a task whose
+ * only success criterion is a model's opinion produces no signal. The fix is to state what
+ * retrieval is expected to return and check it — with `ak-docs bench retrieval`, deterministically.
+ *
+ * The expectations are split across two artifacts on purpose. The task suite is
+ * publication-bound, so it may only carry opaque references (`primary-entrypoint`), never a path.
+ * This file is the local resolution — references to concrete entity ids and document paths for a
+ * repository on the operator's disk — and it must never be published. Putting it under
+ * `docs/study/` would fail the privacy gate, which is the intended outcome rather than a bug.
+ */
+
+export const STUDY_EXPECTATIONS_SCHEMA_VERSION = 1 as const
+export const STUDY_EXPECTATIONS_CONTENT_HASH_ALGO = 'sha256-normalized-v1' as const
+
+const hash = z.string().regex(/^[a-f0-9]{64}$/)
+const identifier = z.string().regex(/^[a-z][a-z0-9-]{0,127}$/)
+const reference = z.string().regex(/^[a-z0-9][a-z0-9._:-]{0,255}$/)
+const target = z.string().min(1).max(512)
+
+const ExpectationsPayloadSchema = z.object({
+  type: z.literal('controlled-study-expectations'),
+  schemaVersion: z.literal(STUDY_EXPECTATIONS_SCHEMA_VERSION),
+  configVersion: reference,
+  /** Self-declared: this artifact resolves references to paths and is never publication-bound. */
+  scope: z.literal('local'),
+  /** The suite these references belong to. A suite that moved on invalidates the resolution. */
+  taskSuiteHash: hash,
+  repositories: z.array(z.object({
+    id: identifier,
+    /** Opaque reference → the entity ids or document paths it stands for in this repository. */
+    targets: z.record(reference, z.array(target).min(1).max(32)),
+  }).strict()).min(1).max(16),
+}).strict()
+
+export const StudyExpectationsV1Schema = ExpectationsPayloadSchema.extend({
+  contentHash: hash,
+  contentHashAlgo: z.literal(STUDY_EXPECTATIONS_CONTENT_HASH_ALGO),
+}).strict()
+
+export type StudyExpectationsV1 = z.infer<typeof StudyExpectationsV1Schema>
+
+export const createStudyExpectations = (input: unknown): StudyExpectationsV1 => {
+  const payload = ExpectationsPayloadSchema.parse(input)
+  const hashable = { ...payload, contentHashAlgo: STUDY_EXPECTATIONS_CONTENT_HASH_ALGO }
+  return StudyExpectationsV1Schema.parse({ ...hashable, contentHash: sha256NormalizedV1(hashable) })
+}
+
+export const parseStudyExpectations = (input: unknown): StudyExpectationsV1 => {
+  const expectations = StudyExpectationsV1Schema.parse(input)
+  if (contentHashForArtifactV1(expectations) !== expectations.contentHash) throw new Error('Invalid study expectations content hash.')
+  if (new Set(expectations.repositories.map((entry) => entry.id)).size !== expectations.repositories.length) {
+    throw new Error('Study expectations contain duplicate repository ids.')
+  }
+  return expectations
+}
+
+/** A reference a task expects that the local file does not resolve. Reported, never ignored. */
+export type UnresolvedExpectation = {
+  readonly taskId: string
+  readonly repositoryId: string
+  readonly reference: string
+  readonly kind: 'entity' | 'document'
+}
+
+export type StudyRetrievalSuite = {
+  readonly suite: RetrievalSuite
+  readonly unresolved: readonly UnresolvedExpectation[]
+  /** Task ids that declare no expectations, so nothing about them is checked here. */
+  readonly withoutExpectations: readonly string[]
+}
+
+export type StudyRetrievalSuiteOptions = {
+  readonly taskSuite: StudyTaskSuiteV1
+  readonly expectations: StudyExpectationsV1
+  /** Check one repository instead of every one the expectations resolve. */
+  readonly repositoryId?: string
+}
+
+const caseId = (task: StudyTaskV1, index: number, total: number): string => (total === 1 ? task.id : `${task.id}-q${index + 1}`)
+
+/**
+ * Turn the study's expectations into a retrieval suite.
+ *
+ * One case per task query, whose expected targets are the resolved references. The result is an
+ * ordinary Open Eval Format suite, so the same command, the same ranking and the same metrics
+ * that gate this repository's retrieval also answer the study's mechanical questions — rather
+ * than a second, study-shaped checker nobody else exercises.
+ */
+export const studyRetrievalSuite = (options: StudyRetrievalSuiteOptions): StudyRetrievalSuite => {
+  const { taskSuite, expectations } = options
+  if (expectations.taskSuiteHash !== taskSuite.contentHash) {
+    throw new Error('Study expectations were written for a different task suite; re-resolve the references against this one.')
+  }
+  const byRepository = new Map(expectations.repositories.map((entry) => [entry.id, entry.targets]))
+  const unresolved: UnresolvedExpectation[] = []
+  const withoutExpectations: string[] = []
+  const cases: RetrievalSuite['cases'] = []
+
+  for (const task of taskSuite.tasks) {
+    if (options.repositoryId !== undefined && task.repositoryId !== options.repositoryId) continue
+    if (!hasRetrievalExpectations(task)) {
+      withoutExpectations.push(task.id)
+      continue
+    }
+    const targets = byRepository.get(task.repositoryId)
+    const resolve = (references: readonly string[], kind: UnresolvedExpectation['kind']): string[] =>
+      references.flatMap((item) => {
+        const resolved = targets?.[item]
+        if (!resolved?.length) {
+          unresolved.push({ taskId: task.id, repositoryId: task.repositoryId, reference: item, kind })
+          return []
+        }
+        return resolved
+      })
+    const expectedTargets = [
+      ...new Set([...resolve(task.expectedEntities ?? [], 'entity'), ...resolve(task.expectedDocuments ?? [], 'document')]),
+    ].sort()
+    if (!expectedTargets.length) continue
+    const queries = taskRetrievalQueries(task)
+    for (const [index, input] of queries.entries()) {
+      cases.push({
+        id: caseId(task, index, queries.length),
+        input,
+        metadata: { expectedTargets, kind: 'question', studyTaskId: task.id, studyRepositoryId: task.repositoryId, studyCategory: task.category },
+      })
+    }
+  }
+
+  return {
+    suite: {
+      evalFormatVersion: EVAL_FORMAT_VERSION,
+      name: `study-${taskSuite.suiteVersion}`,
+      description: 'Mechanical retrieval expectations resolved from the controlled study task suite.',
+      cases,
+    },
+    unresolved: [...unresolved].sort((a, b) => a.taskId.localeCompare(b.taskId) || a.reference.localeCompare(b.reference)),
+    withoutExpectations,
+  }
+}
+
+export type StudyExpectationOutcome = {
+  readonly taskId: string
+  readonly repositoryId: string
+  readonly caseId: string
+  readonly hit: boolean
+  readonly rank: number | null
+  readonly expectedTargets: readonly string[]
+  readonly rankedTargets: readonly string[]
+}
+
+export type StudyExpectationCheck = {
+  /** True when every case hit and every reference resolved: the mechanical verdict, with no model. */
+  readonly ok: boolean
+  readonly checkedTasks: number
+  readonly withoutExpectations: readonly string[]
+  readonly unresolved: readonly UnresolvedExpectation[]
+  readonly outcomes: readonly StudyExpectationOutcome[]
+  readonly result?: RetrievalBenchResultV1
+}
+
+export type CheckStudyExpectationsOptions = StudyRetrievalSuiteOptions & {
+  readonly index: DocBridgeIndexV1
+  readonly limit?: number
+}
+
+/**
+ * Run the study's mechanical expectations through the retrieval benchmark.
+ *
+ * A task with no expectations is reported as unchecked rather than counted as a pass, and an
+ * unresolved reference fails the check: an expectation nobody resolved is an expectation nobody
+ * tested, and calling that a success is exactly the failure this replaces.
+ */
+export const checkStudyExpectations = (options: CheckStudyExpectationsOptions): StudyExpectationCheck => {
+  const { suite, unresolved, withoutExpectations } = studyRetrievalSuite(options)
+  if (!suite.cases.length) {
+    return { ok: false, checkedTasks: 0, withoutExpectations, unresolved, outcomes: [] }
+  }
+  const result = runRetrievalBench({ index: options.index, suite, ...(options.limit === undefined ? {} : { limit: options.limit }) })
+  const byCase = new Map(result.cases.map((entry) => [entry.id, entry]))
+  const outcomes = suite.cases.map((entry): StudyExpectationOutcome => {
+    const outcome = byCase.get(entry.id)
+    return {
+      taskId: String(entry.metadata.studyTaskId ?? entry.id),
+      repositoryId: String(entry.metadata.studyRepositoryId ?? ''),
+      caseId: entry.id,
+      hit: outcome?.hitAt3 ?? false,
+      rank: outcome?.rank ?? null,
+      expectedTargets: outcome?.expectedTargets ?? entry.metadata.expectedTargets,
+      rankedTargets: outcome?.rankedTargets ?? [],
+    }
+  })
+  return {
+    ok: unresolved.length === 0 && outcomes.every((outcome) => outcome.hit),
+    checkedTasks: new Set(outcomes.map((outcome) => outcome.taskId)).size,
+    withoutExpectations,
+    unresolved,
+    outcomes,
+    result,
+  }
+}
+
+export const formatStudyExpectationsText = (check: StudyExpectationCheck): readonly string[] => [
+  `Study expectations: ${check.ok ? 'pass' : 'fail'}`,
+  `Tasks checked: ${check.checkedTasks}${check.withoutExpectations.length ? ` (${check.withoutExpectations.length} without expectations: ${check.withoutExpectations.slice(0, 8).join(', ')}${check.withoutExpectations.length > 8 ? ', …' : ''})` : ''}`,
+  ...(check.result ? [`hit@3: ${(check.result.metrics.hitAt3 * 100).toFixed(1)}% over ${check.result.metrics.caseCount} case(s)`] : ['No case could be built: nothing was checked.']),
+  ...check.unresolved.map((entry) => `  unresolved ${entry.kind} reference "${entry.reference}" for ${entry.taskId} (${entry.repositoryId})`),
+  ...check.outcomes.filter((outcome) => !outcome.hit).map((outcome) => `  miss ${outcome.caseId}: expected ${outcome.expectedTargets.slice(0, 4).join(', ')}`),
+]