@agentskit/doc-bridge 1.7.45 → 1.10.0

package/src/study/protocol.ts

@@ -0,0 +1,271 @@
+import { contentHashForArtifactV1, sha256NormalizedV1 } from '../index-builder/content-hash.js'
+import { z } from 'zod'
+
+export const STUDY_PROTOCOL_SCHEMA_VERSION = 1 as const
+export const STUDY_PROTOCOL_CONTENT_HASH_ALGO = 'sha256-normalized-v1' as const
+
+const hash = z.string().regex(/^[a-f0-9]{64}$/)
+const identifier = z.string().regex(/^[a-z][a-z0-9-]{0,127}$/)
+const reference = z.string().regex(/^[a-z0-9][a-z0-9._:-]{0,255}$/)
+const modelReference = z.string().regex(/^[a-z0-9][a-z0-9._:/-]{0,255}$/)
+export const isSafeStudyText = (value: string): boolean => !/(?:^|[\\/])(?:Users|private|tmp|home)(?:[\\/])|(?:api[_-]?key|token|password|secret)\s*[:=]|-----BEGIN|https?:\/\//i.test(value)
+const safeText = z.string().min(1).max(2_048).refine(
+  isSafeStudyText,
+  'Public study text cannot contain paths, URLs, credentials, or secret material',
+)
+
+const uniqueIds = (values: readonly string[], label: string): void => {
+  if (new Set(values).size !== values.length) throw new Error(`Duplicate ${label} id.`)
+}
+
+const ModelSchema = z
+  .object({
+    id: identifier,
+    role: z.enum(['low-cost', 'reference']),
+    status: z.enum(['planned', 'pinned']),
+    provider: identifier.optional(),
+    model: modelReference.optional(),
+    version: reference.optional(),
+    promptContractHash: hash.optional(),
+  })
+  .strict()
+  .superRefine((value, context) => {
+    if (value.status === 'pinned' && (!value.provider || !value.model || !value.version)) {
+      context.addIssue({ code: z.ZodIssueCode.custom, message: 'Pinned models require provider, model, and version.' })
+    }
+  })
+
+const RepositorySchema = z
+  .object({
+    id: identifier,
+    visibility: z.enum(['public', 'private', 'anonymized']),
+    role: z.enum(['product', 'consumer']),
+  })
+  .strict()
+
+const ScenarioSchema = z.object({
+  id: identifier,
+  label: safeText,
+  source: z.enum(['repository-only', 'deterministic-doc-bridge', 'doc-bridge-registry-agent']),
+  modelIds: z.array(identifier).min(1).max(16),
+  registryAgent: z.enum(['none', 'configured']).optional(),
+  requiresHumanApproval: z.boolean(),
+}).strict()
+
+const MetricSchema = z.object({
+  id: identifier,
+  family: z.enum(['discovery', 'task', 'documentation', 'operations', 'cost']),
+  unit: z.enum(['count', 'milliseconds', 'tokens', 'bytes', 'ratio', 'percent', 'currency']),
+  source: z.enum(['provider', 'runner', 'adjudicator', 'doc-bridge', 'human']),
+  required: z.boolean(),
+  description: safeText,
+}).strict()
+
+const OutcomeSchema = z.object({
+  id: identifier,
+  statement: safeText,
+  checks: z.array(z.object({ id: identifier, command: safeText }).strict()).max(16).default([]),
+  notApplicableReason: safeText.optional(),
+}).strict().superRefine((value, context) => {
+  if (!value.checks.length && !value.notApplicableReason) {
+    context.addIssue({ code: z.ZodIssueCode.custom, path: ['checks'], message: 'An outcome requires an executable check or an explicit not-applicable reason.' })
+  }
+  if (value.checks.length && value.notApplicableReason) {
+    context.addIssue({ code: z.ZodIssueCode.custom, path: ['notApplicableReason'], message: 'An outcome cannot have checks and a not-applicable reason.' })
+  }
+})
+
+const BudgetSchema = z.object({
+  maxTokensPerTask: z.number().int().positive(),
+  maxRuntimeMsPerTask: z.number().int().positive(),
+  maxRuns: z.number().int().positive(),
+  maxNetworkRequests: z.number().int().nonnegative(),
+}).strict()
+
+const PrivacySchema = z.object({
+  mode: z.literal('anonymized'),
+  forbiddenFields: z.array(z.enum(['repository-content', 'paths', 'prompts', 'credentials', 'private-identifiers', 'raw-agent-responses'])).min(1).max(16),
+  publicationRequiresHumanReview: z.literal(true),
+}).strict()
+
+const StoppingSchema = z.object({
+  minControlledRounds: z.number().int().positive(),
+  consecutiveNoMaterialImprovementRounds: z.number().int().positive(),
+  targets: z.array(z.object({
+    metricId: identifier,
+    direction: z.enum(['decrease', 'increase', 'no-regression']),
+    threshold: z.number().finite(),
+  }).strict()).max(32),
+}).strict()
+
+const StudyProtocolPayloadSchema = z
+  .object({
+    type: z.literal('study-protocol'),
+    schemaVersion: z.literal(STUDY_PROTOCOL_SCHEMA_VERSION),
+    protocolVersion: reference,
+    title: safeText,
+    evidenceClasses: z.array(z.enum(['historical', 'controlled'])).min(1).max(2),
+    repositories: z.array(RepositorySchema).min(1).max(64),
+    taskCategories: z.array(z.enum(['discovery', 'architecture', 'documentation', 'implementation'])).min(1).max(4),
+    models: z.array(ModelSchema).min(2).max(16),
+    scenarios: z.array(ScenarioSchema).min(3).max(8),
+    metrics: z.array(MetricSchema).min(1).max(128),
+    outcomes: z.array(OutcomeSchema).min(1).max(64),
+    budget: BudgetSchema,
+    privacy: PrivacySchema,
+    stopping: StoppingSchema,
+  })
+  .strict()
+
+export const StudyProtocolV1Schema = StudyProtocolPayloadSchema.extend({
+  contentHash: hash,
+  contentHashAlgo: z.literal(STUDY_PROTOCOL_CONTENT_HASH_ALGO),
+}).strict()
+
+export type StudyProtocolV1 = z.infer<typeof StudyProtocolV1Schema>
+
+const HistoricalMetricStatusSchema = z.enum(['missing', 'not-analyzed', 'blocked', 'not-applicable'])
+const HistoricalRecordStatusSchema = z.enum(['validated', 'partially-validated', 'not-analyzed', 'blocked', 'not-applicable'])
+
+const HistoricalEvidenceRecordPayloadSchema = z.object({
+  id: identifier,
+  evidenceClass: z.literal('historical'),
+  observedAt: z.string().datetime(),
+  subject: z.object({
+    kind: z.enum(['consumer', 'aggregate']),
+    id: identifier,
+  }).strict(),
+  source: z.object({
+    kind: z.enum(['validation-plan', 'verification-run', 'benchmark', 'study-artifact', 'issue', 'pull-request']),
+    reference,
+  }).strict(),
+  docBridgeVersion: reference.optional(),
+  sourceRevisionHash: hash.optional(),
+  snapshotHash: hash.optional(),
+  reportHash: hash.optional(),
+  workflowRunId: reference.optional(),
+  verificationRunId: reference.optional(),
+  status: HistoricalRecordStatusSchema,
+  metrics: z.record(identifier, z.number().finite()).default({}),
+  missingMetrics: z.array(z.object({
+    metricId: identifier,
+    status: HistoricalMetricStatusSchema,
+    reason: safeText,
+  }).strict()).default([]),
+  limitations: z.array(safeText).min(1).max(32),
+}).strict().superRefine((value, context) => {
+  if (!Object.keys(value.metrics).length && !value.missingMetrics.length) {
+    context.addIssue({ code: z.ZodIssueCode.custom, path: ['metrics'], message: 'A historical record must contain metrics or explicit missing metrics.' })
+  }
+  uniqueIds(value.missingMetrics.map((metric) => metric.metricId), 'missing metric')
+})
+
+export const HistoricalEvidenceRecordV1Schema = HistoricalEvidenceRecordPayloadSchema
+export type HistoricalEvidenceRecordV1 = z.infer<typeof HistoricalEvidenceRecordV1Schema>
+
+const HistoricalEvidenceRegistryPayloadSchema = z.object({
+  type: z.literal('historical-evidence-registry'),
+  schemaVersion: z.literal(STUDY_PROTOCOL_SCHEMA_VERSION),
+  registryVersion: reference,
+  records: z.array(HistoricalEvidenceRecordV1Schema).max(10_000),
+}).strict()
+
+export const HistoricalEvidenceRegistryV1Schema = HistoricalEvidenceRegistryPayloadSchema.extend({
+  contentHash: hash,
+  contentHashAlgo: z.literal(STUDY_PROTOCOL_CONTENT_HASH_ALGO),
+}).strict()
+
+export type HistoricalEvidenceRegistryV1 = z.infer<typeof HistoricalEvidenceRegistryV1Schema>
+
+const assertProtocolReferences = (protocol: StudyProtocolV1): void => {
+  uniqueIds(protocol.repositories.map((item) => item.id), 'repository')
+  uniqueIds(protocol.models.map((item) => item.id), 'model')
+  uniqueIds(protocol.scenarios.map((item) => item.id), 'scenario')
+  uniqueIds(protocol.metrics.map((item) => item.id), 'metric')
+  uniqueIds(protocol.outcomes.map((item) => item.id), 'outcome')
+  uniqueIds(protocol.outcomes.flatMap((item) => item.checks.map((check) => check.id)), 'outcome check')
+  const modelIds = new Set(protocol.models.map((item) => item.id))
+  const metricIds = new Set(protocol.metrics.map((item) => item.id))
+  for (const scenario of protocol.scenarios) {
+    for (const modelId of scenario.modelIds) if (!modelIds.has(modelId)) throw new Error(`Scenario ${scenario.id} references unknown model ${modelId}.`)
+    if (scenario.source === 'doc-bridge-registry-agent' && scenario.registryAgent !== 'configured') throw new Error(`Scenario ${scenario.id} requires a configured Registry agent.`)
+    if (scenario.source !== 'doc-bridge-registry-agent' && scenario.registryAgent === 'configured') throw new Error(`Scenario ${scenario.id} cannot configure a Registry agent.`)
+  }
+  for (const target of protocol.stopping.targets) if (!metricIds.has(target.metricId)) throw new Error(`Stopping target references unknown metric ${target.metricId}.`)
+}
+
+const assertHash = <T extends { readonly contentHash: string }>(artifact: T): T => {
+  const expected = contentHashForArtifactV1(artifact)
+  if (expected !== artifact.contentHash) throw new Error(`Invalid content hash: expected ${expected}.`)
+  return artifact
+}
+
+export const createStudyProtocol = (input: unknown): StudyProtocolV1 => {
+  const payload = StudyProtocolPayloadSchema.parse(input)
+  const hashable = { ...payload, contentHashAlgo: STUDY_PROTOCOL_CONTENT_HASH_ALGO }
+  const protocol = StudyProtocolV1Schema.parse({
+    ...hashable,
+    contentHash: sha256NormalizedV1(hashable),
+  })
+  assertProtocolReferences(protocol)
+  return protocol
+}
+
+export const parseStudyProtocol = (input: unknown): StudyProtocolV1 => {
+  const protocol = StudyProtocolV1Schema.parse(input)
+  assertHash(protocol)
+  assertProtocolReferences(protocol)
+  return protocol
+}
+
+export const createHistoricalEvidenceRegistry = (input: unknown): HistoricalEvidenceRegistryV1 => {
+  const payload = HistoricalEvidenceRegistryPayloadSchema.parse(input)
+  const hashable = { ...payload, contentHashAlgo: STUDY_PROTOCOL_CONTENT_HASH_ALGO }
+  return HistoricalEvidenceRegistryV1Schema.parse({
+    ...hashable,
+    contentHash: sha256NormalizedV1(hashable),
+  })
+}
+
+export const parseHistoricalEvidenceRegistry = (input: unknown): HistoricalEvidenceRegistryV1 => {
+  const registry = HistoricalEvidenceRegistryV1Schema.parse(input)
+  return assertHash(registry)
+}
+
+export const validateHistoricalEvidenceRegistry = (
+  registry: HistoricalEvidenceRegistryV1,
+  protocol: StudyProtocolV1,
+): void => {
+  const repositoryIds = new Set(protocol.repositories.map((item) => item.id))
+  uniqueIds(registry.records.map((record) => record.id), 'historical record')
+  for (const record of registry.records) {
+    if (record.subject.kind === 'consumer' && !repositoryIds.has(record.subject.id)) throw new Error(`Historical record ${record.id} references unknown consumer ${record.subject.id}.`)
+  }
+}
+
+export const formatStudyProtocolText = (protocol: StudyProtocolV1): readonly string[] => [
+  `Protocol: ${protocol.protocolVersion}`,
+  `Repositories: ${protocol.repositories.length}`,
+  `Models: ${protocol.models.length} (${protocol.models.filter((model) => model.status === 'pinned').length} pinned)`,
+  `Scenarios: ${protocol.scenarios.length}`,
+  `Task categories: ${protocol.taskCategories.join(', ')}`,
+  `Metrics: ${protocol.metrics.length}`,
+  `Outcomes mapped: ${protocol.outcomes.length}`,
+  `Budget: ${protocol.budget.maxTokensPerTask} tokens/task, ${protocol.budget.maxRuntimeMsPerTask} ms/task, ${protocol.budget.maxRuns} runs`,
+  `Privacy: ${protocol.privacy.mode}; human publication review: required`,
+  `Content hash: ${protocol.contentHash}`,
+]
+
+export const formatHistoricalEvidenceText = (
+  registry: HistoricalEvidenceRegistryV1,
+): readonly string[] => {
+  const statuses = registry.records.reduce<Record<string, number>>((counts, record) => ({ ...counts, [record.status]: (counts[record.status] ?? 0) + 1 }), {})
+  const missing = registry.records.reduce((count, record) => count + record.missingMetrics.length, 0)
+  return [
+    `Registry: ${registry.registryVersion}`,
+    `Records: ${registry.records.length}`,
+    `Missing measurements: ${missing}`,
+    `Statuses: ${Object.entries(statuses).sort(([a], [b]) => a.localeCompare(b)).map(([status, count]) => `${status}=${count}`).join(', ') || 'none'}`,
+    `Content hash: ${registry.contentHash}`,
+  ]
+}

package/src/study/provider-cli.ts

@@ -0,0 +1,115 @@
+import { accessSync, constants, existsSync, statSync } from 'node:fs'
+import { isAbsolute, resolve } from 'node:path'
+import { z } from 'zod'
+
+import { contentHashForArtifactV1, sha256NormalizedV1 } from '../index-builder/content-hash.js'
+
+export const STUDY_PROVIDER_CLI_SCHEMA_VERSION = 1 as const
+export const STUDY_PROVIDER_CLI_CONTENT_HASH_ALGO = 'sha256-normalized-v1' as const
+
+const hash = z.string().regex(/^[a-f0-9]{64}$/)
+const identifier = z.string().regex(/^[a-z][a-z0-9-]{0,127}$/)
+const reference = z.string().regex(/^[a-z0-9][a-z0-9._:-]{0,255}$/)
+const command = z.string().min(1).max(512).refine((value) => !/[\u0000\r\n]/.test(value), 'Command cannot contain control characters.')
+const argument = z.string().max(2_048).refine((value) => !/[\u0000\r\n]/.test(value), 'Arguments cannot contain control characters.')
+const environmentName = z.string().regex(/^[A-Z][A-Z0-9_]{0,127}$/)
+const pricing = z.object({
+  currency: z.literal('USD'),
+  inputPerMillionUsd: z.number().finite().nonnegative(),
+  cachedInputPerMillionUsd: z.number().finite().nonnegative(),
+  outputPerMillionUsd: z.number().finite().nonnegative(),
+}).strict()
+
+const commandConfig = z.object({
+  command,
+  args: z.array(argument).max(64).default([]),
+  envAllowlist: z.array(environmentName).max(64).default([]),
+  providerNetwork: z.boolean(),
+  maxInputBytes: z.number().int().positive().max(8_000_000),
+  maxOutputBytes: z.number().int().positive().max(256_000),
+  pricing: pricing.optional(),
+}).strict()
+
+const adjudicatorConfig = z.object({
+  id: identifier,
+  modelId: identifier,
+  ...commandConfig.shape,
+}).strict()
+
+const ProviderCliPayloadSchema = z.object({
+  type: z.literal('study-provider-cli-config'),
+  schemaVersion: z.literal(STUDY_PROVIDER_CLI_SCHEMA_VERSION),
+  configVersion: reference,
+  providers: z.array(z.object({
+    modelId: identifier,
+    scenarioIds: z.array(z.enum(['repository-only', 'deterministic-doc-bridge', 'registry-assisted'])).min(1).max(3),
+    ...commandConfig.shape,
+  }).strict()).min(1).max(16),
+  adjudicator: adjudicatorConfig.optional(),
+}).strict()
+
+export const StudyProviderCliConfigV1Schema = ProviderCliPayloadSchema.extend({
+  contentHash: hash,
+  contentHashAlgo: z.literal(STUDY_PROVIDER_CLI_CONTENT_HASH_ALGO),
+}).strict()
+
+export type StudyProviderCliConfigV1 = z.infer<typeof StudyProviderCliConfigV1Schema>
+export type StudyProviderCli = StudyProviderCliConfigV1['providers'][number]
+export type StudyProviderCostPricing = z.infer<typeof pricing>
+export type StudyAdjudicatorCli = z.infer<typeof adjudicatorConfig>
+
+export const createStudyProviderCliConfig = (input: unknown): StudyProviderCliConfigV1 => {
+  const payload = ProviderCliPayloadSchema.parse(input)
+  const hashable = { ...payload, contentHashAlgo: STUDY_PROVIDER_CLI_CONTENT_HASH_ALGO }
+  return StudyProviderCliConfigV1Schema.parse({ ...hashable, contentHash: sha256NormalizedV1(hashable) })
+}
+
+export const parseStudyProviderCliConfig = (input: unknown): StudyProviderCliConfigV1 => {
+  const config = StudyProviderCliConfigV1Schema.parse(input)
+  if (contentHashForArtifactV1(config) !== config.contentHash) throw new Error('Invalid study provider CLI config content hash.')
+  const duplicateKeys = new Set<string>()
+  for (const provider of config.providers) {
+    for (const scenarioId of provider.scenarioIds) {
+      const key = `${provider.modelId}:${scenarioId}`
+      if (duplicateKeys.has(key)) throw new Error(`Duplicate study provider mapping for ${key}.`)
+      duplicateKeys.add(key)
+    }
+  }
+  return config
+}
+
+export const providerForStudyExecution = (
+  config: StudyProviderCliConfigV1,
+  modelId: string,
+  scenarioId: StudyProviderCli['scenarioIds'][number],
+): StudyProviderCli => {
+  const provider = config.providers.find((item) => item.modelId === modelId && item.scenarioIds.includes(scenarioId))
+  if (!provider) throw new Error(`No provider CLI is configured for model ${modelId} and scenario ${scenarioId}.`)
+  return provider
+}
+
+export const calculateStudyCostUsd = (pricing: StudyProviderCostPricing | undefined, usage: { readonly inputTokens?: number; readonly cachedInputTokens?: number; readonly outputTokens?: number }): number | undefined => {
+  if (!pricing || usage.inputTokens === undefined || usage.outputTokens === undefined) return undefined
+  const cachedInputTokens = Math.min(usage.cachedInputTokens ?? 0, usage.inputTokens)
+  const uncachedInputTokens = usage.inputTokens - cachedInputTokens
+  return Number(((uncachedInputTokens * pricing.inputPerMillionUsd + cachedInputTokens * pricing.cachedInputPerMillionUsd + usage.outputTokens * pricing.outputPerMillionUsd) / 1_000_000).toFixed(8))
+}
+
+export const validateStudyProviderCommand = (provider: Pick<StudyProviderCli, 'command'>, cwd: string): void => {
+  if (isAbsolute(provider.command) || provider.command.includes('/')) {
+    const resolvedPath = isAbsolute(provider.command) ? provider.command : resolve(cwd, provider.command)
+    if (!existsSync(resolvedPath) || !statSync(resolvedPath).isFile()) throw new Error(`Provider CLI command is not executable at ${resolvedPath}.`)
+    try { accessSync(resolvedPath, constants.X_OK) } catch { throw new Error(`Provider CLI command is not executable at ${resolvedPath}.`) }
+    return
+  }
+  const pathEntries = (process.env.PATH ?? '').split(':').filter(Boolean)
+  if (pathEntries.some((entry) => { try { accessSync(resolve(entry, provider.command), constants.X_OK); return true } catch { return false } })) return
+  throw new Error(`Provider CLI command ${provider.command} was not found on PATH.`)
+}
+
+export const formatStudyProviderCliText = (config: StudyProviderCliConfigV1): readonly string[] => [
+  `Provider CLI config: ${config.configVersion}`,
+  `Mappings: ${config.providers.length}`,
+  `Hosted providers: ${config.providers.filter((provider) => provider.providerNetwork).length}`,
+  `Content hash: ${config.contentHash}`,
+]

package/src/study/provider-telemetry.ts

@@ -0,0 +1,47 @@
+import { Buffer } from 'node:buffer'
+
+const TOOL_TYPES = new Set([
+  'command_execution',
+  'mcp_tool_call',
+  'web_search_call',
+  'file_search_call',
+  'computer_call',
+])
+
+const byteLength = (value: unknown): number => {
+  if (value === undefined || value === null) return 0
+  const encoded = typeof value === 'string' ? value : JSON.stringify(value)
+  return encoded === undefined ? 0 : Buffer.byteLength(encoded, 'utf8')
+}
+
+const firstPresent = (value: Record<string, unknown>, keys: readonly string[]): unknown => {
+  for (const key of keys) {
+    if (value[key] !== undefined && value[key] !== null) return value[key]
+  }
+  return undefined
+}
+
+export type ProviderToolTelemetry = {
+  readonly observedToolEventCount: number
+  readonly observedToolInputBytes: number
+  readonly observedToolOutputBytes: number
+}
+
+export const measureProviderToolTelemetry = (events: readonly unknown[]): ProviderToolTelemetry => {
+  let observedToolEventCount = 0
+  let observedToolInputBytes = 0
+  let observedToolOutputBytes = 0
+
+  for (const event of events) {
+    if (!event || typeof event !== 'object' || Array.isArray(event)) continue
+    const record = event as { readonly type?: unknown; readonly item?: unknown }
+    if (record.type !== 'item.completed' || !record.item || typeof record.item !== 'object' || Array.isArray(record.item)) continue
+    const item = record.item as Record<string, unknown>
+    if (typeof item.type !== 'string' || !TOOL_TYPES.has(item.type)) continue
+    observedToolEventCount += 1
+    observedToolInputBytes += byteLength(firstPresent(item, ['command', 'arguments', 'input', 'params']))
+    observedToolOutputBytes += byteLength(firstPresent(item, ['aggregated_output', 'output', 'result', 'content']))
+  }
+
+  return { observedToolEventCount, observedToolInputBytes, observedToolOutputBytes }
+}

package/src/study/quality-scorecard.ts

@@ -0,0 +1,164 @@
+export type ScorecardStatus = 'pass' | 'partial' | 'not-analyzed' | 'fail'
+
+export type ScorecardCriterion = {
+  readonly id: string
+  readonly dimension: string
+  readonly status: ScorecardStatus
+  readonly observed: number | null
+  readonly threshold: number | null
+  readonly unit: string
+  readonly required: boolean
+  readonly evidence: readonly string[]
+  readonly reason: string
+}
+
+export type ScorecardDimension = {
+  readonly id: string
+  readonly status: ScorecardStatus
+  readonly criteria: readonly string[]
+}
+
+export type QualityScorecard = {
+  readonly type: 'doc-bridge-quality-scorecard'
+  readonly schemaVersion: 1
+  readonly decision: 'ready' | 'not-ready'
+  readonly criteria: readonly ScorecardCriterion[]
+  readonly dimensions: readonly ScorecardDimension[]
+  readonly summary: {
+    readonly pass: number
+    readonly partial: number
+    readonly notAnalyzed: number
+    readonly fail: number
+  }
+}
+
+export type QualityScorecardInput = {
+  readonly semanticFixture: {
+    readonly precision: number
+    readonly recall: number
+    readonly evidenceRatio: number
+  }
+  readonly semanticTask: {
+    readonly repositoryOnlySuccessRate: number
+    readonly docBridgeSuccessRate: number
+  }
+  readonly acceptanceInstrumentation: {
+    readonly observations: number
+    readonly complete: number
+  }
+  readonly tokenPilot: {
+    readonly reductionPct: number
+    readonly replicates: number
+  }
+  readonly cost: {
+    readonly observedUsd: number | null
+  }
+  readonly generalization: {
+    readonly populations: number
+    readonly models: number
+    readonly replicates: number
+  }
+  readonly documentation: {
+    readonly titleRate: number
+    readonly requiredSectionsRate: number
+    readonly examplesRate: number
+    readonly contradictions: number
+    readonly stale: number
+    readonly structureGaps: number
+    readonly exactDuplicateGroups: number
+    readonly criticalMetadataCoverage: number
+    readonly semanticNotAnalyzed: number
+    readonly semanticReviewed: number
+    readonly semanticCorpus: number
+    readonly semanticContradictionCandidates: number
+    readonly semanticRedundancyCandidates: number
+    readonly semanticMissingCandidates: number
+    readonly semanticClaimCandidates: number
+  }
+  readonly evidence: {
+    readonly contentHashes: number
+    readonly requiredHashes: number
+  }
+  readonly latency: {
+    readonly reductionPct: number
+  }
+  readonly thresholds: {
+    readonly fixturePrecision: number
+    readonly fixtureRecall: number
+    readonly fixtureEvidenceRatio: number
+    readonly taskSuccessRate: number
+    readonly tokenReductionPct: number
+    readonly tokenReplicates: number
+    readonly populationCount: number
+    readonly modelCount: number
+    readonly generalizationReplicates: number
+    readonly documentationRate: number
+    readonly examplesRate: number
+    readonly criticalMetadataCoverage: number
+    readonly semanticNotAnalyzedMaximum: number
+    readonly semanticReviewCoverage: number
+    readonly acceptanceInstrumentationCoverage: number
+    readonly latencyReductionPct: number
+  }
+}
+
+const criterion = (
+  id: string,
+  dimension: string,
+  status: ScorecardStatus,
+  observed: number | null,
+  threshold: number | null,
+  unit: string,
+  required: boolean,
+  evidence: readonly string[],
+  reason: string,
+): ScorecardCriterion => ({ id, dimension, status, observed, threshold, unit, required, evidence, reason })
+
+const dimensionStatus = (criteria: readonly ScorecardCriterion[]): ScorecardStatus => {
+  if (criteria.some((item) => item.status === 'fail')) return 'fail'
+  if (criteria.some((item) => item.status === 'not-analyzed')) return 'not-analyzed'
+  if (criteria.some((item) => item.status === 'partial')) return 'partial'
+  return 'pass'
+}
+
+export const evaluateQualityScorecard = (input: QualityScorecardInput): QualityScorecard => {
+  const t = input.thresholds
+  const criteria = [
+    criterion('semantic-fixture-classification', 'semantic-correctness', input.semanticFixture.precision >= t.fixturePrecision && input.semanticFixture.recall >= t.fixtureRecall && input.semanticFixture.evidenceRatio >= t.fixtureEvidenceRatio ? 'pass' : 'fail', input.semanticFixture.precision, t.fixturePrecision, 'precision', true, ['tests/semantic-benchmark.test.ts'], 'Known structured reconciliation cases meet the fixture threshold.'),
+    criterion('semantic-task-correctness', 'semantic-correctness', Math.min(input.semanticTask.repositoryOnlySuccessRate, input.semanticTask.docBridgeSuccessRate) >= t.taskSuccessRate ? 'pass' : 'fail', input.semanticTask.docBridgeSuccessRate, t.taskSuccessRate, 'success-rate', true, ['docs/study/ab-adjudicated-cost-result-v1.json'], 'Independent task adjudication must demonstrate the configured success threshold in both arms.'),
+    criterion('acceptance-instrumentation', 'semantic-correctness', input.acceptanceInstrumentation.observations > 0 && input.acceptanceInstrumentation.complete / input.acceptanceInstrumentation.observations >= t.acceptanceInstrumentationCoverage ? 'pass' : 'fail', input.acceptanceInstrumentation.observations > 0 ? input.acceptanceInstrumentation.complete / input.acceptanceInstrumentation.observations : null, t.acceptanceInstrumentationCoverage, 'coverage', true, ['docs/study/phase4-public-pilot-ledger-v1.json', 'src/study/execution.ts'], 'Every study observation must record passed, total, and executed acceptance-check counts; the adjudicator separately blocks passes when execution is incomplete.'),
+    criterion('token-reduction-positive', 'token-efficiency', input.tokenPilot.reductionPct >= t.tokenReductionPct ? 'pass' : 'fail', input.tokenPilot.reductionPct, t.tokenReductionPct, 'percent', true, ['docs/study/phase4-public-pilot-result-v1.json'], 'The observed paired token-equivalent delta is positive for the declared pilot.'),
+    criterion('token-reduction-consistent', 'token-efficiency', input.tokenPilot.replicates >= t.tokenReplicates ? 'pass' : 'fail', input.tokenPilot.replicates, t.tokenReplicates, 'replicates', true, ['docs/study/phase4-public-pilot-result-v1.json'], 'A single pilot cannot establish consistent reduction; the configured replicate minimum is required.'),
+    criterion('currency-cost', 'cost', input.cost.observedUsd === null ? 'not-analyzed' : 'pass', input.cost.observedUsd, 0, 'USD', true, ['docs/study/phase4-public-pilot-result-v1.json'], input.cost.observedUsd === null ? 'Provider pricing or observed USD cost is absent; no financial claim is allowed.' : 'Observed USD cost is available.'),
+    criterion('population-generalization', 'generalization', input.generalization.populations >= t.populationCount ? 'pass' : 'fail', input.generalization.populations, t.populationCount, 'populations', true, ['docs/study/phase4-public-pilot-result-v1.json'], 'Enterprise generalization requires multiple independent populations.'),
+    criterion('model-generalization', 'generalization', input.generalization.models >= t.modelCount ? 'pass' : 'fail', input.generalization.models, t.modelCount, 'models', true, ['docs/study/phase4-public-pilot-result-v1.json'], 'The minimum model diversity must be present before generalization claims.'),
+    criterion('replicate-generalization', 'generalization', input.generalization.replicates >= t.generalizationReplicates ? 'pass' : 'fail', input.generalization.replicates, t.generalizationReplicates, 'replicates', true, ['docs/study/phase4-public-pilot-result-v1.json'], 'Each task needs repeated observations before generalization is promoted.'),
+    criterion('documentation-deterministic-quality', 'documentation-quality', input.documentation.titleRate >= t.documentationRate && input.documentation.requiredSectionsRate >= t.documentationRate && input.documentation.contradictions === 0 && input.documentation.stale === 0 && input.documentation.structureGaps === 0 && input.documentation.criticalMetadataCoverage >= t.criticalMetadataCoverage ? 'pass' : 'fail', Math.min(input.documentation.titleRate, input.documentation.requiredSectionsRate, input.documentation.criticalMetadataCoverage), t.documentationRate, 'coverage', true, ['node bin/ak-docs.js audit documentation --json'], 'Structural quality, contradiction, stale-content, and critical metadata gates pass for the declared corpus.'),
+    criterion('documentation-example-coverage', 'documentation-quality', input.documentation.examplesRate >= t.examplesRate ? 'pass' : input.documentation.examplesRate > 0 ? 'partial' : 'fail', input.documentation.examplesRate, t.examplesRate, 'coverage', false, ['node bin/ak-docs.js audit documentation --json'], 'Examples are measured as a quality signal; not every document requires an example, so this criterion is advisory until the corpus declares eligibility.'),
+    criterion('documentation-exact-duplicates', 'documentation-quality', input.documentation.exactDuplicateGroups === 0 ? 'pass' : 'fail', input.documentation.exactDuplicateGroups, 0, 'groups', true, ['node bin/ak-docs.js audit documentation --json'], 'Exact duplicate documentation groups must be absent.'),
+    criterion('documentation-semantic-candidates', 'documentation-quality', input.documentation.semanticContradictionCandidates + input.documentation.semanticRedundancyCandidates + input.documentation.semanticMissingCandidates + input.documentation.semanticClaimCandidates === 0 ? 'pass' : 'fail', input.documentation.semanticContradictionCandidates + input.documentation.semanticRedundancyCandidates + input.documentation.semanticMissingCandidates + input.documentation.semanticClaimCandidates, 0, 'candidates', true, ['semantic-documentation-review', 'semantic-adjudication'], 'Bounded semantic review candidates must be resolved before the reviewed scope is promoted.'),
+    criterion('documentation-semantic-review-coverage', 'documentation-quality', input.documentation.semanticCorpus > 0 && input.documentation.semanticReviewed / input.documentation.semanticCorpus >= t.semanticReviewCoverage ? 'pass' : input.documentation.semanticReviewed > 0 ? 'partial' : 'not-analyzed', input.documentation.semanticCorpus > 0 ? input.documentation.semanticReviewed / input.documentation.semanticCorpus : null, t.semanticReviewCoverage, 'coverage', false, ['semantic-documentation-review', 'node bin/ak-docs.js audit documentation --json'], 'Semantic review coverage is reported separately from deterministic checks; bounded review does not prove full-corpus correctness.'),
+    criterion('documentation-semantic-coverage', 'documentation-quality', input.documentation.semanticNotAnalyzed <= t.semanticNotAnalyzedMaximum ? 'pass' : 'partial', input.documentation.semanticNotAnalyzed, t.semanticNotAnalyzedMaximum, 'documents', false, ['.codex/verification semantic-documentation-review'], 'Natural-language semantics remain bounded to the configured review scope; unreviewed documents stay visible.'),
+    criterion('evidence-reproducibility', 'evidence-quality', input.evidence.contentHashes >= input.evidence.requiredHashes ? 'pass' : 'fail', input.evidence.contentHashes, input.evidence.requiredHashes, 'hashes', true, ['docs/study/*result*.json'], 'All required study artifacts must expose reproducible content hashes.'),
+    criterion('latency-improvement', 'agent-efficiency', input.latency.reductionPct >= t.latencyReductionPct ? 'pass' : 'partial', input.latency.reductionPct, t.latencyReductionPct, 'percent', false, ['docs/study/phase4-public-pilot-result-v1.json'], 'Latency is tracked separately from correctness and token cost.'),
+  ]
+
+  const dimensions = [...new Set(criteria.map((item) => item.dimension))].map((id) => {
+    const items = criteria.filter((item) => item.dimension === id)
+    return { id, status: dimensionStatus(items), criteria: items.map((item) => item.id) }
+  })
+  const summary = {
+    pass: criteria.filter((item) => item.status === 'pass').length,
+    partial: criteria.filter((item) => item.status === 'partial').length,
+    notAnalyzed: criteria.filter((item) => item.status === 'not-analyzed').length,
+    fail: criteria.filter((item) => item.status === 'fail').length,
+  }
+  return {
+    type: 'doc-bridge-quality-scorecard',
+    schemaVersion: 1,
+    decision: criteria.some((item) => item.required && item.status !== 'pass') ? 'not-ready' : 'ready',
+    criteria,
+    dimensions,
+    summary,
+  }
+}