@tangle-network/agent-eval 0.173.3 → 0.174.0

package/dist/types-DQ0e2E7y.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types-DQ0e2E7y.js","names":[],"sources":["../src/ledger-core/deep-freeze.ts","../src/analyst/usage-receipt.ts","../src/analyst/types.ts"],"sourcesContent":["/** Freeze a detached canonical-JSON graph. Canonicalization has already ruled out cycles.\n *\n * Lives outside canonical.ts so the analyst-benchmark implementation digest,\n * which covers canonical.ts, stays bound to the published benchmark evidence. */\nexport function deepFreezeCanonicalJson<T>(value: T): T {\n  if (value && typeof value === 'object' && !Object.isFrozen(value)) {\n    Object.freeze(value)\n    for (const nested of Object.values(value)) deepFreezeCanonicalJson(nested)\n  }\n  return value\n}\n","import type { CostChannel, CostLedgerFilter, CostLedgerHandle } from '../cost-ledger'\nimport type { AnalystUsageReceipt } from './types'\n\nexport const DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS = 5_000\n\n/** Convert one ledger channel's complete call set into one analyst receipt. */\nexport function usageReceiptFromCostLedger(\n  ledger: CostLedgerHandle,\n  filter: CostChannel | CostLedgerFilter = 'analyst',\n): AnalystUsageReceipt {\n  const resolvedFilter = typeof filter === 'string' ? { channel: filter } : filter\n  const summary = ledger.summary(resolvedFilter)\n  const receipts = ledger.list(resolvedFilter)\n  const hasReasoningUsage = receipts.some((receipt) => receipt.reasoningTokens !== undefined)\n  const hasCacheWriteUsage = receipts.some((receipt) => receipt.cacheWriteTokens !== undefined)\n  const cost = summary.costProvenance\n  return {\n    calls: summary.totalCalls + summary.pendingCalls,\n    tokens: summary.usageComplete\n      ? {\n          input: summary.inputTokens,\n          output: summary.outputTokens,\n          ...(hasReasoningUsage ? { reasoning: summary.reasoningTokens ?? 0 } : {}),\n          ...(summary.cachedTokens > 0 ? { cached: summary.cachedTokens } : {}),\n          ...(hasCacheWriteUsage ? { cacheWrite: summary.cacheWriteTokens ?? 0 } : {}),\n        }\n      : null,\n    cost,\n    ...(cost.kind === 'uncaptured' ? { knownCostUsd: summary.totalCostUsd } : {}),\n  }\n}\n\nexport interface SettledUsageReceipt {\n  settled: boolean\n  pendingCalls: number\n  receipt: AnalystUsageReceipt\n}\n\n/** Wait a bounded time for late provider receipts, then take one immutable snapshot. */\nexport async function settleUsageReceiptFromCostLedger(\n  ledger: CostLedgerHandle,\n  options: CostLedgerFilter & { timeoutMs?: number } = {},\n): Promise<SettledUsageReceipt> {\n  const { timeoutMs: requestedTimeoutMs, ...requestedFilter } = options\n  const filter: CostLedgerFilter = {\n    channel: requestedFilter.channel ?? 'analyst',\n    ...(requestedFilter.phase === undefined ? {} : { phase: requestedFilter.phase }),\n    ...(requestedFilter.tags === undefined ? {} : { tags: requestedFilter.tags }),\n  }\n  const timeoutMs = validateUsageSettlementTimeout(requestedTimeoutMs)\n  const initial = ledger.summary(filter)\n  const waitResult =\n    initial.pendingCalls === 0\n      ? true\n      : ledger.waitForIdle\n        ? await ledger.waitForIdle({ timeoutMs })\n        : false\n  const pendingCalls = ledger.summary(filter).pendingCalls\n  return {\n    settled: waitResult && pendingCalls === 0,\n    pendingCalls,\n    receipt: usageReceiptFromCostLedger(ledger, filter),\n  }\n}\n\nexport function validateUsageSettlementTimeout(timeoutMs?: number): number {\n  const resolved = timeoutMs ?? DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS\n  if (!Number.isSafeInteger(resolved) || resolved < 0 || resolved > 2_147_483_647) {\n    throw new TypeError(\n      'settlementTimeoutMs must be a non-negative safe integer no greater than 2147483647',\n    )\n  }\n  return resolved\n}\n\nexport function assertValidAnalystUsageReceipt(\n  receipt: AnalystUsageReceipt,\n  context = 'AnalystContext.recordUsage',\n): void {\n  if (receipt.calls !== null && (!Number.isSafeInteger(receipt.calls) || receipt.calls < 0)) {\n    throw new Error(`${context}: calls must be a non-negative safe integer or null`)\n  }\n  if (receipt.tokens) {\n    assertNonNegativeSafeInteger(receipt.tokens.input, 'tokens.input', context)\n    assertNonNegativeSafeInteger(receipt.tokens.output, 'tokens.output', context)\n    if (receipt.tokens.reasoning !== undefined) {\n      assertNonNegativeSafeInteger(receipt.tokens.reasoning, 'tokens.reasoning', context)\n      if (receipt.tokens.reasoning > receipt.tokens.output) {\n        throw new Error(`${context}: tokens.reasoning must not exceed tokens.output`)\n      }\n    }\n    if (receipt.tokens.cached !== undefined) {\n      assertNonNegativeSafeInteger(receipt.tokens.cached, 'tokens.cached', context)\n    }\n    if (receipt.tokens.cacheWrite !== undefined) {\n      assertNonNegativeSafeInteger(receipt.tokens.cacheWrite, 'tokens.cacheWrite', context)\n    }\n  }\n  if (receipt.cost.kind !== 'uncaptured') {\n    assertNonNegativeFinite(receipt.cost.usd, 'cost.usd', context)\n  } else if (receipt.cost.usd !== null) {\n    throw new Error(`${context}: uncaptured cost.usd must be null`)\n  }\n  if (receipt.knownCostUsd !== undefined) {\n    assertNonNegativeFinite(receipt.knownCostUsd, 'knownCostUsd', context)\n  }\n  if (receipt.partialTokens) {\n    const { input, output } = receipt.partialTokens\n    if (receipt.tokens) {\n      throw new Error(`${context}: partialTokens must be absent when tokens is complete`)\n    }\n    if (input === null && output === null) {\n      throw new Error(`${context}: partialTokens must carry at least one reported side`)\n    }\n    if (input !== null) assertNonNegativeSafeInteger(input, 'partialTokens.input', context)\n    if (output !== null) assertNonNegativeSafeInteger(output, 'partialTokens.output', context)\n  }\n}\n\nfunction assertNonNegativeSafeInteger(value: number, field: string, context: string): void {\n  if (!Number.isSafeInteger(value) || value < 0) {\n    throw new Error(`${context}: ${field} must be a non-negative safe integer`)\n  }\n}\n\nfunction assertNonNegativeFinite(value: number, field: string, context: string): void {\n  if (!Number.isFinite(value) || value < 0) {\n    throw new Error(`${context}: ${field} must be a non-negative finite number`)\n  }\n}\n","/**\n * Analyst contract — the missing orchestration layer over agent-eval's\n * existing analyzers (analyzeTraces, MultiLayerVerifier,\n * SemanticConceptJudge, JudgeFn, ...).\n *\n * Each existing primitive returns its own output shape. The Analyst\n * contract is the single envelope every primitive lifts into, so a\n * registry can run N analysts against a run and a single renderer can\n * compose findings without knowing which analyzer produced them.\n *\n * The contract is intentionally domain-agnostic: nothing here knows\n * about code, voice, RAG, or any particular agent stack. Analysts\n * declare what INPUT KIND they need (a trace store, an artifact dir,\n * a RunRecord, a JudgeInput, or `custom`), and the registry routes\n * the matching input from `AnalystRunInputs`.\n */\n\nimport { createHash } from 'node:crypto'\nimport type { CostLedgerHandle } from '../cost-ledger'\nimport type { RunCostProvenance, RunRecord, RunTokenUsage } from '../run-record'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { JudgeInput } from '../types'\nimport type { ChatClient } from './chat-client'\n\n/**\n * Unified envelope every analyst emits. Schema-versioned so renderers\n * and time-series diffs survive future field additions.\n */\nexport interface AnalystFinding {\n  schema_version: '1.0.0'\n  /**\n   * Stable hash over identity-defining fields (analyst_id + canonical\n   * claim + area + optional subject). Two findings from two runs that\n   * \"are the same finding\" share this id — that's what `diffFindings`\n   * uses to compute appeared/disappeared sets across runs.\n   */\n  finding_id: string\n  analyst_id: string\n  produced_at: string\n  severity: AnalystSeverity\n  /**\n   * Coarse classification. Renderers group by this. Free-form so\n   * domain-specific analysts can introduce categories without a\n   * schema change ('agent-reasoning', 'verification', 'cost',\n   * 'tool-use', 'safety', 'latency', 'data-quality', ...).\n   */\n  area: string\n  claim: string\n  rationale?: string\n  evidence_refs: EvidenceRef[]\n  recommended_action?: string\n  validation_plan?: string\n  /** 0..1 — the analyst's own confidence. Not calibrated across analysts. */\n  confidence: number\n  /**\n   * Optional subject the finding is about — leaf id, agent id, request\n   * id. Included in finding_id when present so per-subject findings\n   * diff cleanly across runs.\n   */\n  subject?: string\n  /** True when this finding was lifted from a judge result rather than observed\n   *  directly in a trace or artifact. Descriptive only: proposal access is\n   *  controlled by `ProposalFinding.proposal_origin`. */\n  derived_from_judge?: boolean\n  /** Analyst-private extras; renderers ignore unless they know the analyst. */\n  metadata?: Record<string, unknown>\n}\n\n/** Finding severity. `AnalystSeverity` derives from this array, so the type\n *  and the schema that validates a finding cannot name different levels. */\nexport const ANALYST_SEVERITIES = ['critical', 'high', 'medium', 'low', 'info'] as const\n\nexport type AnalystSeverity = (typeof ANALYST_SEVERITIES)[number]\n\n/** Data sources that candidate generation may intentionally learn from. */\nexport type ProposalFindingOrigin = 'search' | 'production'\n\n/** A finding explicitly admitted as candidate-generation input. */\nexport type ProposalFinding = AnalystFinding & {\n  readonly proposal_origin: ProposalFindingOrigin\n}\n\nexport interface EvidenceRef {\n  /**\n   * Where the evidence lives. `span` and `event` refer to OTLP trace\n   * elements; `artifact` to a file inside the run's artifact tree;\n   * `finding` to another AnalystFinding (cross-analyst chaining);\n   * `metric` to a named scalar reading the renderer knows how to read.\n   */\n  kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric'\n  uri: string\n  excerpt?: string\n}\n\n// ── Analyst contract ─────────────────────────────────────────────────\n\n/**\n * The discriminator the registry uses to pass the right input.\n * `custom` is the escape hatch — analysts that need something else\n * (e.g. an embedding cache, a partner SDK handle) read it from\n * `AnalystRunInputs.custom[<analyst id>]`.\n */\nexport type AnalystInputKind =\n  | 'trace-store'\n  | 'artifact-dir'\n  | 'run-record'\n  | 'judge-input'\n  | 'custom'\n\nexport interface AnalystCost {\n  /** `deterministic` analysts MUST NOT call the LLM. */\n  kind: 'deterministic' | 'llm'\n  /** Optional declared upper bound; the registry can enforce a budget. */\n  est_usd_per_run?: number\n  /** Models the analyst expects to use (informational). */\n  models?: string[]\n  /** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */\n  settlement_timeout_ms?: number\n}\n\nexport interface AnalystRequirements {\n  /** Min number of shots / samples the analyst needs to produce signal. */\n  min_shots?: number\n  /** Capabilities the runtime must supply (e.g. ['network', 'gpu']). */\n  capabilities?: string[]\n}\n\n/**\n * What's passed to every analyst call. The registry resolves which\n * field the analyst's `inputKind` selects and asserts it's present.\n */\nexport interface AnalystRunInputs {\n  traceStore?: TraceAnalysisStore\n  artifactDir?: string\n  runRecord?: RunRecord\n  judgeInput?: JudgeInput\n  /** Keyed by analyst id; populated by callers that registered custom analysts. */\n  custom?: Record<string, unknown>\n}\n\nexport interface AnalystContext {\n  runId: string\n  /** Stable correlation id so logs from a single registry.run() share a tag. */\n  correlationId: string\n  /** Enforced wall-clock deadline (epoch ms). */\n  deadlineMs?: number\n  /** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */\n  budgetUsd?: number\n  /** Shared paid-call account when the analyst runs inside a larger campaign. */\n  costLedger?: CostLedgerHandle\n  /** Attribution phase used when writing to the shared paid-call account. */\n  costPhase?: string\n  /**\n   * Shared chat client. Analysts that call an LLM go through this so\n   * the operator picks transport (sandbox-sdk | router | cli-bridge |\n   * direct-provider | mock) at the registry boundary without touching\n   * analyst code.\n   */\n  chat?: ChatClient\n  /**\n   * Findings from a prior run the operator wants the analyst to see as\n   * retrieval context. Kinds that take advantage of cross-run memory\n   * (failure-mode \"I saw this cluster last run\", knowledge-gap \"the wiki\n   * page I asked for is still missing\") render these into the actor's\n   * working set. Filtering is the operator's job: pass the slice that\n   * matches the analyst's id, or pass everything and let the kind\n   * filter. Empty / absent means no cross-run context.\n   */\n  priorFindings?: ReadonlyArray<AnalystFinding>\n  /**\n   * Findings emitted by analysts that completed earlier in this registry run.\n   * This is separate from `priorFindings`: upstream findings are dependency\n   * context for the current pass, while prior findings are cross-run memory.\n   * The registry populates this only when `RegistryRunOpts.chainFindings` is on.\n   */\n  upstreamFindings?: ReadonlyArray<AnalystFinding>\n  /**\n   * Report metered work independently of findings. This keeps an empty finding\n   * set from erasing token/cost telemetry. Multiple receipts are accumulated.\n   */\n  recordUsage?: (receipt: AnalystUsageReceipt) => void\n  /** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */\n  tags?: Record<string, string>\n  /** Logger callback — analysts SHOULD prefer this over console.* for testability. */\n  log?: (msg: string, fields?: Record<string, unknown>) => void\n  /** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */\n  signal?: AbortSignal\n  /**\n   * Optional live-execution port. A runtime that owns a sandbox or checkout\n   * fills it so an analyst can execute a bounded probe against the run's\n   * produced state instead of reasoning about it from the trace alone. This\n   * package defines only the port: no field here reaches for an agent loop,\n   * and an absent probe means the analyst works from recorded evidence.\n   */\n  probe?: ExecutionProbe\n}\n\n// ── Live-execution port ─────────────────────────────────────────────\n\n/** One bounded command an analyst asks the probe to run. */\nexport interface ExecutionProbeRequest {\n  command: string\n  /** Working directory inside the probed environment. */\n  cwd?: string\n  /** Hard wall-clock deadline for this one execution. */\n  timeoutMs: number\n  /** Bytes of combined output retained; the prober truncates beyond it. */\n  maxOutputBytes?: number\n  signal?: AbortSignal\n}\n\n/**\n * Typed outcome of one probe execution. `succeeded: false` is a PROBE failure\n * (the environment could not run the command); a command that ran and exited\n * non-zero is a successful observation with a non-zero `exitCode`.\n */\nexport type ExecutionProbeOutcome =\n  | {\n      succeeded: true\n      exitCode: number\n      stdout: string\n      stderr: string\n      durationMs: number\n      /** True when output was cut at `maxOutputBytes`. */\n      truncated: boolean\n    }\n  | { succeeded: false; error: { class: string; message: string } }\n\n/**\n * The seam a runtime fills to let analysts observe produced state live.\n * Implementations own sandboxing, credentials, and cleanup; analysts only\n * submit bounded requests and read typed outcomes.\n */\nexport interface ExecutionProbe {\n  /** One plain sentence naming what is being probed (e.g. a sandbox id). */\n  readonly description: string\n  execute(request: ExecutionProbeRequest): Promise<ExecutionProbeOutcome>\n}\n\n/**\n * The minimal contract. Concrete analysts can refine `TInput` so\n * implementations stay type-safe (e.g. a trace analyst's `TInput` is\n * `TraceAnalysisStore`); the registry passes the right field from\n * `AnalystRunInputs` based on `inputKind`.\n */\nexport interface Analyst<TInput = unknown> {\n  /** Stable identifier — appears in finding_id, telemetry, and registry exclusion lists. */\n  readonly id: string\n  /** Human-readable. One sentence. */\n  readonly description: string\n  readonly inputKind: AnalystInputKind\n  readonly cost: AnalystCost\n  readonly requires?: AnalystRequirements\n  /** Bump on breaking changes to claim wording or area so old finding_ids don't collide. */\n  readonly version: string\n  analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>\n}\n\n/** Metered work performed by one analyst call. */\nexport interface AnalystUsageReceipt {\n  /** Number of model-usage records observed at the provider boundary. */\n  calls: number | null\n  /** Null when the provider did not return token accounting. */\n  tokens: RunTokenUsage | null\n  /** Observed, estimated, or explicitly uncaptured dollar cost. */\n  cost: RunCostProvenance\n  /** Known lower bound when one or more calls have uncaptured cost. */\n  knownCostUsd?: number\n  /**\n   * Token counts the provider reported on only one side. Present exactly when\n   * `tokens` is null and at least one side WAS reported: `RunTokenUsage` has no\n   * nullable side, so a one-sided count cannot live in `tokens` without writing\n   * a zero nobody measured. Read it as a lower bound, never as a total — the\n   * field exists so a null `tokens` cannot hide a real count.\n   */\n  partialTokens?: { input: number | null; output: number | null }\n  /**\n   * True when the token counts were DERIVED by the transport (from character\n   * lengths, say) rather than measured by the model provider. `cost.kind` is\n   * `estimated` both for a rate estimate over exact tokens and for one over\n   * derived tokens; this is the field that separates them.\n   */\n  tokensEstimated?: boolean\n}\n\n// ── finding_id stability ─────────────────────────────────────────────\n\n/**\n * Compute the stable finding_id from the identity-defining fields.\n * Default implementation hashes {analyst_id, area, subject, normalized claim}.\n * Analysts that emit findings whose claim text varies per run (timestamps,\n * counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,\n * or (b) move the variable part into `rationale`/`metadata` and keep the\n * `claim` static.\n */\nexport function computeFindingId(input: {\n  analyst_id: string\n  area: string\n  subject?: string\n  claim: string\n  /** Override the claim for hashing — use when the displayed claim has run-specific bits. */\n  id_basis?: string\n}): string {\n  const basis = JSON.stringify({\n    a: input.analyst_id,\n    r: input.area,\n    s: input.subject ?? '',\n    c: normalizeClaim(input.id_basis ?? input.claim),\n  })\n  return `f_${createHash('sha256').update(basis).digest('hex').slice(0, 20)}`\n}\n\nfunction normalizeClaim(c: string): string {\n  // Lowercase, collapse whitespace, strip trailing punctuation. Goal:\n  // \"Leaf X failed install\" and \"Leaf X failed install.\" hash the same.\n  return c\n    .toLowerCase()\n    .replace(/\\s+/g, ' ')\n    .replace(/[.!?;:,]+$/g, '')\n    .trim()\n}\n\n/**\n * Convenience factory: produce a fully-formed AnalystFinding with the\n * id computed automatically. Analyst code stays terse.\n */\nexport function makeFinding(\n  init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {\n    id_basis?: string\n    produced_at?: string\n  },\n): AnalystFinding {\n  const { id_basis, produced_at, ...rest } = init\n  return {\n    schema_version: '1.0.0',\n    finding_id: computeFindingId({\n      analyst_id: rest.analyst_id,\n      area: rest.area,\n      subject: rest.subject,\n      claim: rest.claim,\n      id_basis,\n    }),\n    produced_at: produced_at ?? new Date().toISOString(),\n    ...rest,\n  }\n}\n\n/** Build a finding whose source is explicitly allowed during candidate generation. */\nexport function makeProposalFinding(\n  init: Omit<ProposalFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {\n    id_basis?: string\n    produced_at?: string\n  },\n): ProposalFinding {\n  const { proposal_origin, ...finding } = init\n  return { ...makeFinding(finding), proposal_origin }\n}\n\n// ── Registry result envelope ────────────────────────────────────────\n\nexport interface AnalystRunSummary {\n  analyst_id: string\n  status: 'ok' | 'skipped' | 'failed'\n  /** Why skipped — missing input, budget exceeded, capability unmet. */\n  reason?: string\n  findings_count: number\n  latency_ms: number\n  /** Additive model usage and cost provenance for this analyst. */\n  usage: AnalystUsageReceipt\n  /** When `status='failed'`: the error class + message, never the full stack. */\n  error?: { class: string; message: string }\n}\n\nexport interface AnalystRunResult {\n  run_id: string\n  correlation_id: string\n  started_at: string\n  ended_at: string\n  findings: AnalystFinding[]\n  per_analyst: AnalystRunSummary[]\n  /** Total LLM cost in USD across all analysts in this registry.run(). */\n  total_cost_usd: number\n  /**\n   * Provenance for `total_cost_usd`. When uncaptured, the numeric field is only\n   * the known subtotal and must not be treated as the run's total spend.\n   */\n  total_cost_provenance?: RunCostProvenance\n}\n\n// ── Streaming event envelope ────────────────────────────────────────\n\n/**\n * Events emitted by `AnalystRegistry.runStream(...)` in real time as\n * the registry executes. UIs subscribe via `for await (const ev of\n * registry.runStream(...))`; `registry.run(...)` is a thin collector\n * over the same stream, so the two surfaces share their invariants.\n *\n * Per-finding events are intentionally omitted — analyzers are batch\n * operations (a recursive engine returns the full `findings:json[]` at the\n * end of the responder), so streaming inside one analyst would only\n * emit partial JSON consumers can't render. The kind-completion event\n * is the right granularity; subscribers wanting per-finding rendering\n * iterate `event.findings` themselves.\n */\nexport type AnalystRunEvent =\n  | {\n      type: 'run-started'\n      run_id: string\n      correlation_id: string\n      started_at: string\n      /** The ordered list of analyst ids the registry will run. */\n      analyst_ids: ReadonlyArray<string>\n    }\n  | {\n      type: 'analyst-skipped'\n      summary: AnalystRunSummary\n    }\n  | {\n      type: 'analyst-started'\n      analyst_id: string\n      started_at: string\n    }\n  | {\n      type: 'analyst-completed'\n      /** `summary.status` is `'ok'` for clean completion or `'failed'` for thrown analysts. */\n      summary: AnalystRunSummary\n      findings: ReadonlyArray<AnalystFinding>\n    }\n  | {\n      type: 'run-completed'\n      result: AnalystRunResult\n    }\n"],"mappings":";;;;;;AAIA,SAAgB,wBAA2B,OAAa;CACtD,IAAI,SAAS,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;EACjE,OAAO,OAAO,KAAK;EACnB,KAAK,MAAM,UAAU,OAAO,OAAO,KAAK,GAAG,wBAAwB,MAAM;CAC3E;CACA,OAAO;AACT;;ACJA,SAAgB,2BACd,QACA,SAAyC,WACpB;CACrB,MAAM,iBAAiB,OAAO,WAAW,WAAW,EAAE,SAAS,OAAO,IAAI;CAC1E,MAAM,UAAU,OAAO,QAAQ,cAAc;CAC7C,MAAM,WAAW,OAAO,KAAK,cAAc;CAC3C,MAAM,oBAAoB,SAAS,MAAM,YAAY,QAAQ,oBAAoB,KAAA,CAAS;CAC1F,MAAM,qBAAqB,SAAS,MAAM,YAAY,QAAQ,qBAAqB,KAAA,CAAS;CAC5F,MAAM,OAAO,QAAQ;CACrB,OAAO;EACL,OAAO,QAAQ,aAAa,QAAQ;EACpC,QAAQ,QAAQ,gBACZ;GACE,OAAO,QAAQ;GACf,QAAQ,QAAQ;GAChB,GAAI,oBAAoB,EAAE,WAAW,QAAQ,mBAAmB,EAAE,IAAI,CAAC;GACvE,GAAI,QAAQ,eAAe,IAAI,EAAE,QAAQ,QAAQ,aAAa,IAAI,CAAC;GACnE,GAAI,qBAAqB,EAAE,YAAY,QAAQ,oBAAoB,EAAE,IAAI,CAAC;EAC5E,IACA;EACJ;EACA,GAAI,KAAK,SAAS,eAAe,EAAE,cAAc,QAAQ,aAAa,IAAI,CAAC;CAC7E;AACF;;AASA,eAAsB,iCACpB,QACA,UAAqD,CAAC,GACxB;CAC9B,MAAM,EAAE,WAAW,oBAAoB,GAAG,oBAAoB;CAC9D,MAAM,SAA2B;EAC/B,SAAS,gBAAgB,WAAW;EACpC,GAAI,gBAAgB,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,gBAAgB,MAAM;EAC9E,GAAI,gBAAgB,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM,gBAAgB,KAAK;CAC7E;CACA,MAAM,YAAY,+BAA+B,kBAAkB;CAEnE,MAAM,aADU,OAAO,QAAQ,MAEvB,CAAC,CAAC,iBAAiB,IACrB,OACA,OAAO,cACL,MAAM,OAAO,YAAY,EAAE,UAAU,CAAC,IACtC;CACR,MAAM,eAAe,OAAO,QAAQ,MAAM,CAAC,CAAC;CAC5C,OAAO;EACL,SAAS,cAAc,iBAAiB;EACxC;EACA,SAAS,2BAA2B,QAAQ,MAAM;CACpD;AACF;AAEA,SAAgB,+BAA+B,WAA4B;CACzE,MAAM,WAAW,aAAA;CACjB,IAAI,CAAC,OAAO,cAAc,QAAQ,KAAK,WAAW,KAAK,WAAW,YAChE,MAAM,IAAI,UACR,oFACF;CAEF,OAAO;AACT;AAEA,SAAgB,+BACd,SACA,UAAU,8BACJ;CACN,IAAI,QAAQ,UAAU,SAAS,CAAC,OAAO,cAAc,QAAQ,KAAK,KAAK,QAAQ,QAAQ,IACrF,MAAM,IAAI,MAAM,GAAG,QAAQ,oDAAoD;CAEjF,IAAI,QAAQ,QAAQ;EAClB,6BAA6B,QAAQ,OAAO,OAAO,gBAAgB,OAAO;EAC1E,6BAA6B,QAAQ,OAAO,QAAQ,iBAAiB,OAAO;EAC5E,IAAI,QAAQ,OAAO,cAAc,KAAA,GAAW;GAC1C,6BAA6B,QAAQ,OAAO,WAAW,oBAAoB,OAAO;GAClF,IAAI,QAAQ,OAAO,YAAY,QAAQ,OAAO,QAC5C,MAAM,IAAI,MAAM,GAAG,QAAQ,iDAAiD;EAEhF;EACA,IAAI,QAAQ,OAAO,WAAW,KAAA,GAC5B,6BAA6B,QAAQ,OAAO,QAAQ,iBAAiB,OAAO;EAE9E,IAAI,QAAQ,OAAO,eAAe,KAAA,GAChC,6BAA6B,QAAQ,OAAO,YAAY,qBAAqB,OAAO;CAExF;CACA,IAAI,QAAQ,KAAK,SAAS,cACxB,wBAAwB,QAAQ,KAAK,KAAK,YAAY,OAAO;MACxD,IAAI,QAAQ,KAAK,QAAQ,MAC9B,MAAM,IAAI,MAAM,GAAG,QAAQ,mCAAmC;CAEhE,IAAI,QAAQ,iBAAiB,KAAA,GAC3B,wBAAwB,QAAQ,cAAc,gBAAgB,OAAO;CAEvE,IAAI,QAAQ,eAAe;EACzB,MAAM,EAAE,OAAO,WAAW,QAAQ;EAClC,IAAI,QAAQ,QACV,MAAM,IAAI,MAAM,GAAG,QAAQ,uDAAuD;EAEpF,IAAI,UAAU,QAAQ,WAAW,MAC/B,MAAM,IAAI,MAAM,GAAG,QAAQ,sDAAsD;EAEnF,IAAI,UAAU,MAAM,6BAA6B,OAAO,uBAAuB,OAAO;EACtF,IAAI,WAAW,MAAM,6BAA6B,QAAQ,wBAAwB,OAAO;CAC3F;AACF;AAEA,SAAS,6BAA6B,OAAe,OAAe,SAAuB;CACzF,IAAI,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,GAC1C,MAAM,IAAI,MAAM,GAAG,QAAQ,IAAI,MAAM,qCAAqC;AAE9E;AAEA,SAAS,wBAAwB,OAAe,OAAe,SAAuB;CACpF,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,MAAM,GAAG,QAAQ,IAAI,MAAM,sCAAsC;AAE/E;;;;;;;;;;;;;;;;;;;;;AC3DA,MAAa,qBAAqB;CAAC;CAAY;CAAQ;CAAU;CAAO;AAAM;;;;;;;;;AAiO9E,SAAgB,iBAAiB,OAOtB;CACT,MAAM,QAAQ,KAAK,UAAU;EAC3B,GAAG,MAAM;EACT,GAAG,MAAM;EACT,GAAG,MAAM,WAAW;EACpB,GAAG,eAAe,MAAM,YAAY,MAAM,KAAK;CACjD,CAAC;CACD,OAAO,KAAK,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,EAAE;AAC1E;AAEA,SAAS,eAAe,GAAmB;CAGzC,OAAO,EACJ,YAAY,CAAC,CACb,QAAQ,QAAQ,GAAG,CAAC,CACpB,QAAQ,eAAe,EAAE,CAAC,CAC1B,KAAK;AACV;;;;;AAMA,SAAgB,YACd,MAIgB;CAChB,MAAM,EAAE,UAAU,aAAa,GAAG,SAAS;CAC3C,OAAO;EACL,gBAAgB;EAChB,YAAY,iBAAiB;GAC3B,YAAY,KAAK;GACjB,MAAM,KAAK;GACX,SAAS,KAAK;GACd,OAAO,KAAK;GACZ;EACF,CAAC;EACD,aAAa,gCAAe,IAAI,KAAK,EAAA,CAAE,YAAY;EACnD,GAAG;CACL;AACF;;AAGA,SAAgB,oBACd,MAIiB;CACjB,MAAM,EAAE,iBAAiB,GAAG,YAAY;CACxC,OAAO;EAAE,GAAG,YAAY,OAAO;EAAG;CAAgB;AACpD"}

package/dist/{types-BDV4PiMR.d.ts → types-Dd1ejaeI.d.ts}

@@ -1,5 +1,5 @@
 import { s as RunSplitTag } from "./run-record-DTv1MdjK.js";
-import { R as Scenario, d as DispatchContext } from "./types-Ba5UQyVD.js";
+import { R as Scenario, d as DispatchContext } from "./types-BJz2CPTM.js";
 //#region src/benchmarks/types.d.ts
 type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
 type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
@@ -90,4 +90,4 @@ declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
 declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
 //#endregion
 export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
-//# sourceMappingURL=types-BDV4PiMR.d.ts.map
+//# sourceMappingURL=types-Dd1ejaeI.d.ts.map

package/dist/{types-BDV4PiMR.d.ts.map → types-Dd1ejaeI.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"types-BDV4PiMR.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;EACA;EACA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
+{"version":3,"file":"types-Dd1ejaeI.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;EACA;EACA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}

package/docs/campaign-proposers.md

@@ -20,6 +20,48 @@ Use it when one surface must get better.
 Use it when two or more methods must be compared at equal budget.
 Runnable versions: [`examples/self-improve-optimizer`](../examples/self-improve-optimizer/) and [`examples/compare-optimization-methods`](../examples/compare-optimization-methods/).
 
+## Read An Improvement Result
+
+`selfImprove({ method })` executes the complete method once and measures its selected surface on final cases.
+The method may select the unchanged baseline; that result returns `gateDecision: 'hold'` and an empty diff.
+Agent Eval does not score train and selection cases again or choose a different surface after the method finishes.
+
+The result type has two modes:
+
+| Mode | Result | Search evidence | Cost |
+|---|---|---|---|
+| `proposer` | `SelfImproveProposerResult` | Native `raw.generations`, `generationsExplored`, and optional `searchHistory` | Shared `cost` ledger summary |
+| `method` | `SelfImproveMethodResult` | Actual `raw.method` and its optional `searchHistory` | Combined method and final `cost`; receipt breakdown in `ledgerCost` |
+
+Both types are exported from the package root and `/contract`.
+`SelfImproveResult` is their union; branch on `result.mode` before reading mode-specific fields.
+Calls with a concrete `method` or `proposer` infer the corresponding result type.
+Method mode has no native generation count or fabricated native search measurements.
+Its durable `method-provenance.json` uses schema `tangle.method-improvement` and records partition, measurement, and cost-receipt digests.
+Proposer mode retains `LoopProvenanceRecord`.
+
+When method holdout is deferred, `baseline` and `winner.compositeMean` are `null`, `lift` is absent, and the decision is `hold`.
+The selected surface remains available in `winner.surface`.
+Method cost preserves the larger of reported search spend and newly recorded search receipts, then adds final measurements without counting receipts twice.
+Underreported spending and incomplete receipts remain explicit; `raw.method.cost` retains the original report.
+Inspect `cost.accountingComplete` and `cost.incompleteReasons` before treating the known subtotal as complete spending.
+The shared dollar limit controls calls admitted through the cost ledger; arbitrary off-ledger callbacks must enforce their own spending limits.
+
+Native generation records report `ci95: null` because search does not estimate candidate uncertainty.
+Final comparisons retain their independently computed statistics.
+Every final case and replica must have complete execution and judge results before comparison.
+
+## Bind Cached Measurements To Their Evaluator
+
+Candidate surface content is part of native search and final measurement identity.
+Pass a stable `dispatchRef` for execution behavior outside that surface, such as the worker revision and tool configuration.
+Change it when that behavior changes; function names cannot identify captured state.
+Set `judgeVersion` when a judge's scoring behavior changes.
+
+To reuse `premeasuredBaseline` in proposer mode, measure the same train cases, seed, replicas, execution revision, and judges.
+The standalone campaign must use `dispatchRef: surfaceDispatchRef(baselineSurface, dispatchRef)` from `/campaign`.
+Agent Eval refuses a prior baseline whose evaluator manifest differs.
+
 ## Adapt A Third-Party Text Optimizer
 
 `externalTextOptimizationMethod()` is the general adapter for a package that already owns text or component search.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.173.3",
+  "version": "0.174.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {

package/dist/agent-profile-B9_GGsG8.d.ts

@@ -1,84 +0,0 @@
-import { AgentProfile, AgentProfile as AgentProfile$1, HarnessType, HarnessType as HarnessType$1 } from "@tangle-network/agent-interface";
-//#region src/agent-profile.d.ts
-/**
- * The agentic coding harnesses an eval sweeps by default — the ones we care about
- * ranking. This is the SINGLE source of that list; consumers import it instead of
- * re-declaring their own (a re-declared list is how the fleet drifts). Pass an
- * explicit `harnesses` (e.g. `harnessTypeSchema.options` for literally every known
- * harness) to widen beyond these.
- */
-declare const CODING_HARNESSES: readonly HarnessType[];
-interface ProfileAxisSpec {
-  /** The domain profile to sweep. Its prompt/tools/skills are held fixed; only the
-   *  harness and model vary. `model.default` is the fallback model. */
-  base: AgentProfile;
-  /** Harnesses to cross. Default: {@link CODING_HARNESSES}. */
-  harnesses?: readonly HarnessType[];
-  /** Models to cross. Default: `[base.model.default]` — one model, i.e. today's
-   *  single-model behaviour, so omitting this never changes an existing run. */
-  models?: readonly string[];
-  /** Force every (harness, model) pair verbatim, even ones the harness can't run —
-   *  for deliberately testing failure modes. Default (false): SNAP instead — a
-   *  vendor-locked harness runs only the swept models in its family, or its native
-   *  default when it supports none, so no harness is dropped and none gets a
-   *  guaranteed-failing foreign-model cell. */
-  keepIncompatible?: boolean;
-}
-/** Model sentinel for a vendor-locked harness that supports none of the swept models:
- *  it carries no provider prefix, so `harnessSupportsModel` accepts it and the harness
- *  resolves it to its own native default model at runtime (e.g. kimi-code → its Kimi
- *  model). Lets `expandProfileAxes` snap-instead-of-drop without a per-harness flagship
- *  table that would rot as router catalogs change. */
-declare const HARNESS_NATIVE_MODEL = "default";
-/**
- * Expand a base profile across the harness × model matrix into the `AgentProfile[]`
- * that `runProfileMatrix` / `selfImprove` score — the ONE place "which harnesses ×
- * which models do we evaluate" lives, so no product hand-rolls its own harness list
- * or column→profile mapping (the pattern that let those copies drift and silently
- * break the harness pivot).
- *
- * Each cell clones `base`, sets the canonical top-level `harness` and `model.default`,
- * and stamps `metadata.harness` + `metadata.harnessModel` for matrix grouping. Both
- * metadata fields are hash-bearing, so every cell gets a distinct `agentProfileId` row
- * and results join back by harness/model via {@link harnessAxisOf} with no
- * hand-recomputed key. A vendor-locked harness snaps to its family's swept models — or
- * its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none — so every
- * requested harness runs; `keepIncompatible` forces every pair verbatim.
- *
- * Omit `harnesses`/`models` to sweep the full default set — the "turn it on for
- * everything we care about" switch, identical in shape whether one harness or all.
- */
-declare function expandProfileAxes(spec: ProfileAxisSpec): AgentProfile[];
-/**
- * Read the (harness, model) a matrix cell ran under, off a profile or a result row's
- * profile — the join-back for a `byHarness` pivot. Returns undefined when the profile
- * wasn't produced by {@link expandProfileAxes}. Callers group `result.byProfile` by
- * this instead of recomputing an id (recomputing the wrong key is what broke the pivot
- * in the hand-rolled copies).
- */
-declare function harnessAxisOf(profile: Pick<AgentProfile, 'metadata'>): {
-  harness: HarnessType;
-  model: string;
-} | undefined;
-/**
- * Collision-resistant, path-safe, human-readable profile id for eval artifacts.
- * Scorecard joins still use `agentProfileHash`; this id is for run ids, matrix
- * keys, and directory names where two profiles must not collapse onto one row.
- * The suffix is the first 64 bits of the behaviour hash, enough for ordinary
- * eval matrices while keeping filenames readable.
- */
-declare function agentProfileId(profile: AgentProfile): string;
-/**
- * Deterministic behaviour identity for the canonical
- * `@tangle-network/agent-interface` AgentProfile.
- *
- * `name` and `description` are labels and do not affect the hash. Profile
- * `version`, prompt, model hints, tools, resources, hooks, modes, permissions,
- * and extensions do affect the hash. Resource array order is hash-bearing
- * because mount order can change agent behaviour. Undefined fields are treated
- * as absent; explicit `null` fields remain hash-bearing.
- */
-declare function agentProfileHash(profile: AgentProfile): string;
-//#endregion
-export { ProfileAxisSpec as a, expandProfileAxes as c, HarnessType$1 as i, harnessAxisOf as l, CODING_HARNESSES as n, agentProfileHash as o, HARNESS_NATIVE_MODEL as r, agentProfileId as s, AgentProfile$1 as t };
-//# sourceMappingURL=agent-profile-B9_GGsG8.d.ts.map

package/dist/agent-profile-B9_GGsG8.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"agent-profile-B9_GGsG8.d.ts","names":[],"sources":["../src/agent-profile.ts"],"mappings":";;;;;;;;;cAea,2BAA2B;UAOvB;;;EAGf,MAAM;;EAEN,qBAAqB;;;EAGrB;;;;;;EAMA;;;;;;;cAQW;;;;;;;;;;;;;;;;;;;iBAoBG,kBAAkB,MAAM,kBAAkB;;;;;;;;iBAwD1C,cACd,SAAS,KAAK;EACX,SAAS;EAAa;;;;;;;;;iBAiBX,eAAe,SAAS;;;;;;;;;;;iBAmDxB,iBAAiB,SAAS"}

package/dist/backend-integrity-CeuTgqsd.d.ts

@@ -1,280 +0,0 @@
-import { t as AgentEvalError } from "./errors-DEE6u6ot.js";
-import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
-import { a as RunRecord } from "./run-record-DTv1MdjK.js";
-import { Y as RawProviderSink, p as ChatClient } from "./types-gvRsyJLh.js";
-import { a as CheckerIdentity, n as VerdictCertification, t as DefaultVerdict, x as VerificationStrategySource } from "./verdict-E4eRNf7-.js";
-//#region src/artifact-validator.d.ts
-/**
- * Artifact validators.
- *
- * Generic "score a produced artifact" primitive. Tax uses it for PDF form
- * correctness, research for sourced briefs, browser for task assertions, coding
- * for social posts. One interface, many validators.
- *
- * A validator receives an `Artifact` (file on disk, JSON blob, text, binary)
- * plus a `ValidationContext` (scenario id, the turns that produced it) and
- * returns a `ValidationResult` with pass/fail + 0..1 score + structured
- * issues.
- */
-interface Artifact {
-  /** Logical kind — validators type-guard on this */
-  kind: 'file' | 'json' | 'text' | 'binary' | string;
-  /** Filesystem-style path, optional */
-  path?: string;
-  /** String content for text/json/file kinds */
-  content?: string;
-  /** Binary content (if kind === 'binary') */
-  bytes?: Uint8Array;
-  /** Caller-supplied metadata (mimeType, sha256, size, etc.) */
-  metadata?: Record<string, unknown>;
-}
-//#endregion
-//#region src/completion-verifier.d.ts
-/** What kind of produced state can satisfy a requirement structurally. */
-type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any';
-interface CompletionRequirement {
-  /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */
-  reqId: string;
-  /** Human-readable description of the required deliverable. */
-  title: string;
-  /** Optional kind/category hint, matched against a produced item's kind. */
-  category?: string;
-  /** What produced state satisfies this requirement. Defaults to 'any'. */
-  satisfiedBy?: SatisfiedBy;
-}
-interface TaskGold {
-  taskId: string;
-  requirements: CompletionRequirement[];
-}
-interface ProducedProposal {
-  id: string;
-  title: string;
-  status: 'pending' | 'approved' | 'rejected';
-  /** Optional persisted body — when present, enables a correctness check. */
-  content?: string;
-}
-/** Everything observable about what a run actually produced. */
-interface ProducedState {
-  /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */
-  artifacts: Artifact[];
-  /** Proposals / filings the agent created. */
-  proposals: ProducedProposal[];
-  /** Names of tools the agent invoked. */
-  toolCalls: string[];
-}
-interface RequirementCheck {
-  reqId: string;
-  title: string;
-  /** A produced item of the right kind matched the requirement, non-empty. */
-  structurallyPresent: boolean;
-  /**
-   * Whether the matched item actually fulfils the requirement. `null` when
-   * not structurally present, when the matched item carries no content
-   * to assess, or when the correctness check itself failed (`unmeasured`).
-   */
-  correct: boolean | null;
-  /** structurallyPresent && !unmeasured && correct !== false. */
-  satisfied: boolean;
-  /**
-   * Set when the correctness check itself errored (LLM call failure or an
-   * unparseable response after retry). The requirement's fulfilment is
-   * UNKNOWN — `correct` stays null, `satisfied` is false, and
-   * `completionVerdict` excludes the row from `completionRate`'s
-   * denominator. Never folded into a zero: a synthetic zero is
-   * indistinguishable from a real failure (see `JudgeParseError`).
-   */
-  unmeasured?: true;
-  /** Why the correctness check could not be measured (present iff `unmeasured`). */
-  unmeasuredReason?: string;
-  /** Human-readable evidence for the verdict. */
-  evidence: string[];
-}
-/** Extends the substrate verdict spine: `valid` = `fullyComplete` and
- *  `score` = `completionRate` — derived in `completionVerdict()`, the one
- *  place those equalities hold by construction. */
-interface CompletionVerdict extends DefaultVerdict {
-  taskId: string;
-  requirements: RequirementCheck[];
-  /** satisfied / MEASURABLE requirements (unmeasured rows leave the denominator). */
-  completionRate: number;
-  /** Every measurable requirement satisfied (false when anything is unmeasured). */
-  fullyComplete: boolean;
-  /** Requirements whose correctness check errored — reported, never scored as zero. */
-  unmeasuredCount: number;
-}
-/**
- * Construct a `CompletionVerdict` from the per-requirement checks, deriving
- * `completionRate` / `fullyComplete` and the spine fields (`valid` =
- * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
- * requirements — a verdict over nothing is a misconfiguration, mirroring
- * `verifyCompletion`'s gold-spec guard.
- */
-declare function completionVerdict(input: {
-  taskId: string;
-  requirements: RequirementCheck[];
-  /** What certified the correctness stage, when anything did. Omitted =
-   *  an uncertified verdict — the honest default for a bare checker. */
-  certification?: VerdictCertification;
-}): CompletionVerdict;
-/**
- * What a correctness checker declares about itself so the completion
- * verdict can carry a certification: which strategy member it discharges,
- * its exact identity, and the steps its answers rest on unverified.
- */
-interface CorrectnessCheckerAttestation {
-  strategy: VerificationStrategySource;
-  checker: CheckerIdentity;
-  assumptions: string[];
-}
-/**
- * Decides whether a produced item's content actually fulfils a requirement.
- * Injected so the structural verifier stays pure and unit-testable; the
- * production implementation is `createLlmCorrectnessChecker`.
- *
- * `attestation` is optional metadata on the function value: a checker that
- * carries one yields certified completion verdicts; a bare function yields
- * the same verdict uncertified. A plain arrow function remains a valid
- * checker.
- */
-interface CorrectnessChecker {
-  (requirement: CompletionRequirement, content: string): Promise<{
-    correct: boolean;
-    reason: string;
-  }>;
-  attestation?: CorrectnessCheckerAttestation;
-}
-/**
- * Verify whether a run completed the task. `checkCorrectness` is injected —
- * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.
- *
- * Throws on a gold spec with no requirements: an eval task that requires
- * nothing is a misconfiguration, not a vacuously-complete task.
- */
-declare function verifyCompletion(gold: TaskGold, state: ProducedState, checkCorrectness: CorrectnessChecker): Promise<CompletionVerdict>;
-interface LlmCorrectnessCheckerOpts {
-  model?: string;
-  /** Optional ledger for direct use. */
-  costLedger?: CostLedgerHandle;
-  costPhase?: string;
-  costTags?: Record<string, string>;
-  signal?: AbortSignal;
-  /** Max chars of artifact content sent to the checker. */
-  maxContentChars?: number;
-  /**
-   * Checker LLM calls per requirement before giving up (parse failures and
-   * call errors both consume attempts). The failure then surfaces as an
-   * `unmeasured` requirement, never a zero.
-   */
-  maxAttempts?: number;
-  /**
-   * Forensic capture of every checker request/response/error — without it a
-   * checker failure is unauditable (the agent-turn raws never contain the
-   * checker's own calls). Same sink contract as `LlmClient`.
-   */
-  rawSink?: RawProviderSink;
-}
-/**
- * Production `CorrectnessChecker` — one LLM call per matched artifact,
- * deterministic (temperature 0), structured JSON out. Judges fulfilment
- * only: a plan, a gesture, or a description of what should be done does not
- * fulfil a requirement — the artifact must BE the deliverable.
- */
-declare function createLlmCorrectnessChecker(chat: ChatClient, opts?: LlmCorrectnessCheckerOpts): CorrectnessChecker;
-//#endregion
-//#region src/produced-state.d.ts
-/** A tool the agent invoked. */
-interface ToolCallEventLike {
-  type: 'tool_call';
-  toolName: string;
-}
-/**
- * An artifact the agent produced. `content` is the enriched field — the
- * runtime's base `artifact` event carries only metadata; the completion
- * oracle needs the body to verify the deliverable, so the runtime emits it.
- */
-interface ArtifactEventLike {
-  type: 'artifact';
-  artifactId: string;
-  name?: string;
-  mimeType?: string;
-  uri?: string;
-  content?: string;
-}
-/** A proposal / filing the agent created. */
-interface ProposalEventLike {
-  type: 'proposal_created';
-  proposalId: string;
-  title: string;
-  status?: 'pending' | 'approved' | 'rejected';
-  content?: string;
-}
-/**
- * The subset of runtime stream events `extractProducedState` consumes.
- * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;
- * the `{ type: string }` catch-all keeps the input permissive so callers can
- * pass the whole unfiltered telemetry stream — unrecognized events are skipped.
- */
-type RuntimeEventLike = ToolCallEventLike | ArtifactEventLike | ProposalEventLike | {
-  type: string;
-};
-/**
- * Normalize a run's runtime event stream into `ProducedState`.
- *
- * Pure and total — unrecognized event types are skipped. `toolCalls` is
- * deduplicated by name in first-seen order (completion cares about a tool's
- * presence, not its call count). An artifact with neither a name nor a uri
- * still yields an entry keyed by its `artifactId` so it is never silently
- * dropped; an artifact with no `content` yields empty content, which the
- * completion oracle's structural check then rejects on its own.
- */
-declare function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState;
-//#endregion
-//#region src/integrity/backend-integrity.d.ts
-interface BackendIntegrityReport {
-  /** Total records inspected. */
-  totalRecords: number;
-  /** Records with input=0 AND output=0 (a stub fingerprint). */
-  stubRecords: number;
-  /** Records with nonzero token usage (real LLM activity). */
-  realRecords: number;
-  /** Records where output>0 but costUsd=0 (real LLM, broken cost ledger). */
-  uncostedRecords: number;
-  /** Sum of input tokens across all records. */
-  totalInputTokens: number;
-  /** Sum of output tokens across all records. */
-  totalOutputTokens: number;
-  /** Sum of costUsd across all records. */
-  totalCostUsd: number;
-  /** Worst-case integrity verdict. */
-  verdict: 'real' | 'mixed' | 'stub';
-  /** Human-readable diagnosis suitable for terminal output. */
-  diagnosis: string;
-}
-/**
- * Error thrown when an integrity assertion fails. Caller can pattern-match
- * by `code === 'AGENT_EVAL_BACKEND_STUB'` to differentiate from other
- * errors.
- */
-declare class BackendIntegrityError extends AgentEvalError {
-  readonly report: BackendIntegrityReport;
-  constructor(message: string, report: BackendIntegrityReport);
-}
-/**
- * Inspect a batch of RunRecords and return an integrity report. Pure
- * function — no I/O, no logging. The caller decides what to do with the
- * verdict (print warning, throw, gate CI, etc.).
- */
-declare function summarizeBackendIntegrity(records: ReadonlyArray<RunRecord>): BackendIntegrityReport;
-/**
- * Throw BackendIntegrityError if the verdict is 'stub' — i.e. every record
- * shows zero LLM activity. Non-strict callers can pass `{ allowMixed: false }`
- * to also reject mixed verdicts (recommended for CI gates).
- *
- * Real backends pass through silently.
- */
-declare function assertRealBackend(records: ReadonlyArray<RunRecord>, opts?: {
-  allowMixed?: boolean;
-}): BackendIntegrityReport;
-//#endregion
-export { SatisfiedBy as _, ArtifactEventLike as a, createLlmCorrectnessChecker as b, ToolCallEventLike as c, CompletionVerdict as d, CorrectnessChecker as f, RequirementCheck as g, ProducedState as h, summarizeBackendIntegrity as i, extractProducedState as l, ProducedProposal as m, BackendIntegrityReport as n, ProposalEventLike as o, LlmCorrectnessCheckerOpts as p, assertRealBackend as r, RuntimeEventLike as s, BackendIntegrityError as t, CompletionRequirement as u, TaskGold as v, verifyCompletion as x, completionVerdict as y };
-//# sourceMappingURL=backend-integrity-CeuTgqsd.d.ts.map

package/dist/backend-integrity-CeuTgqsd.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"backend-integrity-CeuTgqsd.d.ts","names":[],"sources":["../src/artifact-validator.ts","../src/completion-verifier.ts","../src/produced-state.ts","../src/integrity/backend-integrity.ts"],"mappings":";;;;;;;;;;;;;;;;;;UAaiB;;EAEf;;EAEA;;EAEA;;EAEA,QAAQ;;EAER,WAAW;;;;;KCsBD;UAEK;;EAEf;;EAEA;;EAEA;;EAEA,cAAc;;UAGC;EACf;EACA,cAAc;;UAGC;EACf;EACA;EACA;;EAEA;;;UAIe;;EAEf,WAAW;;EAEX,WAAW;;EAEX;;UAGe;EACf;EACA;;EAEA;;;;;;EAMA;;EAEA;;;;;;;;;EASA;;EAEA;;EAEA;;;;;UAMe,0BAA0B;EACzC;EACA,cAAc;;EAEd;;EAEA;;EAEA;;;;;;;;;iBAUc,kBAAkB;EAChC;EACA,cAAc;;;EAGd,gBAAgB;IACd;;;;;;UAqCa;EACf,UAAU;EACV,SAAS;EACT;;;;;;;;;;;;UAae;GAEb,aAAa,uBACb,kBACC;IAAU;IAAkB;;EAC/B,cAAc;;;;;;;;;iBAsLM,iBACpB,MAAM,UACN,OAAO,eACP,kBAAkB,qBACjB,QAAQ;UAyGM;EACf;;EAEA,aAAa;EACb;EACA,WAAW;EACX,SAAS;;EAET;;;;;;EAMA;;;;;;EAMA,UAAU;;;;;;;;iBA2CI,4BACd,MAAM,YACN,OAAM,4BACL;;;;UCnhBc;EACf;EACA;;;;;;;UAQe;EACf;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EAIA;;;;;;;;KASU,mBACR,oBACA,oBACA;EACE;;;;;;;;;;;;iBAmBU,qBAAqB,iBAAiB,qBAAqB;;;UCpD1D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;cAQW,8BAA8B;WAGvB,QAAQ;EAF1B,YACE,iBACgB,QAAQ;;;;;;;iBAWZ,0BACd,SAAS,cAAc,aACtB;;;;;;;;iBAuHa,kBACd,SAAS,cAAc,YACvB;EAAQ;IACP"}