@tangle-network/agent-eval 0.173.3 → 0.175.0

package/dist/agent-profile-B9_GGsG8.d.ts

@@ -1,84 +0,0 @@
-import { AgentProfile, AgentProfile as AgentProfile$1, HarnessType, HarnessType as HarnessType$1 } from "@tangle-network/agent-interface";
-//#region src/agent-profile.d.ts
-/**
- * The agentic coding harnesses an eval sweeps by default — the ones we care about
- * ranking. This is the SINGLE source of that list; consumers import it instead of
- * re-declaring their own (a re-declared list is how the fleet drifts). Pass an
- * explicit `harnesses` (e.g. `harnessTypeSchema.options` for literally every known
- * harness) to widen beyond these.
- */
-declare const CODING_HARNESSES: readonly HarnessType[];
-interface ProfileAxisSpec {
-  /** The domain profile to sweep. Its prompt/tools/skills are held fixed; only the
-   *  harness and model vary. `model.default` is the fallback model. */
-  base: AgentProfile;
-  /** Harnesses to cross. Default: {@link CODING_HARNESSES}. */
-  harnesses?: readonly HarnessType[];
-  /** Models to cross. Default: `[base.model.default]` — one model, i.e. today's
-   *  single-model behaviour, so omitting this never changes an existing run. */
-  models?: readonly string[];
-  /** Force every (harness, model) pair verbatim, even ones the harness can't run —
-   *  for deliberately testing failure modes. Default (false): SNAP instead — a
-   *  vendor-locked harness runs only the swept models in its family, or its native
-   *  default when it supports none, so no harness is dropped and none gets a
-   *  guaranteed-failing foreign-model cell. */
-  keepIncompatible?: boolean;
-}
-/** Model sentinel for a vendor-locked harness that supports none of the swept models:
- *  it carries no provider prefix, so `harnessSupportsModel` accepts it and the harness
- *  resolves it to its own native default model at runtime (e.g. kimi-code → its Kimi
- *  model). Lets `expandProfileAxes` snap-instead-of-drop without a per-harness flagship
- *  table that would rot as router catalogs change. */
-declare const HARNESS_NATIVE_MODEL = "default";
-/**
- * Expand a base profile across the harness × model matrix into the `AgentProfile[]`
- * that `runProfileMatrix` / `selfImprove` score — the ONE place "which harnesses ×
- * which models do we evaluate" lives, so no product hand-rolls its own harness list
- * or column→profile mapping (the pattern that let those copies drift and silently
- * break the harness pivot).
- *
- * Each cell clones `base`, sets the canonical top-level `harness` and `model.default`,
- * and stamps `metadata.harness` + `metadata.harnessModel` for matrix grouping. Both
- * metadata fields are hash-bearing, so every cell gets a distinct `agentProfileId` row
- * and results join back by harness/model via {@link harnessAxisOf} with no
- * hand-recomputed key. A vendor-locked harness snaps to its family's swept models — or
- * its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none — so every
- * requested harness runs; `keepIncompatible` forces every pair verbatim.
- *
- * Omit `harnesses`/`models` to sweep the full default set — the "turn it on for
- * everything we care about" switch, identical in shape whether one harness or all.
- */
-declare function expandProfileAxes(spec: ProfileAxisSpec): AgentProfile[];
-/**
- * Read the (harness, model) a matrix cell ran under, off a profile or a result row's
- * profile — the join-back for a `byHarness` pivot. Returns undefined when the profile
- * wasn't produced by {@link expandProfileAxes}. Callers group `result.byProfile` by
- * this instead of recomputing an id (recomputing the wrong key is what broke the pivot
- * in the hand-rolled copies).
- */
-declare function harnessAxisOf(profile: Pick<AgentProfile, 'metadata'>): {
-  harness: HarnessType;
-  model: string;
-} | undefined;
-/**
- * Collision-resistant, path-safe, human-readable profile id for eval artifacts.
- * Scorecard joins still use `agentProfileHash`; this id is for run ids, matrix
- * keys, and directory names where two profiles must not collapse onto one row.
- * The suffix is the first 64 bits of the behaviour hash, enough for ordinary
- * eval matrices while keeping filenames readable.
- */
-declare function agentProfileId(profile: AgentProfile): string;
-/**
- * Deterministic behaviour identity for the canonical
- * `@tangle-network/agent-interface` AgentProfile.
- *
- * `name` and `description` are labels and do not affect the hash. Profile
- * `version`, prompt, model hints, tools, resources, hooks, modes, permissions,
- * and extensions do affect the hash. Resource array order is hash-bearing
- * because mount order can change agent behaviour. Undefined fields are treated
- * as absent; explicit `null` fields remain hash-bearing.
- */
-declare function agentProfileHash(profile: AgentProfile): string;
-//#endregion
-export { ProfileAxisSpec as a, expandProfileAxes as c, HarnessType$1 as i, harnessAxisOf as l, CODING_HARNESSES as n, agentProfileHash as o, HARNESS_NATIVE_MODEL as r, agentProfileId as s, AgentProfile$1 as t };
-//# sourceMappingURL=agent-profile-B9_GGsG8.d.ts.map

package/dist/agent-profile-B9_GGsG8.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"agent-profile-B9_GGsG8.d.ts","names":[],"sources":["../src/agent-profile.ts"],"mappings":";;;;;;;;;cAea,2BAA2B;UAOvB;;;EAGf,MAAM;;EAEN,qBAAqB;;;EAGrB;;;;;;EAMA;;;;;;;cAQW;;;;;;;;;;;;;;;;;;;iBAoBG,kBAAkB,MAAM,kBAAkB;;;;;;;;iBAwD1C,cACd,SAAS,KAAK;EACX,SAAS;EAAa;;;;;;;;;iBAiBX,eAAe,SAAS;;;;;;;;;;;iBAmDxB,iBAAiB,SAAS"}

package/dist/backend-integrity-CeuTgqsd.d.ts

@@ -1,280 +0,0 @@
-import { t as AgentEvalError } from "./errors-DEE6u6ot.js";
-import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
-import { a as RunRecord } from "./run-record-DTv1MdjK.js";
-import { Y as RawProviderSink, p as ChatClient } from "./types-gvRsyJLh.js";
-import { a as CheckerIdentity, n as VerdictCertification, t as DefaultVerdict, x as VerificationStrategySource } from "./verdict-E4eRNf7-.js";
-//#region src/artifact-validator.d.ts
-/**
- * Artifact validators.
- *
- * Generic "score a produced artifact" primitive. Tax uses it for PDF form
- * correctness, research for sourced briefs, browser for task assertions, coding
- * for social posts. One interface, many validators.
- *
- * A validator receives an `Artifact` (file on disk, JSON blob, text, binary)
- * plus a `ValidationContext` (scenario id, the turns that produced it) and
- * returns a `ValidationResult` with pass/fail + 0..1 score + structured
- * issues.
- */
-interface Artifact {
-  /** Logical kind — validators type-guard on this */
-  kind: 'file' | 'json' | 'text' | 'binary' | string;
-  /** Filesystem-style path, optional */
-  path?: string;
-  /** String content for text/json/file kinds */
-  content?: string;
-  /** Binary content (if kind === 'binary') */
-  bytes?: Uint8Array;
-  /** Caller-supplied metadata (mimeType, sha256, size, etc.) */
-  metadata?: Record<string, unknown>;
-}
-//#endregion
-//#region src/completion-verifier.d.ts
-/** What kind of produced state can satisfy a requirement structurally. */
-type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any';
-interface CompletionRequirement {
-  /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */
-  reqId: string;
-  /** Human-readable description of the required deliverable. */
-  title: string;
-  /** Optional kind/category hint, matched against a produced item's kind. */
-  category?: string;
-  /** What produced state satisfies this requirement. Defaults to 'any'. */
-  satisfiedBy?: SatisfiedBy;
-}
-interface TaskGold {
-  taskId: string;
-  requirements: CompletionRequirement[];
-}
-interface ProducedProposal {
-  id: string;
-  title: string;
-  status: 'pending' | 'approved' | 'rejected';
-  /** Optional persisted body — when present, enables a correctness check. */
-  content?: string;
-}
-/** Everything observable about what a run actually produced. */
-interface ProducedState {
-  /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */
-  artifacts: Artifact[];
-  /** Proposals / filings the agent created. */
-  proposals: ProducedProposal[];
-  /** Names of tools the agent invoked. */
-  toolCalls: string[];
-}
-interface RequirementCheck {
-  reqId: string;
-  title: string;
-  /** A produced item of the right kind matched the requirement, non-empty. */
-  structurallyPresent: boolean;
-  /**
-   * Whether the matched item actually fulfils the requirement. `null` when
-   * not structurally present, when the matched item carries no content
-   * to assess, or when the correctness check itself failed (`unmeasured`).
-   */
-  correct: boolean | null;
-  /** structurallyPresent && !unmeasured && correct !== false. */
-  satisfied: boolean;
-  /**
-   * Set when the correctness check itself errored (LLM call failure or an
-   * unparseable response after retry). The requirement's fulfilment is
-   * UNKNOWN — `correct` stays null, `satisfied` is false, and
-   * `completionVerdict` excludes the row from `completionRate`'s
-   * denominator. Never folded into a zero: a synthetic zero is
-   * indistinguishable from a real failure (see `JudgeParseError`).
-   */
-  unmeasured?: true;
-  /** Why the correctness check could not be measured (present iff `unmeasured`). */
-  unmeasuredReason?: string;
-  /** Human-readable evidence for the verdict. */
-  evidence: string[];
-}
-/** Extends the substrate verdict spine: `valid` = `fullyComplete` and
- *  `score` = `completionRate` — derived in `completionVerdict()`, the one
- *  place those equalities hold by construction. */
-interface CompletionVerdict extends DefaultVerdict {
-  taskId: string;
-  requirements: RequirementCheck[];
-  /** satisfied / MEASURABLE requirements (unmeasured rows leave the denominator). */
-  completionRate: number;
-  /** Every measurable requirement satisfied (false when anything is unmeasured). */
-  fullyComplete: boolean;
-  /** Requirements whose correctness check errored — reported, never scored as zero. */
-  unmeasuredCount: number;
-}
-/**
- * Construct a `CompletionVerdict` from the per-requirement checks, deriving
- * `completionRate` / `fullyComplete` and the spine fields (`valid` =
- * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
- * requirements — a verdict over nothing is a misconfiguration, mirroring
- * `verifyCompletion`'s gold-spec guard.
- */
-declare function completionVerdict(input: {
-  taskId: string;
-  requirements: RequirementCheck[];
-  /** What certified the correctness stage, when anything did. Omitted =
-   *  an uncertified verdict — the honest default for a bare checker. */
-  certification?: VerdictCertification;
-}): CompletionVerdict;
-/**
- * What a correctness checker declares about itself so the completion
- * verdict can carry a certification: which strategy member it discharges,
- * its exact identity, and the steps its answers rest on unverified.
- */
-interface CorrectnessCheckerAttestation {
-  strategy: VerificationStrategySource;
-  checker: CheckerIdentity;
-  assumptions: string[];
-}
-/**
- * Decides whether a produced item's content actually fulfils a requirement.
- * Injected so the structural verifier stays pure and unit-testable; the
- * production implementation is `createLlmCorrectnessChecker`.
- *
- * `attestation` is optional metadata on the function value: a checker that
- * carries one yields certified completion verdicts; a bare function yields
- * the same verdict uncertified. A plain arrow function remains a valid
- * checker.
- */
-interface CorrectnessChecker {
-  (requirement: CompletionRequirement, content: string): Promise<{
-    correct: boolean;
-    reason: string;
-  }>;
-  attestation?: CorrectnessCheckerAttestation;
-}
-/**
- * Verify whether a run completed the task. `checkCorrectness` is injected —
- * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.
- *
- * Throws on a gold spec with no requirements: an eval task that requires
- * nothing is a misconfiguration, not a vacuously-complete task.
- */
-declare function verifyCompletion(gold: TaskGold, state: ProducedState, checkCorrectness: CorrectnessChecker): Promise<CompletionVerdict>;
-interface LlmCorrectnessCheckerOpts {
-  model?: string;
-  /** Optional ledger for direct use. */
-  costLedger?: CostLedgerHandle;
-  costPhase?: string;
-  costTags?: Record<string, string>;
-  signal?: AbortSignal;
-  /** Max chars of artifact content sent to the checker. */
-  maxContentChars?: number;
-  /**
-   * Checker LLM calls per requirement before giving up (parse failures and
-   * call errors both consume attempts). The failure then surfaces as an
-   * `unmeasured` requirement, never a zero.
-   */
-  maxAttempts?: number;
-  /**
-   * Forensic capture of every checker request/response/error — without it a
-   * checker failure is unauditable (the agent-turn raws never contain the
-   * checker's own calls). Same sink contract as `LlmClient`.
-   */
-  rawSink?: RawProviderSink;
-}
-/**
- * Production `CorrectnessChecker` — one LLM call per matched artifact,
- * deterministic (temperature 0), structured JSON out. Judges fulfilment
- * only: a plan, a gesture, or a description of what should be done does not
- * fulfil a requirement — the artifact must BE the deliverable.
- */
-declare function createLlmCorrectnessChecker(chat: ChatClient, opts?: LlmCorrectnessCheckerOpts): CorrectnessChecker;
-//#endregion
-//#region src/produced-state.d.ts
-/** A tool the agent invoked. */
-interface ToolCallEventLike {
-  type: 'tool_call';
-  toolName: string;
-}
-/**
- * An artifact the agent produced. `content` is the enriched field — the
- * runtime's base `artifact` event carries only metadata; the completion
- * oracle needs the body to verify the deliverable, so the runtime emits it.
- */
-interface ArtifactEventLike {
-  type: 'artifact';
-  artifactId: string;
-  name?: string;
-  mimeType?: string;
-  uri?: string;
-  content?: string;
-}
-/** A proposal / filing the agent created. */
-interface ProposalEventLike {
-  type: 'proposal_created';
-  proposalId: string;
-  title: string;
-  status?: 'pending' | 'approved' | 'rejected';
-  content?: string;
-}
-/**
- * The subset of runtime stream events `extractProducedState` consumes.
- * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;
- * the `{ type: string }` catch-all keeps the input permissive so callers can
- * pass the whole unfiltered telemetry stream — unrecognized events are skipped.
- */
-type RuntimeEventLike = ToolCallEventLike | ArtifactEventLike | ProposalEventLike | {
-  type: string;
-};
-/**
- * Normalize a run's runtime event stream into `ProducedState`.
- *
- * Pure and total — unrecognized event types are skipped. `toolCalls` is
- * deduplicated by name in first-seen order (completion cares about a tool's
- * presence, not its call count). An artifact with neither a name nor a uri
- * still yields an entry keyed by its `artifactId` so it is never silently
- * dropped; an artifact with no `content` yields empty content, which the
- * completion oracle's structural check then rejects on its own.
- */
-declare function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState;
-//#endregion
-//#region src/integrity/backend-integrity.d.ts
-interface BackendIntegrityReport {
-  /** Total records inspected. */
-  totalRecords: number;
-  /** Records with input=0 AND output=0 (a stub fingerprint). */
-  stubRecords: number;
-  /** Records with nonzero token usage (real LLM activity). */
-  realRecords: number;
-  /** Records where output>0 but costUsd=0 (real LLM, broken cost ledger). */
-  uncostedRecords: number;
-  /** Sum of input tokens across all records. */
-  totalInputTokens: number;
-  /** Sum of output tokens across all records. */
-  totalOutputTokens: number;
-  /** Sum of costUsd across all records. */
-  totalCostUsd: number;
-  /** Worst-case integrity verdict. */
-  verdict: 'real' | 'mixed' | 'stub';
-  /** Human-readable diagnosis suitable for terminal output. */
-  diagnosis: string;
-}
-/**
- * Error thrown when an integrity assertion fails. Caller can pattern-match
- * by `code === 'AGENT_EVAL_BACKEND_STUB'` to differentiate from other
- * errors.
- */
-declare class BackendIntegrityError extends AgentEvalError {
-  readonly report: BackendIntegrityReport;
-  constructor(message: string, report: BackendIntegrityReport);
-}
-/**
- * Inspect a batch of RunRecords and return an integrity report. Pure
- * function — no I/O, no logging. The caller decides what to do with the
- * verdict (print warning, throw, gate CI, etc.).
- */
-declare function summarizeBackendIntegrity(records: ReadonlyArray<RunRecord>): BackendIntegrityReport;
-/**
- * Throw BackendIntegrityError if the verdict is 'stub' — i.e. every record
- * shows zero LLM activity. Non-strict callers can pass `{ allowMixed: false }`
- * to also reject mixed verdicts (recommended for CI gates).
- *
- * Real backends pass through silently.
- */
-declare function assertRealBackend(records: ReadonlyArray<RunRecord>, opts?: {
-  allowMixed?: boolean;
-}): BackendIntegrityReport;
-//#endregion
-export { SatisfiedBy as _, ArtifactEventLike as a, createLlmCorrectnessChecker as b, ToolCallEventLike as c, CompletionVerdict as d, CorrectnessChecker as f, RequirementCheck as g, ProducedState as h, summarizeBackendIntegrity as i, extractProducedState as l, ProducedProposal as m, BackendIntegrityReport as n, ProposalEventLike as o, LlmCorrectnessCheckerOpts as p, assertRealBackend as r, RuntimeEventLike as s, BackendIntegrityError as t, CompletionRequirement as u, TaskGold as v, verifyCompletion as x, completionVerdict as y };
-//# sourceMappingURL=backend-integrity-CeuTgqsd.d.ts.map

package/dist/backend-integrity-CeuTgqsd.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"backend-integrity-CeuTgqsd.d.ts","names":[],"sources":["../src/artifact-validator.ts","../src/completion-verifier.ts","../src/produced-state.ts","../src/integrity/backend-integrity.ts"],"mappings":";;;;;;;;;;;;;;;;;;UAaiB;;EAEf;;EAEA;;EAEA;;EAEA,QAAQ;;EAER,WAAW;;;;;KCsBD;UAEK;;EAEf;;EAEA;;EAEA;;EAEA,cAAc;;UAGC;EACf;EACA,cAAc;;UAGC;EACf;EACA;EACA;;EAEA;;;UAIe;;EAEf,WAAW;;EAEX,WAAW;;EAEX;;UAGe;EACf;EACA;;EAEA;;;;;;EAMA;;EAEA;;;;;;;;;EASA;;EAEA;;EAEA;;;;;UAMe,0BAA0B;EACzC;EACA,cAAc;;EAEd;;EAEA;;EAEA;;;;;;;;;iBAUc,kBAAkB;EAChC;EACA,cAAc;;;EAGd,gBAAgB;IACd;;;;;;UAqCa;EACf,UAAU;EACV,SAAS;EACT;;;;;;;;;;;;UAae;GAEb,aAAa,uBACb,kBACC;IAAU;IAAkB;;EAC/B,cAAc;;;;;;;;;iBAsLM,iBACpB,MAAM,UACN,OAAO,eACP,kBAAkB,qBACjB,QAAQ;UAyGM;EACf;;EAEA,aAAa;EACb;EACA,WAAW;EACX,SAAS;;EAET;;;;;;EAMA;;;;;;EAMA,UAAU;;;;;;;;iBA2CI,4BACd,MAAM,YACN,OAAM,4BACL;;;;UCnhBc;EACf;EACA;;;;;;;UAQe;EACf;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EAIA;;;;;;;;KASU,mBACR,oBACA,oBACA;EACE;;;;;;;;;;;;iBAmBU,qBAAqB,iBAAiB,qBAAqB;;;UCpD1D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;cAQW,8BAA8B;WAGvB,QAAQ;EAF1B,YACE,iBACgB,QAAQ;;;;;;;iBAWZ,0BACd,SAAS,cAAc,aACtB;;;;;;;;iBAuHa,kBACd,SAAS,cAAc,YACvB;EAAQ;IACP"}

package/dist/benchmark-BjLGkfnN.d.ts

@@ -1,236 +0,0 @@
-import { c as AnalystRunInputs, f as AnalystUsageReceipt, i as AnalystFinding, p as EvidenceRef, w as TraceAnalysisStore } from "./types-DN2WdT5S.js";
-import { c as RegistryRunOpts, n as AnalystRegistry } from "./registry-7pOUBrtX.js";
-//#region src/analyst/benchmark-scoring.d.ts
-declare function scoreAnalystFindings(testCase: Pick<AnalystBenchmarkCase, 'id' | 'expectedIssues' | 'labeledEvidence'>, findings: readonly AnalystFinding[]): AnalystFindingScore;
-//#endregion
-//#region src/analyst/benchmark.d.ts
-interface AnalystEvidenceExpectation {
-  uri: string;
-  kind?: EvidenceRef['kind'];
-}
-interface AnalystIssueExpectation {
-  id: string;
-  findingIds?: readonly string[];
-  areas?: readonly string[];
-  subjects?: readonly string[];
-  evidence?: readonly AnalystEvidenceExpectation[];
-  evidenceMode?: 'any' | 'all';
-  /** Exact evidence location for the first unrecoverable or causal step. */
-  criticalEvidence?: readonly AnalystEvidenceExpectation[];
-}
-type AnalystBenchmarkLabelState = 'positive' | 'trusted-negative' | 'unlabeled';
-interface AnalystBenchmarkCase<TInput = unknown> {
-  id: string;
-  /** Independent source unit used for resampling, such as a task or incident. */
-  clusterId: string;
-  /** Whether labels prove an issue, prove no issue, or leave the outcome unknown. */
-  labelState: AnalystBenchmarkLabelState;
-  input: TInput;
-  expectedIssues: readonly AnalystIssueExpectation[];
-  /** Complete set of labeled locations used to measure label-location agreement. */
-  labeledEvidence?: readonly AnalystEvidenceExpectation[];
-  tags?: readonly string[];
-  metadata?: Record<string, unknown>;
-}
-interface AnalystFindingScore {
-  expectedIssueCount: number;
-  matchedIssueIds: string[];
-  missedIssueIds: string[];
-  supportedFindingIndexes: number[];
-  unsupportedFindingIndexes: number[];
-  unlabeledEvidence: EvidenceRef[];
-  issueRecall: number;
-  findingPrecision: number;
-  f1: number;
-  criticalStepAccuracy: number | null;
-  /** Share of findings that cite at least one evidence location. */
-  citationCoverage: number | null;
-  /** Share of citations that include a non-empty source excerpt. */
-  citationExcerptCoverage: number | null;
-  /** Share of citations that agree with a labeled case location. */
-  citationLabelAgreement: number | null;
-  predictionOnLabelEmptyCase: boolean;
-}
-interface AnalystEvidenceResolutionError {
-  evidence: EvidenceRef;
-  class: string;
-  message: string;
-}
-interface AnalystEvidenceResolution {
-  checked: number;
-  resolved: number;
-  unresolvedEvidence: EvidenceRef[];
-  errors: AnalystEvidenceResolutionError[];
-  /** Null when no citations were checked or any resolution attempt failed. */
-  validity: number | null;
-}
-type AnalystEvidenceResolver<TInput = unknown> = (input: {
-  caseId: string;
-  caseInput: TInput;
-  evidence: EvidenceRef;
-  signal?: AbortSignal;
-}) => boolean | Promise<boolean>;
-/**
- * Resolve canonical `trace://<trace>/span/<span>` evidence against a trace store.
- * Other evidence kinds and URI schemes require a caller-supplied resolver.
- */
-declare function traceStoreEvidenceResolver<TInput>(getStore: (input: TInput) => TraceAnalysisStore): AnalystEvidenceResolver<TInput>;
-interface AnalystBenchmarkOutput {
-  findings: readonly AnalystFinding[];
-  usage?: AnalystUsageReceipt;
-  metadata?: Record<string, unknown>;
-  /**
-   * End-to-end duration measured by an external runner before import.
-   * Use null when the source explicitly did not capture duration.
-   */
-  observedLatencyMs?: number | null;
-  /** Marks a completed transport as a failed analyst run while retaining usage and metadata. */
-  error?: AnalystBenchmarkError;
-}
-interface AnalystBenchmarkError {
-  class: string;
-  message: string;
-  code?: string;
-  status?: number;
-}
-interface AnalystBenchmarkRunner<TInput = unknown> {
-  id: string;
-  analyze(input: TInput, context: {
-    caseId: string;
-    repetition: number;
-    signal?: AbortSignal;
-  }): AnalystBenchmarkOutput | Promise<AnalystBenchmarkOutput>;
-}
-interface AnalystBenchmarkObservation {
-  runnerId: string;
-  caseId: string;
-  clusterId: string;
-  labelState: AnalystBenchmarkLabelState;
-  repetition: number;
-  executionIndex: number;
-  latencyMs: number | null;
-  latencySource: 'benchmark-clock' | 'runner-reported' | 'uncaptured';
-  findings: readonly AnalystFinding[];
-  score: AnalystFindingScore;
-  evidenceResolution?: AnalystEvidenceResolution;
-  caseTags: readonly string[];
-  caseMetadata?: Record<string, unknown>;
-  usage?: AnalystUsageReceipt;
-  runnerMetadata?: Record<string, unknown>;
-  error?: AnalystBenchmarkError;
-}
-interface AnalystLatencyDistribution {
-  min: number;
-  mean: number;
-  p50: number;
-  p95: number;
-  max: number;
-}
-interface AnalystBenchmarkSummary {
-  runnerId: string;
-  plannedRuns: number;
-  completedRuns: number;
-  failedRuns: number;
-  issueBearingRuns: number;
-  trustedNegativeRuns: number;
-  unlabeledRuns: number;
-  /** Pooled across all labeled issues and findings. */
-  issueRecall: number | null;
-  /** Pooled across all labeled issues and findings. */
-  findingPrecision: number | null;
-  /** Harmonic mean of the pooled precision and recall. */
-  f1: number | null;
-  /** Mean of per-case recall over issue-bearing runs. */
-  macroIssueRecall: number | null;
-  /** Mean of per-case precision over issue-bearing runs. */
-  macroFindingPrecision: number | null;
-  /** Mean of per-case F1 over issue-bearing runs. */
-  macroF1: number | null;
-  criticalStepAccuracy: number | null;
-  citationCoverage: number | null;
-  citationExcerptCoverage: number | null;
-  citationLabelAgreement: number | null;
-  citationResolution: number | null;
-  citationResolutionUnknownRuns: number;
-  unresolvedCitations: number;
-  citationResolutionErrors: number;
-  trustedNegativeFalsePositiveRate: number | null;
-  trustedNegativeFailureRate: number | null;
-  unlabeledPredictionRate: number | null;
-  unlabeledFailureRate: number | null;
-  /** Primary repeatability measure over complete finding identity and evidence. */
-  predictionAgreement: number | null;
-  /** Repeated cases contributing equally to predictionAgreement. */
-  predictionAgreementCases: number;
-  /** Secondary repeatability detail over matched expected labels. */
-  matchedLabelAgreement: number | null;
-  /** Positive repeated cases contributing equally to matchedLabelAgreement. */
-  matchedLabelAgreementCases: number;
-  latencyMs: AnalystLatencyDistribution | null;
-  benchmarkClockLatencyRuns: number;
-  runnerReportedLatencyRuns: number;
-  latencyUnknownRuns: number;
-  calls: number;
-  callsUnknownRuns: number;
-  inputTokens: number;
-  outputTokens: number;
-  reasoningTokens: number;
-  cachedTokens: number;
-  cacheWriteTokens: number;
-  tokenUsageUnknownRuns: number;
-  reasoningTokenUsageUnknownRuns: number;
-  cachedTokenUsageUnknownRuns: number;
-  cacheWriteTokenUsageUnknownRuns: number;
-  knownCostUsd: number;
-  costUnknownRuns: number;
-}
-interface AnalystBenchmarkDatasetRef {
-  id: string;
-  revision: string;
-  split?: string;
-}
-interface AnalystBenchmarkDescriptor {
-  id?: string;
-  dataset?: AnalystBenchmarkDatasetRef;
-  command?: string;
-  environment?: Record<string, string>;
-  metadata?: Record<string, unknown>;
-}
-interface AnalystBenchmarkProvenance extends AnalystBenchmarkDescriptor {
-  startedAt: string;
-  endedAt: string;
-  caseCount: number;
-  runnerIds: string[];
-  repetitions: number;
-  maxConcurrency: number;
-  runnerOrderSeed: number;
-}
-interface AnalystBenchmarkResult {
-  provenance: AnalystBenchmarkProvenance;
-  observations: AnalystBenchmarkObservation[];
-  summaries: AnalystBenchmarkSummary[];
-}
-interface RunAnalystBenchmarkOptions<TInput> {
-  cases: readonly AnalystBenchmarkCase<TInput>[];
-  runners: readonly AnalystBenchmarkRunner<TInput>[];
-  repetitions?: number;
-  maxConcurrency?: number;
-  runnerOrderSeed?: number;
-  resolveEvidence?: AnalystEvidenceResolver<TInput>;
-  benchmark?: AnalystBenchmarkDescriptor;
-  /** Previously persisted rows. Exact case, runner, repetition, and execution identities are required. */
-  initialObservations?: readonly AnalystBenchmarkObservation[];
-  onObservation?: (observation: AnalystBenchmarkObservation) => void | Promise<void>;
-  signal?: AbortSignal;
-}
-declare function runAnalystBenchmark<TInput>(options: RunAnalystBenchmarkOptions<TInput>): Promise<AnalystBenchmarkResult>;
-declare function registryBenchmarkRunner(options: {
-  id: string;
-  registry: AnalystRegistry;
-  runOptions?: Omit<RegistryRunOpts, 'signal'>;
-  /** Count any selected analyst failure as a failed benchmark run. */
-  failOnAnalystFailure?: boolean;
-}): AnalystBenchmarkRunner<AnalystRunInputs>;
-//#endregion
-export { scoreAnalystFindings as C, traceStoreEvidenceResolver as S, AnalystIssueExpectation as _, AnalystBenchmarkLabelState as a, registryBenchmarkRunner as b, AnalystBenchmarkProvenance as c, AnalystBenchmarkSummary as d, AnalystEvidenceExpectation as f, AnalystFindingScore as g, AnalystEvidenceResolver as h, AnalystBenchmarkError as i, AnalystBenchmarkResult as l, AnalystEvidenceResolutionError as m, AnalystBenchmarkDatasetRef as n, AnalystBenchmarkObservation as o, AnalystEvidenceResolution as p, AnalystBenchmarkDescriptor as r, AnalystBenchmarkOutput as s, AnalystBenchmarkCase as t, AnalystBenchmarkRunner as u, AnalystLatencyDistribution as v, runAnalystBenchmark as x, RunAnalystBenchmarkOptions as y };
-//# sourceMappingURL=benchmark-BjLGkfnN.d.ts.map

package/dist/benchmark-BjLGkfnN.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"benchmark-BjLGkfnN.d.ts","names":[],"sources":["../src/analyst/benchmark-scoring.ts","../src/analyst/benchmark.ts"],"mappings":";;;iBASgB,qBACd,UAAU,KAAK,oEACf,mBAAmB,mBAClB;;;UCIc;EACf;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA,oBAAoB;EACpB;;EAEA,4BAA4B;;KAGlB;UAEK,qBAAqB;EACpC;;EAEA;;EAEA,YAAY;EACZ,OAAO;EACP,yBAAyB;;EAEzB,2BAA2B;EAC3B;EACA,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA,mBAAmB;EACnB;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;UAGe;EACf;EACA;EACA,oBAAoB;EACpB,QAAQ;;EAER;;KAGU,wBAAwB,qBAAqB;EACvD;EACA,WAAW;EACX,UAAU;EACV,SAAS;gBACK;;;;;iBAMA,2BAA2B,QACzC,WAAW,OAAO,WAAW,qBAC5B,wBAAwB;UAoBV;EACf,mBAAmB;EACnB,QAAQ;EACR,WAAW;;;;;EAKX;;EAEA,QAAQ;;UAGO;EACf;EACA;EACA;EACA;;UAGe,uBAAuB;EACtC;EACA,QACE,OAAO,QACP;IAAW;IAAgB;IAAoB,SAAS;MACvD,yBAAyB,QAAQ;;UAGrB;EACf;EACA;EACA;EACA,YAAY;EACZ;EACA;EACA;EACA;EACA,mBAAmB;EACnB,OAAO;EACP,qBAAqB;EACrB;EACA,eAAe;EACf,QAAQ;EACR,iBAAiB;EACjB,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;EACA,WAAW;EACX;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA,UAAU;EACV;EACA,cAAc;EACd,WAAW;;UAGI,mCAAmC;EAClD;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,YAAY;EACZ,cAAc;EACd,WAAW;;UAGI,2BAA2B;EAC1C,gBAAgB,qBAAqB;EACrC,kBAAkB,uBAAuB;EACzC;EACA;EACA;EACA,kBAAkB,wBAAwB;EAC1C,YAAY;;EAEZ,+BAA+B;EAC/B,iBAAiB,aAAa,uCAAuC;EACrE,SAAS;;iBAGW,oBAAoB,QACxC,SAAS,2BAA2B,UACnC,QAAQ;iBAyDK,wBAAwB;EACtC;EACA,UAAU;EACV,aAAa,KAAK;;EAElB;IACE,uBAAuB"}