@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/pre-registration-mAnCugl9.d.ts

@@ -1,523 +0,0 @@
-import { A as AgentEvalError } from './errors-CzMUYo7b.js';
-import { R as RunRecord } from './run-record-e7vj1uZQ.js';
-import { TCloud } from '@tangle-network/tcloud';
-import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
-
-/**
- * Backend-integrity guard: distinguish "agent failed" from "eval ran against
- * a stub / unconfigured backend." Without this guard a canonical eval can
- * silently report `0/N passed` and look like an agent-quality problem when
- * the LLM was never actually called — the failure mode we just hit running
- * the 4-vertical parallel eval (legal-sandbox-stub returned hard-coded 33-104
- * char strings; gtm/creative defaulted to a cli-bridge that wasn't running).
- *
- * The shape:
- *
- *   const report = summarizeBackendIntegrity(records)
- *   assertRealBackend(records)   // throws BackendIntegrityError if 100% stub
- *
- * A record is "stub-mode" if its `tokenUsage.input === 0 && tokenUsage.output === 0`.
- * (`costUsd` alone is unreliable — some backends successfully call LLMs but
- *  don't propagate pricing, producing real tokens with $0 cost.)
- *
- * Verdicts:
- *   - `real`   — at least one record has nonzero token usage
- *   - `stub`   — every record is stub-mode (eval ran blind)
- *   - `mixed`  — some records real, some stub (partial backend failure;
- *                often the 429-cascade or auth-half-failed case)
- */
-
-interface BackendIntegrityReport {
-    /** Total records inspected. */
-    totalRecords: number;
-    /** Records with input=0 AND output=0 (a stub fingerprint). */
-    stubRecords: number;
-    /** Records with nonzero token usage (real LLM activity). */
-    realRecords: number;
-    /** Records where output>0 but costUsd=0 (real LLM, broken cost ledger). */
-    uncostedRecords: number;
-    /** Sum of input tokens across all records. */
-    totalInputTokens: number;
-    /** Sum of output tokens across all records. */
-    totalOutputTokens: number;
-    /** Sum of costUsd across all records. */
-    totalCostUsd: number;
-    /** Worst-case integrity verdict. */
-    verdict: 'real' | 'mixed' | 'stub';
-    /** Human-readable diagnosis suitable for terminal output. */
-    diagnosis: string;
-}
-/**
- * Error thrown when an integrity assertion fails. Caller can pattern-match
- * by `code === 'AGENT_EVAL_BACKEND_STUB'` to differentiate from other
- * errors.
- */
-declare class BackendIntegrityError extends AgentEvalError {
-    readonly report: BackendIntegrityReport;
-    constructor(message: string, report: BackendIntegrityReport);
-}
-/**
- * Inspect a batch of RunRecords and return an integrity report. Pure
- * function — no I/O, no logging. The caller decides what to do with the
- * verdict (print warning, throw, gate CI, etc.).
- */
-declare function summarizeBackendIntegrity(records: ReadonlyArray<RunRecord>): BackendIntegrityReport;
-/**
- * Throw BackendIntegrityError if the verdict is 'stub' — i.e. every record
- * shows zero LLM activity. Non-strict callers can pass `{ allowMixed: false }`
- * to also reject mixed verdicts (recommended for CI gates).
- *
- * Real backends pass through silently.
- */
-declare function assertRealBackend(records: ReadonlyArray<RunRecord>, opts?: {
-    allowMixed?: boolean;
-}): BackendIntegrityReport;
-
-/**
- * Artifact validators.
- *
- * Generic "score a produced artifact" primitive. Tax uses it for PDF form
- * correctness, research for sourced briefs, browser for task assertions, coding
- * for social posts. One interface, many validators; all plug into
- * `BenchmarkRunner` the same way.
- *
- * A validator receives an `Artifact` (file on disk, JSON blob, text, binary)
- * plus a `ValidationContext` (scenario id, the turns that produced it) and
- * returns a `ValidationResult` with pass/fail + 0..1 score + structured
- * issues.
- */
-interface Artifact {
-    /** Logical kind — validators type-guard on this */
-    kind: 'file' | 'json' | 'text' | 'binary' | string;
-    /** Filesystem-style path, optional */
-    path?: string;
-    /** String content for text/json/file kinds */
-    content?: string;
-    /** Binary content (if kind === 'binary') */
-    bytes?: Uint8Array;
-    /** Caller-supplied metadata (mimeType, sha256, size, etc.) */
-    metadata?: Record<string, unknown>;
-}
-interface ValidationContext {
-    scenarioId: string;
-    turnIndex?: number;
-    /** Prior artifacts for multi-artifact scenarios */
-    priorArtifacts?: Artifact[];
-    /** Free-form hints the validator uses for domain-specific checks */
-    hints?: Record<string, unknown>;
-}
-interface ValidationIssue {
-    severity: 'error' | 'warning' | 'info';
-    message: string;
-    /** Optional path into the artifact (e.g. JSON path or byte offset) */
-    locus?: string;
-}
-interface ValidationResult {
-    pass: boolean;
-    /** 0–1 normalized score. Validators should be monotonic in pass-ness. */
-    score: number;
-    issues: ValidationIssue[];
-    /** Diagnostic payload for reporters */
-    evidence?: Record<string, unknown>;
-}
-interface ArtifactValidator {
-    /** Stable identifier for the validator; appears in reports. */
-    name: string;
-    /** Optional description for human-facing reports. */
-    description?: string;
-    /** Called once per artifact; validators are expected to be pure + idempotent. */
-    validate(artifact: Artifact, context: ValidationContext): Promise<ValidationResult>;
-}
-/**
- * Run every validator on the same artifact; aggregate pass as AND, score as
- * (weighted) mean, issues concatenated. Weights default to 1 each.
- */
-declare function composeValidators(validators: ArtifactValidator[], options?: {
-    name?: string;
-    weights?: number[];
-}): ArtifactValidator;
-/** Pass if the artifact body matches a provided regex. */
-declare function regexMatch(name: string, pattern: RegExp): ArtifactValidator;
-/** Pass if JSON parses and every required key is present. */
-declare function jsonHasKeys(name: string, requiredPaths: string[]): ArtifactValidator;
-/** Pass if min ≤ byte length ≤ max. */
-declare function byteLengthRange(name: string, min: number, max: number): ArtifactValidator;
-/** Pass if the artifact contains every required substring (case-insensitive by default). */
-declare function containsAll(name: string, required: string[], options?: {
-    caseSensitive?: boolean;
-}): ArtifactValidator;
-
-/**
- * Completion verifier — the task-completion oracle.
- *
- * Answers the only eval question that is not a proxy: did the agent actually
- * COMPLETE the task — produce every required deliverable, persisted and
- * correct — rather than describe what should be done. A fluent transcript
- * that never produces the artifact scores zero here.
- *
- * Per requirement, a two-stage check:
- *   1. Structural — a produced item (vault artifact / approved proposal /
- *      tool call) of the right kind is matched against the requirement and
- *      carries non-empty content. Deterministic; no LLM.
- *   2. Correctness — only if structurally present AND the matched item
- *      carries content, one targeted check decides whether that item
- *      actually fulfils the requirement. A hallucinated artifact fails here;
- *      an absent one already failed stage 1.
- *
- * `completionRate` is satisfied / total. Quality dimensions are meaningless
- * on an incomplete task — callers gate on `fullyComplete` / `completionRate`
- * before scoring quality.
- */
-
-/** What kind of produced state can satisfy a requirement structurally. */
-type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any';
-interface CompletionRequirement {
-    /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */
-    reqId: string;
-    /** Human-readable description of the required deliverable. */
-    title: string;
-    /** Optional kind/category hint, matched against a produced item's kind. */
-    category?: string;
-    /** What produced state satisfies this requirement. Defaults to 'any'. */
-    satisfiedBy?: SatisfiedBy;
-}
-interface TaskGold {
-    taskId: string;
-    requirements: CompletionRequirement[];
-}
-interface ProducedProposal {
-    id: string;
-    title: string;
-    status: 'pending' | 'approved' | 'rejected';
-    /** Optional persisted body — when present, enables a correctness check. */
-    content?: string;
-}
-/** Everything observable about what a run actually produced. */
-interface ProducedState {
-    /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */
-    artifacts: Artifact[];
-    /** Proposals / filings the agent created. */
-    proposals: ProducedProposal[];
-    /** Names of tools the agent invoked. */
-    toolCalls: string[];
-}
-interface RequirementCheck {
-    reqId: string;
-    title: string;
-    /** A produced item of the right kind matched the requirement, non-empty. */
-    structurallyPresent: boolean;
-    /**
-     * Whether the matched item actually fulfils the requirement. `null` when
-     * not structurally present, or when the matched item carries no content
-     * to assess.
-     */
-    correct: boolean | null;
-    /** structurallyPresent && correct !== false. */
-    satisfied: boolean;
-    /** Human-readable evidence for the verdict. */
-    evidence: string[];
-}
-/** Extends the substrate verdict spine: `valid` = `fullyComplete` and
- *  `score` = `completionRate` — derived in `completionVerdict()`, the one
- *  place those equalities hold by construction. */
-interface CompletionVerdict extends DefaultVerdict {
-    taskId: string;
-    requirements: RequirementCheck[];
-    /** satisfied / total requirements. */
-    completionRate: number;
-    /** Every requirement satisfied. */
-    fullyComplete: boolean;
-}
-/**
- * Construct a `CompletionVerdict` from the per-requirement checks, deriving
- * `completionRate` / `fullyComplete` and the spine fields (`valid` =
- * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
- * requirements — a verdict over nothing is a misconfiguration, mirroring
- * `verifyCompletion`'s gold-spec guard.
- */
-declare function completionVerdict(input: {
-    taskId: string;
-    requirements: RequirementCheck[];
-}): CompletionVerdict;
-/**
- * Decides whether a produced item's content actually fulfils a requirement.
- * Injected so the structural verifier stays pure and unit-testable; the
- * production implementation is `createLlmCorrectnessChecker`.
- */
-type CorrectnessChecker = (requirement: CompletionRequirement, content: string) => Promise<{
-    correct: boolean;
-    reason: string;
-}>;
-/**
- * Verify whether a run completed the task. `checkCorrectness` is injected —
- * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.
- *
- * Throws on a gold spec with no requirements: an eval task that requires
- * nothing is a misconfiguration, not a vacuously-complete task.
- */
-declare function verifyCompletion(gold: TaskGold, state: ProducedState, checkCorrectness: CorrectnessChecker): Promise<CompletionVerdict>;
-interface LlmCorrectnessCheckerOpts {
-    model?: string;
-    /** Max chars of artifact content sent to the checker. */
-    maxContentChars?: number;
-}
-/** Parse the correctness checker's model response. Fails loud on a bad shape. */
-declare function parseCorrectnessResponse(raw: string): {
-    correct: boolean;
-    reason: string;
-};
-/**
- * Production `CorrectnessChecker` — one LLM call per matched artifact,
- * deterministic (temperature 0), structured JSON out. Judges fulfilment
- * only: a plan, a gesture, or a description of what should be done does not
- * fulfil a requirement — the artifact must BE the deliverable.
- */
-declare function createLlmCorrectnessChecker(tc: TCloud, opts?: LlmCorrectnessCheckerOpts): CorrectnessChecker;
-/**
- * Deterministic `CorrectnessChecker` — the no-LLM counterpart to
- * `createLlmCorrectnessChecker`. A produced item fulfils a requirement when its
- * content is substantive (≥ `minContentLength` chars) AND recalls ≥ `minRecall`
- * of the requirement title's significant tokens. No network.
- *
- * Polarity-blind: token recall credits a negation that contains the
- * requirement's tokens ("I will NOT produce the comparison" recalls every token
- * of "produce the comparison"). The structural match stage is ALSO lexical, so
- * pairing the two collapses to a single gameable gate. Use this only as an
- * opt-in structural pre-filter or for tasks whose requirements have no polarity
- * to invert; for produced-state grading the correctness checker MUST be semantic
- * (`createLlmCorrectnessChecker`). See the anti-game fixtures in the test suite.
- */
-declare function createTokenRecallChecker(opts?: {
-    minRecall?: number;
-    minContentLength?: number;
-}): CorrectnessChecker;
-
-/**
- * Produced-state extraction — normalize a run's runtime event stream into the
- * typed `ProducedState` the completion oracle consumes.
- *
- * `ProducedState` answers "what did the agent actually produce" — vault
- * artifacts, proposals, tool calls. The runtime emits these as a stream of
- * events; this module is the single normalization point from that stream to
- * the shape `verifyCompletion` expects.
- *
- * Input is structurally typed (`RuntimeEventLike`) so this module does not
- * depend on agent-runtime — agent-runtime's `RuntimeStreamEvent` satisfies it
- * structurally. The `content` on `ArtifactEventLike` and the whole
- * `proposal_created` variant are the runtime-side enrichments this contract
- * requires; the runtime emits them, this module consumes them.
- */
-
-/** A tool the agent invoked. */
-interface ToolCallEventLike {
-    type: 'tool_call';
-    toolName: string;
-}
-/**
- * An artifact the agent produced. `content` is the enriched field — the
- * runtime's base `artifact` event carries only metadata; the completion
- * oracle needs the body to verify the deliverable, so the runtime emits it.
- */
-interface ArtifactEventLike {
-    type: 'artifact';
-    artifactId: string;
-    name?: string;
-    mimeType?: string;
-    uri?: string;
-    content?: string;
-}
-/** A proposal / filing the agent created. */
-interface ProposalEventLike {
-    type: 'proposal_created';
-    proposalId: string;
-    title: string;
-    status?: 'pending' | 'approved' | 'rejected';
-    content?: string;
-}
-/**
- * The subset of runtime stream events `extractProducedState` consumes.
- * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;
- * the `{ type: string }` catch-all keeps the input permissive so callers can
- * pass the whole unfiltered telemetry stream — unrecognized events are skipped.
- */
-type RuntimeEventLike = ToolCallEventLike | ArtifactEventLike | ProposalEventLike | {
-    type: string;
-};
-/**
- * Normalize a run's runtime event stream into `ProducedState`.
- *
- * Pure and total — unrecognized event types are skipped. `toolCalls` is
- * deduplicated by name in first-seen order (completion cares about a tool's
- * presence, not its call count). An artifact with neither a name nor a uri
- * still yields an entry keyed by its `artifactId` so it is never silently
- * dropped; an artifact with no `content` yields empty content, which the
- * completion oracle's structural check then rejects on its own.
- */
-declare function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState;
-
-/**
- * @stable
- *
- * AgentProfile — the eval harness's unit of variation.
- *
- * A profile pins everything that changes agent behaviour for a benchmark
- * cell: the model, the active skills, the prompt version, the available
- * tools. Vary the profile — swap a model, add a skill — and re-run the suite
- * to benchmark the change. The scorecard keys a cell on
- * `(scenarioId, profileHash)`, so the model is not a separate axis: it lives
- * inside the profile, and two profiles with the same model but different
- * skills are different cells.
- *
- * `agentProfileHash` is the profile's behaviour identity. Two profiles that
- * produce the same agent behaviour share a hash (and a scorecard cell);
- * reordering `skills` or `tools` does not change it; the human-facing `id`
- * label does not affect it.
- */
-interface AgentProfile {
-    /** Human-facing label, e.g. `sonnet-legal-skills-v3`. Not part of the hash. */
-    id: string;
-    /** Model snapshot id this profile pins, e.g. `claude-sonnet-4-6@2025-04-15`. */
-    model: string;
-    /** Skill ids/versions active in this profile — the primary behaviour lever. */
-    skills?: string[];
-    /** Prompt version identifier. */
-    promptVersion?: string;
-    /** Tool ids available to the agent. */
-    tools?: string[];
-    /** Any other behaviour-bearing knobs that should fingerprint into the hash. */
-    metadata?: Record<string, string | number | boolean>;
-}
-/**
- * Deterministic behaviour identity of a profile — a sha256 over the
- * behaviour-bearing fields. `skills` and `tools` are order-insensitive; the
- * `id` label is excluded. Throws on a profile with no `model` — an unkeyable
- * profile must fail loud rather than collapse into a blank-model cell.
- */
-declare function agentProfileHash(profile: AgentProfile): string;
-
-/**
- * Pre-registered hypotheses — declare what you're testing BEFORE the
- * run, check it AFTER. Prevents p-hacking, optional stopping, and the
- * "we ran until it looked good" failure mode.
- *
- * Manifest is a plain JSON-friendly object. Sign it with a content hash
- * + timestamp; the registered record becomes immutable. Post-run,
- * evaluate the manifest against observed results — the library refuses
- * to let you re-interpret a different metric as the declared one.
- */
-interface HypothesisManifest {
-    id: string;
-    /** Human prose — goes into the audit trail. */
-    hypothesis: string;
-    /** Metric the hypothesis claims to move. */
-    metric: string;
-    /** 'increase' = candidate should score higher than baseline; 'decrease' = lower. */
-    direction: 'increase' | 'decrease';
-    /** Minimum effect size to count (same units as the metric). */
-    minEffect: number;
-    /** Alpha threshold. */
-    alpha: number;
-    /** Target statistical power at which sample size was pre-computed. */
-    power: number;
-    /** Declared N per arm before running. */
-    preRegisteredN: number;
-    /** ISO8601 timestamp the manifest was registered. */
-    registeredAt: string;
-    /** Optional identifiers to tie into the trace corpus. */
-    baselineLabel?: string;
-    candidateLabel?: string;
-}
-/**
- * Identifier for the hashing scheme used to produce `contentHash`.
- *
- * `'sha256-content'` — sha256 hex over the canonicalized manifest with
- * the `contentHash` and `algo` fields stripped. Held as a string union
- * so future schemes can be added without breaking parsers; SignedManifest
- * values without `algo` deserialize cleanly because the field is optional.
- */
-type SignedManifestAlgo = 'sha256-content';
-interface SignedManifest extends HypothesisManifest {
-    /** sha256 hex of canonicalized manifest (everything except contentHash and algo). */
-    contentHash: string;
-    /**
-     * Algorithm string describing how `contentHash` was produced.
-     *
-     * Optional on the type so serialized manifests without it still parse,
-     * but ALWAYS populated by {@link signManifest}. Consumers that want to
-     * enforce a known algorithm should reject manifests where this field
-     * is missing or unrecognized.
-     */
-    algo?: SignedManifestAlgo;
-}
-interface HypothesisResult {
-    manifest: SignedManifest;
-    observedN: number;
-    observedEffect: number;
-    observedPValue: number;
-    /** True iff the observed effect hits the pre-declared direction with
-     *  magnitude ≥ minEffect AND p < alpha. */
-    confirmed: boolean;
-    /** Enumerated reasons the hypothesis was rejected (each a machine-tag). */
-    rejectionReasons: Array<'wrong_direction' | 'effect_too_small' | 'not_significant' | 'undersampled'>;
-    notes?: string;
-}
-/**
- * Deterministic JSON canonicalization — sort object keys recursively.
- *
- * Two semantically-equal objects produce byte-identical canonicalized output;
- * this is what makes a content-hash stable across encoders, key insertion
- * orders, and runtime versions. Exported for any consumer that needs the same
- * canonicalization guarantee outside the manifest-signing path (e.g., signing
- * an artifact bundle, hashing a dataset version, etc.).
- */
-declare function canonicalize(v: unknown): unknown;
-/**
- * SHA-256 hex (full 64 chars) over the canonicalized JSON encoding of `obj`.
- *
- * The same primitive `signManifest` and `verifyManifest` are built on, exposed
- * directly so consumers signing arbitrary structured content (artifact bundles,
- * production packets, dataset manifests, etc.) don't have to re-derive
- * canonicalize+sha256 from scratch.
- *
- * Stable across:
- *   - object key insertion order (canonicalization sorts keys recursively)
- *   - encoder choice (UTF-8 via TextEncoder, fixed)
- *   - runtime (uses the Web Crypto subtle digest, present in Node ≥18 and browsers)
- *
- * Named `hashJson` to disambiguate from `prompt-registry.ts`'s `hashContent`,
- * which takes a string input and returns a truncated 12-char prompt id.
- * Use `hashJson` when you mean "canonicalize then hash."
- *
- * @example
- *   const hash = await hashJson({ id: '1', kind: 'spec' })
- *   // 'a3f1...' (64 hex chars)
- */
-declare function hashJson<T>(obj: T): Promise<string>;
-/**
- * Sign a manifest with a SHA-256 content hash.
- *
- * The hash covers the canonicalized manifest with the `contentHash`
- * and `algo` fields stripped; this lets verifiers re-sign the rest and
- * compare. Returned manifest always carries `algo: 'sha256-content'`
- * so downstream consumers can identify the scheme; manifests without
- * `algo` still verify because it is stripped before hashing on both sides.
- */
-declare function signManifest(m: HypothesisManifest): Promise<SignedManifest>;
-/**
- * Verify that a signed manifest has not been tampered with.
- *
- * Strips `contentHash` and `algo` before re-signing so manifests without
- * `algo` verify identically to ones that carry it.
- */
-declare function verifyManifest(m: SignedManifest): Promise<boolean>;
-/**
- * Evaluate a pre-registered hypothesis against observed results.
- * Mechanical — no re-interpretation permitted.
- */
-declare function evaluateHypothesis(manifest: SignedManifest, observed: {
-    n: number;
-    effect: number;
-    pValue: number;
-}): Promise<HypothesisResult>;
-
-export { type AgentProfile as A, type BackendIntegrityReport as B, type CompletionRequirement as C, hashJson as D, jsonHasKeys as E, parseCorrectnessResponse as F, regexMatch as G, type HypothesisManifest as H, signManifest as I, summarizeBackendIntegrity as J, verifyCompletion as K, type LlmCorrectnessCheckerOpts as L, verifyManifest as M, type ProducedState as P, type RuntimeEventLike as R, type SignedManifest as S, type TaskGold as T, type ValidationContext as V, type CompletionVerdict as a, type CorrectnessChecker as b, type Artifact as c, type ArtifactEventLike as d, type ArtifactValidator as e, BackendIntegrityError as f, type HypothesisResult as g, type ProducedProposal as h, type ProposalEventLike as i, type RequirementCheck as j, type SatisfiedBy as k, type SignedManifestAlgo as l, type ToolCallEventLike as m, type ValidationIssue as n, type ValidationResult as o, agentProfileHash as p, assertRealBackend as q, byteLengthRange as r, canonicalize as s, completionVerdict as t, composeValidators as u, containsAll as v, createLlmCorrectnessChecker as w, createTokenRecallChecker as x, evaluateHypothesis as y, extractProducedState as z };